suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectional NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "detect.h"
28 #include "pkt-var.h"
29 #include "conf.h"
30 
31 #include "threads.h"
32 #include "threadvars.h"
33 #include "tm-threads.h"
34 
35 #include "util-print.h"
36 #include "util-unittest.h"
37 
38 #include "util-debug.h"
39 
40 #include "output.h"
41 #include "util-privs.h"
42 #include "util-buffer.h"
43 #include "util-device.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir)
53 {
54  char timebuf[64];
55  char srcip[46] = {0}, dstip[46] = {0};
56  Port sp, dp;
57 
58  JsonBuilder *js = jb_new_object();
59  if (unlikely(js == NULL))
60  return NULL;
61 
62  SCTime_t ts = TimeGet();
63 
64  CreateIsoTimeString(ts, timebuf, sizeof(timebuf));
65 
66  /* reverse header direction if the flow started out wrong */
67  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
68 
69  if (FLOW_IS_IPV4(f)) {
70  if (dir == 0) {
71  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
72  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
73  } else {
74  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
75  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
76  }
77  } else if (FLOW_IS_IPV6(f)) {
78  if (dir == 0) {
79  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
80  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
81  } else {
82  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
83  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
84  }
85  }
86 
87  if (dir == 0) {
88  sp = f->sp;
89  dp = f->dp;
90  } else {
91  sp = f->dp;
92  dp = f->sp;
93  }
94 
95  /* time */
96  jb_set_string(js, "timestamp", timebuf);
97 
98  CreateEveFlowId(js, (const Flow *)f);
99 
100 #if 0 // TODO
101  /* sensor id */
102  if (sensor_id >= 0)
103  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
104 #endif
105 
106  /* input interface */
107  if (f->livedev) {
108  jb_set_string(js, "in_iface", f->livedev->dev);
109  }
110 
111  JB_SET_STRING(js, "event_type", "netflow");
112 
113  /* vlan */
114  if (f->vlan_idx > 0) {
115  jb_open_array(js, "vlan");
116  jb_append_uint(js, f->vlan_id[0]);
117  if (f->vlan_idx > 1) {
118  jb_append_uint(js, f->vlan_id[1]);
119  }
120  if (f->vlan_idx > 2) {
121  jb_append_uint(js, f->vlan_id[2]);
122  }
123  jb_close(js);
124  }
125 
126  /* tuple */
127  jb_set_string(js, "src_ip", srcip);
128  switch(f->proto) {
129  case IPPROTO_ICMP:
130  break;
131  case IPPROTO_UDP:
132  case IPPROTO_TCP:
133  case IPPROTO_SCTP:
134  jb_set_uint(js, "src_port", sp);
135  break;
136  }
137  jb_set_string(js, "dest_ip", dstip);
138  switch(f->proto) {
139  case IPPROTO_ICMP:
140  break;
141  case IPPROTO_UDP:
142  case IPPROTO_TCP:
143  case IPPROTO_SCTP:
144  jb_set_uint(js, "dest_port", dp);
145  break;
146  }
147 
148  if (SCProtoNameValid(f->proto)) {
149  jb_set_string(js, "proto", known_proto[f->proto]);
150  } else {
151  char proto[4];
152  snprintf(proto, sizeof(proto), "%"PRIu8"", f->proto);
153  jb_set_string(js, "proto", proto);
154  }
155 
156  switch (f->proto) {
157  case IPPROTO_ICMP:
158  case IPPROTO_ICMPV6: {
159  uint8_t type = f->icmp_s.type;
160  uint8_t code = f->icmp_s.code;
161  if (dir == 1) {
162  type = f->icmp_d.type;
163  code = f->icmp_d.code;
164 
165  }
166  jb_set_uint(js, "icmp_type", type);
167  jb_set_uint(js, "icmp_code", code);
168  break;
169  }
170  case IPPROTO_ESP:
171  jb_set_uint(js, "spi", f->esp.spi);
172  break;
173  }
174  return js;
175 }
176 
177 /* JSON format logging */
178 static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f)
179 {
180  jb_set_string(js, "app_proto",
182 
183  jb_open_object(js, "netflow");
184 
185  jb_set_uint(js, "pkts", f->todstpktcnt);
186  jb_set_uint(js, "bytes", f->todstbytecnt);
187 
188  char timebuf1[64], timebuf2[64];
189 
190  CreateIsoTimeString(f->startts, timebuf1, sizeof(timebuf1));
191  CreateIsoTimeString(f->lastts, timebuf2, sizeof(timebuf2));
192 
193  jb_set_string(js, "start", timebuf1);
194  jb_set_string(js, "end", timebuf2);
195 
196  uint64_t age = (SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts));
197  jb_set_uint(js, "age", age);
198 
199  jb_set_uint(js, "min_ttl", f->min_ttl_toserver);
200  jb_set_uint(js, "max_ttl", f->max_ttl_toserver);
201 
202  /* Close netflow. */
203  jb_close(js);
204 
205  /* TCP */
206  if (f->proto == IPPROTO_TCP) {
207  jb_open_object(js, "tcp");
208 
209  TcpSession *ssn = f->protoctx;
210 
211  char hexflags[3];
212  snprintf(hexflags, sizeof(hexflags), "%02x",
213  ssn ? ssn->client.tcp_flags : 0);
214  jb_set_string(js, "tcp_flags", hexflags);
215 
216  EveTcpFlags(ssn ? ssn->client.tcp_flags : 0, js);
217 
218  jb_close(js);
219  }
220 }
221 
222 static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f)
223 {
224  jb_set_string(js, "app_proto",
226 
227  jb_open_object(js, "netflow");
228 
229  jb_set_uint(js, "pkts", f->tosrcpktcnt);
230  jb_set_uint(js, "bytes", f->tosrcbytecnt);
231 
232  char timebuf1[64], timebuf2[64];
233 
234  CreateIsoTimeString(f->startts, timebuf1, sizeof(timebuf1));
235  CreateIsoTimeString(f->lastts, timebuf2, sizeof(timebuf2));
236 
237  jb_set_string(js, "start", timebuf1);
238  jb_set_string(js, "end", timebuf2);
239 
240  uint64_t age = (SCTIME_SECS(f->lastts) - SCTIME_SECS(f->startts));
241  jb_set_uint(js, "age", age);
242 
243  /* To client is zero if we did not see any packet */
244  if (f->tosrcpktcnt) {
245  jb_set_uint(js, "min_ttl", f->min_ttl_toclient);
246  jb_set_uint(js, "max_ttl", f->max_ttl_toclient);
247  }
248 
249  /* Close netflow. */
250  jb_close(js);
251 
252  /* TCP */
253  if (f->proto == IPPROTO_TCP) {
254  jb_open_object(js, "tcp");
255 
256  TcpSession *ssn = f->protoctx;
257 
258  char hexflags[3];
259  snprintf(hexflags, sizeof(hexflags), "%02x",
260  ssn ? ssn->server.tcp_flags : 0);
261  jb_set_string(js, "tcp_flags", hexflags);
262 
263  EveTcpFlags(ssn ? ssn->server.tcp_flags : 0, js);
264 
265  jb_close(js);
266  }
267 }
268 
269 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
270 {
271  SCEnter();
272  OutputJsonThreadCtx *jhl = thread_data;
273 
274  JsonBuilder *jb = CreateEveHeaderFromNetFlow(f, 0);
275  if (unlikely(jb == NULL))
276  return TM_ECODE_OK;
277  NetFlowLogEveToServer(jb, f);
278  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER);
279  OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl);
280  jb_free(jb);
281 
282  /* only log a response record if we actually have seen response packets */
283  if (f->tosrcpktcnt) {
284  jb = CreateEveHeaderFromNetFlow(f, 1);
285  if (unlikely(jb == NULL))
286  return TM_ECODE_OK;
287  NetFlowLogEveToClient(jb, f);
288  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT);
289  OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl);
290  jb_free(jb);
291  }
293 }
294 
296 {
297  /* register as child of eve-log */
298  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog", "eve-log.netflow",
300 }
tm-threads.h
ts
uint64_t ts
Definition: source-erf-file.c:55
OutputJsonLogInitSub
OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
Definition: output-json-common.c:73
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:171
CreateIsoTimeString
void CreateIsoTimeString(const SCTime_t ts, char *str, size_t size)
Definition: util-time.c:209
OutputJsonThreadCtx_::ctx
OutputJsonCtx * ctx
Definition: output-json.h:90
Flow_::startts
SCTime_t startts
Definition: flow.h:495
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:84
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
Flow_::proto
uint8_t proto
Definition: flow.h:378
JsonLogThreadInit
TmEcode JsonLogThreadInit(ThreadVars *t, const void *initdata, void **data)
Definition: output-json-common.c:90
threads.h
Flow_
Flow data structure.
Definition: flow.h:356
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:75
OutputJsonBuilderBuffer
void OutputJsonBuilderBuffer(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx)
Definition: output-json.c:958
output-json-netflow.h
SCProtoNameValid
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
Definition: util-proto-name.c:453
util-privs.h
Flow_::icmp_d
struct Flow_::@116::@122 icmp_d
known_proto
const char * known_proto[256]
Definition: util-proto-name.c:40
Flow_::max_ttl_toserver
uint8_t max_ttl_toserver
Definition: flow.h:473
proto
uint8_t proto
Definition: decode-template.h:0
Flow_::dp
Port dp
Definition: flow.h:372
Flow_::protoctx
void * protoctx
Definition: flow.h:446
Flow_::icmp_s
struct Flow_::@114::@120 icmp_s
util-unittest.h
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
OutputJsonThreadCtx_
Definition: output-json.h:89
Flow_::tosrcbytecnt
uint64_t tosrcbytecnt
Definition: flow.h:500
Flow_::dst
FlowAddress dst
Definition: flow.h:359
OutputRegisterFlowSubModule
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
Register a flow output sub-module.
Definition: output.c:466
Flow_::esp
struct Flow_::@114::@121 esp
Flow_::min_ttl_toserver
uint8_t min_ttl_toserver
Definition: flow.h:472
util-device.h
util-debug.h
JB_SET_STRING
#define JB_SET_STRING(jb, key, val)
Definition: rust.h:26
Flow_::todstpktcnt
uint32_t todstpktcnt
Definition: flow.h:497
output-json.h
Flow_::lastts
SCTime_t lastts
Definition: flow.h:415
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
PrintInet
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:230
EveAddCommonOptions
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js, enum OutputJsonLogDirection dir)
Definition: output-json.c:398
pkt-var.h
EveTcpFlags
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
Definition: output-json.c:439
util-time.h
LiveDevice_::dev
char * dev
Definition: util-device.h:51
Flow_::todstbytecnt
uint64_t todstbytecnt
Definition: flow.h:499
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:169
LOGGER_JSON_NETFLOW
@ LOGGER_JSON_NETFLOW
Definition: suricata-common.h:487
type
uint16_t type
Definition: decode-vlan.c:107
TimeGet
SCTime_t TimeGet(void)
Definition: util-time.c:152
stream-tcp-private.h
conf.h
Port
uint16_t Port
Definition: decode.h:214
SCTime_t
Definition: util-time.h:40
Flow_::vlan_idx
uint8_t vlan_idx
Definition: flow.h:382
Flow_::min_ttl_toclient
uint8_t min_ttl_toclient
Definition: flow.h:474
LOG_DIR_FLOW_TOCLIENT
@ LOG_DIR_FLOW_TOCLIENT
Definition: output-json.h:39
util-proto-name.h
Flow_::src
FlowAddress src
Definition: flow.h:359
suricata-common.h
SCTIME_SECS
#define SCTIME_SECS(t)
Definition: util-time.h:57
Flow_::max_ttl_toclient
uint8_t max_ttl_toclient
Definition: flow.h:475
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:297
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
Flow_::livedev
struct LiveDevice_ * livedev
Definition: flow.h:403
threadvars.h
CreateEveFlowId
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
Definition: output-json.c:689
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:296
LOG_DIR_FLOW_TOSERVER
@ LOG_DIR_FLOW_TOSERVER
Definition: output-json.h:40
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:456
util-logopenfile.h
Flow_::flags
uint32_t flags
Definition: flow.h:426
util-buffer.h
Flow_::vlan_id
uint16_t vlan_id[VLAN_MAX_LAYERS]
Definition: flow.h:380
IPPROTO_SCTP
#define IPPROTO_SCTP
Definition: decode.h:1194
Flow_::sp
Port sp
Definition: flow.h:361
JsonLogThreadDeinit
TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data)
Definition: output-json-common.c:123
TcpSession_
Definition: stream-tcp-private.h:283
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:457
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:455
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:111
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
TcpStream_::tcp_flags
uint8_t tcp_flags
Definition: stream-tcp-private.h:111
FlowAddress_::address
union FlowAddress_::@113 address
Flow_::tosrcpktcnt
uint32_t tosrcpktcnt
Definition: flow.h:498
output.h
JsonNetFlowLogRegister
void JsonNetFlowLogRegister(void)
Definition: output-json-netflow.c:295