suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-device.h"
45 #include "util-proto-name.h"
46 #include "util-logopenfile.h"
47 #include "util-time.h"
48 #include "output-json.h"
49 #include "output-json-netflow.h"
50 
51 #include "stream-tcp-private.h"
52 
53 static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir)
54 {
55  char timebuf[64];
56  char srcip[46] = {0}, dstip[46] = {0};
57  Port sp, dp;
58 
59  JsonBuilder *js = jb_new_object();
60  if (unlikely(js == NULL))
61  return NULL;
62 
63  struct timeval tv;
64  memset(&tv, 0x00, sizeof(tv));
65  TimeGet(&tv);
66 
67  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
68 
69  /* reverse header direction if the flow started out wrong */
70  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
71 
72  if (FLOW_IS_IPV4(f)) {
73  if (dir == 0) {
74  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
75  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
76  } else {
77  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
78  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
79  }
80  } else if (FLOW_IS_IPV6(f)) {
81  if (dir == 0) {
82  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
83  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
84  } else {
85  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
86  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
87  }
88  }
89 
90  if (dir == 0) {
91  sp = f->sp;
92  dp = f->dp;
93  } else {
94  sp = f->dp;
95  dp = f->sp;
96  }
97 
98  /* time */
99  jb_set_string(js, "timestamp", timebuf);
100 
101  CreateEveFlowId(js, (const Flow *)f);
102 
103 #if 0 // TODO
104  /* sensor id */
105  if (sensor_id >= 0)
106  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
107 #endif
108 
109  /* input interface */
110  if (f->livedev) {
111  jb_set_string(js, "in_iface", f->livedev->dev);
112  }
113 
114  JB_SET_STRING(js, "event_type", "netflow");
115 
116  /* vlan */
117  if (f->vlan_idx > 0) {
118  jb_open_array(js, "vlan");
119  jb_append_uint(js, f->vlan_id[0]);
120  if (f->vlan_idx > 1) {
121  jb_append_uint(js, f->vlan_id[1]);
122  }
123  jb_close(js);
124  }
125 
126  /* tuple */
127  jb_set_string(js, "src_ip", srcip);
128  switch(f->proto) {
129  case IPPROTO_ICMP:
130  break;
131  case IPPROTO_UDP:
132  case IPPROTO_TCP:
133  case IPPROTO_SCTP:
134  jb_set_uint(js, "src_port", sp);
135  break;
136  }
137  jb_set_string(js, "dest_ip", dstip);
138  switch(f->proto) {
139  case IPPROTO_ICMP:
140  break;
141  case IPPROTO_UDP:
142  case IPPROTO_TCP:
143  case IPPROTO_SCTP:
144  jb_set_uint(js, "dest_port", dp);
145  break;
146  }
147 
148  if (SCProtoNameValid(f->proto)) {
149  jb_set_string(js, "proto", known_proto[f->proto]);
150  } else {
151  char proto[4];
152  snprintf(proto, sizeof(proto), "%"PRIu8"", f->proto);
153  jb_set_string(js, "proto", proto);
154  }
155 
156  switch (f->proto) {
157  case IPPROTO_ICMP:
158  case IPPROTO_ICMPV6: {
159  uint8_t type = f->icmp_s.type;
160  uint8_t code = f->icmp_s.code;
161  if (dir == 1) {
162  type = f->icmp_d.type;
163  code = f->icmp_d.code;
164 
165  }
166  jb_set_uint(js, "icmp_type", type);
167  jb_set_uint(js, "icmp_code", code);
168  break;
169  }
170  case IPPROTO_ESP:
171  jb_set_uint(js, "spi", f->esp.spi);
172  break;
173  }
174  return js;
175 }
176 
177 /* JSON format logging */
178 static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f)
179 {
180  jb_set_string(js, "app_proto",
182 
183  jb_open_object(js, "netflow");
184 
185  jb_set_uint(js, "pkts", f->todstpktcnt);
186  jb_set_uint(js, "bytes", f->todstbytecnt);
187 
188  char timebuf1[64], timebuf2[64];
189 
190  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
191  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
192 
193  jb_set_string(js, "start", timebuf1);
194  jb_set_string(js, "end", timebuf2);
195 
196  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
197  jb_set_uint(js, "age", age);
198 
199  jb_set_uint(js, "min_ttl", f->min_ttl_toserver);
200  jb_set_uint(js, "max_ttl", f->max_ttl_toserver);
201 
202  /* Close netflow. */
203  jb_close(js);
204 
205  /* TCP */
206  if (f->proto == IPPROTO_TCP) {
207  jb_open_object(js, "tcp");
208 
209  TcpSession *ssn = f->protoctx;
210 
211  char hexflags[3];
212  snprintf(hexflags, sizeof(hexflags), "%02x",
213  ssn ? ssn->client.tcp_flags : 0);
214  jb_set_string(js, "tcp_flags", hexflags);
215 
216  EveTcpFlags(ssn ? ssn->client.tcp_flags : 0, js);
217 
218  jb_close(js);
219  }
220 }
221 
222 static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f)
223 {
224  jb_set_string(js, "app_proto",
226 
227  jb_open_object(js, "netflow");
228 
229  jb_set_uint(js, "pkts", f->tosrcpktcnt);
230  jb_set_uint(js, "bytes", f->tosrcbytecnt);
231 
232  char timebuf1[64], timebuf2[64];
233 
234  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
235  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
236 
237  jb_set_string(js, "start", timebuf1);
238  jb_set_string(js, "end", timebuf2);
239 
240  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
241  jb_set_uint(js, "age", age);
242 
243  /* To client is zero if we did not see any packet */
244  if (f->tosrcpktcnt) {
245  jb_set_uint(js, "min_ttl", f->min_ttl_toclient);
246  jb_set_uint(js, "max_ttl", f->max_ttl_toclient);
247  }
248 
249  /* Close netflow. */
250  jb_close(js);
251 
252  /* TCP */
253  if (f->proto == IPPROTO_TCP) {
254  jb_open_object(js, "tcp");
255 
256  TcpSession *ssn = f->protoctx;
257 
258  char hexflags[3];
259  snprintf(hexflags, sizeof(hexflags), "%02x",
260  ssn ? ssn->server.tcp_flags : 0);
261  jb_set_string(js, "tcp_flags", hexflags);
262 
263  EveTcpFlags(ssn ? ssn->server.tcp_flags : 0, js);
264 
265  jb_close(js);
266  }
267 }
268 
269 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
270 {
271  SCEnter();
272  OutputJsonThreadCtx *jhl = thread_data;
273 
274  JsonBuilder *jb = CreateEveHeaderFromNetFlow(f, 0);
275  if (unlikely(jb == NULL))
276  return TM_ECODE_OK;
277  NetFlowLogEveToServer(jb, f);
278  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb);
279  OutputJsonBuilderBuffer(jb, jhl);
280  jb_free(jb);
281 
282  /* only log a response record if we actually have seen response packets */
283  if (f->tosrcpktcnt) {
284  jb = CreateEveHeaderFromNetFlow(f, 1);
285  if (unlikely(jb == NULL))
286  return TM_ECODE_OK;
287  NetFlowLogEveToClient(jb, f);
288  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb);
289  OutputJsonBuilderBuffer(jb, jhl);
290  jb_free(jb);
291  }
293 }
294 
296 {
297  /* register as child of eve-log */
298  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog", "eve-log.netflow",
299  OutputJsonLogInitSub, JsonNetFlowLogger, JsonLogThreadInit, JsonLogThreadDeinit, NULL);
300 }
tm-threads.h
FlowAddress_::address
union FlowAddress_::@115 address
OutputJsonLogInitSub
OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
Definition: output-json-common.c:88
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:158
CreateIsoTimeString
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:215
OutputJsonThreadCtx_::ctx
OutputJsonCtx * ctx
Definition: output-json.h:90
Flow_::startts
struct timeval startts
Definition: flow.h:500
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:84
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
Flow_::proto
uint8_t proto
Definition: flow.h:375
JsonLogThreadInit
TmEcode JsonLogThreadInit(ThreadVars *t, const void *initdata, void **data)
Definition: output-json-common.c:105
threads.h
Flow_
Flow data structure.
Definition: flow.h:353
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
OutputJsonBuilderBuffer
int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
Definition: output-json.c:960
output-json-netflow.h
Flow_::vlan_id
uint16_t vlan_id[2]
Definition: flow.h:377
SCProtoNameValid
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
Definition: util-proto-name.c:454
util-privs.h
known_proto
const char * known_proto[256]
Definition: util-proto-name.c:39
EveAddCommonOptions
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js)
Definition: output-json.c:414
Flow_::max_ttl_toserver
uint8_t max_ttl_toserver
Definition: flow.h:478
proto
uint8_t proto
Definition: decode-template.h:0
Flow_::dp
Port dp
Definition: flow.h:369
Flow_::protoctx
void * protoctx
Definition: flow.h:451
Flow_::icmp_s
struct Flow_::@116::@122 icmp_s
util-unittest.h
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:82
OutputJsonThreadCtx_
Definition: output-json.h:89
Flow_::tosrcbytecnt
uint64_t tosrcbytecnt
Definition: flow.h:505
Flow_::dst
FlowAddress dst
Definition: flow.h:356
Flow_::min_ttl_toserver
uint8_t min_ttl_toserver
Definition: flow.h:477
util-device.h
util-debug.h
JB_SET_STRING
#define JB_SET_STRING(jb, key, val)
Definition: rust.h:37
type
uint8_t type
Definition: decode-icmpv4.h:0
Flow_::todstpktcnt
uint32_t todstpktcnt
Definition: flow.h:502
output-json.h
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:298
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:56
PrintInet
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:274
pkt-var.h
EveTcpFlags
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
Definition: output-json.c:455
util-time.h
LiveDevice_::dev
char * dev
Definition: util-device.h:40
Flow_::todstbytecnt
uint64_t todstbytecnt
Definition: flow.h:504
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:156
LOGGER_JSON_NETFLOW
@ LOGGER_JSON_NETFLOW
Definition: suricata-common.h:486
stream-tcp-private.h
conf.h
Port
uint16_t Port
Definition: decode.h:237
Flow_::vlan_idx
uint8_t vlan_idx
Definition: flow.h:385
Flow_::min_ttl_toclient
uint8_t min_ttl_toclient
Definition: flow.h:479
util-proto-name.h
Flow_::icmp_d
struct Flow_::@118::@124 icmp_d
Flow_::src
FlowAddress src
Definition: flow.h:356
Flow_::lastts
struct timeval lastts
Definition: flow.h:414
suricata-common.h
Flow_::max_ttl_toclient
uint8_t max_ttl_toclient
Definition: flow.h:480
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:284
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:31
Flow_::livedev
struct LiveDevice_ * livedev
Definition: flow.h:406
threadvars.h
Flow_::esp
struct Flow_::@116::@123 esp
CreateEveFlowId
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
Definition: output-json.c:727
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:283
OutputRegisterFlowSubModule
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:588
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:461
util-logopenfile.h
Flow_::flags
uint32_t flags
Definition: flow.h:431
util-buffer.h
TimeGet
void TimeGet(struct timeval *tv)
Definition: util-time.c:155
Flow_::sp
Port sp
Definition: flow.h:358
JsonLogThreadDeinit
TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data)
Definition: output-json-common.c:138
TcpSession_
Definition: stream-tcp-private.h:271
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:462
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:460
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:108
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:302
code
uint8_t code
Definition: decode-icmpv4.h:1
TcpStream_::tcp_flags
uint8_t tcp_flags
Definition: stream-tcp-private.h:111
Flow_::tosrcpktcnt
uint32_t tosrcpktcnt
Definition: flow.h:503
debug.h
output.h
JsonNetFlowLogRegister
void JsonNetFlowLogRegister(void)
Definition: output-json-netflow.c:295