suricata
output-json-netflow.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Unidirectiontal NetFlow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-netflow.h"
49 
50 #include "stream-tcp-private.h"
51 
52 static JsonBuilder *CreateEveHeaderFromNetFlow(const Flow *f, int dir)
53 {
54  char timebuf[64];
55  char srcip[46] = {0}, dstip[46] = {0};
56  Port sp, dp;
57 
58  JsonBuilder *js = jb_new_object();
59  if (unlikely(js == NULL))
60  return NULL;
61 
62  struct timeval tv;
63  memset(&tv, 0x00, sizeof(tv));
64  TimeGet(&tv);
65 
66  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
67 
68  /* reverse header direction if the flow started out wrong */
69  dir ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
70 
71  if (FLOW_IS_IPV4(f)) {
72  if (dir == 0) {
73  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
74  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
75  } else {
76  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
77  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
78  }
79  } else if (FLOW_IS_IPV6(f)) {
80  if (dir == 0) {
81  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
82  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
83  } else {
84  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
85  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
86  }
87  }
88 
89  if (dir == 0) {
90  sp = f->sp;
91  dp = f->dp;
92  } else {
93  sp = f->dp;
94  dp = f->sp;
95  }
96 
97  /* time */
98  jb_set_string(js, "timestamp", timebuf);
99 
100  CreateEveFlowId(js, (const Flow *)f);
101 
102 #if 0 // TODO
103  /* sensor id */
104  if (sensor_id >= 0)
105  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
106 #endif
107 
108  /* input interface */
109  if (f->livedev) {
110  jb_set_string(js, "in_iface", f->livedev->dev);
111  }
112 
113  JB_SET_STRING(js, "event_type", "netflow");
114 
115  /* vlan */
116  if (f->vlan_idx > 0) {
117  jb_open_array(js, "vlan");
118  jb_append_uint(js, f->vlan_id[0]);
119  if (f->vlan_idx > 1) {
120  jb_append_uint(js, f->vlan_id[1]);
121  }
122  jb_close(js);
123  }
124 
125  /* tuple */
126  jb_set_string(js, "src_ip", srcip);
127  switch(f->proto) {
128  case IPPROTO_ICMP:
129  break;
130  case IPPROTO_UDP:
131  case IPPROTO_TCP:
132  case IPPROTO_SCTP:
133  jb_set_uint(js, "src_port", sp);
134  break;
135  }
136  jb_set_string(js, "dest_ip", dstip);
137  switch(f->proto) {
138  case IPPROTO_ICMP:
139  break;
140  case IPPROTO_UDP:
141  case IPPROTO_TCP:
142  case IPPROTO_SCTP:
143  jb_set_uint(js, "dest_port", dp);
144  break;
145  }
146 
147  if (SCProtoNameValid(f->proto)) {
148  jb_set_string(js, "proto", known_proto[f->proto]);
149  } else {
150  char proto[4];
151  snprintf(proto, sizeof(proto), "%"PRIu8"", f->proto);
152  jb_set_string(js, "proto", proto);
153  }
154 
155  switch (f->proto) {
156  case IPPROTO_ICMP:
157  case IPPROTO_ICMPV6: {
158  uint8_t type = f->icmp_s.type;
159  uint8_t code = f->icmp_s.code;
160  if (dir == 1) {
161  type = f->icmp_d.type;
162  code = f->icmp_d.code;
163 
164  }
165  jb_set_uint(js, "icmp_type", type);
166  jb_set_uint(js, "icmp_code", code);
167  break;
168  }
169  case IPPROTO_ESP:
170  jb_set_uint(js, "spi", f->esp.spi);
171  break;
172  }
173  return js;
174 }
175 
176 /* JSON format logging */
177 static void NetFlowLogEveToServer(JsonBuilder *js, Flow *f)
178 {
179  jb_set_string(js, "app_proto",
181 
182  jb_open_object(js, "netflow");
183 
184  jb_set_uint(js, "pkts", f->todstpktcnt);
185  jb_set_uint(js, "bytes", f->todstbytecnt);
186 
187  char timebuf1[64], timebuf2[64];
188 
189  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
190  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
191 
192  jb_set_string(js, "start", timebuf1);
193  jb_set_string(js, "end", timebuf2);
194 
195  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
196  jb_set_uint(js, "age", age);
197 
198  jb_set_uint(js, "min_ttl", f->min_ttl_toserver);
199  jb_set_uint(js, "max_ttl", f->max_ttl_toserver);
200 
201  /* Close netflow. */
202  jb_close(js);
203 
204  /* TCP */
205  if (f->proto == IPPROTO_TCP) {
206  jb_open_object(js, "tcp");
207 
208  TcpSession *ssn = f->protoctx;
209 
210  char hexflags[3];
211  snprintf(hexflags, sizeof(hexflags), "%02x",
212  ssn ? ssn->client.tcp_flags : 0);
213  jb_set_string(js, "tcp_flags", hexflags);
214 
215  EveTcpFlags(ssn ? ssn->client.tcp_flags : 0, js);
216 
217  jb_close(js);
218  }
219 }
220 
221 static void NetFlowLogEveToClient(JsonBuilder *js, Flow *f)
222 {
223  jb_set_string(js, "app_proto",
225 
226  jb_open_object(js, "netflow");
227 
228  jb_set_uint(js, "pkts", f->tosrcpktcnt);
229  jb_set_uint(js, "bytes", f->tosrcbytecnt);
230 
231  char timebuf1[64], timebuf2[64];
232 
233  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
234  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
235 
236  jb_set_string(js, "start", timebuf1);
237  jb_set_string(js, "end", timebuf2);
238 
239  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
240  jb_set_uint(js, "age", age);
241 
242  /* To client is zero if we did not see any packet */
243  if (f->tosrcpktcnt) {
244  jb_set_uint(js, "min_ttl", f->min_ttl_toclient);
245  jb_set_uint(js, "max_ttl", f->max_ttl_toclient);
246  }
247 
248  /* Close netflow. */
249  jb_close(js);
250 
251  /* TCP */
252  if (f->proto == IPPROTO_TCP) {
253  jb_open_object(js, "tcp");
254 
255  TcpSession *ssn = f->protoctx;
256 
257  char hexflags[3];
258  snprintf(hexflags, sizeof(hexflags), "%02x",
259  ssn ? ssn->server.tcp_flags : 0);
260  jb_set_string(js, "tcp_flags", hexflags);
261 
262  EveTcpFlags(ssn ? ssn->server.tcp_flags : 0, js);
263 
264  jb_close(js);
265  }
266 }
267 
268 static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
269 {
270  SCEnter();
271  OutputJsonThreadCtx *jhl = thread_data;
272 
273  JsonBuilder *jb = CreateEveHeaderFromNetFlow(f, 0);
274  if (unlikely(jb == NULL))
275  return TM_ECODE_OK;
276  NetFlowLogEveToServer(jb, f);
277  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb);
278  OutputJsonBuilderBuffer(jb, jhl);
279  jb_free(jb);
280 
281  /* only log a response record if we actually have seen response packets */
282  if (f->tosrcpktcnt) {
283  jb = CreateEveHeaderFromNetFlow(f, 1);
284  if (unlikely(jb == NULL))
285  return TM_ECODE_OK;
286  NetFlowLogEveToClient(jb, f);
287  EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb);
288  OutputJsonBuilderBuffer(jb, jhl);
289  jb_free(jb);
290  }
292 }
293 
295 {
296  /* register as child of eve-log */
297  OutputRegisterFlowSubModule(LOGGER_JSON_NETFLOW, "eve-log", "JsonNetFlowLog", "eve-log.netflow",
298  OutputJsonLogInitSub, JsonNetFlowLogger, JsonLogThreadInit, JsonLogThreadDeinit, NULL);
299 }
tm-threads.h
OutputJsonLogInitSub
OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
Definition: output-json-common.c:88
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:160
CreateIsoTimeString
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:213
OutputJsonThreadCtx_::ctx
OutputJsonCtx * ctx
Definition: output-json.h:90
Flow_::startts
struct timeval startts
Definition: flow.h:500
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:84
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
Flow_::proto
uint8_t proto
Definition: flow.h:375
JsonLogThreadInit
TmEcode JsonLogThreadInit(ThreadVars *t, const void *initdata, void **data)
Definition: output-json-common.c:105
threads.h
Flow_
Flow data structure.
Definition: flow.h:353
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
OutputJsonBuilderBuffer
int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
Definition: output-json.c:975
output-json-netflow.h
Flow_::vlan_id
uint16_t vlan_id[2]
Definition: flow.h:377
SCProtoNameValid
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
Definition: util-proto-name.c:195
util-privs.h
known_proto
const char * known_proto[256]
Definition: util-proto-name.c:34
EveAddCommonOptions
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js)
Definition: output-json.c:427
Flow_::max_ttl_toserver
uint8_t max_ttl_toserver
Definition: flow.h:478
proto
uint8_t proto
Definition: decode-template.h:0
Flow_::dp
Port dp
Definition: flow.h:369
Flow_::protoctx
void * protoctx
Definition: flow.h:451
util-unittest.h
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
OutputJsonThreadCtx_
Definition: output-json.h:89
Flow_::tosrcbytecnt
uint64_t tosrcbytecnt
Definition: flow.h:505
Flow_::dst
FlowAddress dst
Definition: flow.h:356
Flow_::min_ttl_toserver
uint8_t min_ttl_toserver
Definition: flow.h:477
util-debug.h
JB_SET_STRING
#define JB_SET_STRING(jb, key, val)
Definition: rust.h:34
type
uint8_t type
Definition: decode-icmpv4.h:0
Flow_::todstpktcnt
uint32_t todstpktcnt
Definition: flow.h:502
output-json.h
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
PrintInet
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:271
pkt-var.h
EveTcpFlags
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
Definition: output-json.c:472
util-time.h
LiveDevice_::dev
char * dev
Definition: util-device.h:41
Flow_::todstbytecnt
uint64_t todstbytecnt
Definition: flow.h:504
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:158
LOGGER_JSON_NETFLOW
@ LOGGER_JSON_NETFLOW
Definition: suricata-common.h:483
stream-tcp-private.h
conf.h
Port
uint16_t Port
Definition: decode.h:243
Flow_::vlan_idx
uint8_t vlan_idx
Definition: flow.h:385
Flow_::min_ttl_toclient
uint8_t min_ttl_toclient
Definition: flow.h:479
util-proto-name.h
Flow_::src
FlowAddress src
Definition: flow.h:356
Flow_::lastts
struct timeval lastts
Definition: flow.h:414
suricata-common.h
Flow_::max_ttl_toclient
uint8_t max_ttl_toclient
Definition: flow.h:480
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
Flow_::icmp_d
struct Flow_::@114::@120 icmp_d
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
Flow_::icmp_s
struct Flow_::@112::@118 icmp_s
Flow_::livedev
struct LiveDevice_ * livedev
Definition: flow.h:406
threadvars.h
CreateEveFlowId
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
Definition: output-json.c:744
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
OutputRegisterFlowSubModule
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:585
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:461
util-logopenfile.h
Flow_::flags
uint32_t flags
Definition: flow.h:431
FlowAddress_::address
union FlowAddress_::@111 address
util-buffer.h
TimeGet
void TimeGet(struct timeval *tv)
Definition: util-time.c:153
Flow_::sp
Port sp
Definition: flow.h:358
JsonLogThreadDeinit
TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data)
Definition: output-json-common.c:138
TcpSession_
Definition: stream-tcp-private.h:260
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:462
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:460
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:110
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
code
uint8_t code
Definition: decode-icmpv4.h:1
TcpStream_::tcp_flags
uint8_t tcp_flags
Definition: stream-tcp-private.h:99
Flow_::tosrcpktcnt
uint32_t tosrcpktcnt
Definition: flow.h:503
debug.h
output.h
JsonNetFlowLogRegister
void JsonNetFlowLogRegister(void)
Definition: output-json-netflow.c:294
Flow_::esp
struct Flow_::@112::@119 esp