suricata
output-json-flow.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements Flow JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 #include "util-proto-name.h"
45 #include "util-logopenfile.h"
46 #include "util-time.h"
47 #include "output-json.h"
48 #include "output-json-flow.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "flow-storage.h"
53 
54 static JsonBuilder *CreateEveHeaderFromFlow(const Flow *f)
55 {
56  char timebuf[64];
57  char srcip[46] = {0}, dstip[46] = {0};
58  Port sp, dp;
59 
60  JsonBuilder *jb = jb_new_object();
61  if (unlikely(jb == NULL)) {
62  return NULL;
63  }
64 
65  struct timeval tv;
66  memset(&tv, 0x00, sizeof(tv));
67  TimeGet(&tv);
68 
69  CreateIsoTimeString(&tv, timebuf, sizeof(timebuf));
70 
71  if ((f->flags & FLOW_DIR_REVERSED) == 0) {
72  if (FLOW_IS_IPV4(f)) {
73  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), srcip, sizeof(srcip));
74  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), dstip, sizeof(dstip));
75  } else if (FLOW_IS_IPV6(f)) {
76  PrintInet(AF_INET6, (const void *)&(f->src.address), srcip, sizeof(srcip));
77  PrintInet(AF_INET6, (const void *)&(f->dst.address), dstip, sizeof(dstip));
78  }
79  sp = f->sp;
80  dp = f->dp;
81  } else {
82  if (FLOW_IS_IPV4(f)) {
83  PrintInet(AF_INET, (const void *)&(f->dst.addr_data32[0]), srcip, sizeof(srcip));
84  PrintInet(AF_INET, (const void *)&(f->src.addr_data32[0]), dstip, sizeof(dstip));
85  } else if (FLOW_IS_IPV6(f)) {
86  PrintInet(AF_INET6, (const void *)&(f->dst.address), srcip, sizeof(srcip));
87  PrintInet(AF_INET6, (const void *)&(f->src.address), dstip, sizeof(dstip));
88  }
89  sp = f->dp;
90  dp = f->sp;
91  }
92 
93  /* time */
94  jb_set_string(jb, "timestamp", timebuf);
95 
96  CreateEveFlowId(jb, (const Flow *)f);
97 
98 #if 0 // TODO
99  /* sensor id */
100  if (sensor_id >= 0)
101  json_object_set_new(js, "sensor_id", json_integer(sensor_id));
102 #endif
103 
104  /* input interface */
105  if (f->livedev) {
106  jb_set_string(jb, "in_iface", f->livedev->dev);
107  }
108 
109  JB_SET_STRING(jb, "event_type", "flow");
110 
111  /* vlan */
112  if (f->vlan_idx > 0) {
113  jb_open_array(jb, "vlan");
114  jb_append_uint(jb, f->vlan_id[0]);
115  if (f->vlan_idx > 1) {
116  jb_append_uint(jb, f->vlan_id[1]);
117  }
118  jb_close(jb);
119  }
120 
121  /* tuple */
122  jb_set_string(jb, "src_ip", srcip);
123  switch(f->proto) {
124  case IPPROTO_ICMP:
125  break;
126  case IPPROTO_UDP:
127  case IPPROTO_TCP:
128  case IPPROTO_SCTP:
129  jb_set_uint(jb, "src_port", sp);
130  break;
131  }
132  jb_set_string(jb, "dest_ip", dstip);
133  switch(f->proto) {
134  case IPPROTO_ICMP:
135  break;
136  case IPPROTO_UDP:
137  case IPPROTO_TCP:
138  case IPPROTO_SCTP:
139  jb_set_uint(jb, "dest_port", dp);
140  break;
141  }
142 
143  if (SCProtoNameValid(f->proto)) {
144  jb_set_string(jb, "proto", known_proto[f->proto]);
145  } else {
146  char proto[4];
147  snprintf(proto, sizeof(proto), "%"PRIu8"", f->proto);
148  jb_set_string(jb, "proto", proto);
149  }
150 
151  switch (f->proto) {
152  case IPPROTO_ICMP:
153  case IPPROTO_ICMPV6:
154  jb_set_uint(jb, "icmp_type", f->icmp_s.type);
155  jb_set_uint(jb, "icmp_code", f->icmp_s.code);
156  if (f->tosrcpktcnt) {
157  jb_set_uint(jb, "response_icmp_type", f->icmp_d.type);
158  jb_set_uint(jb, "response_icmp_code", f->icmp_d.code);
159  }
160  break;
161  case IPPROTO_ESP:
162  jb_set_uint(jb, "spi", f->esp.spi);
163  break;
164  }
165  return jb;
166 }
167 
168 void EveAddAppProto(Flow *f, JsonBuilder *js)
169 {
170  if (f->alproto) {
171  jb_set_string(js, "app_proto", AppProtoToString(f->alproto));
172  }
173  if (f->alproto_ts && f->alproto_ts != f->alproto) {
174  jb_set_string(js, "app_proto_ts", AppProtoToString(f->alproto_ts));
175  }
176  if (f->alproto_tc && f->alproto_tc != f->alproto) {
177  jb_set_string(js, "app_proto_tc", AppProtoToString(f->alproto_tc));
178  }
179  if (f->alproto_orig != f->alproto && f->alproto_orig != ALPROTO_UNKNOWN) {
180  jb_set_string(js, "app_proto_orig", AppProtoToString(f->alproto_orig));
181  }
182  if (f->alproto_expect != f->alproto && f->alproto_expect != ALPROTO_UNKNOWN) {
183  jb_set_string(js, "app_proto_expected",
185  }
186 
187 }
188 
189 void EveAddFlow(Flow *f, JsonBuilder *js)
190 {
192  if (fc) {
193  jb_set_uint(js, "pkts_toserver", f->todstpktcnt + fc->todstpktcnt);
194  jb_set_uint(js, "pkts_toclient", f->tosrcpktcnt + fc->tosrcpktcnt);
195  jb_set_uint(js, "bytes_toserver", f->todstbytecnt + fc->todstbytecnt);
196  jb_set_uint(js, "bytes_toclient", f->tosrcbytecnt + fc->tosrcbytecnt);
197 
198  jb_open_object(js, "bypassed");
199  jb_set_uint(js, "pkts_toserver", fc->todstpktcnt);
200  jb_set_uint(js, "pkts_toclient", fc->tosrcpktcnt);
201  jb_set_uint(js, "bytes_toserver", fc->todstbytecnt);
202  jb_set_uint(js, "bytes_toclient", fc->tosrcbytecnt);
203  jb_close(js);
204  } else {
205  jb_set_uint(js, "pkts_toserver", f->todstpktcnt);
206  jb_set_uint(js, "pkts_toclient", f->tosrcpktcnt);
207  jb_set_uint(js, "bytes_toserver", f->todstbytecnt);
208  jb_set_uint(js, "bytes_toclient", f->tosrcbytecnt);
209  }
210 
211  char timebuf1[64];
212  CreateIsoTimeString(&f->startts, timebuf1, sizeof(timebuf1));
213  jb_set_string(js, "start", timebuf1);
214 }
215 
216 /* Eve format logging */
217 static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f)
218 {
219  EveAddAppProto(f, jb);
220  jb_open_object(jb, "flow");
221  EveAddFlow(f, jb);
222 
223  char timebuf2[64];
224  CreateIsoTimeString(&f->lastts, timebuf2, sizeof(timebuf2));
225  jb_set_string(jb, "end", timebuf2);
226 
227  int32_t age = f->lastts.tv_sec - f->startts.tv_sec;
228  jb_set_uint(jb, "age", age);
229 
231  JB_SET_TRUE(jb, "emergency");
232  const char *state = NULL;
234  state = "new";
236  state = "established";
238  state = "closed";
240  state = "bypassed";
241  int flow_state = f->flow_state;
242  switch (flow_state) {
244  JB_SET_STRING(jb, "bypass", "local");
245  break;
246 #ifdef CAPTURE_OFFLOAD
247  case FLOW_STATE_CAPTURE_BYPASSED:
248  JB_SET_STRING(jb, "bypass", "capture");
249  break;
250 #endif
251  default:
253  "Invalid flow state: %d, contact developers",
254  flow_state);
255  }
256  }
257 
258  jb_set_string(jb, "state", state);
259 
260  const char *reason = NULL;
262  reason = "forced";
264  reason = "shutdown";
266  reason = "timeout";
267  else
268  reason = "unknown";
269 
270  jb_set_string(jb, "reason", reason);
271 
272  jb_set_bool(jb, "alerted", FlowHasAlerts(f));
273  if (f->flags & FLOW_WRONG_THREAD)
274  JB_SET_TRUE(jb, "wrong_thread");
275 
276  /* Close flow. */
277  jb_close(jb);
278 
279  EveAddCommonOptions(&aft->ctx->cfg, NULL, f, jb);
280 
281  /* TCP */
282  if (f->proto == IPPROTO_TCP) {
283  jb_open_object(jb, "tcp");
284 
285  TcpSession *ssn = f->protoctx;
286 
287  char hexflags[3];
288  snprintf(hexflags, sizeof(hexflags), "%02x",
289  ssn ? ssn->tcp_packet_flags : 0);
290  jb_set_string(jb, "tcp_flags", hexflags);
291 
292  snprintf(hexflags, sizeof(hexflags), "%02x",
293  ssn ? ssn->client.tcp_flags : 0);
294  jb_set_string(jb, "tcp_flags_ts", hexflags);
295 
296  snprintf(hexflags, sizeof(hexflags), "%02x",
297  ssn ? ssn->server.tcp_flags : 0);
298  jb_set_string(jb, "tcp_flags_tc", hexflags);
299 
300  EveTcpFlags(ssn ? ssn->tcp_packet_flags : 0, jb);
301 
302  if (ssn) {
303  const char *tcp_state = StreamTcpStateAsString(ssn->state);
304  if (tcp_state != NULL)
305  jb_set_string(jb, "state", tcp_state);
306  }
307 
308  /* Close tcp. */
309  jb_close(jb);
310  }
311 }
312 
313 static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
314 {
315  SCEnter();
316  OutputJsonThreadCtx *thread = thread_data;
317 
318  /* reset */
319  MemBufferReset(thread->buffer);
320 
321  JsonBuilder *jb = CreateEveHeaderFromFlow(f);
322  if (unlikely(jb == NULL)) {
324  }
325 
326  EveFlowLogJSON(thread, jb, f);
327 
328  OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
329  jb_free(jb);
330 
332 }
333 
335 {
336  /* register as child of eve-log */
337  OutputRegisterFlowSubModule(LOGGER_JSON_FLOW, "eve-log", "JsonFlowLog", "eve-log.flow",
339 }
tm-threads.h
OutputJsonLogInitSub
OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
Definition: output-json-common.c:50
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:157
SC_ERR_INVALID_VALUE
@ SC_ERR_INVALID_VALUE
Definition: util-error.h:160
CreateIsoTimeString
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:213
OutputJsonThreadCtx_::ctx
OutputJsonCtx * ctx
Definition: output-json.h:90
Flow_::startts
struct timeval startts
Definition: flow.h:497
stream-tcp.h
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:84
GetFlowBypassInfoID
FlowStorageId GetFlowBypassInfoID(void)
Definition: flow-util.c:220
FLOW_END_FLAG_STATE_NEW
#define FLOW_END_FLAG_STATE_NEW
Definition: flow.h:233
FlowBypassInfo_
Definition: flow.h:526
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
Flow_::proto
uint8_t proto
Definition: flow.h:372
JsonLogThreadInit
TmEcode JsonLogThreadInit(ThreadVars *t, const void *initdata, void **data)
Definition: output-json-common.c:75
threads.h
Flow_
Flow data structure.
Definition: flow.h:350
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:30
FLOW_END_FLAG_STATE_ESTABLISHED
#define FLOW_END_FLAG_STATE_ESTABLISHED
Definition: flow.h:234
Flow_::alproto_orig
AppProto alproto_orig
Definition: flow.h:463
Flow_::vlan_id
uint16_t vlan_id[2]
Definition: flow.h:374
FLOW_END_FLAG_STATE_BYPASSED
#define FLOW_END_FLAG_STATE_BYPASSED
Definition: flow.h:240
SCProtoNameValid
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
Definition: util-proto-name.c:110
FlowBypassInfo_::tosrcbytecnt
uint64_t tosrcbytecnt
Definition: flow.h:531
OutputJsonBuilderBuffer
int OutputJsonBuilderBuffer(JsonBuilder *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:974
util-privs.h
EveAddCommonOptions
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js)
Definition: output-json.c:426
proto
uint8_t proto
Definition: decode-template.h:0
Flow_::dp
Port dp
Definition: flow.h:366
Flow_::protoctx
void * protoctx
Definition: flow.h:448
JsonFlowLogRegister
void JsonFlowLogRegister(void)
Definition: output-json-flow.c:334
util-unittest.h
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
Flow_::flow_state
FlowStateType flow_state
Definition: flow.h:419
OutputJsonThreadCtx_
Definition: output-json.h:89
LOGGER_JSON_FLOW
@ LOGGER_JSON_FLOW
Definition: suricata-common.h:481
Flow_::tosrcbytecnt
uint64_t tosrcbytecnt
Definition: flow.h:502
Flow_::dst
FlowAddress dst
Definition: flow.h:353
FLOW_STATE_LOCAL_BYPASSED
@ FLOW_STATE_LOCAL_BYPASSED
Definition: flow.h:509
util-debug.h
JB_SET_STRING
#define JB_SET_STRING(jb, key, val)
Definition: rust.h:33
output-json-flow.h
FlowBypassInfo_::todstbytecnt
uint64_t todstbytecnt
Definition: flow.h:533
Flow_::todstpktcnt
uint32_t todstpktcnt
Definition: flow.h:499
output-json.h
FlowAddress_::address
union FlowAddress_::@118 address
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
PrintInet
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:271
Flow_::flow_end_flags
uint8_t flow_end_flags
Definition: flow.h:454
pkt-var.h
EveTcpFlags
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
Definition: output-json.c:471
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:262
FlowBypassInfo_::todstpktcnt
uint64_t todstpktcnt
Definition: flow.h:532
FLOW_WRONG_THREAD
#define FLOW_WRONG_THREAD
Definition: flow.h:108
util-time.h
JB_SET_TRUE
#define JB_SET_TRUE(jb, key)
Definition: rust.h:34
LiveDevice_::dev
char * dev
Definition: util-device.h:41
TcpSession_::tcp_packet_flags
uint8_t tcp_packet_flags
Definition: stream-tcp-private.h:267
Flow_::todstbytecnt
uint64_t todstbytecnt
Definition: flow.h:501
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:155
stream-tcp-private.h
conf.h
Port
uint16_t Port
Definition: decode.h:242
FLOW_END_FLAG_EMERGENCY
#define FLOW_END_FLAG_EMERGENCY
Definition: flow.h:236
Flow_::vlan_idx
uint8_t vlan_idx
Definition: flow.h:382
OutputJsonThreadCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json.h:91
util-proto-name.h
FlowGetStorageById
void * FlowGetStorageById(Flow *f, FlowStorageId id)
Definition: flow-storage.c:39
FlowBypassInfo_::tosrcpktcnt
uint64_t tosrcpktcnt
Definition: flow.h:530
Flow_::alproto_expect
AppProto alproto_expect
Definition: flow.h:466
Flow_::src
FlowAddress src
Definition: flow.h:353
Flow_::lastts
struct timeval lastts
Definition: flow.h:411
MemBufferReset
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
flow-storage.h
OutputJsonThreadCtx_::buffer
MemBuffer * buffer
Definition: output-json.h:92
suricata-common.h
Flow_::icmp_d
struct Flow_::@121::@127 icmp_d
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
EveAddAppProto
void EveAddAppProto(Flow *f, JsonBuilder *js)
Definition: output-json-flow.c:168
FLOW_END_FLAG_STATE_CLOSED
#define FLOW_END_FLAG_STATE_CLOSED
Definition: flow.h:235
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
Flow_::livedev
struct LiveDevice_ * livedev
Definition: flow.h:403
threadvars.h
CreateEveFlowId
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
Definition: output-json.c:743
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
OutputRegisterFlowSubModule
void OutputRegisterFlowSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, FlowLogger FlowLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a flow output sub-module.
Definition: output.c:585
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:458
util-logopenfile.h
Flow_::flags
uint32_t flags
Definition: flow.h:428
util-buffer.h
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
TimeGet
void TimeGet(struct timeval *tv)
Definition: util-time.c:153
FLOW_END_FLAG_SHUTDOWN
#define FLOW_END_FLAG_SHUTDOWN
Definition: flow.h:239
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:6293
known_proto
char * known_proto[256]
Definition: util-proto-name.c:33
FlowHasAlerts
int FlowHasAlerts(const Flow *f)
Check if flow has alerts.
Definition: flow.c:180
FLOW_END_FLAG_TIMEOUT
#define FLOW_END_FLAG_TIMEOUT
Definition: flow.h:237
Flow_::esp
struct Flow_::@119::@126 esp
Flow_::sp
Port sp
Definition: flow.h:355
JsonLogThreadDeinit
TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data)
Definition: output-json-common.c:109
TcpSession_
Definition: stream-tcp-private.h:260
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:459
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:457
FLOW_END_FLAG_FORCED
#define FLOW_END_FLAG_FORCED
Definition: flow.h:238
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:110
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
TcpStream_::tcp_flags
uint8_t tcp_flags
Definition: stream-tcp-private.h:99
Flow_::icmp_s
struct Flow_::@119::@125 icmp_s
Flow_::tosrcpktcnt
uint32_t tosrcpktcnt
Definition: flow.h:500
debug.h
output.h
EveAddFlow
void EveAddFlow(Flow *f, JsonBuilder *js)
Definition: output-json-flow.c:189