suricata
output-json-alert.c
Go to the documentation of this file.
1 /* Copyright (C) 2013-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Logs alerts in JSON format.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "tm-threads.h"
35 #include "threadvars.h"
36 #include "util-debug.h"
37 
38 #include "util-misc.h"
39 #include "util-unittest.h"
40 #include "util-unittest-helper.h"
41 
42 #include "detect-parse.h"
43 #include "detect-engine.h"
44 #include "detect-engine-mpm.h"
45 #include "detect-reference.h"
46 #include "detect-metadata.h"
47 #include "app-layer-parser.h"
48 #include "app-layer-dnp3.h"
49 #include "app-layer-dns-common.h"
50 #include "app-layer-htp.h"
51 #include "app-layer-htp-xff.h"
52 #include "app-layer-ftp.h"
54 #include "util-syslog.h"
55 #include "util-logopenfile.h"
56 
57 #include "output.h"
58 #include "output-json.h"
59 #include "output-json-alert.h"
60 #include "output-json-dnp3.h"
61 #include "output-json-dns.h"
62 #include "output-json-http.h"
63 #include "output-json-tls.h"
64 #include "output-json-ssh.h"
65 #include "output-json-smtp.h"
67 #include "output-json-nfs.h"
68 #include "output-json-smb.h"
69 #include "output-json-flow.h"
70 #include "output-json-sip.h"
71 
72 #include "util-byte.h"
73 #include "util-privs.h"
74 #include "util-print.h"
75 #include "util-proto-name.h"
76 #include "util-optimize.h"
77 #include "util-buffer.h"
78 #include "util-crypt.h"
79 #include "util-validate.h"
80 
81 #define MODULE_NAME "JsonAlertLog"
82 
83 #define LOG_JSON_PAYLOAD BIT_U16(0)
84 #define LOG_JSON_PACKET BIT_U16(1)
85 #define LOG_JSON_PAYLOAD_BASE64 BIT_U16(2)
86 #define LOG_JSON_TAGGED_PACKETS BIT_U16(3)
87 #define LOG_JSON_APP_LAYER BIT_U16(4)
88 #define LOG_JSON_FLOW BIT_U16(5)
89 #define LOG_JSON_HTTP_BODY BIT_U16(6)
90 #define LOG_JSON_HTTP_BODY_BASE64 BIT_U16(7)
91 #define LOG_JSON_RULE_METADATA BIT_U16(8)
92 #define LOG_JSON_RULE BIT_U16(9)
93 
94 #define METADATA_DEFAULTS ( LOG_JSON_FLOW | \
95  LOG_JSON_APP_LAYER | \
96  LOG_JSON_RULE_METADATA)
97 
98 #define JSON_BODY_LOGGING (LOG_JSON_HTTP_BODY | LOG_JSON_HTTP_BODY_BASE64)
99 
100 #define JSON_STREAM_BUFFER_SIZE 4096
101 
102 typedef struct AlertJsonOutputCtx_ {
104  uint16_t flags;
110 
111 typedef struct JsonAlertLogThread_ {
112  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
118 
119 /* Callback function to pack payload contents from a stream into a buffer
120  * so we can report them in JSON output. */
121 static int AlertJsonDumpStreamSegmentCallback(const Packet *p, void *data, const uint8_t *buf, uint32_t buflen)
122 {
123  MemBuffer *payload = (MemBuffer *)data;
124  MemBufferWriteRaw(payload, buf, buflen);
125 
126  return 1;
127 }
128 
129 static void AlertJsonTls(const Flow *f, json_t *js)
130 {
131  SSLState *ssl_state = (SSLState *)FlowGetAppState(f);
132  if (ssl_state) {
133  json_t *tjs = json_object();
134  if (unlikely(tjs == NULL))
135  return;
136 
137  JsonTlsLogJSONBasic(tjs, ssl_state);
138  JsonTlsLogJSONExtended(tjs, ssl_state);
139 
140  json_object_set_new(js, "tls", tjs);
141  }
142 
143  return;
144 }
145 
146 static void AlertJsonSsh(const Flow *f, json_t *js)
147 {
148  SshState *ssh_state = (SshState *)FlowGetAppState(f);
149  if (ssh_state) {
150  json_t *tjs = json_object();
151  if (unlikely(tjs == NULL))
152  return;
153 
154  JsonSshLogJSON(tjs, ssh_state);
155 
156  json_object_set_new(js, "ssh", tjs);
157  }
158 
159  return;
160 }
161 
162 static void AlertJsonDnp3(const Flow *f, const uint64_t tx_id, json_t *js)
163 {
164  DNP3State *dnp3_state = (DNP3State *)FlowGetAppState(f);
165  if (dnp3_state) {
167  dnp3_state, tx_id);
168  if (tx) {
169  json_t *dnp3js = json_object();
170  if (likely(dnp3js != NULL)) {
171  if (tx->has_request && tx->request_done) {
172  json_t *request = JsonDNP3LogRequest(tx);
173  if (request != NULL) {
174  json_object_set_new(dnp3js, "request", request);
175  }
176  }
177  if (tx->has_response && tx->response_done) {
178  json_t *response = JsonDNP3LogResponse(tx);
179  if (response != NULL) {
180  json_object_set_new(dnp3js, "response", response);
181  }
182  }
183  json_object_set_new(js, "dnp3", dnp3js);
184  }
185  }
186  }
187 
188  return;
189 }
190 
191 static void AlertJsonDns(const Flow *f, const uint64_t tx_id, json_t *js)
192 {
193  RSDNSState *dns_state = (RSDNSState *)FlowGetAppState(f);
194  if (dns_state) {
195  void *txptr = AppLayerParserGetTx(f->proto, ALPROTO_DNS,
196  dns_state, tx_id);
197  if (txptr) {
198  json_t *dnsjs = json_object();
199  if (unlikely(dnsjs == NULL)) {
200  return;
201  }
202  json_t *qjs = JsonDNSLogQuery(txptr, tx_id);
203  if (qjs != NULL) {
204  json_object_set_new(dnsjs, "query", qjs);
205  }
206  json_t *ajs = JsonDNSLogAnswer(txptr, tx_id);
207  if (ajs != NULL) {
208  json_object_set_new(dnsjs, "answer", ajs);
209  }
210  json_object_set_new(js, "dns", dnsjs);
211  }
212  }
213  return;
214 }
215 
216 static void AlertJsonSourceTarget(const Packet *p, const PacketAlert *pa,
217  json_t *js, json_t* ajs)
218 {
219  json_t *sjs = json_object();
220  if (sjs == NULL) {
221  return;
222  }
223 
224  json_t *tjs = json_object();
225  if (tjs == NULL) {
226  json_decref(sjs);
227  return;
228  }
229 
230  if (pa->s->flags & SIG_FLAG_DEST_IS_TARGET) {
231  json_object_set(sjs, "ip", json_object_get(js, "src_ip"));
232  json_object_set(tjs, "ip", json_object_get(js, "dest_ip"));
233  switch (p->proto) {
234  case IPPROTO_ICMP:
235  case IPPROTO_ICMPV6:
236  break;
237  case IPPROTO_UDP:
238  case IPPROTO_TCP:
239  case IPPROTO_SCTP:
240  json_object_set(sjs, "port", json_object_get(js, "src_port"));
241  json_object_set(tjs, "port", json_object_get(js, "dest_port"));
242  break;
243  }
244  } else if (pa->s->flags & SIG_FLAG_SRC_IS_TARGET) {
245  json_object_set(sjs, "ip", json_object_get(js, "dest_ip"));
246  json_object_set(tjs, "ip", json_object_get(js, "src_ip"));
247  switch (p->proto) {
248  case IPPROTO_ICMP:
249  case IPPROTO_ICMPV6:
250  break;
251  case IPPROTO_UDP:
252  case IPPROTO_TCP:
253  case IPPROTO_SCTP:
254  json_object_set(sjs, "port", json_object_get(js, "dest_port"));
255  json_object_set(tjs, "port", json_object_get(js, "src_port"));
256  break;
257  }
258  }
259  json_object_set_new(ajs, "source", sjs);
260  json_object_set_new(ajs, "target", tjs);
261 }
262 
263 static void AlertJsonMetadata(AlertJsonOutputCtx *json_output_ctx, const PacketAlert *pa, json_t *ajs)
264 {
265  if (pa->s->metadata) {
266  const DetectMetadata* kv = pa->s->metadata;
267  json_t *mjs = json_object();
268  if (unlikely(mjs == NULL)) {
269  return;
270  }
271  while (kv) {
272  json_t *jkey = json_object_get(mjs, kv->key);
273  if (jkey == NULL) {
274  jkey = json_array();
275  if (unlikely(jkey == NULL))
276  break;
277  json_array_append_new(jkey, json_string(kv->value));
278  json_object_set_new(mjs, kv->key, jkey);
279  } else {
280  json_array_append_new(jkey, json_string(kv->value));
281  }
282 
283  kv = kv->next;
284  }
285 
286  if (json_object_size(mjs) == 0) {
287  json_decref(mjs);
288  } else {
289  json_object_set_new(ajs, "metadata", mjs);
290  }
291  }
292 }
293 
294 
295 void AlertJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, json_t *js,
296  uint16_t flags)
297 {
298  AlertJsonOutputCtx *json_output_ctx = (AlertJsonOutputCtx *)ctx;
299  const char *action = "allowed";
300  /* use packet action if rate_filter modified the action */
304  action = "blocked";
305  }
306  } else {
308  action = "blocked";
309  } else if ((pa->action & ACTION_DROP) && EngineModeIsIPS()) {
310  action = "blocked";
311  }
312  }
313 
314  /* Add tx_id to root element for correlation with other events. */
315  json_object_del(js, "tx_id");
316  if (pa->flags & PACKET_ALERT_FLAG_TX)
317  json_object_set_new(js, "tx_id", json_integer(pa->tx_id));
318 
319  json_t *ajs = json_object();
320  if (ajs == NULL) {
321  return;
322  }
323 
324  json_object_set_new(ajs, "action", json_string(action));
325  json_object_set_new(ajs, "gid", json_integer(pa->s->gid));
326  json_object_set_new(ajs, "signature_id", json_integer(pa->s->id));
327  json_object_set_new(ajs, "rev", json_integer(pa->s->rev));
328  json_object_set_new(ajs, "signature",
329  SCJsonString((pa->s->msg) ? pa->s->msg : ""));
330  json_object_set_new(ajs, "category",
331  SCJsonString((pa->s->class_msg) ? pa->s->class_msg : ""));
332  json_object_set_new(ajs, "severity", json_integer(pa->s->prio));
333 
334  if (p->tenant_id > 0)
335  json_object_set_new(ajs, "tenant_id", json_integer(p->tenant_id));
336 
337  if (pa->s->flags & SIG_FLAG_HAS_TARGET) {
338  AlertJsonSourceTarget(p, pa, js, ajs);
339  }
340 
341  if ((json_output_ctx != NULL) && (flags & LOG_JSON_RULE_METADATA)) {
342  AlertJsonMetadata(json_output_ctx, pa, ajs);
343  }
344 
345  /* alert */
346  json_object_set_new(js, "alert", ajs);
347 }
348 
349 static void AlertJsonTunnel(const Packet *p, json_t *js)
350 {
351  json_t *tunnel = json_object();
352  if (tunnel == NULL)
353  return;
354 
355  if (p->root == NULL) {
356  json_decref(tunnel);
357  return;
358  }
359 
360  /* get a lock to access root packet fields */
361  SCMutex *m = &p->root->tunnel_mutex;
362 
363  SCMutexLock(m);
364  JsonFiveTuple((const Packet *)p->root, 0, tunnel);
365  SCMutexUnlock(m);
366 
367  json_object_set_new(tunnel, "depth", json_integer(p->recursion_level));
368 
369  json_object_set_new(js, "tunnel", tunnel);
370 }
371 
372 static void AlertAddPayload(AlertJsonOutputCtx *json_output_ctx, json_t *js, const Packet *p)
373 {
374  if (json_output_ctx->flags & LOG_JSON_PAYLOAD_BASE64) {
375  unsigned long len = p->payload_len * 2 + 1;
376  uint8_t encoded[len];
377  if (Base64Encode(p->payload, p->payload_len, encoded, &len) == SC_BASE64_OK) {
378  json_object_set_new(js, "payload", json_string((char *)encoded));
379  }
380  }
381 
382  if (json_output_ctx->flags & LOG_JSON_PAYLOAD) {
383  uint8_t printable_buf[p->payload_len + 1];
384  uint32_t offset = 0;
385  PrintStringsToBuffer(printable_buf, &offset,
386  p->payload_len + 1,
387  p->payload, p->payload_len);
388  printable_buf[p->payload_len] = '\0';
389  json_object_set_new(js, "payload_printable", json_string((char *)printable_buf));
390  }
391 }
392 
393 static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
394 {
395  MemBuffer *payload = aft->payload_buffer;
396  AlertJsonOutputCtx *json_output_ctx = aft->json_output_ctx;
397  json_t *hjs = NULL;
398 
399  int i;
400 
401  if (p->alerts.cnt == 0 && !(p->flags & PKT_HAS_TAG))
402  return TM_ECODE_OK;
403 
404  json_t *js = CreateJSONHeader(p, LOG_DIR_PACKET, "alert");
405  if (unlikely(js == NULL))
406  return TM_ECODE_OK;
407 
408  JsonAddCommonOptions(&json_output_ctx->cfg, p, p->flow, js);
409 
410  for (i = 0; i < p->alerts.cnt; i++) {
411  const PacketAlert *pa = &p->alerts.alerts[i];
412  if (unlikely(pa->s == NULL)) {
413  continue;
414  }
415 
417 
418  /* alert */
419  AlertJsonHeader(json_output_ctx, p, pa, js, json_output_ctx->flags);
420 
421  if (IS_TUNNEL_PKT(p)) {
422  AlertJsonTunnel(p, js);
423  }
424 
425  if (json_output_ctx->flags & LOG_JSON_APP_LAYER && p->flow != NULL) {
427  switch (proto) {
428  case ALPROTO_HTTP:
429  hjs = JsonHttpAddMetadata(p->flow, pa->tx_id);
430  if (hjs) {
431  if (json_output_ctx->flags & LOG_JSON_HTTP_BODY) {
433  }
434  if (json_output_ctx->flags & LOG_JSON_HTTP_BODY_BASE64) {
435  JsonHttpLogJSONBodyBase64(hjs, p->flow, pa->tx_id);
436  }
437  json_object_set_new(js, "http", hjs);
438  }
439  break;
440  case ALPROTO_TLS:
441  AlertJsonTls(p->flow, js);
442  break;
443  case ALPROTO_SSH:
444  AlertJsonSsh(p->flow, js);
445  break;
446  case ALPROTO_SMTP:
447  hjs = JsonSMTPAddMetadata(p->flow, pa->tx_id);
448  if (hjs) {
449  json_object_set_new(js, "smtp", hjs);
450  }
451 
452  hjs = JsonEmailAddMetadata(p->flow, pa->tx_id);
453  if (hjs) {
454  json_object_set_new(js, "email", hjs);
455  }
456  break;
457  case ALPROTO_NFS:
458  hjs = JsonNFSAddMetadataRPC(p->flow, pa->tx_id);
459  if (hjs)
460  json_object_set_new(js, "rpc", hjs);
461  hjs = JsonNFSAddMetadata(p->flow, pa->tx_id);
462  if (hjs)
463  json_object_set_new(js, "nfs", hjs);
464  break;
465  case ALPROTO_SMB:
466  hjs = JsonSMBAddMetadata(p->flow, pa->tx_id);
467  if (hjs)
468  json_object_set_new(js, "smb", hjs);
469  break;
470  case ALPROTO_SIP:
471  hjs = JsonSIPAddMetadata(p->flow, pa->tx_id);
472  if (hjs)
473  json_object_set_new(js, "sip", hjs);
474  break;
475  case ALPROTO_FTPDATA:
476  hjs = JsonFTPDataAddMetadata(p->flow);
477  if (hjs)
478  json_object_set_new(js, "ftp-data", hjs);
479  break;
480  case ALPROTO_DNP3:
481  AlertJsonDnp3(p->flow, pa->tx_id, js);
482  break;
483  case ALPROTO_DNS:
484  AlertJsonDns(p->flow, pa->tx_id, js);
485  break;
486  default:
487  break;
488  }
489  }
490 
491  if (p->flow) {
492  if (json_output_ctx->flags & LOG_JSON_FLOW) {
493  hjs = json_object();
494  if (hjs != NULL) {
495  JsonAddFlow(p->flow, js, hjs);
496  json_object_set_new(js, "flow", hjs);
497  }
498  } else {
499  json_object_set_new(js, "app_proto",
500  json_string(AppProtoToString(p->flow->alproto)));
501  }
502  }
503 
504  /* payload */
505  if (json_output_ctx->flags & (LOG_JSON_PAYLOAD | LOG_JSON_PAYLOAD_BASE64)) {
506  int stream = (p->proto == IPPROTO_TCP) ?
508  1 : 0) : 0;
509 
510  /* Is this a stream? If so, pack part of it into the payload field */
511  if (stream) {
512  uint8_t flag;
513 
514  MemBufferReset(payload);
515 
516  if (p->flowflags & FLOW_PKT_TOSERVER) {
517  flag = FLOW_PKT_TOCLIENT;
518  } else {
519  flag = FLOW_PKT_TOSERVER;
520  }
521 
522  StreamSegmentForEach((const Packet *)p, flag,
523  AlertJsonDumpStreamSegmentCallback,
524  (void *)payload);
525  if (payload->offset) {
526  if (json_output_ctx->flags & LOG_JSON_PAYLOAD_BASE64) {
527  unsigned long len = json_output_ctx->payload_buffer_size * 2;
528  uint8_t encoded[len];
529  Base64Encode(payload->buffer, payload->offset, encoded, &len);
530  json_object_set_new(js, "payload", json_string((char *)encoded));
531  }
532 
533  if (json_output_ctx->flags & LOG_JSON_PAYLOAD) {
534  uint8_t printable_buf[payload->offset + 1];
535  uint32_t offset = 0;
536  PrintStringsToBuffer(printable_buf, &offset,
537  sizeof(printable_buf),
538  payload->buffer, payload->offset);
539  json_object_set_new(js, "payload_printable",
540  json_string((char *)printable_buf));
541  }
542  } else if (p->payload_len) {
543  /* Fallback on packet payload */
544  AlertAddPayload(json_output_ctx, js, p);
545  }
546  } else {
547  /* This is a single packet and not a stream */
548  AlertAddPayload(json_output_ctx, js, p);
549  }
550 
551  json_object_set_new(js, "stream", json_integer(stream));
552  }
553 
554  /* base64-encoded full packet */
555  if (json_output_ctx->flags & LOG_JSON_PACKET) {
556  JsonPacket(p, js, 0);
557  }
558 
559  /* signature text */
560  if (json_output_ctx->flags & LOG_JSON_RULE) {
561  hjs = json_object_get(js, "alert");
562  if (json_is_object(hjs))
563  json_object_set_new(hjs, "rule", json_string(pa->s->sig_str));
564  }
565 
566  HttpXFFCfg *xff_cfg = json_output_ctx->xff_cfg != NULL ?
567  json_output_ctx->xff_cfg : json_output_ctx->parent_xff_cfg;;
568 
569  /* xff header */
570  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
571  int have_xff_ip = 0;
572  char buffer[XFF_MAXLEN];
573 
574  if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
575  if (pa->flags & PACKET_ALERT_FLAG_TX) {
576  have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
577  } else {
578  have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
579  }
580  }
581 
582  if (have_xff_ip) {
583  if (xff_cfg->flags & XFF_EXTRADATA) {
584  json_object_set_new(js, "xff", json_string(buffer));
585  }
586  else if (xff_cfg->flags & XFF_OVERWRITE) {
587  if (p->flowflags & FLOW_PKT_TOCLIENT) {
588  json_object_set(js, "dest_ip", json_string(buffer));
589  } else {
590  json_object_set(js, "src_ip", json_string(buffer));
591  }
592  }
593  }
594  }
595 
596  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
597  json_object_del(js, "alert");
598  }
599  json_object_clear(js);
600  json_decref(js);
601 
602  if ((p->flags & PKT_HAS_TAG) && (json_output_ctx->flags &
605  json_t *packetjs = CreateJSONHeader(p, LOG_DIR_PACKET, "packet");
606  if (unlikely(packetjs != NULL)) {
607  JsonPacket(p, packetjs, 0);
608  OutputJSONBuffer(packetjs, aft->file_ctx, &aft->json_buffer);
609  json_decref(packetjs);
610  }
611  }
612 
613  return TM_ECODE_OK;
614 }
615 
616 static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
617 {
618  int i;
619  char timebuf[64];
620  json_t *js;
621 
622  if (p->alerts.cnt == 0)
623  return TM_ECODE_OK;
624 
625  CreateIsoTimeString(&p->ts, timebuf, sizeof(timebuf));
626 
627  for (i = 0; i < p->alerts.cnt; i++) {
629 
630  const PacketAlert *pa = &p->alerts.alerts[i];
631  if (unlikely(pa->s == NULL)) {
632  continue;
633  }
634 
635  const char *action = "allowed";
637  action = "blocked";
638  } else if ((pa->action & ACTION_DROP) && EngineModeIsIPS()) {
639  action = "blocked";
640  }
641 
642  js = json_object();
643  if (js == NULL)
644  return TM_ECODE_OK;
645 
646  json_t *ajs = json_object();
647  if (ajs == NULL) {
648  json_decref(js);
649  return TM_ECODE_OK;
650  }
651 
652  /* time & tx */
653  json_object_set_new(js, "timestamp", json_string(timebuf));
654 
655  /* tuple */
656  //json_object_set_new(js, "srcip", json_string(srcip));
657  //json_object_set_new(js, "sp", json_integer(p->sp));
658  //json_object_set_new(js, "dstip", json_string(dstip));
659  //json_object_set_new(js, "dp", json_integer(p->dp));
660  //json_object_set_new(js, "proto", json_integer(proto));
661 
662  json_object_set_new(ajs, "action", json_string(action));
663  json_object_set_new(ajs, "gid", json_integer(pa->s->gid));
664  json_object_set_new(ajs, "signature_id", json_integer(pa->s->id));
665  json_object_set_new(ajs, "rev", json_integer(pa->s->rev));
666  json_object_set_new(ajs, "signature",
667  json_string((pa->s->msg) ? pa->s->msg : ""));
668  json_object_set_new(ajs, "category",
669  json_string((pa->s->class_msg) ? pa->s->class_msg : ""));
670  json_object_set_new(ajs, "severity", json_integer(pa->s->prio));
671 
672  if (p->tenant_id > 0)
673  json_object_set_new(ajs, "tenant_id", json_integer(p->tenant_id));
674 
675  /* alert */
676  json_object_set_new(js, "alert", ajs);
677  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
678  json_object_clear(js);
679  json_decref(js);
680  }
681 
682  return TM_ECODE_OK;
683 }
684 
685 static int JsonAlertLogger(ThreadVars *tv, void *thread_data, const Packet *p)
686 {
687  JsonAlertLogThread *aft = thread_data;
688 
689  if (PKT_IS_IPV4(p) || PKT_IS_IPV6(p)) {
690  return AlertJson(tv, aft, p);
691  } else if (p->alerts.cnt > 0) {
692  return AlertJsonDecoderEvent(tv, aft, p);
693  }
694  return 0;
695 }
696 
697 static int JsonAlertLogCondition(ThreadVars *tv, const Packet *p)
698 {
699  if (p->alerts.cnt || (p->flags & PKT_HAS_TAG)) {
700  return TRUE;
701  }
702  return FALSE;
703 }
704 
705 static TmEcode JsonAlertLogThreadInit(ThreadVars *t, const void *initdata, void **data)
706 {
708  if (unlikely(aft == NULL))
709  return TM_ECODE_FAILED;
710  memset(aft, 0, sizeof(JsonAlertLogThread));
711  if(initdata == NULL)
712  {
713  SCLogDebug("Error getting context for EveLogAlert. \"initdata\" argument NULL");
714  SCFree(aft);
715  return TM_ECODE_FAILED;
716  }
717 
719  if (aft->json_buffer == NULL) {
720  SCFree(aft);
721  return TM_ECODE_FAILED;
722  }
723 
724  /** Use the Output Context (file pointer and mutex) */
725  AlertJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
726  aft->file_ctx = json_output_ctx->file_ctx;
727  aft->json_output_ctx = json_output_ctx;
728 
729  aft->payload_buffer = MemBufferCreateNew(json_output_ctx->payload_buffer_size);
730  if (aft->payload_buffer == NULL) {
732  SCFree(aft);
733  return TM_ECODE_FAILED;
734  }
735 
736  *data = (void *)aft;
737  return TM_ECODE_OK;
738 }
739 
740 static TmEcode JsonAlertLogThreadDeinit(ThreadVars *t, void *data)
741 {
742  JsonAlertLogThread *aft = (JsonAlertLogThread *)data;
743  if (aft == NULL) {
744  return TM_ECODE_OK;
745  }
746 
749 
750  /* clear memory */
751  memset(aft, 0, sizeof(JsonAlertLogThread));
752 
753  SCFree(aft);
754  return TM_ECODE_OK;
755 }
756 
757 static void JsonAlertLogDeInitCtx(OutputCtx *output_ctx)
758 {
759  AlertJsonOutputCtx *json_output_ctx = (AlertJsonOutputCtx *) output_ctx->data;
760  if (json_output_ctx != NULL) {
761  HttpXFFCfg *xff_cfg = json_output_ctx->xff_cfg;
762  if (xff_cfg != NULL) {
763  SCFree(xff_cfg);
764  }
765  LogFileFreeCtx(json_output_ctx->file_ctx);
766  SCFree(json_output_ctx);
767  }
768  SCFree(output_ctx);
769 }
770 
771 static void JsonAlertLogDeInitCtxSub(OutputCtx *output_ctx)
772 {
773  SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
774 
775  AlertJsonOutputCtx *json_output_ctx = (AlertJsonOutputCtx *) output_ctx->data;
776 
777  if (json_output_ctx != NULL) {
778  HttpXFFCfg *xff_cfg = json_output_ctx->xff_cfg;
779  if (xff_cfg != NULL) {
780  SCFree(xff_cfg);
781  }
782  SCFree(json_output_ctx);
783  }
784  SCFree(output_ctx);
785 }
786 
787 static void SetFlag(const ConfNode *conf, const char *name, uint16_t flag, uint16_t *out_flags)
788 {
789  DEBUG_VALIDATE_BUG_ON(conf == NULL);
790  const char *setting = ConfNodeLookupChildValue(conf, name);
791  if (setting != NULL) {
792  if (ConfValIsTrue(setting)) {
793  *out_flags |= flag;
794  } else {
795  *out_flags &= ~flag;
796  }
797  }
798 }
799 
800 #define DEFAULT_LOG_FILENAME "alert.json"
801 
802 static void JsonAlertLogSetupMetadata(AlertJsonOutputCtx *json_output_ctx,
803  ConfNode *conf)
804 {
805  static bool warn_no_meta = false;
807  uint16_t flags = METADATA_DEFAULTS;
808 
809  if (conf != NULL) {
810  /* Check for metadata to enable/disable. */
811  ConfNode *metadata = ConfNodeLookupChild(conf, "metadata");
812  if (metadata != NULL) {
813  if (metadata->val != NULL && ConfValIsFalse(metadata->val)) {
814  flags &= ~METADATA_DEFAULTS;
815  } else if (ConfNodeHasChildren(metadata)) {
816  ConfNode *rule_metadata = ConfNodeLookupChild(metadata, "rule");
817  if (rule_metadata) {
818  SetFlag(rule_metadata, "raw", LOG_JSON_RULE, &flags);
819  SetFlag(rule_metadata, "metadata", LOG_JSON_RULE_METADATA,
820  &flags);
821  }
822  SetFlag(metadata, "flow", LOG_JSON_FLOW, &flags);
823  SetFlag(metadata, "app-layer", LOG_JSON_APP_LAYER, &flags);
824  }
825  }
826 
827  /* Non-metadata toggles. */
828  SetFlag(conf, "payload", LOG_JSON_PAYLOAD_BASE64, &flags);
829  SetFlag(conf, "packet", LOG_JSON_PACKET, &flags);
830  SetFlag(conf, "tagged-packets", LOG_JSON_TAGGED_PACKETS, &flags);
831  SetFlag(conf, "payload-printable", LOG_JSON_PAYLOAD, &flags);
832  SetFlag(conf, "http-body-printable", LOG_JSON_HTTP_BODY, &flags);
833  SetFlag(conf, "http-body", LOG_JSON_HTTP_BODY_BASE64, &flags);
834 
835  /* Check for obsolete configuration flags to enable specific
836  * protocols. These are now just aliases for enabling
837  * app-layer logging. */
838  SetFlag(conf, "http", LOG_JSON_APP_LAYER, &flags);
839  SetFlag(conf, "tls", LOG_JSON_APP_LAYER, &flags);
840  SetFlag(conf, "ssh", LOG_JSON_APP_LAYER, &flags);
841  SetFlag(conf, "smtp", LOG_JSON_APP_LAYER, &flags);
842  SetFlag(conf, "dnp3", LOG_JSON_APP_LAYER, &flags);
843 
844  /* And check for obsolete configuration flags for enabling
845  * app-layer and flow as these have been moved under the
846  * metadata key. */
847  SetFlag(conf, "app-layer", LOG_JSON_APP_LAYER, &flags);
848  SetFlag(conf, "flow", LOG_JSON_FLOW, &flags);
849 
850  const char *payload_buffer_value = ConfNodeLookupChildValue(conf, "payload-buffer-size");
851 
852  if (payload_buffer_value != NULL) {
853  uint32_t value;
854  if (ParseSizeStringU32(payload_buffer_value, &value) < 0) {
855  SCLogError(SC_ERR_ALERT_PAYLOAD_BUFFER, "Error parsing "
856  "payload-buffer-size - %s. Killing engine",
857  payload_buffer_value);
858  exit(EXIT_FAILURE);
859  } else {
860  payload_buffer_size = value;
861  }
862  }
863 
864  if (!warn_no_meta && flags & JSON_BODY_LOGGING) {
865  if (((flags & LOG_JSON_APP_LAYER) == 0)) {
866  SCLogWarning(SC_WARN_ALERT_CONFIG, "HTTP body logging has been configured, however, "
867  "metadata logging has not been enabled. HTTP body logging will be disabled.");
868  flags &= ~JSON_BODY_LOGGING;
869  warn_no_meta = true;
870  }
871  }
872 
873  json_output_ctx->payload_buffer_size = payload_buffer_size;
874  }
875 
876  if (flags & LOG_JSON_RULE_METADATA) {
878  }
879 
880  json_output_ctx->flags |= flags;
881 }
882 
883 static HttpXFFCfg *JsonAlertLogGetXffCfg(ConfNode *conf)
884 {
885  HttpXFFCfg *xff_cfg = NULL;
886  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
887  xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
888  if (likely(xff_cfg != NULL)) {
889  HttpXFFGetCfg(conf, xff_cfg);
890  }
891  }
892  return xff_cfg;
893 }
894 
895 /**
896  * \brief Create a new LogFileCtx for "fast" output style.
897  * \param conf The configuration node for this output.
898  * \return A LogFileCtx pointer on success, NULL on failure.
899  */
900 static OutputInitResult JsonAlertLogInitCtx(ConfNode *conf)
901 {
902  OutputInitResult result = { NULL, false };
903  AlertJsonOutputCtx *json_output_ctx = NULL;
904  LogFileCtx *logfile_ctx = LogFileNewCtx();
905  if (logfile_ctx == NULL) {
906  SCLogDebug("AlertFastLogInitCtx2: Could not create new LogFileCtx");
907  return result;
908  }
909 
910  if (SCConfLogOpenGeneric(conf, logfile_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
911  LogFileFreeCtx(logfile_ctx);
912  return result;
913  }
914 
915  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
916  if (unlikely(output_ctx == NULL)) {
917  LogFileFreeCtx(logfile_ctx);
918  return result;
919  }
920 
921  json_output_ctx = SCMalloc(sizeof(AlertJsonOutputCtx));
922  if (unlikely(json_output_ctx == NULL)) {
923  LogFileFreeCtx(logfile_ctx);
924  SCFree(output_ctx);
925  return result;
926  }
927  memset(json_output_ctx, 0, sizeof(AlertJsonOutputCtx));
928 
929  json_output_ctx->file_ctx = logfile_ctx;
930 
931  JsonAlertLogSetupMetadata(json_output_ctx, conf);
932  json_output_ctx->xff_cfg = JsonAlertLogGetXffCfg(conf);
933 
934  output_ctx->data = json_output_ctx;
935  output_ctx->DeInit = JsonAlertLogDeInitCtx;
936 
937  result.ctx = output_ctx;
938  result.ok = true;
939  return result;
940 }
941 
942 /**
943  * \brief Create a new LogFileCtx for "fast" output style.
944  * \param conf The configuration node for this output.
945  * \return A LogFileCtx pointer on success, NULL on failure.
946  */
947 static OutputInitResult JsonAlertLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
948 {
949  OutputInitResult result = { NULL, false };
950  OutputJsonCtx *ajt = parent_ctx->data;
951  AlertJsonOutputCtx *json_output_ctx = NULL;
952 
953  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
954  if (unlikely(output_ctx == NULL))
955  return result;
956 
957  json_output_ctx = SCMalloc(sizeof(AlertJsonOutputCtx));
958  if (unlikely(json_output_ctx == NULL)) {
959  goto error;
960  }
961  memset(json_output_ctx, 0, sizeof(AlertJsonOutputCtx));
962 
963  json_output_ctx->file_ctx = ajt->file_ctx;
964  json_output_ctx->cfg = ajt->cfg;
965 
966  JsonAlertLogSetupMetadata(json_output_ctx, conf);
967  json_output_ctx->xff_cfg = JsonAlertLogGetXffCfg(conf);
968  if (json_output_ctx->xff_cfg == NULL) {
969  json_output_ctx->parent_xff_cfg = ajt->xff_cfg;
970  }
971 
972  output_ctx->data = json_output_ctx;
973  output_ctx->DeInit = JsonAlertLogDeInitCtxSub;
974 
975  result.ctx = output_ctx;
976  result.ok = true;
977  return result;
978 
979 error:
980  if (json_output_ctx != NULL) {
981  SCFree(json_output_ctx);
982  }
983  if (output_ctx != NULL) {
984  SCFree(output_ctx);
985  }
986 
987  return result;
988 }
989 
991 {
993  JsonAlertLogInitCtx, JsonAlertLogger, JsonAlertLogCondition,
994  JsonAlertLogThreadInit, JsonAlertLogThreadDeinit, NULL);
996  "eve-log.alert", JsonAlertLogInitCtxSub, JsonAlertLogger,
997  JsonAlertLogCondition, JsonAlertLogThreadInit, JsonAlertLogThreadDeinit,
998  NULL);
999 }
#define PACKET_ALERT_FLAG_STREAM_MATCH
Definition: decode.h:284
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define SCMutex
json_t * JsonSMTPAddMetadata(const Flow *f, uint64_t tx_id)
void JsonFiveTuple(const Packet *p, enum OutputJsonLogDirection dir, json_t *js)
Add five tuple from packet to JSON object.
Definition: output-json.c:455
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:44
int StreamSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback CallbackFunc, void *data)
Run callback for all segments.
Definition: stream.c:39
json_t * CreateJSONHeader(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type)
Definition: output-json.c:710
#define SCLogDebug(...)
Definition: util-debug.h:335
#define MemBufferWriteRaw(dst, raw_buffer, raw_buffer_len)
Write a raw buffer to the MemBuffer dst.
Definition: util-buffer.h:133
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:809
DetectMetadata * metadata
Definition: detect.h:587
void AlertJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, json_t *js, uint16_t flags)
int HttpXFFGetIP(const Flow *f, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any. The caller needs to lock the flow.
struct Flow_ * flow
Definition: decode.h:445
json_t * JsonDNP3LogResponse(DNP3Transaction *dnp3tx)
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:286
#define ACTION_REJECT_DST
json_t * JsonDNP3LogRequest(DNP3Transaction *dnp3tx)
#define JSON_BODY_LOGGING
#define PACKET_TEST_ACTION(p, a)
Definition: decode.h:857
uint32_t flags
Definition: detect.h:523
uint8_t proto
Definition: flow.h:344
char * msg
Definition: detect.h:580
uint32_t id
Definition: detect.h:555
#define FALSE
json_t * JsonSMBAddMetadata(const Flow *f, uint64_t tx_id)
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
void JsonTlsLogJSONExtended(json_t *tjs, SSLState *state)
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
#define LOG_JSON_PAYLOAD
int prio
Definition: detect.h:558
Per flow DNP3 state.
#define ACTION_REJECT
uint8_t action
Definition: decode.h:271
#define LOG_JSON_RULE
uint64_t offset
#define IS_TUNNEL_PKT(p)
Definition: decode.h:881
#define PKT_IS_IPV6(p)
Definition: decode.h:252
const char * value
const char * key
DNP3 transaction.
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1063
#define METADATA_DEFAULTS
void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, json_t *js)
Definition: output-json.c:390
uint16_t AppProto
char * val
Definition: conf.h:34
#define MODULE_NAME
int EngineModeIsIPS(void)
Definition: suricata.c:247
#define SIG_FLAG_DEST_IS_TARGET
Definition: detect.h:251
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
AlertJsonOutputCtx * json_output_ctx
#define LOG_JSON_PACKET
#define TRUE
#define SCMutexLock(mut)
#define PKT_IS_IPV4(p)
Definition: decode.h:251
json_t * JsonHttpAddMetadata(const Flow *f, uint64_t tx_id)
Signature metadata list.
HttpXFFCfg * xff_cfg
Definition: output-json.h:82
#define XFF_MAXLEN
#define SIG_FLAG_SRC_IS_TARGET
Definition: detect.h:249
#define LOG_JSON_PAYLOAD_BASE64
#define JSON_STREAM_BUFFER_SIZE
PacketAlert alerts[PACKET_ALERT_MAX]
Definition: decode.h:294
const struct Signature_ * s
Definition: decode.h:273
SSLv[2.0|3.[0|1|2|3]] state structure.
#define SCCalloc(nm, a)
Definition: util-mem.h:253
void JsonAddFlow(Flow *f, json_t *js, json_t *hjs)
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
void OutputRegisterPacketModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output module.
Definition: output.c:163
#define LOG_JSON_FLOW
#define SCMutexUnlock(mut)
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
uint8_t proto
Definition: decode.h:430
json_t * JsonNFSAddMetadataRPC(const Flow *f, uint64_t tx_id)
void OutputRegisterPacketSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output sub-module.
Definition: output.c:204
json_t * JsonEmailAddMetadata(const Flow *f, uint32_t tx_id)
json_t * JsonSIPAddMetadata(const Flow *f, uint64_t tx_id)
#define ACTION_REJECT_BOTH
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
uint8_t recursion_level
Definition: decode.h:433
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow...
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define DEFAULT_LOG_FILENAME
OutputJsonCommonSettings cfg
Definition: output-json.h:81
LogFileCtx * file_ctx
Definition: output-json.h:79
void JsonHttpLogJSONBodyBase64(json_t *js, Flow *f, uint64_t tx_id)
uint8_t flowflags
Definition: decode.h:439
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:591
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
uint8_t * buffer
Definition: util-buffer.h:28
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
json_t * SCJsonString(const char *val)
Definition: output-json.c:107
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
void PrintStringsToBuffer(uint8_t *dst_buf, uint32_t *dst_buf_offset_ptr, uint32_t dst_buf_size, const uint8_t *src_buf, const uint32_t src_buf_len)
Definition: util-print.c:224
uint32_t gid
Definition: detect.h:556
uint8_t proto
void JsonAlertLogRegister(void)
uint64_t tx_id
Definition: decode.h:274
json_t * JsonNFSAddMetadata(const Flow *f, uint64_t tx_id)
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
void JsonHttpLogJSONBodyPrintable(json_t *js, Flow *f, uint64_t tx_id)
char * class_msg
Definition: detect.h:583
json_t * JsonFTPDataAddMetadata(const Flow *f)
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:222
SCMutex tunnel_mutex
Definition: decode.h:587
#define SIG_FLAG_HAS_TARGET
Definition: detect.h:253
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define XFF_OVERWRITE
char * sig_str
Definition: detect.h:589
#define LOG_JSON_TAGGED_PACKETS
#define SCFree(a)
Definition: util-mem.h:322
void JsonTlsLogJSONBasic(json_t *js, SSLState *ssl_state)
uint16_t tx_id
void JsonSshLogJSON(json_t *tjs, SshState *ssh_state)
uint8_t flags
Definition: decode.h:272
#define LOG_JSON_HTTP_BODY
HttpXFFCfg * parent_xff_cfg
void * data
Definition: tm-modules.h:81
#define PKT_HAS_TAG
Definition: decode.h:1089
SCMutex m
Definition: flow-hash.h:105
uint32_t rev
Definition: detect.h:557
PacketAlerts alerts
Definition: decode.h:555
#define LOG_JSON_RULE_METADATA
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1068
int Base64Encode(const unsigned char *in, unsigned long inlen, unsigned char *out, unsigned long *outlen)
Definition: util-crypt.c:272
struct RSDNSState_ RSDNSState
#define XFF_DISABLED
void JsonPacket(const Packet *p, json_t *js, unsigned long max_length)
Jsonify a packet.
Definition: output-json.c:408
json_t * JsonDNSLogAnswer(void *txptr, uint64_t tx_id)
#define PACKET_ALERT_FLAG_STATE_MATCH
Definition: decode.h:282
bool ConfNodeHasChildren(const ConfNode *node)
Check if a node has any children.
Definition: conf.c:796
json_t * JsonDNSLogQuery(void *txptr, uint64_t tx_id)
#define LOG_JSON_HTTP_BODY_BASE64
struct DetectMetadata_ * next
uint16_t cnt
Definition: decode.h:293
void DetectEngineSetParseMetadata(void)
uint32_t tenant_id
Definition: decode.h:594
OutputJsonCommonSettings cfg
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
struct JsonAlertLogThread_ JsonAlertLogThread
struct timeval ts
Definition: decode.h:451
int ParseSizeStringU32(const char *size, uint32_t *res)
Definition: util-misc.c:186
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
#define PACKET_ALERT_RATE_FILTER_MODIFIED
Definition: decode.h:288
#define ACTION_DROP
uint32_t flags
Definition: decode.h:443
uint16_t payload_len
Definition: decode.h:541
#define likely(expr)
Definition: util-optimize.h:32
#define LOG_JSON_APP_LAYER
uint32_t offset
Definition: util-buffer.h:30
Flow data structure.
Definition: flow.h:325
uint8_t * payload
Definition: decode.h:540
#define XFF_EXTRADATA
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
struct AlertJsonOutputCtx_ AlertJsonOutputCtx
struct Packet_ * root
Definition: decode.h:577
#define DEBUG_VALIDATE_BUG_ON(exp)