suricata
output-json-anomaly.c
Go to the documentation of this file.
1 /* Copyright (C) 2019 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Jeff Lucovsky <jeff@lucovsky.org>
22  *
23  * Logs anomalies in JSON format.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 #include "app-layer.h"
33 #include "app-layer-events.h"
34 #include "app-layer-parser.h"
35 
36 #include "threads.h"
37 #include "tm-threads.h"
38 #include "threadvars.h"
39 #include "util-debug.h"
40 
41 #include "util-misc.h"
42 
43 #include "detect-parse.h"
44 #include "detect-engine.h"
45 #include "util-logopenfile.h"
46 
47 #include "output.h"
48 #include "output-json.h"
49 #include "output-json-anomaly.h"
50 
51 #include "util-byte.h"
52 #include "util-enum.h"
53 #include "util-privs.h"
54 #include "util-print.h"
55 #include "util-proto-name.h"
56 #include "util-optimize.h"
57 #include "util-buffer.h"
58 #include "util-crypt.h"
59 #include "util-validate.h"
60 
61 #define MODULE_NAME "JsonAnomalyLog"
62 
63 #define ANOMALY_EVENT_TYPE "anomaly"
64 #define LOG_JSON_DECODE_TYPE BIT_U16(0)
65 #define LOG_JSON_STREAM_TYPE BIT_U16(1)
66 #define LOG_JSON_APPLAYER_TYPE BIT_U16(2)
67 #define LOG_JSON_PACKETHDR BIT_U16(3)
68 
69 #define LOG_JSON_PACKET_TYPE (LOG_JSON_DECODE_TYPE | LOG_JSON_STREAM_TYPE)
70 #define ANOMALY_DEFAULTS LOG_JSON_APPLAYER_TYPE
71 
72 #define TX_ID_UNUSED UINT64_MAX
73 
74 typedef struct AnomalyJsonOutputCtx_ {
76  uint16_t flags;
77  OutputJsonCommonSettings cfg;
79 
80 typedef struct JsonAnomalyLogThread_ {
81  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
86 
87 static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft,
88  const Packet *p)
89 {
90  bool is_ip_pkt = PKT_IS_IPV4(p) || PKT_IS_IPV6(p);
91 
92  char timebuf[64];
93  CreateIsoTimeString(&p->ts, timebuf, sizeof(timebuf));
94 
95  uint16_t log_type = aft->json_output_ctx->flags;
96  bool log_stream = log_type & LOG_JSON_STREAM_TYPE;
97  bool log_decode = log_type & LOG_JSON_DECODE_TYPE;
98  for (int i = 0; i < p->events.cnt; i++) {
99  uint8_t event_code = p->events.events[i];
100  bool is_decode = EVENT_IS_DECODER_PACKET_ERROR(event_code);
101  if (is_decode && !log_decode)
102  continue;
103  if (!is_decode && !log_stream)
104  continue;
105 
107 
108  json_t *js = CreateJSONHeader(p, LOG_DIR_PACKET, ANOMALY_EVENT_TYPE);
109 
110  if (unlikely(js == NULL)) {
111  return TM_ECODE_OK;
112  }
113 
114  json_t *ajs = json_object();
115  if (unlikely(ajs == NULL)) {
116  json_decref(js);
117  return TM_ECODE_OK;
118  }
119 
120  if (!is_ip_pkt) {
121  json_object_set_new(js, "timestamp", json_string(timebuf));
122  } else {
123  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
124  }
125 
127  JsonPacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32);
128  }
129 
130  if (event_code < DECODE_EVENT_MAX) {
131  const char *event = DEvents[event_code].event_name;
132  json_object_set_new(ajs, "type",
133  EVENT_IS_DECODER_PACKET_ERROR(event_code) ?
134  json_string("decode") : json_string("stream"));
135  json_object_set_new(ajs, "event", json_string(event));
136  } else {
137  json_object_set_new(ajs, "type", json_string("unknown"));
138  json_object_set_new(ajs, "code", json_integer(event_code));
139  }
140 
141  /* anomaly */
142  json_object_set_new(js, ANOMALY_EVENT_TYPE, ajs);
143  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
144 
145  json_object_clear(js);
146  json_decref(js);
147  }
148 
149  return TM_ECODE_OK;
150 }
151 
152 static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft,
153  const Packet *p, AppLayerDecoderEvents *decoder_events,
154  bool is_pktlayer, const char *layer, uint64_t tx_id)
155 {
156  const char *alprotoname = AppLayerGetProtoName(p->flow->alproto);
157 
158  SCLogDebug("decoder_events %p event_count %d (last logged %d) %s",
159  decoder_events, decoder_events->cnt,
160  decoder_events->event_last_logged,
161  tx_id != TX_ID_UNUSED ? "tx" : "no-tx");
162 
163  for (int i = decoder_events->event_last_logged; i < decoder_events->cnt; i++) {
165 
166  json_t *js;
167  if (tx_id != TX_ID_UNUSED) {
168  js = CreateJSONHeaderWithTxId(p, LOG_DIR_PACKET,
169  ANOMALY_EVENT_TYPE, tx_id);
170  } else {
171  js = CreateJSONHeader(p, LOG_DIR_PACKET, ANOMALY_EVENT_TYPE);
172  }
173  if (unlikely(js == NULL)) {
174  return TM_ECODE_OK;
175  }
176 
177  json_t *ajs = json_object();
178  if (unlikely(ajs == NULL)) {
179  json_decref(js);
180  return TM_ECODE_OK;
181  }
182 
183  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
184 
185  json_object_set_new(js, "app_proto", json_string(alprotoname));
186 
187  const char *event_name = NULL;
188  uint8_t event_code = decoder_events->events[i];
190  int r;
191  if (is_pktlayer) {
192  r = AppLayerGetEventInfoById(event_code, &event_name, &event_type);
193  } else {
195  event_code, &event_name, &event_type);
196  }
197  if (r == 0) {
198  json_object_set_new(ajs, "type", json_string("applayer"));
199  json_object_set_new(ajs, "event", json_string(event_name));
200  } else {
201  json_object_set_new(ajs, "type", json_string("unknown"));
202  json_object_set_new(ajs, "code", json_integer(event_code));
203  }
204 
205  /* log event number (and total) */
206  char event_no_buf[16];
207  snprintf(event_no_buf, sizeof(event_no_buf), "%d (of %d)",
208  i+1, decoder_events->cnt);
209  json_object_set_new(ajs, "event_no", json_string(event_no_buf));
210  json_object_set_new(ajs, "layer", json_string(layer));
211 
212  /* anomaly */
213  json_object_set_new(js, ANOMALY_EVENT_TYPE, ajs);
214  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
215 
216  json_object_clear(js);
217  json_decref(js);
218 
219  /* Current implementation assumes a single owner for this value */
220  decoder_events->event_last_logged++;
221  }
222 
223  return TM_ECODE_OK;
224 }
225 
226 static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet *p,
227  Flow *f, void *state, void *tx, uint64_t tx_id)
228 {
229  JsonAnomalyLogThread *aft = thread_data;
231  return TM_ECODE_OK;
232  }
233 
234  AppLayerDecoderEvents *decoder_events;
235  decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx);
236  if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) {
237  SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id);
238  AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false,
239  "proto_parser", tx_id);
240  }
241  return TM_ECODE_OK;
242 }
243 
244 static inline bool AnomalyHasParserEvents(const Packet *p)
245 {
246  return (p->flow && p->flow->alparser &&
248 }
249 
250 static inline bool AnomalyHasPacketAppLayerEvents(const Packet *p)
251 {
252  return p->app_layer_events && p->app_layer_events->cnt;
253 }
254 
255 static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *p)
256 {
257 
258  int rc = TM_ECODE_OK;
259 
260  /* decode or stream */
262  if (p->events.cnt) {
263  rc = AnomalyDecodeEventJson(tv, aft, p);
264  }
265  }
266 
267  /* applayer */
269  /* app layer proto detect events */
270  if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) {
271  rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events,
272  true, "proto_detect", TX_ID_UNUSED);
273  }
274 
275  /* parser state events */
276  if (rc == TM_ECODE_OK && AnomalyHasParserEvents(p)) {
277  SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto);
279  if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) {
280  rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events,
281  false, "parser", TX_ID_UNUSED);
282  }
283  }
284  }
285 
286  return rc;
287 }
288 
289 static int JsonAnomalyLogger(ThreadVars *tv, void *thread_data, const Packet *p)
290 {
291  JsonAnomalyLogThread *aft = thread_data;
292  return AnomalyJson(tv, aft, p);
293 }
294 
295 static int JsonAnomalyLogCondition(ThreadVars *tv, const Packet *p)
296 {
297  return p->events.cnt > 0 ||
298  (p->app_layer_events && p->app_layer_events->cnt > 0) ||
299  AnomalyHasParserEvents(p);
300 }
301 
302 static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, void **data)
303 {
305  if (unlikely(aft == NULL)) {
306  return TM_ECODE_FAILED;
307  }
308 
309  if (initdata == NULL) {
310  SCLogDebug("Error getting context for EveLogAnomaly. \"initdata\" argument NULL");
311  SCFree(aft);
312  return TM_ECODE_FAILED;
313  }
314 
315  aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
316  if (aft->json_buffer == NULL) {
317  SCFree(aft);
318  return TM_ECODE_FAILED;
319  }
320 
321  /** Use the Output Context (file pointer and mutex) */
322  AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
323  aft->file_ctx = json_output_ctx->file_ctx;
324  aft->json_output_ctx = json_output_ctx;
325 
326  *data = (void *)aft;
327  return TM_ECODE_OK;
328 }
329 
330 static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data)
331 {
333  if (aft == NULL) {
334  return TM_ECODE_OK;
335  }
336 
338 
339  /* clear memory */
340  memset(aft, 0, sizeof(JsonAnomalyLogThread));
341 
342  SCFree(aft);
343  return TM_ECODE_OK;
344 }
345 
346 static void JsonAnomalyLogDeInitCtxSub(OutputCtx *output_ctx)
347 {
348  SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
349 
350  AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data;
351 
352  if (json_output_ctx != NULL) {
353  SCFree(json_output_ctx);
354  }
355  SCFree(output_ctx);
356 }
357 
358 #define DEFAULT_LOG_FILENAME "anomaly.json"
359 static void SetFlag(const ConfNode *conf, const char *name, uint16_t flag, uint16_t *out_flags)
360 {
361  DEBUG_VALIDATE_BUG_ON(conf == NULL);
362  const char *setting = ConfNodeLookupChildValue(conf, name);
363  if (setting != NULL) {
364  if (ConfValIsTrue(setting)) {
365  *out_flags |= flag;
366  } else {
367  *out_flags &= ~flag;
368  }
369  }
370 }
371 
372 static void JsonAnomalyLogConf(AnomalyJsonOutputCtx *json_output_ctx,
373  ConfNode *conf)
374 {
375  static bool _warn_no_flags = false;
376  static bool _warn_no_packet = false;
377  uint16_t flags = ANOMALY_DEFAULTS;
378  if (conf != NULL) {
379  /* Check for metadata to enable/disable. */
380  ConfNode *typeconf = ConfNodeLookupChild(conf, "types");
381  if (typeconf != NULL) {
382  SetFlag(typeconf, "applayer", LOG_JSON_APPLAYER_TYPE, &flags);
383  SetFlag(typeconf, "stream", LOG_JSON_STREAM_TYPE, &flags);
384  SetFlag(typeconf, "decode", LOG_JSON_DECODE_TYPE, &flags);
385  }
386  SetFlag(conf, "packethdr", LOG_JSON_PACKETHDR, &flags);
387  }
388  if (((flags & (LOG_JSON_DECODE_TYPE | LOG_JSON_PACKETHDR)) == LOG_JSON_PACKETHDR) && !_warn_no_packet) {
389  SCLogWarning(SC_WARN_ANOMALY_CONFIG, "Anomaly logging configured to include packet headers, however decode "
390  "type logging has not been selected. Packet headers will not be logged.");
391  _warn_no_packet = true;
392  flags &= ~LOG_JSON_PACKETHDR;
393  }
394 
395  if (flags == 0 && !_warn_no_flags) {
396  SCLogWarning(SC_WARN_ANOMALY_CONFIG, "Anomaly logging has been configured; however, no logging types "
397  "have been selected. Select one or more logging types.");
398  _warn_no_flags = true;
399  }
400  json_output_ctx->flags |= flags;
401 }
402 
403 /**
404  * \brief Create a new LogFileCtx for "fast" output style.
405  * \param conf The configuration node for this output.
406  * \return A LogFileCtx pointer on success, NULL on failure.
407  */
408 static OutputInitResult JsonAnomalyLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
409 {
410  OutputInitResult result = { NULL, false };
411  OutputJsonCtx *ajt = parent_ctx->data;
412  AnomalyJsonOutputCtx *json_output_ctx = NULL;
413 
414  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
415  if (unlikely(output_ctx == NULL))
416  return result;
417 
418  json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx));
419  if (unlikely(json_output_ctx == NULL)) {
420  goto error;
421  }
422 
423  json_output_ctx->file_ctx = ajt->file_ctx;
424  JsonAnomalyLogConf(json_output_ctx, conf);
425  json_output_ctx->cfg = ajt->cfg;
426 
427  output_ctx->data = json_output_ctx;
428  output_ctx->DeInit = JsonAnomalyLogDeInitCtxSub;
429 
430  result.ctx = output_ctx;
431  result.ok = true;
432  return result;
433 
434 error:
435  SCFree(output_ctx);
436 
437  return result;
438 }
439 
441 {
443  "eve-log.anomaly", JsonAnomalyLogInitCtxSub, JsonAnomalyLogger,
444  JsonAnomalyLogCondition, JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit,
445  NULL);
446 
448  "eve-log.anomaly", JsonAnomalyLogInitCtxSub, ALPROTO_UNKNOWN,
449  JsonAnomalyTxLogger, JsonAnomalyLogThreadInit,
450  JsonAnomalyLogThreadDeinit, NULL);
451 }
AppLayerDecoderEvents * AppLayerParserGetDecoderEvents(AppLayerParserState *pstate)
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
enum AppLayerEventType_ AppLayerEventType
#define SCLogDebug(...)
Definition: util-debug.h:335
#define LOG_JSON_APPLAYER_TYPE
struct Flow_ * flow
Definition: decode.h:445
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
uint32_t event_type
uint8_t proto
Definition: flow.h:344
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
#define PKT_IS_IPV6(p)
Definition: decode.h:252
#define LOG_JSON_PACKET_TYPE
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
uint8_t events[PACKET_ENGINE_EVENT_MAX]
Definition: decode.h:307
struct AnomalyJsonOutputCtx_ AnomalyJsonOutputCtx
AppLayerDecoderEvents * AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alproto, void *tx)
int AppLayerParserGetEventInfoById(uint8_t ipproto, AppProto alproto, int event_id, const char **event_name, AppLayerEventType *event_type)
#define ANOMALY_EVENT_TYPE
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
#define ANOMALY_DEFAULTS
#define PKT_IS_IPV4(p)
Definition: decode.h:251
#define SCCalloc(nm, a)
Definition: util-mem.h:253
Data structure to store app layer decoder events.
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
void OutputRegisterPacketSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output sub-module.
Definition: output.c:204
OutputJsonCommonSettings cfg
void JsonAnomalyLogRegister(void)
PacketEngineEvents events
Definition: decode.h:565
#define LOG_JSON_PACKETHDR
const struct DecodeEvents_ DEvents[]
Definition: decode-events.c:29
#define MODULE_NAME
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
void OutputRegisterTxSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output.c:402
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define LOG_JSON_STREAM_TYPE
Definition: conf.h:32
int AppLayerGetEventInfoById(int event_id, const char **event_name, AppLayerEventType *event_type)
OutputCtx * ctx
Definition: output.h:42
struct JsonAnomalyLogThread_ JsonAnomalyLogThread
const char * event_name
#define SCFree(a)
Definition: util-mem.h:322
uint16_t tx_id
AnomalyJsonOutputCtx * json_output_ctx
void * data
Definition: tm-modules.h:81
#define LOG_JSON_DECODE_TYPE
#define EVENT_IS_DECODER_PACKET_ERROR(e)
const char * AppLayerGetProtoName(AppProto alproto)
Given the internal protocol id, returns a string representation of the protocol.
Definition: app-layer.c:766
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:451
AppProto alproto
application level protocol
Definition: flow.h:409
#define GET_PKT_LEN(p)
Definition: decode.h:224
bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
Flow data structure.
Definition: flow.h:325
AppLayerParserState * alparser
Definition: flow.h:437
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
#define DEBUG_VALIDATE_BUG_ON(exp)
#define TX_ID_UNUSED