suricata
output-json-anomaly.c
Go to the documentation of this file.
1 /* Copyright (C) 2019 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Jeff Lucovsky <jeff@lucovsky.org>
22  *
23  * Logs anomalies in JSON format.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 #include "app-layer.h"
33 #include "app-layer-events.h"
34 #include "app-layer-parser.h"
35 
36 #include "threads.h"
37 #include "tm-threads.h"
38 #include "threadvars.h"
39 #include "util-debug.h"
40 
41 #include "util-misc.h"
42 
43 #include "detect-parse.h"
44 #include "detect-engine.h"
45 #include "util-logopenfile.h"
46 
47 #include "output.h"
48 #include "output-json.h"
49 #include "output-json-anomaly.h"
50 
51 #include "util-byte.h"
52 #include "util-enum.h"
53 #include "util-privs.h"
54 #include "util-print.h"
55 #include "util-proto-name.h"
56 #include "util-optimize.h"
57 #include "util-buffer.h"
58 #include "util-crypt.h"
59 #include "util-validate.h"
60 
61 #define MODULE_NAME "JsonAnomalyLog"
62 
63 #define ANOMALY_EVENT_TYPE "anomaly"
64 #define LOG_JSON_DECODE_TYPE BIT_U16(0)
65 #define LOG_JSON_STREAM_TYPE BIT_U16(1)
66 #define LOG_JSON_APPLAYER_TYPE BIT_U16(2)
67 #define LOG_JSON_PACKETHDR BIT_U16(3)
68 
69 #define LOG_JSON_PACKET_TYPE (LOG_JSON_DECODE_TYPE | LOG_JSON_STREAM_TYPE)
70 #define ANOMALY_DEFAULTS LOG_JSON_APPLAYER_TYPE
71 
72 #define TX_ID_UNUSED UINT64_MAX
73 
74 typedef struct AnomalyJsonOutputCtx_ {
76  uint16_t flags;
79 
80 typedef struct JsonAnomalyLogThread_ {
81  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
86 
87 static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft,
88  const Packet *p)
89 {
90  bool is_ip_pkt = PKT_IS_IPV4(p) || PKT_IS_IPV6(p);
91 
92  char timebuf[64];
93  CreateIsoTimeString(&p->ts, timebuf, sizeof(timebuf));
94 
95  uint16_t log_type = aft->json_output_ctx->flags;
96  bool log_stream = log_type & LOG_JSON_STREAM_TYPE;
97  bool log_decode = log_type & LOG_JSON_DECODE_TYPE;
98  for (int i = 0; i < p->events.cnt; i++) {
99  uint8_t event_code = p->events.events[i];
100  bool is_decode = EVENT_IS_DECODER_PACKET_ERROR(event_code);
101  if (is_decode && !log_decode)
102  continue;
103  if (!is_decode && !log_stream)
104  continue;
105 
107 
109 
110  if (unlikely(js == NULL)) {
111  return TM_ECODE_OK;
112  }
113 
114  json_t *ajs = json_object();
115  if (unlikely(ajs == NULL)) {
116  json_decref(js);
117  return TM_ECODE_OK;
118  }
119 
120  if (!is_ip_pkt) {
121  json_object_set_new(js, "timestamp", json_string(timebuf));
122  } else {
123  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
124  }
125 
127  JsonPacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32);
128  }
129 
130  if (event_code < DECODE_EVENT_MAX) {
131  const char *event = DEvents[event_code].event_name;
132  json_object_set_new(ajs, "type",
133  EVENT_IS_DECODER_PACKET_ERROR(event_code) ?
134  json_string("decode") : json_string("stream"));
135  json_object_set_new(ajs, "event", json_string(event));
136  } else {
137  json_object_set_new(ajs, "type", json_string("unknown"));
138  json_object_set_new(ajs, "code", json_integer(event_code));
139  }
140 
141  /* anomaly */
142  json_object_set_new(js, ANOMALY_EVENT_TYPE, ajs);
143  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
144 
145  json_object_clear(js);
146  json_decref(js);
147  }
148 
149  return TM_ECODE_OK;
150 }
151 
152 static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft,
153  const Packet *p, AppLayerDecoderEvents *decoder_events,
154  bool is_pktlayer, const char *layer, uint64_t tx_id)
155 {
156  const char *alprotoname = AppLayerGetProtoName(p->flow->alproto);
157 
158  SCLogDebug("decoder_events %p event_count %d (last logged %d) %s",
159  decoder_events, decoder_events->cnt,
160  decoder_events->event_last_logged,
161  tx_id != TX_ID_UNUSED ? "tx" : "no-tx");
162 
163  for (int i = decoder_events->event_last_logged; i < decoder_events->cnt; i++) {
165 
166  json_t *js;
167  if (tx_id != TX_ID_UNUSED) {
169  ANOMALY_EVENT_TYPE, tx_id);
170  } else {
172  }
173  if (unlikely(js == NULL)) {
174  return TM_ECODE_OK;
175  }
176 
177  json_t *ajs = json_object();
178  if (unlikely(ajs == NULL)) {
179  json_decref(js);
180  return TM_ECODE_OK;
181  }
182 
183  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
184 
185  json_object_set_new(js, "app_proto", json_string(alprotoname));
186 
187  const char *event_name = NULL;
188  uint8_t event_code = decoder_events->events[i];
190  int r;
191  if (is_pktlayer) {
192  r = AppLayerGetEventInfoById(event_code, &event_name, &event_type);
193  } else {
195  event_code, &event_name, &event_type);
196  }
197  if (r == 0) {
198  json_object_set_new(ajs, "type", json_string("applayer"));
199  json_object_set_new(ajs, "event", json_string(event_name));
200  } else {
201  json_object_set_new(ajs, "type", json_string("unknown"));
202  json_object_set_new(ajs, "code", json_integer(event_code));
203  }
204 
205  json_object_set_new(ajs, "layer", json_string(layer));
206 
207  /* anomaly */
208  json_object_set_new(js, ANOMALY_EVENT_TYPE, ajs);
209  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
210 
211  json_object_clear(js);
212  json_decref(js);
213 
214  /* Current implementation assumes a single owner for this value */
215  decoder_events->event_last_logged++;
216  }
217 
218  return TM_ECODE_OK;
219 }
220 
221 static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet *p,
222  Flow *f, void *state, void *tx, uint64_t tx_id)
223 {
224  JsonAnomalyLogThread *aft = thread_data;
226  return TM_ECODE_OK;
227  }
228 
229  AppLayerDecoderEvents *decoder_events;
230  decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx);
231  if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) {
232  SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id);
233  AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false,
234  "proto_parser", tx_id);
235  }
236  return TM_ECODE_OK;
237 }
238 
239 static inline bool AnomalyHasParserEvents(const Packet *p)
240 {
241  return (p->flow && p->flow->alparser &&
243 }
244 
245 static inline bool AnomalyHasPacketAppLayerEvents(const Packet *p)
246 {
247  return p->app_layer_events && p->app_layer_events->cnt;
248 }
249 
250 static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *p)
251 {
252 
253  int rc = TM_ECODE_OK;
254 
255  /* decode or stream */
257  if (p->events.cnt) {
258  rc = AnomalyDecodeEventJson(tv, aft, p);
259  }
260  }
261 
262  /* applayer */
264  /* app layer proto detect events */
265  if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) {
266  rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events,
267  true, "proto_detect", TX_ID_UNUSED);
268  }
269 
270  /* parser state events */
271  if (rc == TM_ECODE_OK && AnomalyHasParserEvents(p)) {
272  SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto);
274  if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) {
275  rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events,
276  false, "parser", TX_ID_UNUSED);
277  }
278  }
279  }
280 
281  return rc;
282 }
283 
284 static int JsonAnomalyLogger(ThreadVars *tv, void *thread_data, const Packet *p)
285 {
286  JsonAnomalyLogThread *aft = thread_data;
287  return AnomalyJson(tv, aft, p);
288 }
289 
290 static int JsonAnomalyLogCondition(ThreadVars *tv, const Packet *p)
291 {
292  return p->events.cnt > 0 ||
293  (p->app_layer_events && p->app_layer_events->cnt > 0) ||
294  AnomalyHasParserEvents(p);
295 }
296 
297 static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, void **data)
298 {
300  if (unlikely(aft == NULL)) {
301  return TM_ECODE_FAILED;
302  }
303 
304  if (initdata == NULL) {
305  SCLogDebug("Error getting context for EveLogAnomaly. \"initdata\" argument NULL");
306  SCFree(aft);
307  return TM_ECODE_FAILED;
308  }
309 
311  if (aft->json_buffer == NULL) {
312  SCFree(aft);
313  return TM_ECODE_FAILED;
314  }
315 
316  /** Use the Output Context (file pointer and mutex) */
317  AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
318  aft->file_ctx = json_output_ctx->file_ctx;
319  aft->json_output_ctx = json_output_ctx;
320 
321  *data = (void *)aft;
322  return TM_ECODE_OK;
323 }
324 
325 static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data)
326 {
328  if (aft == NULL) {
329  return TM_ECODE_OK;
330  }
331 
333 
334  /* clear memory */
335  memset(aft, 0, sizeof(JsonAnomalyLogThread));
336 
337  SCFree(aft);
338  return TM_ECODE_OK;
339 }
340 
341 static void JsonAnomalyLogDeInitCtxSub(OutputCtx *output_ctx)
342 {
343  SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
344 
345  AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data;
346 
347  if (json_output_ctx != NULL) {
348  SCFree(json_output_ctx);
349  }
350  SCFree(output_ctx);
351 }
352 
353 #define DEFAULT_LOG_FILENAME "anomaly.json"
354 static void SetFlag(const ConfNode *conf, const char *name, uint16_t flag, uint16_t *out_flags)
355 {
356  DEBUG_VALIDATE_BUG_ON(conf == NULL);
357  const char *setting = ConfNodeLookupChildValue(conf, name);
358  if (setting != NULL) {
359  if (ConfValIsTrue(setting)) {
360  *out_flags |= flag;
361  } else {
362  *out_flags &= ~flag;
363  }
364  }
365 }
366 
367 static void JsonAnomalyLogConf(AnomalyJsonOutputCtx *json_output_ctx,
368  ConfNode *conf)
369 {
370  static bool warn_no_flags = false;
371  static bool warn_no_packet = false;
372  uint16_t flags = ANOMALY_DEFAULTS;
373  if (conf != NULL) {
374  /* Check for metadata to enable/disable. */
375  ConfNode *typeconf = ConfNodeLookupChild(conf, "types");
376  if (typeconf != NULL) {
377  SetFlag(typeconf, "applayer", LOG_JSON_APPLAYER_TYPE, &flags);
378  SetFlag(typeconf, "stream", LOG_JSON_STREAM_TYPE, &flags);
379  SetFlag(typeconf, "decode", LOG_JSON_DECODE_TYPE, &flags);
380  }
381  SetFlag(conf, "packethdr", LOG_JSON_PACKETHDR, &flags);
382  }
383  if (((flags & (LOG_JSON_DECODE_TYPE | LOG_JSON_PACKETHDR)) == LOG_JSON_PACKETHDR) && !warn_no_packet) {
384  SCLogWarning(SC_WARN_ANOMALY_CONFIG, "Anomaly logging configured to include packet headers, however decode "
385  "type logging has not been selected. Packet headers will not be logged.");
386  warn_no_packet = true;
387  flags &= ~LOG_JSON_PACKETHDR;
388  }
389 
390  if (flags == 0 && !warn_no_flags) {
391  SCLogWarning(SC_WARN_ANOMALY_CONFIG, "Anomaly logging has been configured; however, no logging types "
392  "have been selected. Select one or more logging types.");
393  warn_no_flags = true;
394  }
395  json_output_ctx->flags |= flags;
396 }
397 
398 /**
399  * \brief Create a new LogFileCtx for "fast" output style.
400  * \param conf The configuration node for this output.
401  * \return A LogFileCtx pointer on success, NULL on failure.
402  */
403 static OutputInitResult JsonAnomalyLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
404 {
405  OutputInitResult result = { NULL, false };
406  OutputJsonCtx *ajt = parent_ctx->data;
407  AnomalyJsonOutputCtx *json_output_ctx = NULL;
408 
409  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
410  if (unlikely(output_ctx == NULL))
411  return result;
412 
413  json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx));
414  if (unlikely(json_output_ctx == NULL)) {
415  goto error;
416  }
417 
418  json_output_ctx->file_ctx = ajt->file_ctx;
419  JsonAnomalyLogConf(json_output_ctx, conf);
420  json_output_ctx->cfg = ajt->cfg;
421 
422  output_ctx->data = json_output_ctx;
423  output_ctx->DeInit = JsonAnomalyLogDeInitCtxSub;
424 
425  result.ctx = output_ctx;
426  result.ok = true;
427  return result;
428 
429 error:
430  SCFree(output_ctx);
431 
432  return result;
433 }
434 
436 {
438  "eve-log.anomaly", JsonAnomalyLogInitCtxSub, JsonAnomalyLogger,
439  JsonAnomalyLogCondition, JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit,
440  NULL);
441 
443  "eve-log.anomaly", JsonAnomalyLogInitCtxSub, ALPROTO_UNKNOWN,
444  JsonAnomalyTxLogger, JsonAnomalyLogThreadInit,
445  JsonAnomalyLogThreadDeinit, NULL);
446 }
AppLayerDecoderEvents * AppLayerParserGetDecoderEvents(AppLayerParserState *pstate)
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
enum AppLayerEventType_ AppLayerEventType
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:44
json_t * CreateJSONHeader(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type)
Definition: output-json.c:710
#define SCLogDebug(...)
Definition: util-debug.h:335
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:809
#define LOG_JSON_APPLAYER_TYPE
struct Flow_ * flow
Definition: decode.h:445
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
uint32_t event_type
uint8_t proto
Definition: flow.h:344
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
json_t * CreateJSONHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, uint64_t tx_id)
Definition: output-json.c:783
#define PKT_IS_IPV6(p)
Definition: decode.h:252
#define LOG_JSON_PACKET_TYPE
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
uint8_t events[PACKET_ENGINE_EVENT_MAX]
Definition: decode.h:307
struct AnomalyJsonOutputCtx_ AnomalyJsonOutputCtx
AppLayerDecoderEvents * AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alproto, void *tx)
int AppLayerParserGetEventInfoById(uint8_t ipproto, AppProto alproto, int event_id, const char **event_name, AppLayerEventType *event_type)
#define ANOMALY_EVENT_TYPE
void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, json_t *js)
Definition: output-json.c:390
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
#define ANOMALY_DEFAULTS
#define PKT_IS_IPV4(p)
Definition: decode.h:251
#define SCCalloc(nm, a)
Definition: util-mem.h:253
Data structure to store app layer decoder events.
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
void OutputRegisterPacketSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output sub-module.
Definition: output.c:204
OutputJsonCommonSettings cfg
void JsonAnomalyLogRegister(void)
OutputJsonCommonSettings cfg
Definition: output-json.h:81
LogFileCtx * file_ctx
Definition: output-json.h:79
PacketEngineEvents events
Definition: decode.h:565
#define LOG_JSON_PACKETHDR
const struct DecodeEvents_ DEvents[]
Definition: decode-events.c:29
#define MODULE_NAME
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
void OutputRegisterTxSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output.c:402
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define LOG_JSON_STREAM_TYPE
Definition: conf.h:32
int AppLayerGetEventInfoById(int event_id, const char **event_name, AppLayerEventType *event_type)
OutputCtx * ctx
Definition: output.h:42
struct JsonAnomalyLogThread_ JsonAnomalyLogThread
const char * event_name
#define SCFree(a)
Definition: util-mem.h:322
uint16_t tx_id
AnomalyJsonOutputCtx * json_output_ctx
void * data
Definition: tm-modules.h:81
#define LOG_JSON_DECODE_TYPE
void JsonPacket(const Packet *p, json_t *js, unsigned long max_length)
Jsonify a packet.
Definition: output-json.c:408
#define EVENT_IS_DECODER_PACKET_ERROR(e)
const char * AppLayerGetProtoName(AppProto alproto)
Given the internal protocol id, returns a string representation of the protocol.
Definition: app-layer.c:766
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:451
AppProto alproto
application level protocol
Definition: flow.h:409
#define GET_PKT_LEN(p)
Definition: decode.h:224
bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
Flow data structure.
Definition: flow.h:325
AppLayerParserState * alparser
Definition: flow.h:437
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
#define DEBUG_VALIDATE_BUG_ON(exp)
#define TX_ID_UNUSED