suricata
output-json-anomaly.c
Go to the documentation of this file.
1 /* Copyright (C) 2019 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Jeff Lucovsky <jeff@lucovsky.org>
22  *
23  * Logs anomalies in JSON format.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 #include "app-layer.h"
33 #include "app-layer-events.h"
34 #include "app-layer-parser.h"
35 
36 #include "threads.h"
37 #include "tm-threads.h"
38 #include "threadvars.h"
39 #include "util-debug.h"
40 
41 #include "util-misc.h"
42 
43 #include "detect-parse.h"
44 #include "detect-engine.h"
45 #include "util-logopenfile.h"
46 
47 #include "output.h"
48 #include "output-json.h"
49 #include "output-json-anomaly.h"
50 
51 #include "util-byte.h"
52 #include "util-enum.h"
53 #include "util-privs.h"
54 #include "util-print.h"
55 #include "util-proto-name.h"
56 #include "util-optimize.h"
57 #include "util-buffer.h"
58 #include "util-crypt.h"
59 #include "util-validate.h"
60 
61 #define MODULE_NAME "JsonAnomalyLog"
62 
63 #ifdef HAVE_LIBJANSSON
64 
65 #define ANOMALY_EVENT_TYPE "anomaly"
66 #define LOG_JSON_PACKETHDR BIT_U16(0)
67 #define TX_ID_UNUSED UINT64_MAX
68 
69 typedef struct AnomalyJsonOutputCtx_ {
70  LogFileCtx* file_ctx;
71  uint16_t flags;
72  OutputJsonCommonSettings cfg;
73 } AnomalyJsonOutputCtx;
74 
75 typedef struct JsonAnomalyLogThread_ {
76  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
77  LogFileCtx* file_ctx;
78  MemBuffer *json_buffer;
79  AnomalyJsonOutputCtx* json_output_ctx;
80 } JsonAnomalyLogThread;
81 
82 static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft,
83  const Packet *p)
84 {
85  bool is_ip_pkt = PKT_IS_IPV4(p) || PKT_IS_IPV6(p);
86 
87  char timebuf[64];
88  CreateIsoTimeString(&p->ts, timebuf, sizeof(timebuf));
89 
90  for (int i = 0; i < p->events.cnt; i++) {
91  MemBufferReset(aft->json_buffer);
92 
93  json_t *js = CreateJSONHeader(p, LOG_DIR_PACKET, ANOMALY_EVENT_TYPE);
94 
95  if (unlikely(js == NULL)) {
96  return TM_ECODE_OK;
97  }
98 
99  json_t *ajs = json_object();
100  if (unlikely(ajs == NULL)) {
101  json_decref(js);
102  return TM_ECODE_OK;
103  }
104 
105  if (!is_ip_pkt) {
106  json_object_set_new(js, "timestamp", json_string(timebuf));
107  } else {
108  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
109  }
110 
111  if (aft->json_output_ctx->flags & LOG_JSON_PACKETHDR) {
112  JsonPacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32);
113  }
114 
115  uint8_t event_code = p->events.events[i];
116  if (event_code < DECODE_EVENT_MAX) {
117  const char *event = DEvents[event_code].event_name;
118  json_object_set_new(ajs, "type",
119  EVENT_IS_DECODER_PACKET_ERROR(event_code) ?
120  json_string("packet") : json_string("stream"));
121  json_object_set_new(ajs, "event", json_string(event));
122  } else {
123  json_object_set_new(ajs, "type", json_string("unknown"));
124  json_object_set_new(ajs, "code", json_integer(event_code));
125  }
126 
127  /* anomaly */
128  json_object_set_new(js, ANOMALY_EVENT_TYPE, ajs);
129  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
130 
131  json_object_clear(js);
132  json_decref(js);
133  }
134 
135  return TM_ECODE_OK;
136 }
137 
138 static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft,
139  const Packet *p, AppLayerDecoderEvents *decoder_events,
140  bool is_pktlayer, const char *layer, uint64_t tx_id)
141 {
142  const char *alprotoname = AppLayerGetProtoName(p->flow->alproto);
143 
144  SCLogDebug("decoder_events %p event_count %d (last logged %d) %s",
145  decoder_events, decoder_events->cnt,
146  decoder_events->event_last_logged,
147  tx_id != TX_ID_UNUSED ? "tx" : "no-tx");
148 
149  for (int i = decoder_events->event_last_logged; i < decoder_events->cnt; i++) {
150  MemBufferReset(aft->json_buffer);
151 
152  json_t *js;
153  if (tx_id != TX_ID_UNUSED) {
154  js = CreateJSONHeaderWithTxId(p, LOG_DIR_PACKET,
155  ANOMALY_EVENT_TYPE, tx_id);
156  } else {
157  js = CreateJSONHeader(p, LOG_DIR_PACKET, ANOMALY_EVENT_TYPE);
158  }
159  if (unlikely(js == NULL)) {
160  return TM_ECODE_OK;
161  }
162 
163  json_t *ajs = json_object();
164  if (unlikely(ajs == NULL)) {
165  json_decref(js);
166  return TM_ECODE_OK;
167  }
168 
169  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
170 
171  json_object_set_new(js, "app_proto", json_string(alprotoname));
172 
173  const char *event_name = NULL;
174  uint8_t event_code = decoder_events->events[i];
176  int r;
177  if (is_pktlayer) {
178  r = AppLayerGetEventInfoById(event_code, &event_name, &event_type);
179  } else {
181  event_code, &event_name, &event_type);
182  }
183  if (r == 0) {
184  json_object_set_new(ajs, "type", json_string("applayer"));
185  json_object_set_new(ajs, "event", json_string(event_name));
186  } else {
187  json_object_set_new(ajs, "type", json_string("unknown"));
188  json_object_set_new(ajs, "code", json_integer(event_code));
189  }
190 
191  /* log event number (and total) */
192  char event_no_buf[16];
193  snprintf(event_no_buf, sizeof(event_no_buf), "%d (of %d)",
194  i+1, decoder_events->cnt);
195  json_object_set_new(ajs, "event_no", json_string(event_no_buf));
196  json_object_set_new(ajs, "layer", json_string(layer));
197 
198  /* anomaly */
199  json_object_set_new(js, ANOMALY_EVENT_TYPE, ajs);
200  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
201 
202  json_object_clear(js);
203  json_decref(js);
204 
205  /* Current implementation assumes a single owner for this value */
206  decoder_events->event_last_logged++;
207  }
208 
209  return TM_ECODE_OK;
210 }
211 
212 static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet *p,
213  Flow *f, void *state, void *tx, uint64_t tx_id)
214 {
215  AppLayerDecoderEvents *decoder_events;
216  decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx);
217  if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) {
218  SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id);
219  JsonAnomalyLogThread *aft = thread_data;
220  AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false,
221  "proto_parser", tx_id);
222  }
223  return TM_ECODE_OK;
224 }
225 
226 static inline bool AnomalyHasParserEvents(const Packet *p)
227 {
228  return (p->flow && p->flow->alparser &&
230 }
231 
232 static inline bool AnomalyHasPacketAppLayerEvents(const Packet *p)
233 {
234  return p->app_layer_events && p->app_layer_events->cnt;
235 }
236 
237 static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *p)
238 {
239 
240  int rc = TM_ECODE_OK;
241 
242  if (p->events.cnt) {
243  rc = AnomalyDecodeEventJson(tv, aft, p);
244  }
245 
246  /* app layer events */
247  if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) {
248  rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events,
249  true, "proto_detect", TX_ID_UNUSED);
250  }
251 
252  /* parser state events */
253  if (rc == TM_ECODE_OK && AnomalyHasParserEvents(p)) {
254  SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto);
256  if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) {
257  rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events,
258  false, "parser", TX_ID_UNUSED);
259  }
260  }
261 
262  return rc;
263 }
264 
265 static int JsonAnomalyLogger(ThreadVars *tv, void *thread_data, const Packet *p)
266 {
267  JsonAnomalyLogThread *aft = thread_data;
268  return AnomalyJson(tv, aft, p);
269 }
270 
271 static int JsonAnomalyLogCondition(ThreadVars *tv, const Packet *p)
272 {
273  return p->events.cnt > 0 ||
274  (p->app_layer_events && p->app_layer_events->cnt > 0) ||
275  AnomalyHasParserEvents(p);
276 }
277 
278 static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, void **data)
279 {
280  JsonAnomalyLogThread *aft = SCCalloc(1, sizeof(JsonAnomalyLogThread));
281  if (unlikely(aft == NULL)) {
282  return TM_ECODE_FAILED;
283  }
284 
285  if (initdata == NULL) {
286  SCLogDebug("Error getting context for EveLogAnomaly. \"initdata\" argument NULL");
287  SCFree(aft);
288  return TM_ECODE_FAILED;
289  }
290 
291  aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
292  if (aft->json_buffer == NULL) {
293  SCFree(aft);
294  return TM_ECODE_FAILED;
295  }
296 
297  /** Use the Output Context (file pointer and mutex) */
298  AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
299  aft->file_ctx = json_output_ctx->file_ctx;
300  aft->json_output_ctx = json_output_ctx;
301 
302  *data = (void *)aft;
303  return TM_ECODE_OK;
304 }
305 
306 static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data)
307 {
308  JsonAnomalyLogThread *aft = (JsonAnomalyLogThread *)data;
309  if (aft == NULL) {
310  return TM_ECODE_OK;
311  }
312 
313  MemBufferFree(aft->json_buffer);
314 
315  /* clear memory */
316  memset(aft, 0, sizeof(JsonAnomalyLogThread));
317 
318  SCFree(aft);
319  return TM_ECODE_OK;
320 }
321 
322 static void JsonAnomalyLogDeInitCtxSub(OutputCtx *output_ctx)
323 {
324  SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
325 
326  AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data;
327 
328  if (json_output_ctx != NULL) {
329  SCFree(json_output_ctx);
330  }
331  SCFree(output_ctx);
332 }
333 
334 #define DEFAULT_LOG_FILENAME "anomaly.json"
335 static void SetFlag(const ConfNode *conf, const char *name, uint16_t flag, uint16_t *out_flags)
336 {
337  DEBUG_VALIDATE_BUG_ON(conf == NULL);
338  const char *setting = ConfNodeLookupChildValue(conf, name);
339  if (setting != NULL) {
340  if (ConfValIsTrue(setting)) {
341  *out_flags |= flag;
342  } else {
343  *out_flags &= ~flag;
344  }
345  }
346 }
347 
348 static void JsonAnomalyLogConf(AnomalyJsonOutputCtx *json_output_ctx,
349  ConfNode *conf)
350 {
351  uint16_t flags = 0;
352  if (conf != NULL) {
353  SetFlag(conf, "packethdr", LOG_JSON_PACKETHDR, &flags);
354  }
355  json_output_ctx->flags |= flags;
356 }
357 
358 /**
359  * \brief Create a new LogFileCtx for "fast" output style.
360  * \param conf The configuration node for this output.
361  * \return A LogFileCtx pointer on success, NULL on failure.
362  */
363 static OutputInitResult JsonAnomalyLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
364 {
365  OutputInitResult result = { NULL, false };
366  OutputJsonCtx *ajt = parent_ctx->data;
367  AnomalyJsonOutputCtx *json_output_ctx = NULL;
368 
369  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
370  if (unlikely(output_ctx == NULL))
371  return result;
372 
373  json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx));
374  if (unlikely(json_output_ctx == NULL)) {
375  goto error;
376  }
377 
378  json_output_ctx->file_ctx = ajt->file_ctx;
379  JsonAnomalyLogConf(json_output_ctx, conf);
380  json_output_ctx->cfg = ajt->cfg;
381 
382  output_ctx->data = json_output_ctx;
383  output_ctx->DeInit = JsonAnomalyLogDeInitCtxSub;
384 
385  result.ctx = output_ctx;
386  result.ok = true;
387  return result;
388 
389 error:
390  SCFree(output_ctx);
391 
392  return result;
393 }
394 
395 void JsonAnomalyLogRegister (void)
396 {
398  "eve-log.anomaly", JsonAnomalyLogInitCtxSub, JsonAnomalyLogger,
399  JsonAnomalyLogCondition, JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit,
400  NULL);
401 
403  "eve-log.anomaly", JsonAnomalyLogInitCtxSub, ALPROTO_UNKNOWN,
404  JsonAnomalyTxLogger, JsonAnomalyLogThreadInit,
405  JsonAnomalyLogThreadDeinit, NULL);
406 }
407 
408 #else
409 
411 {
412 }
413 
414 #endif
AppLayerDecoderEvents * AppLayerParserGetDecoderEvents(AppLayerParserState *pstate)
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
enum AppLayerEventType_ AppLayerEventType
uint16_t flags
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:445
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
uint32_t event_type
uint8_t proto
Definition: flow.h:344
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
#define PKT_IS_IPV6(p)
Definition: decode.h:252
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
uint8_t events[PACKET_ENGINE_EVENT_MAX]
Definition: decode.h:307
AppLayerDecoderEvents * AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alproto, void *tx)
int AppLayerParserGetEventInfoById(uint8_t ipproto, AppProto alproto, int event_id, const char **event_name, AppLayerEventType *event_type)
#define PKT_IS_IPV4(p)
Definition: decode.h:251
#define SCCalloc(nm, a)
Definition: util-mem.h:253
Data structure to store app layer decoder events.
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
void OutputRegisterPacketSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output sub-module.
Definition: output.c:202
void JsonAnomalyLogRegister(void)
PacketEngineEvents events
Definition: decode.h:565
const struct DecodeEvents_ DEvents[]
Definition: decode-events.c:29
#define MODULE_NAME
void OutputRegisterTxSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output.c:400
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
Definition: conf.h:32
int AppLayerGetEventInfoById(int event_id, const char **event_name, AppLayerEventType *event_type)
OutputCtx * ctx
Definition: output.h:42
const char * event_name
#define SCFree(a)
Definition: util-mem.h:322
uint16_t tx_id
void * data
Definition: tm-modules.h:81
#define EVENT_IS_DECODER_PACKET_ERROR(e)
const char * AppLayerGetProtoName(AppProto alproto)
Given the internal protocol id, returns a string representation of the protocol.
Definition: app-layer.c:766
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:451
AppProto alproto
application level protocol
Definition: flow.h:409
#define GET_PKT_LEN(p)
Definition: decode.h:224
bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
Flow data structure.
Definition: flow.h:325
AppLayerParserState * alparser
Definition: flow.h:437
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
#define DEBUG_VALIDATE_BUG_ON(exp)