suricata
output-json-stats.c
Go to the documentation of this file.
1 /* Copyright (C) 2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Implements JSON stats counters logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 #include "detect-engine.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 
37 #include "util-print.h"
38 #include "util-unittest.h"
39 
40 #include "util-debug.h"
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 
45 #include "util-logopenfile.h"
46 #include "util-crypt.h"
47 
48 #include "output-json.h"
49 #include "output-json-stats.h"
50 
51 #define MODULE_NAME "JsonStatsLog"
52 
53 #ifdef HAVE_LIBJANSSON
54 
55 extern bool stats_decoder_events;
56 const char *stats_decoder_events_prefix;
57 
58 /**
59  * specify which engine info will be printed in stats log.
60  * ALL means both last reload and ruleset stats.
61  */
62 typedef enum OutputEngineInfo_ {
63  OUTPUT_ENGINE_LAST_RELOAD = 0,
64  OUTPUT_ENGINE_RULESET,
65  OUTPUT_ENGINE_ALL,
66 } OutputEngineInfo;
67 
68 typedef struct OutputStatsCtx_ {
69  LogFileCtx *file_ctx;
70  uint32_t flags; /** Store mode */
71 } OutputStatsCtx;
72 
73 typedef struct JsonStatsLogThread_ {
74  OutputStatsCtx *statslog_ctx;
75  MemBuffer *buffer;
76 } JsonStatsLogThread;
77 
78 static json_t *EngineStats2Json(const DetectEngineCtx *de_ctx,
79  const OutputEngineInfo output)
80 {
81  struct timeval last_reload;
82  char timebuf[64];
83  const SigFileLoaderStat *sig_stat = NULL;
84 
85  json_t *jdata = json_object();
86  if (jdata == NULL) {
87  return NULL;
88  }
89 
90  if (output == OUTPUT_ENGINE_LAST_RELOAD || output == OUTPUT_ENGINE_ALL) {
91  last_reload = de_ctx->last_reload;
92  CreateIsoTimeString(&last_reload, timebuf, sizeof(timebuf));
93  json_object_set_new(jdata, "last_reload", json_string(timebuf));
94  }
95 
96  sig_stat = &de_ctx->sig_stat;
97  if ((output == OUTPUT_ENGINE_RULESET || output == OUTPUT_ENGINE_ALL) &&
98  sig_stat != NULL)
99  {
100  json_object_set_new(jdata, "rules_loaded",
101  json_integer(sig_stat->good_sigs_total));
102  json_object_set_new(jdata, "rules_failed",
103  json_integer(sig_stat->bad_sigs_total));
104  }
105 
106  return jdata;
107 }
108 
109 static TmEcode OutputEngineStats2Json(json_t **jdata, const OutputEngineInfo output)
110 {
112  if (de_ctx == NULL) {
113  goto err1;
114  }
115  /* Since we need to deference de_ctx pointer, we don't want to lost it. */
116  DetectEngineCtx *list = de_ctx;
117 
118  json_t *js_tenant_list = json_array();
119  json_t *js_tenant = NULL;
120 
121  if (js_tenant_list == NULL) {
122  goto err2;
123  }
124 
125  while(list) {
126  js_tenant = json_object();
127  if (js_tenant == NULL) {
128  goto err3;
129  }
130  json_object_set_new(js_tenant, "id", json_integer(list->tenant_id));
131 
132  json_t *js_stats = EngineStats2Json(list, output);
133  if (js_stats == NULL) {
134  goto err4;
135  }
136  json_object_update(js_tenant, js_stats);
137  json_array_append_new(js_tenant_list, js_tenant);
138  json_decref(js_stats);
139  list = list->next;
140  }
141 
142  DetectEngineDeReference(&de_ctx);
143  *jdata = js_tenant_list;
144  return TM_ECODE_OK;
145 
146 err4:
147  json_object_clear(js_tenant);
148  json_decref(js_tenant);
149 
150 err3:
151  json_object_clear(js_tenant_list);
152  json_decref(js_tenant_list);
153 
154 err2:
155  DetectEngineDeReference(&de_ctx);
156 
157 err1:
158  json_object_set_new(*jdata, "message", json_string("Unable to get info"));
159  return TM_ECODE_FAILED;
160 }
161 
162 TmEcode OutputEngineStatsReloadTime(json_t **jdata) {
163  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_LAST_RELOAD);
164 }
165 
166 TmEcode OutputEngineStatsRuleset(json_t **jdata) {
167  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_RULESET);
168 }
169 
170 static json_t *OutputStats2Json(json_t *js, const char *key)
171 {
172  void *iter;
173 
174  const char *dot = index(key, '.');
175  if (dot == NULL)
176  return NULL;
177  if (strlen(dot) > 2) {
178  if (*(dot + 1) == '.' && *(dot + 2) != '\0')
179  dot = index(dot + 2, '.');
180  }
181 
182  size_t predot_len = (dot - key) + 1;
183  char s[predot_len];
184  strlcpy(s, key, predot_len);
185 
186  iter = json_object_iter_at(js, s);
187  const char *s2 = index(dot+1, '.');
188 
189  json_t *value = json_object_iter_value(iter);
190  if (value == NULL) {
191  value = json_object();
192 
193  if (!strncmp(s, "detect", 6)) {
194  json_t *js_engine = NULL;
195 
196  TmEcode ret = OutputEngineStats2Json(&js_engine, OUTPUT_ENGINE_ALL);
197  if (ret == TM_ECODE_OK && js_engine) {
198  json_object_set_new(value, "engines", js_engine);
199  }
200  }
201  json_object_set_new(js, s, value);
202  }
203  if (s2 != NULL) {
204  return OutputStats2Json(value, &key[dot-key+1]);
205  }
206  return value;
207 }
208 
209 /** \brief turn StatsTable into a json object
210  * \param flags JSON_STATS_* flags for controlling output
211  */
212 json_t *StatsToJSON(const StatsTable *st, uint8_t flags)
213 {
214  const char delta_suffix[] = "_delta";
215  struct timeval tval;
216  gettimeofday(&tval, NULL);
217 
218  json_t *js_stats = json_object();
219  if (unlikely(js_stats == NULL)) {
220  return NULL;
221  }
222 
223  /* Uptime, in seconds. */
224  double up_time_d = difftime(tval.tv_sec, st->start_time);
225  json_object_set_new(js_stats, "uptime",
226  json_integer((int)up_time_d));
227 
228  uint32_t u = 0;
229  if (flags & JSON_STATS_TOTALS) {
230  for (u = 0; u < st->nstats; u++) {
231  if (st->stats[u].name == NULL)
232  continue;
233  const char *name = st->stats[u].name;
234  const char *shortname = name;
235  if (rindex(name, '.') != NULL) {
236  shortname = &name[rindex(name, '.') - name + 1];
237  }
238  json_t *js_type = OutputStats2Json(js_stats, name);
239  if (js_type != NULL) {
240  json_object_set_new(js_type, shortname,
241  json_integer(st->stats[u].value));
242 
243  if (flags & JSON_STATS_DELTAS) {
244  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
245  snprintf(deltaname, sizeof(deltaname), "%s%s", shortname,
246  delta_suffix);
247  json_object_set_new(js_type, deltaname,
248  json_integer(st->stats[u].value - st->stats[u].pvalue));
249  }
250  }
251  }
252  }
253 
254  /* per thread stats - stored in a "threads" object. */
255  if (st->tstats != NULL && (flags & JSON_STATS_THREADS)) {
256  /* for each thread (store) */
257  json_t *threads = json_object();
258  if (unlikely(threads == NULL)) {
259  json_decref(js_stats);
260  return NULL;
261  }
262  uint32_t x;
263  for (x = 0; x < st->ntstats; x++) {
264  uint32_t offset = x * st->nstats;
265 
266  /* for each counter */
267  for (u = offset; u < (offset + st->nstats); u++) {
268  if (st->tstats[u].name == NULL)
269  continue;
270 
271  char str[256];
272  snprintf(str, sizeof(str), "%s.%s", st->tstats[u].tm_name, st->tstats[u].name);
273  char *shortname = &str[rindex(str, '.') - str + 1];
274  json_t *js_type = OutputStats2Json(threads, str);
275 
276  if (js_type != NULL) {
277  json_object_set_new(js_type, shortname, json_integer(st->tstats[u].value));
278 
279  if (flags & JSON_STATS_DELTAS) {
280  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
281  snprintf(deltaname, sizeof(deltaname), "%s%s",
282  shortname, delta_suffix);
283  json_object_set_new(js_type, deltaname,
284  json_integer(st->tstats[u].value - st->tstats[u].pvalue));
285  }
286  }
287  }
288  }
289  json_object_set_new(js_stats, "threads", threads);
290  }
291  return js_stats;
292 }
293 
294 static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
295 {
296  SCEnter();
297  JsonStatsLogThread *aft = (JsonStatsLogThread *)thread_data;
298 
299  struct timeval tval;
300  gettimeofday(&tval, NULL);
301 
302  json_t *js = json_object();
303  if (unlikely(js == NULL))
304  return 0;
305  char timebuf[64];
306  CreateIsoTimeString(&tval, timebuf, sizeof(timebuf));
307  json_object_set_new(js, "timestamp", json_string(timebuf));
308  json_object_set_new(js, "event_type", json_string("stats"));
309 
310  json_t *js_stats = StatsToJSON(st, aft->statslog_ctx->flags);
311  if (js_stats == NULL) {
312  json_decref(js);
313  return 0;
314  }
315 
316  json_object_set_new(js, "stats", js_stats);
317 
318  OutputJSONBuffer(js, aft->statslog_ctx->file_ctx, &aft->buffer);
319  MemBufferReset(aft->buffer);
320 
321  json_object_clear(js_stats);
322  json_object_del(js, "stats");
323  json_object_clear(js);
324  json_decref(js);
325 
326  SCReturnInt(0);
327 }
328 
329 static TmEcode JsonStatsLogThreadInit(ThreadVars *t, const void *initdata, void **data)
330 {
331  JsonStatsLogThread *aft = SCMalloc(sizeof(JsonStatsLogThread));
332  if (unlikely(aft == NULL))
333  return TM_ECODE_FAILED;
334  memset(aft, 0, sizeof(JsonStatsLogThread));
335 
336  if(initdata == NULL)
337  {
338  SCLogDebug("Error getting context for EveLogStats. \"initdata\" argument NULL");
339  SCFree(aft);
340  return TM_ECODE_FAILED;
341  }
342 
343  /* Use the Ouptut Context (file pointer and mutex) */
344  aft->statslog_ctx = ((OutputCtx *)initdata)->data;
345 
346  aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
347  if (aft->buffer == NULL) {
348  SCFree(aft);
349  return TM_ECODE_FAILED;
350  }
351 
352  *data = (void *)aft;
353  return TM_ECODE_OK;
354 }
355 
356 static TmEcode JsonStatsLogThreadDeinit(ThreadVars *t, void *data)
357 {
358  JsonStatsLogThread *aft = (JsonStatsLogThread *)data;
359  if (aft == NULL) {
360  return TM_ECODE_OK;
361  }
362 
363  MemBufferFree(aft->buffer);
364 
365  /* clear memory */
366  memset(aft, 0, sizeof(JsonStatsLogThread));
367 
368  SCFree(aft);
369  return TM_ECODE_OK;
370 }
371 
372 static void OutputStatsLogDeinit(OutputCtx *output_ctx)
373 {
374 
375  OutputStatsCtx *stats_ctx = output_ctx->data;
376  LogFileCtx *logfile_ctx = stats_ctx->file_ctx;
377  LogFileFreeCtx(logfile_ctx);
378  SCFree(stats_ctx);
379  SCFree(output_ctx);
380 }
381 
382 #define DEFAULT_LOG_FILENAME "stats.json"
383 static OutputInitResult OutputStatsLogInit(ConfNode *conf)
384 {
385  OutputInitResult result = { NULL, false };
386  LogFileCtx *file_ctx = LogFileNewCtx();
387  if(file_ctx == NULL) {
388  SCLogError(SC_ERR_STATS_LOG_GENERIC, "couldn't create new file_ctx");
389  return result;
390  }
391 
392  if (stats_decoder_events &&
393  strcmp(stats_decoder_events_prefix, "decoder") == 0) {
394  SCLogWarning(SC_WARN_EVE_MISSING_EVENTS, "json stats will not display "
395  "all decoder events correctly. See #2225. Set a prefix in "
396  "stats.decoder-events-prefix. In 5.0 the prefix will default "
397  "to 'decoder.event'.");
398  }
399 
400  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
401  LogFileFreeCtx(file_ctx);
402  return result;
403  }
404 
405  OutputStatsCtx *stats_ctx = SCMalloc(sizeof(OutputStatsCtx));
406  if (unlikely(stats_ctx == NULL)) {
407  LogFileFreeCtx(file_ctx);
408  return result;
409  }
410  stats_ctx->flags = JSON_STATS_TOTALS;
411 
412  if (conf != NULL) {
413  const char *totals = ConfNodeLookupChildValue(conf, "totals");
414  const char *threads = ConfNodeLookupChildValue(conf, "threads");
415  const char *deltas = ConfNodeLookupChildValue(conf, "deltas");
416  SCLogDebug("totals %s threads %s deltas %s", totals, threads, deltas);
417 
418  if (totals != NULL && ConfValIsFalse(totals)) {
419  stats_ctx->flags &= ~JSON_STATS_TOTALS;
420  }
421  if (threads != NULL && ConfValIsTrue(threads)) {
422  stats_ctx->flags |= JSON_STATS_THREADS;
423  }
424  if (deltas != NULL && ConfValIsTrue(deltas)) {
425  stats_ctx->flags |= JSON_STATS_DELTAS;
426  }
427  SCLogDebug("stats_ctx->flags %08x", stats_ctx->flags);
428  }
429 
430  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
431  if (unlikely(output_ctx == NULL)) {
432  LogFileFreeCtx(file_ctx);
433  SCFree(stats_ctx);
434  return result;
435  }
436 
437  stats_ctx->file_ctx = file_ctx;
438 
439  output_ctx->data = stats_ctx;
440  output_ctx->DeInit = OutputStatsLogDeinit;
441 
442  result.ctx = output_ctx;
443  result.ok = true;
444  return result;
445 }
446 
447 static void OutputStatsLogDeinitSub(OutputCtx *output_ctx)
448 {
449  OutputStatsCtx *stats_ctx = output_ctx->data;
450  SCFree(stats_ctx);
451  SCFree(output_ctx);
452 }
453 
454 static OutputInitResult OutputStatsLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
455 {
456  OutputInitResult result = { NULL, false };
457  OutputJsonCtx *ajt = parent_ctx->data;
458  OutputStatsCtx *stats_ctx = SCMalloc(sizeof(OutputStatsCtx));
459  if (unlikely(stats_ctx == NULL))
460  return result;
461 
462  if (stats_decoder_events &&
463  strcmp(stats_decoder_events_prefix, "decoder") == 0) {
464  SCLogWarning(SC_WARN_EVE_MISSING_EVENTS, "eve.stats will not display "
465  "all decoder events correctly. See #2225. Set a prefix in "
466  "stats.decoder-events-prefix. In 5.0 the prefix will default "
467  "to 'decoder.event'.");
468  }
469 
470  stats_ctx->flags = JSON_STATS_TOTALS;
471 
472  if (conf != NULL) {
473  const char *totals = ConfNodeLookupChildValue(conf, "totals");
474  const char *threads = ConfNodeLookupChildValue(conf, "threads");
475  const char *deltas = ConfNodeLookupChildValue(conf, "deltas");
476  SCLogDebug("totals %s threads %s deltas %s", totals, threads, deltas);
477 
478  if ((totals != NULL && ConfValIsFalse(totals)) &&
479  (threads != NULL && ConfValIsFalse(threads))) {
480  SCFree(stats_ctx);
482  "Cannot disable both totals and threads in stats logging");
483  return result;
484  }
485 
486  if (totals != NULL && ConfValIsFalse(totals)) {
487  stats_ctx->flags &= ~JSON_STATS_TOTALS;
488  }
489  if (threads != NULL && ConfValIsTrue(threads)) {
490  stats_ctx->flags |= JSON_STATS_THREADS;
491  }
492  if (deltas != NULL && ConfValIsTrue(deltas)) {
493  stats_ctx->flags |= JSON_STATS_DELTAS;
494  }
495  SCLogDebug("stats_ctx->flags %08x", stats_ctx->flags);
496  }
497 
498  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
499  if (unlikely(output_ctx == NULL)) {
500  SCFree(stats_ctx);
501  return result;
502  }
503 
504  stats_ctx->file_ctx = ajt->file_ctx;
505 
506  output_ctx->data = stats_ctx;
507  output_ctx->DeInit = OutputStatsLogDeinitSub;
508 
509  result.ctx = output_ctx;
510  result.ok = true;
511  return result;
512 }
513 
514 void JsonStatsLogRegister(void) {
515  /* register as separate module */
517  OutputStatsLogInit, JsonStatsLogger, JsonStatsLogThreadInit,
518  JsonStatsLogThreadDeinit, NULL);
519 
520  /* also register as child of eve-log */
522  "eve-log.stats", OutputStatsLogInitSub, JsonStatsLogger,
523  JsonStatsLogThreadInit, JsonStatsLogThreadDeinit, NULL);
524 }
525 
526 #else
527 
529 {
530 }
531 
532 #endif
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
const char * stats_decoder_events_prefix
Definition: counters.c:103
uint16_t flags
#define SCLogDebug(...)
Definition: util-debug.h:335
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
Signature loader statistics.
Definition: detect.h:718
void OutputRegisterStatsModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, StatsLogger StatsLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a stats data output module.
Definition: output.c:744
void DetectEngineDeReference(DetectEngineCtx **de_ctx)
void OutputRegisterStatsSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, StatsLogger StatsLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a stats data output sub-module.
Definition: output.c:783
#define unlikely(expr)
Definition: util-optimize.h:35
DetectEngineCtx * DetectEngineGetCurrent(void)
bool stats_decoder_events
Definition: counters.c:102
uint64_t pvalue
Definition: output-stats.h:33
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
uint64_t offset
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
#define rindex
Definition: win32-misc.h:30
main detection engine ctx
Definition: detect.h:756
#define str(s)
#define SCCalloc(nm, a)
Definition: util-mem.h:253
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define JSON_STATS_TOTALS
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
#define JSON_STATS_THREADS
void JsonStatsLogRegister(void)
struct timeval last_reload
Definition: detect.h:932
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
struct DetectEngineCtx_ * next
Definition: detect.h:892
#define MODULE_NAME
#define SCEnter(...)
Definition: util-debug.h:337
StatsRecord * tstats
Definition: output-stats.h:38
SigFileLoaderStat sig_stat
Definition: detect.h:935
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:591
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
const char * tm_name
Definition: output-stats.h:31
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
#define SCReturnInt(x)
Definition: util-debug.h:341
#define index
Definition: win32-misc.h:29
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define DEFAULT_LOG_FILENAME
#define SCMalloc(a)
Definition: util-mem.h:222
StatsRecord * stats
Definition: output-stats.h:37
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define SCFree(a)
Definition: util-mem.h:322
uint64_t value
Definition: output-stats.h:32
void * data
Definition: tm-modules.h:81
time_t start_time
Definition: output-stats.h:41
#define JSON_STATS_DELTAS
Per thread variable structure.
Definition: threadvars.h:57
const char * name
Definition: output-stats.h:30
uint32_t ntstats
Definition: output-stats.h:40
uint32_t nstats
Definition: output-stats.h:39
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82