suricata
output-json-stats.c
Go to the documentation of this file.
1 /* Copyright (C) 2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Implements JSON stats counters logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 #include "detect-engine.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 
37 #include "util-print.h"
38 #include "util-unittest.h"
39 
40 #include "util-debug.h"
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 
45 #include "util-logopenfile.h"
46 #include "util-crypt.h"
47 
48 #include "output-json.h"
49 #include "output-json-stats.h"
50 
51 #define MODULE_NAME "JsonStatsLog"
52 
53 extern bool stats_decoder_events;
55 
56 /**
57  * specify which engine info will be printed in stats log.
58  * ALL means both last reload and ruleset stats.
59  */
60 typedef enum OutputEngineInfo_ {
65 
66 typedef struct OutputStatsCtx_ {
68  uint32_t flags; /** Store mode */
70 
71 typedef struct JsonStatsLogThread_ {
75 
76 static json_t *EngineStats2Json(const DetectEngineCtx *de_ctx,
77  const OutputEngineInfo output)
78 {
79  struct timeval last_reload;
80  char timebuf[64];
81  const SigFileLoaderStat *sig_stat = NULL;
82 
83  json_t *jdata = json_object();
84  if (jdata == NULL) {
85  return NULL;
86  }
87 
88  if (output == OUTPUT_ENGINE_LAST_RELOAD || output == OUTPUT_ENGINE_ALL) {
89  last_reload = de_ctx->last_reload;
90  CreateIsoTimeString(&last_reload, timebuf, sizeof(timebuf));
91  json_object_set_new(jdata, "last_reload", json_string(timebuf));
92  }
93 
94  sig_stat = &de_ctx->sig_stat;
95  if ((output == OUTPUT_ENGINE_RULESET || output == OUTPUT_ENGINE_ALL) &&
96  sig_stat != NULL)
97  {
98  json_object_set_new(jdata, "rules_loaded",
99  json_integer(sig_stat->good_sigs_total));
100  json_object_set_new(jdata, "rules_failed",
101  json_integer(sig_stat->bad_sigs_total));
102  }
103 
104  return jdata;
105 }
106 
107 static TmEcode OutputEngineStats2Json(json_t **jdata, const OutputEngineInfo output)
108 {
110  if (de_ctx == NULL) {
111  goto err1;
112  }
113  /* Since we need to deference de_ctx pointer, we don't want to lost it. */
114  DetectEngineCtx *list = de_ctx;
115 
116  json_t *js_tenant_list = json_array();
117  json_t *js_tenant = NULL;
118 
119  if (js_tenant_list == NULL) {
120  goto err2;
121  }
122 
123  while(list) {
124  js_tenant = json_object();
125  if (js_tenant == NULL) {
126  goto err3;
127  }
128  json_object_set_new(js_tenant, "id", json_integer(list->tenant_id));
129 
130  json_t *js_stats = EngineStats2Json(list, output);
131  if (js_stats == NULL) {
132  goto err4;
133  }
134  json_object_update(js_tenant, js_stats);
135  json_array_append_new(js_tenant_list, js_tenant);
136  json_decref(js_stats);
137  list = list->next;
138  }
139 
140  DetectEngineDeReference(&de_ctx);
141  *jdata = js_tenant_list;
142  return TM_ECODE_OK;
143 
144 err4:
145  json_object_clear(js_tenant);
146  json_decref(js_tenant);
147 
148 err3:
149  json_object_clear(js_tenant_list);
150  json_decref(js_tenant_list);
151 
152 err2:
153  DetectEngineDeReference(&de_ctx);
154 
155 err1:
156  json_object_set_new(*jdata, "message", json_string("Unable to get info"));
157  return TM_ECODE_FAILED;
158 }
159 
161  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_LAST_RELOAD);
162 }
163 
165  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_RULESET);
166 }
167 
168 static json_t *OutputStats2Json(json_t *js, const char *key)
169 {
170  void *iter;
171 
172  const char *dot = strchr(key, '.');
173  if (dot == NULL)
174  return NULL;
175  if (strlen(dot) > 2) {
176  if (*(dot + 1) == '.' && *(dot + 2) != '\0')
177  dot = strchr(dot + 2, '.');
178  }
179 
180  size_t predot_len = (dot - key) + 1;
181  char s[predot_len];
182  strlcpy(s, key, predot_len);
183 
184  iter = json_object_iter_at(js, s);
185  const char *s2 = strchr(dot+1, '.');
186 
187  json_t *value = json_object_iter_value(iter);
188  if (value == NULL) {
189  value = json_object();
190 
191  if (!strncmp(s, "detect", 6)) {
192  json_t *js_engine = NULL;
193 
194  TmEcode ret = OutputEngineStats2Json(&js_engine, OUTPUT_ENGINE_ALL);
195  if (ret == TM_ECODE_OK && js_engine) {
196  json_object_set_new(value, "engines", js_engine);
197  }
198  }
199  json_object_set_new(js, s, value);
200  }
201  if (s2 != NULL) {
202  return OutputStats2Json(value, &key[dot-key+1]);
203  }
204  return value;
205 }
206 
207 /** \brief turn StatsTable into a json object
208  * \param flags JSON_STATS_* flags for controlling output
209  */
210 json_t *StatsToJSON(const StatsTable *st, uint8_t flags)
211 {
212  const char delta_suffix[] = "_delta";
213  struct timeval tval;
214  gettimeofday(&tval, NULL);
215 
216  json_t *js_stats = json_object();
217  if (unlikely(js_stats == NULL)) {
218  return NULL;
219  }
220 
221  /* Uptime, in seconds. */
222  double up_time_d = difftime(tval.tv_sec, st->start_time);
223  json_object_set_new(js_stats, "uptime",
224  json_integer((int)up_time_d));
225 
226  uint32_t u = 0;
227  if (flags & JSON_STATS_TOTALS) {
228  for (u = 0; u < st->nstats; u++) {
229  if (st->stats[u].name == NULL)
230  continue;
231  const char *name = st->stats[u].name;
232  const char *shortname = name;
233  if (strrchr(name, '.') != NULL) {
234  shortname = &name[strrchr(name, '.') - name + 1];
235  }
236  json_t *js_type = OutputStats2Json(js_stats, name);
237  if (js_type != NULL) {
238  json_object_set_new(js_type, shortname,
239  json_integer(st->stats[u].value));
240 
241  if (flags & JSON_STATS_DELTAS) {
242  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
243  snprintf(deltaname, sizeof(deltaname), "%s%s", shortname,
244  delta_suffix);
245  json_object_set_new(js_type, deltaname,
246  json_integer(st->stats[u].value - st->stats[u].pvalue));
247  }
248  }
249  }
250  }
251 
252  /* per thread stats - stored in a "threads" object. */
253  if (st->tstats != NULL && (flags & JSON_STATS_THREADS)) {
254  /* for each thread (store) */
255  json_t *threads = json_object();
256  if (unlikely(threads == NULL)) {
257  json_decref(js_stats);
258  return NULL;
259  }
260  uint32_t x;
261  for (x = 0; x < st->ntstats; x++) {
262  uint32_t offset = x * st->nstats;
263 
264  /* for each counter */
265  for (u = offset; u < (offset + st->nstats); u++) {
266  if (st->tstats[u].name == NULL)
267  continue;
268 
269  char str[256];
270  snprintf(str, sizeof(str), "%s.%s", st->tstats[u].tm_name, st->tstats[u].name);
271  char *shortname = &str[strrchr(str, '.') - str + 1];
272  json_t *js_type = OutputStats2Json(threads, str);
273 
274  if (js_type != NULL) {
275  json_object_set_new(js_type, shortname, json_integer(st->tstats[u].value));
276 
277  if (flags & JSON_STATS_DELTAS) {
278  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
279  snprintf(deltaname, sizeof(deltaname), "%s%s",
280  shortname, delta_suffix);
281  json_object_set_new(js_type, deltaname,
282  json_integer(st->tstats[u].value - st->tstats[u].pvalue));
283  }
284  }
285  }
286  }
287  json_object_set_new(js_stats, "threads", threads);
288  }
289  return js_stats;
290 }
291 
292 static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
293 {
294  SCEnter();
295  JsonStatsLogThread *aft = (JsonStatsLogThread *)thread_data;
296 
297  struct timeval tval;
298  gettimeofday(&tval, NULL);
299 
300  json_t *js = json_object();
301  if (unlikely(js == NULL))
302  return 0;
303  char timebuf[64];
304  CreateIsoTimeString(&tval, timebuf, sizeof(timebuf));
305  json_object_set_new(js, "timestamp", json_string(timebuf));
306  json_object_set_new(js, "event_type", json_string("stats"));
307 
308  json_t *js_stats = StatsToJSON(st, aft->statslog_ctx->flags);
309  if (js_stats == NULL) {
310  json_decref(js);
311  return 0;
312  }
313 
314  json_object_set_new(js, "stats", js_stats);
315 
316  OutputJSONBuffer(js, aft->statslog_ctx->file_ctx, &aft->buffer);
317  MemBufferReset(aft->buffer);
318 
319  json_object_clear(js_stats);
320  json_object_del(js, "stats");
321  json_object_clear(js);
322  json_decref(js);
323 
324  SCReturnInt(0);
325 }
326 
327 static TmEcode JsonStatsLogThreadInit(ThreadVars *t, const void *initdata, void **data)
328 {
330  if (unlikely(aft == NULL))
331  return TM_ECODE_FAILED;
332  memset(aft, 0, sizeof(JsonStatsLogThread));
333 
334  if(initdata == NULL)
335  {
336  SCLogDebug("Error getting context for EveLogStats. \"initdata\" argument NULL");
337  SCFree(aft);
338  return TM_ECODE_FAILED;
339  }
340 
341  /* Use the Ouptut Context (file pointer and mutex) */
342  aft->statslog_ctx = ((OutputCtx *)initdata)->data;
343 
345  if (aft->buffer == NULL) {
346  SCFree(aft);
347  return TM_ECODE_FAILED;
348  }
349 
350  *data = (void *)aft;
351  return TM_ECODE_OK;
352 }
353 
354 static TmEcode JsonStatsLogThreadDeinit(ThreadVars *t, void *data)
355 {
356  JsonStatsLogThread *aft = (JsonStatsLogThread *)data;
357  if (aft == NULL) {
358  return TM_ECODE_OK;
359  }
360 
361  MemBufferFree(aft->buffer);
362 
363  /* clear memory */
364  memset(aft, 0, sizeof(JsonStatsLogThread));
365 
366  SCFree(aft);
367  return TM_ECODE_OK;
368 }
369 
370 static void OutputStatsLogDeinit(OutputCtx *output_ctx)
371 {
372 
373  OutputStatsCtx *stats_ctx = output_ctx->data;
374  LogFileCtx *logfile_ctx = stats_ctx->file_ctx;
375  LogFileFreeCtx(logfile_ctx);
376  SCFree(stats_ctx);
377  SCFree(output_ctx);
378 }
379 
380 #define DEFAULT_LOG_FILENAME "stats.json"
381 static OutputInitResult OutputStatsLogInit(ConfNode *conf)
382 {
383  OutputInitResult result = { NULL, false };
384 
385  if (!StatsEnabled()) {
387  "stats.json: stats are disabled globally: set stats.enabled to true. "
388  "See %s%s/configuration/suricata-yaml.html#stats", DOC_URL, DOC_VERSION);
389  return result;
390  }
391 
393  if(file_ctx == NULL) {
394  SCLogError(SC_ERR_STATS_LOG_GENERIC, "couldn't create new file_ctx");
395  return result;
396  }
397 
398  if (stats_decoder_events &&
399  strcmp(stats_decoder_events_prefix, "decoder") == 0) {
400  SCLogWarning(SC_WARN_EVE_MISSING_EVENTS, "json stats will not display "
401  "all decoder events correctly. See #2225. Set a prefix in "
402  "stats.decoder-events-prefix.");
403  }
404 
405  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
406  LogFileFreeCtx(file_ctx);
407  return result;
408  }
409 
410  OutputStatsCtx *stats_ctx = SCMalloc(sizeof(OutputStatsCtx));
411  if (unlikely(stats_ctx == NULL)) {
412  LogFileFreeCtx(file_ctx);
413  return result;
414  }
415  stats_ctx->flags = JSON_STATS_TOTALS;
416 
417  if (conf != NULL) {
418  const char *totals = ConfNodeLookupChildValue(conf, "totals");
419  const char *threads = ConfNodeLookupChildValue(conf, "threads");
420  const char *deltas = ConfNodeLookupChildValue(conf, "deltas");
421  SCLogDebug("totals %s threads %s deltas %s", totals, threads, deltas);
422 
423  if (totals != NULL && ConfValIsFalse(totals)) {
424  stats_ctx->flags &= ~JSON_STATS_TOTALS;
425  }
426  if (threads != NULL && ConfValIsTrue(threads)) {
427  stats_ctx->flags |= JSON_STATS_THREADS;
428  }
429  if (deltas != NULL && ConfValIsTrue(deltas)) {
430  stats_ctx->flags |= JSON_STATS_DELTAS;
431  }
432  SCLogDebug("stats_ctx->flags %08x", stats_ctx->flags);
433  }
434 
435  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
436  if (unlikely(output_ctx == NULL)) {
437  LogFileFreeCtx(file_ctx);
438  SCFree(stats_ctx);
439  return result;
440  }
441 
442  stats_ctx->file_ctx = file_ctx;
443 
444  output_ctx->data = stats_ctx;
445  output_ctx->DeInit = OutputStatsLogDeinit;
446 
447  result.ctx = output_ctx;
448  result.ok = true;
449  return result;
450 }
451 
452 static void OutputStatsLogDeinitSub(OutputCtx *output_ctx)
453 {
454  OutputStatsCtx *stats_ctx = output_ctx->data;
455  SCFree(stats_ctx);
456  SCFree(output_ctx);
457 }
458 
459 static OutputInitResult OutputStatsLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
460 {
461  OutputInitResult result = { NULL, false };
462  OutputJsonCtx *ajt = parent_ctx->data;
463 
464  if (!StatsEnabled()) {
466  "eve.stats: stats are disabled globally: set stats.enabled to true. "
467  "See %s%s/configuration/suricata-yaml.html#stats", DOC_URL, DOC_VERSION);
468  return result;
469  }
470 
471  OutputStatsCtx *stats_ctx = SCMalloc(sizeof(OutputStatsCtx));
472  if (unlikely(stats_ctx == NULL))
473  return result;
474 
475  if (stats_decoder_events &&
476  strcmp(stats_decoder_events_prefix, "decoder") == 0) {
477  SCLogWarning(SC_WARN_EVE_MISSING_EVENTS, "eve.stats will not display "
478  "all decoder events correctly. See #2225. Set a prefix in "
479  "stats.decoder-events-prefix.");
480  }
481 
482  stats_ctx->flags = JSON_STATS_TOTALS;
483 
484  if (conf != NULL) {
485  const char *totals = ConfNodeLookupChildValue(conf, "totals");
486  const char *threads = ConfNodeLookupChildValue(conf, "threads");
487  const char *deltas = ConfNodeLookupChildValue(conf, "deltas");
488  SCLogDebug("totals %s threads %s deltas %s", totals, threads, deltas);
489 
490  if ((totals != NULL && ConfValIsFalse(totals)) &&
491  (threads != NULL && ConfValIsFalse(threads))) {
492  SCFree(stats_ctx);
494  "Cannot disable both totals and threads in stats logging");
495  return result;
496  }
497 
498  if (totals != NULL && ConfValIsFalse(totals)) {
499  stats_ctx->flags &= ~JSON_STATS_TOTALS;
500  }
501  if (threads != NULL && ConfValIsTrue(threads)) {
502  stats_ctx->flags |= JSON_STATS_THREADS;
503  }
504  if (deltas != NULL && ConfValIsTrue(deltas)) {
505  stats_ctx->flags |= JSON_STATS_DELTAS;
506  }
507  SCLogDebug("stats_ctx->flags %08x", stats_ctx->flags);
508  }
509 
510  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
511  if (unlikely(output_ctx == NULL)) {
512  SCFree(stats_ctx);
513  return result;
514  }
515 
516  stats_ctx->file_ctx = ajt->file_ctx;
517 
518  output_ctx->data = stats_ctx;
519  output_ctx->DeInit = OutputStatsLogDeinitSub;
520 
521  result.ctx = output_ctx;
522  result.ok = true;
523  return result;
524 }
525 
527  /* register as separate module */
529  OutputStatsLogInit, JsonStatsLogger, JsonStatsLogThreadInit,
530  JsonStatsLogThreadDeinit, NULL);
531 
532  /* also register as child of eve-log */
534  "eve-log.stats", OutputStatsLogInitSub, JsonStatsLogger,
535  JsonStatsLogThreadInit, JsonStatsLogThreadDeinit, NULL);
536 }
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:44
#define SCLogDebug(...)
Definition: util-debug.h:335
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:809
json_t * StatsToJSON(const StatsTable *st, uint8_t flags)
turn StatsTable into a json object
OutputEngineInfo_
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
Signature loader statistics.
Definition: detect.h:723
void OutputRegisterStatsModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, StatsLogger StatsLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a stats data output module.
Definition: output.c:746
void DetectEngineDeReference(DetectEngineCtx **de_ctx)
bool StatsEnabled(void)
Definition: counters.c:119
void OutputRegisterStatsSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, StatsLogger StatsLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a stats data output sub-module.
Definition: output.c:785
#define unlikely(expr)
Definition: util-optimize.h:35
DetectEngineCtx * DetectEngineGetCurrent(void)
bool stats_decoder_events
Definition: counters.c:102
uint64_t pvalue
Definition: output-stats.h:33
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
uint64_t offset
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
#define DEFAULT_LOG_FILENAME
main detection engine ctx
Definition: detect.h:761
OutputStatsCtx * statslog_ctx
#define str(s)
#define SCCalloc(nm, a)
Definition: util-mem.h:253
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define JSON_STATS_TOTALS
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:187
#define JSON_STATS_THREADS
void JsonStatsLogRegister(void)
struct timeval last_reload
Definition: detect.h:938
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
LogFileCtx * file_ctx
Definition: output-json.h:79
struct DetectEngineCtx_ * next
Definition: detect.h:898
#define MODULE_NAME
#define SCEnter(...)
Definition: util-debug.h:337
StatsRecord * tstats
Definition: output-stats.h:38
SigFileLoaderStat sig_stat
Definition: detect.h:941
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:591
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
const char * tm_name
Definition: output-stats.h:31
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
enum OutputEngineInfo_ OutputEngineInfo
#define SCReturnInt(x)
Definition: util-debug.h:341
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
TmEcode OutputEngineStatsRuleset(json_t **jdata)
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:222
StatsRecord * stats
Definition: output-stats.h:37
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define SCFree(a)
Definition: util-mem.h:322
uint64_t value
Definition: output-stats.h:32
const char * stats_decoder_events_prefix
LogFileCtx * file_ctx
void * data
Definition: tm-modules.h:81
TmEcode OutputEngineStatsReloadTime(json_t **jdata)
time_t start_time
Definition: output-stats.h:41
#define JSON_STATS_DELTAS
#define DOC_URL
Definition: suricata.h:86
Per thread variable structure.
Definition: threadvars.h:57
const char * name
Definition: output-stats.h:30
uint32_t ntstats
Definition: output-stats.h:40
uint32_t nstats
Definition: output-stats.h:39
#define DOC_VERSION
Definition: suricata.h:91
struct OutputStatsCtx_ OutputStatsCtx
struct JsonStatsLogThread_ JsonStatsLogThread
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82