suricata
output-json-stats.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Implements JSON stats counters logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 #include "detect-engine.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 
37 #include "util-print.h"
38 #include "util-unittest.h"
39 
40 #include "util-debug.h"
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 
45 #include "util-logopenfile.h"
46 #include "util-crypt.h"
47 
48 #include "output-json.h"
49 #include "output-json-stats.h"
50 
51 #define MODULE_NAME "JsonStatsLog"
52 
53 extern bool stats_decoder_events;
54 extern const char *stats_decoder_events_prefix;
55 
56 /**
57  * specify which engine info will be printed in stats log.
58  * ALL means both last reload and ruleset stats.
59  */
60 typedef enum OutputEngineInfo_ {
65 
66 typedef struct OutputStatsCtx_ {
68  uint32_t flags; /** Store mode */
70 
71 typedef struct JsonStatsLogThread_ {
76 
77 static json_t *EngineStats2Json(const DetectEngineCtx *de_ctx,
78  const OutputEngineInfo output)
79 {
80  struct timeval last_reload;
81  char timebuf[64];
82  const SigFileLoaderStat *sig_stat = NULL;
83 
84  json_t *jdata = json_object();
85  if (jdata == NULL) {
86  return NULL;
87  }
88 
89  if (output == OUTPUT_ENGINE_LAST_RELOAD || output == OUTPUT_ENGINE_ALL) {
90  last_reload = de_ctx->last_reload;
91  CreateIsoTimeString(&last_reload, timebuf, sizeof(timebuf));
92  json_object_set_new(jdata, "last_reload", json_string(timebuf));
93  }
94 
95  sig_stat = &de_ctx->sig_stat;
96  if ((output == OUTPUT_ENGINE_RULESET || output == OUTPUT_ENGINE_ALL) &&
97  sig_stat != NULL)
98  {
99  json_object_set_new(jdata, "rules_loaded",
100  json_integer(sig_stat->good_sigs_total));
101  json_object_set_new(jdata, "rules_failed",
102  json_integer(sig_stat->bad_sigs_total));
103  }
104 
105  return jdata;
106 }
107 
108 static TmEcode OutputEngineStats2Json(json_t **jdata, const OutputEngineInfo output)
109 {
111  if (de_ctx == NULL) {
112  goto err1;
113  }
114  /* Since we need to deference de_ctx pointer, we don't want to lost it. */
115  DetectEngineCtx *list = de_ctx;
116 
117  json_t *js_tenant_list = json_array();
118  json_t *js_tenant = NULL;
119 
120  if (js_tenant_list == NULL) {
121  goto err2;
122  }
123 
124  while(list) {
125  js_tenant = json_object();
126  if (js_tenant == NULL) {
127  goto err3;
128  }
129  json_object_set_new(js_tenant, "id", json_integer(list->tenant_id));
130 
131  json_t *js_stats = EngineStats2Json(list, output);
132  if (js_stats == NULL) {
133  goto err4;
134  }
135  json_object_update(js_tenant, js_stats);
136  json_array_append_new(js_tenant_list, js_tenant);
137  json_decref(js_stats);
138  list = list->next;
139  }
140 
142  *jdata = js_tenant_list;
143  return TM_ECODE_OK;
144 
145 err4:
146  json_object_clear(js_tenant);
147  json_decref(js_tenant);
148 
149 err3:
150  json_object_clear(js_tenant_list);
151  json_decref(js_tenant_list);
152 
153 err2:
155 
156 err1:
157  json_object_set_new(*jdata, "message", json_string("Unable to get info"));
158  return TM_ECODE_FAILED;
159 }
160 
162  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_LAST_RELOAD);
163 }
164 
166  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_RULESET);
167 }
168 
169 static json_t *OutputStats2Json(json_t *js, const char *key)
170 {
171  void *iter;
172 
173  const char *dot = strchr(key, '.');
174  if (dot == NULL)
175  return NULL;
176  if (strlen(dot) > 2) {
177  if (*(dot + 1) == '.' && *(dot + 2) != '\0')
178  dot = strchr(dot + 2, '.');
179  }
180 
181  size_t predot_len = (dot - key) + 1;
182  char s[predot_len];
183  strlcpy(s, key, predot_len);
184 
185  iter = json_object_iter_at(js, s);
186  const char *s2 = strchr(dot+1, '.');
187 
188  json_t *value = json_object_iter_value(iter);
189  if (value == NULL) {
190  value = json_object();
191 
192  if (!strncmp(s, "detect", 6)) {
193  json_t *js_engine = NULL;
194 
195  TmEcode ret = OutputEngineStats2Json(&js_engine, OUTPUT_ENGINE_ALL);
196  if (ret == TM_ECODE_OK && js_engine) {
197  json_object_set_new(value, "engines", js_engine);
198  }
199  }
200  json_object_set_new(js, s, value);
201  }
202  if (s2 != NULL) {
203  return OutputStats2Json(value, &key[dot-key+1]);
204  }
205  return value;
206 }
207 
208 /** \brief turn StatsTable into a json object
209  * \param flags JSON_STATS_* flags for controlling output
210  */
211 json_t *StatsToJSON(const StatsTable *st, uint8_t flags)
212 {
213  const char delta_suffix[] = "_delta";
214  struct timeval tval;
215  gettimeofday(&tval, NULL);
216 
217  json_t *js_stats = json_object();
218  if (unlikely(js_stats == NULL)) {
219  return NULL;
220  }
221 
222  /* Uptime, in seconds. */
223  double up_time_d = difftime(tval.tv_sec, st->start_time);
224  json_object_set_new(js_stats, "uptime",
225  json_integer((int)up_time_d));
226 
227  uint32_t u = 0;
228  if (flags & JSON_STATS_TOTALS) {
229  for (u = 0; u < st->nstats; u++) {
230  if (st->stats[u].name == NULL)
231  continue;
232  const char *name = st->stats[u].name;
233  const char *shortname = name;
234  if (strrchr(name, '.') != NULL) {
235  shortname = &name[strrchr(name, '.') - name + 1];
236  }
237  json_t *js_type = OutputStats2Json(js_stats, name);
238  if (js_type != NULL) {
239  json_object_set_new(js_type, shortname,
240  json_integer(st->stats[u].value));
241 
242  if (flags & JSON_STATS_DELTAS) {
243  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
244  snprintf(deltaname, sizeof(deltaname), "%s%s", shortname,
245  delta_suffix);
246  json_object_set_new(js_type, deltaname,
247  json_integer(st->stats[u].value - st->stats[u].pvalue));
248  }
249  }
250  }
251  }
252 
253  /* per thread stats - stored in a "threads" object. */
254  if (st->tstats != NULL && (flags & JSON_STATS_THREADS)) {
255  /* for each thread (store) */
256  json_t *threads = json_object();
257  if (unlikely(threads == NULL)) {
258  json_decref(js_stats);
259  return NULL;
260  }
261  uint32_t x;
262  for (x = 0; x < st->ntstats; x++) {
263  uint32_t offset = x * st->nstats;
264 
265  /* for each counter */
266  for (u = offset; u < (offset + st->nstats); u++) {
267  if (st->tstats[u].name == NULL)
268  continue;
269 
270  char str[256];
271  snprintf(str, sizeof(str), "%s.%s", st->tstats[u].tm_name, st->tstats[u].name);
272  char *shortname = &str[strrchr(str, '.') - str + 1];
273  json_t *js_type = OutputStats2Json(threads, str);
274 
275  if (js_type != NULL) {
276  json_object_set_new(js_type, shortname, json_integer(st->tstats[u].value));
277 
278  if (flags & JSON_STATS_DELTAS) {
279  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
280  snprintf(deltaname, sizeof(deltaname), "%s%s",
281  shortname, delta_suffix);
282  json_object_set_new(js_type, deltaname,
283  json_integer(st->tstats[u].value - st->tstats[u].pvalue));
284  }
285  }
286  }
287  }
288  json_object_set_new(js_stats, "threads", threads);
289  }
290  return js_stats;
291 }
292 
293 static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
294 {
295  SCEnter();
296  JsonStatsLogThread *aft = (JsonStatsLogThread *)thread_data;
297 
298  struct timeval tval;
299  gettimeofday(&tval, NULL);
300 
301  json_t *js = json_object();
302  if (unlikely(js == NULL))
303  return 0;
304  char timebuf[64];
305  CreateIsoTimeString(&tval, timebuf, sizeof(timebuf));
306  json_object_set_new(js, "timestamp", json_string(timebuf));
307  json_object_set_new(js, "event_type", json_string("stats"));
308 
309  json_t *js_stats = StatsToJSON(st, aft->statslog_ctx->flags);
310  if (js_stats == NULL) {
311  json_decref(js);
312  return 0;
313  }
314 
315  json_object_set_new(js, "stats", js_stats);
316 
317  OutputJSONBuffer(js, aft->file_ctx, &aft->buffer);
318  MemBufferReset(aft->buffer);
319 
320  json_object_clear(js_stats);
321  json_object_del(js, "stats");
322  json_object_clear(js);
323  json_decref(js);
324 
325  SCReturnInt(0);
326 }
327 
328 static TmEcode JsonStatsLogThreadInit(ThreadVars *t, const void *initdata, void **data)
329 {
331  if (unlikely(aft == NULL))
332  return TM_ECODE_FAILED;
333 
334  if(initdata == NULL)
335  {
336  SCLogDebug("Error getting context for EveLogStats. \"initdata\" argument NULL");
337  goto error_exit;
338  }
339 
341  if (aft->buffer == NULL) {
342  goto error_exit;
343  }
344 
345  /* Use the Output Context (file pointer and mutex) */
346  aft->statslog_ctx = ((OutputCtx *)initdata)->data;
347 
349  if (!aft->file_ctx) {
350  goto error_exit;
351  }
352 
353  *data = (void *)aft;
354  return TM_ECODE_OK;
355 
356 error_exit:
357  if (aft->buffer != NULL) {
358  MemBufferFree(aft->buffer);
359  }
360  SCFree(aft);
361  return TM_ECODE_FAILED;
362 }
363 
364 static TmEcode JsonStatsLogThreadDeinit(ThreadVars *t, void *data)
365 {
366  JsonStatsLogThread *aft = (JsonStatsLogThread *)data;
367  if (aft == NULL) {
368  return TM_ECODE_OK;
369  }
370 
371  MemBufferFree(aft->buffer);
372 
373  /* clear memory */
374  memset(aft, 0, sizeof(JsonStatsLogThread));
375 
376  SCFree(aft);
377  return TM_ECODE_OK;
378 }
379 
380 static void OutputStatsLogDeinitSub(OutputCtx *output_ctx)
381 {
382  OutputStatsCtx *stats_ctx = output_ctx->data;
383  SCFree(stats_ctx);
384  SCFree(output_ctx);
385 }
386 
387 static OutputInitResult OutputStatsLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
388 {
389  OutputInitResult result = { NULL, false };
390  OutputJsonCtx *ajt = parent_ctx->data;
391 
392  if (!StatsEnabled()) {
394  "eve.stats: stats are disabled globally: set stats.enabled to true. "
395  "See %s/configuration/suricata-yaml.html#stats", GetDocURL());
396  return result;
397  }
398 
399  OutputStatsCtx *stats_ctx = SCMalloc(sizeof(OutputStatsCtx));
400  if (unlikely(stats_ctx == NULL))
401  return result;
402 
403  if (stats_decoder_events &&
404  strcmp(stats_decoder_events_prefix, "decoder") == 0) {
405  SCLogWarning(SC_WARN_EVE_MISSING_EVENTS, "eve.stats will not display "
406  "all decoder events correctly. See #2225. Set a prefix in "
407  "stats.decoder-events-prefix.");
408  }
409 
410  stats_ctx->flags = JSON_STATS_TOTALS;
411 
412  if (conf != NULL) {
413  const char *totals = ConfNodeLookupChildValue(conf, "totals");
414  const char *threads = ConfNodeLookupChildValue(conf, "threads");
415  const char *deltas = ConfNodeLookupChildValue(conf, "deltas");
416  SCLogDebug("totals %s threads %s deltas %s", totals, threads, deltas);
417 
418  if ((totals != NULL && ConfValIsFalse(totals)) &&
419  (threads != NULL && ConfValIsFalse(threads))) {
420  SCFree(stats_ctx);
422  "Cannot disable both totals and threads in stats logging");
423  return result;
424  }
425 
426  if (totals != NULL && ConfValIsFalse(totals)) {
427  stats_ctx->flags &= ~JSON_STATS_TOTALS;
428  }
429  if (threads != NULL && ConfValIsTrue(threads)) {
430  stats_ctx->flags |= JSON_STATS_THREADS;
431  }
432  if (deltas != NULL && ConfValIsTrue(deltas)) {
433  stats_ctx->flags |= JSON_STATS_DELTAS;
434  }
435  SCLogDebug("stats_ctx->flags %08x", stats_ctx->flags);
436  }
437 
438  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
439  if (unlikely(output_ctx == NULL)) {
440  SCFree(stats_ctx);
441  return result;
442  }
443 
444  stats_ctx->file_ctx = ajt->file_ctx;
445 
446  output_ctx->data = stats_ctx;
447  output_ctx->DeInit = OutputStatsLogDeinitSub;
448 
449  result.ctx = output_ctx;
450  result.ok = true;
451  return result;
452 }
453 
455  /* register as child of eve-log */
457  "eve-log.stats", OutputStatsLogInitSub, JsonStatsLogger,
458  JsonStatsLogThreadInit, JsonStatsLogThreadDeinit, NULL);
459 }
tm-threads.h
DetectEngineCtx_::tenant_id
int tenant_id
Definition: detect.h:771
StatsTable_::ntstats
uint32_t ntstats
Definition: output-stats.h:40
detect-engine.h
OutputRegisterStatsSubModule
void OutputRegisterStatsSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, StatsLogger StatsLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a stats data output sub-module.
Definition: output.c:745
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
DetectEngineDeReference
void DetectEngineDeReference(DetectEngineCtx **de_ctx)
Definition: detect-engine.c:3933
CreateIsoTimeString
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:213
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
LOGGER_JSON_STATS
@ LOGGER_JSON_STATS
Definition: suricata-common.h:487
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
JSON_OUTPUT_BUFFER_SIZE
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:63
threads.h
JsonStatsLogThread
struct JsonStatsLogThread_ JsonStatsLogThread
OutputJsonCtx_
Definition: output-json.h:101
LogFileCtx_
Definition: util-logopenfile.h:60
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:767
DetectEngineGetCurrent
DetectEngineCtx * DetectEngineGetCurrent(void)
Definition: detect-engine.c:3232
StatsRecord_::value
uint64_t value
Definition: output-stats.h:32
util-privs.h
JSON_STATS_TOTALS
#define JSON_STATS_TOTALS
Definition: output-json-stats.h:29
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:81
JsonStatsLogRegister
void JsonStatsLogRegister(void)
Definition: output-json-stats.c:454
util-unittest.h
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:565
OutputCtx_::data
void * data
Definition: tm-modules.h:81
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
OutputCtx_
Definition: tm-modules.h:78
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
OutputEngineInfo_
OutputEngineInfo_
Definition: output-json-stats.c:60
stats_decoder_events_prefix
const char * stats_decoder_events_prefix
Definition: counters.c:104
SC_ERR_STATS_LOG_GENERIC
@ SC_ERR_STATS_LOG_GENERIC
Definition: util-error.h:311
util-debug.h
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:44
output-json.h
DetectEngineCtx_::last_reload
struct timeval last_reload
Definition: detect.h:945
JsonStatsLogThread_::buffer
MemBuffer * buffer
Definition: output-json-stats.c:74
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
util-crypt.h
StatsRecord_::name
const char * name
Definition: output-stats.h:30
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
pkt-var.h
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:659
OutputInitResult_::ok
bool ok
Definition: output.h:45
ThreadVars_::id
int id
Definition: threadvars.h:87
SigFileLoaderStat_::bad_sigs_total
int bad_sigs_total
Definition: detect.h:734
OutputEngineStatsReloadTime
TmEcode OutputEngineStatsReloadTime(json_t **jdata)
Definition: output-json-stats.c:161
StatsTable_
Definition: output-stats.h:36
StatsToJSON
json_t * StatsToJSON(const StatsTable *st, uint8_t flags)
turn StatsTable into a json object
Definition: output-json-stats.c:211
conf.h
TmEcode
TmEcode
Definition: tm-threads-common.h:79
StatsTable_::start_time
time_t start_time
Definition: output-stats.h:41
MemBuffer_
Definition: util-buffer.h:27
JsonStatsLogThread_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-stats.c:73
StatsTable_::stats
StatsRecord * stats
Definition: output-stats.h:37
MemBufferReset
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
JsonStatsLogThread_
Definition: output-json-stats.c:71
OutputInitResult_
Definition: output.h:43
DetectEngineCtx_::sig_stat
SigFileLoaderStat sig_stat
Definition: detect.h:948
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
output-json-stats.h
OUTPUT_ENGINE_LAST_RELOAD
@ OUTPUT_ENGINE_LAST_RELOAD
Definition: output-json-stats.c:61
DetectEngineCtx_::next
struct DetectEngineCtx_ * next
Definition: detect.h:905
OUTPUT_ENGINE_ALL
@ OUTPUT_ENGINE_ALL
Definition: output-json-stats.c:63
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
StatsEnabled
bool StatsEnabled(void)
Definition: counters.c:120
JSON_STATS_DELTAS
#define JSON_STATS_DELTAS
Definition: output-json-stats.h:31
StatsRecord_::tm_name
const char * tm_name
Definition: output-stats.h:31
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
JSON_STATS_THREADS
#define JSON_STATS_THREADS
Definition: output-json-stats.h:30
StatsTable_::tstats
StatsRecord * tstats
Definition: output-stats.h:38
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
threadvars.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
OutputStatsCtx_::flags
uint32_t flags
Definition: output-json-stats.c:68
str
#define str(s)
Definition: suricata-common.h:273
SCLogWarning
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:244
SCFree
#define SCFree(p)
Definition: util-mem.h:61
SigFileLoaderStat_::good_sigs_total
int good_sigs_total
Definition: detect.h:733
ConfNode_
Definition: conf.h:32
util-logopenfile.h
util-buffer.h
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:590
OutputStatsCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-stats.c:67
SC_WARN_EVE_MISSING_EVENTS
@ SC_WARN_EVE_MISSING_EVENTS
Definition: util-error.h:351
OutputJsonCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json.h:102
OutputJSONBuffer
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:949
OutputEngineInfo
enum OutputEngineInfo_ OutputEngineInfo
GetDocURL
const char * GetDocURL(void)
Definition: suricata.c:1022
SC_ERR_JSON_STATS_LOG_NEGATED
@ SC_ERR_JSON_STATS_LOG_NEGATED
Definition: util-error.h:306
OUTPUT_ENGINE_RULESET
@ OUTPUT_ENGINE_RULESET
Definition: output-json-stats.c:62
StatsRecord_::pvalue
uint64_t pvalue
Definition: output-stats.h:33
OutputEngineStatsRuleset
TmEcode OutputEngineStatsRuleset(json_t **jdata)
Definition: output-json-stats.c:165
JsonStatsLogThread_::statslog_ctx
OutputStatsCtx * statslog_ctx
Definition: output-json-stats.c:72
stats_decoder_events
bool stats_decoder_events
Definition: counters.c:103
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
MODULE_NAME
#define MODULE_NAME
Definition: output-json-stats.c:51
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
OutputStatsCtx
struct OutputStatsCtx_ OutputStatsCtx
SigFileLoaderStat_
Signature loader statistics.
Definition: detect.h:729
StatsTable_::nstats
uint32_t nstats
Definition: output-stats.h:39
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
debug.h
output.h
OutputStatsCtx_
Definition: output-json-stats.c:66
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842