suricata
output-json-stats.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Implements JSON stats counters logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "detect.h"
28 #include "pkt-var.h"
29 #include "conf.h"
30 #include "detect-engine.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-time.h"
38 #include "util-unittest.h"
39 
40 #include "util-debug.h"
41 #include "output.h"
42 #include "util-privs.h"
43 #include "util-buffer.h"
44 
45 #include "util-logopenfile.h"
46 
47 #include "output-json.h"
48 #include "output-json-stats.h"
49 
50 #define MODULE_NAME "JsonStatsLog"
51 
52 extern bool stats_decoder_events;
53 extern const char *stats_decoder_events_prefix;
54 
55 /**
56  * specify which engine info will be printed in stats log.
57  * ALL means both last reload and ruleset stats.
58  */
59 typedef enum OutputEngineInfo_ {
64 
65 typedef struct OutputStatsCtx_ {
67  uint8_t flags; /** Store mode */
69 
70 typedef struct JsonStatsLogThread_ {
75 
76 static json_t *EngineStats2Json(const DetectEngineCtx *de_ctx,
77  const OutputEngineInfo output)
78 {
79  char timebuf[64];
80  const SigFileLoaderStat *sig_stat = NULL;
81 
82  json_t *jdata = json_object();
83  if (jdata == NULL) {
84  return NULL;
85  }
86 
87  if (output == OUTPUT_ENGINE_LAST_RELOAD || output == OUTPUT_ENGINE_ALL) {
89  CreateIsoTimeString(last_reload, timebuf, sizeof(timebuf));
90  json_object_set_new(jdata, "last_reload", json_string(timebuf));
91  }
92 
93  sig_stat = &de_ctx->sig_stat;
94  if ((output == OUTPUT_ENGINE_RULESET || output == OUTPUT_ENGINE_ALL) &&
95  sig_stat != NULL)
96  {
97  json_object_set_new(jdata, "rules_loaded",
98  json_integer(sig_stat->good_sigs_total));
99  json_object_set_new(jdata, "rules_failed",
100  json_integer(sig_stat->bad_sigs_total));
101  }
102 
103  return jdata;
104 }
105 
106 static TmEcode OutputEngineStats2Json(json_t **jdata, const OutputEngineInfo output)
107 {
109  if (de_ctx == NULL) {
110  goto err1;
111  }
112  /* Since we need to deference de_ctx pointer, we don't want to lost it. */
113  DetectEngineCtx *list = de_ctx;
114 
115  json_t *js_tenant_list = json_array();
116  json_t *js_tenant = NULL;
117 
118  if (js_tenant_list == NULL) {
119  goto err2;
120  }
121 
122  while(list) {
123  js_tenant = json_object();
124  if (js_tenant == NULL) {
125  goto err3;
126  }
127  json_object_set_new(js_tenant, "id", json_integer(list->tenant_id));
128 
129  json_t *js_stats = EngineStats2Json(list, output);
130  if (js_stats == NULL) {
131  goto err4;
132  }
133  json_object_update(js_tenant, js_stats);
134  json_array_append_new(js_tenant_list, js_tenant);
135  json_decref(js_stats);
136  list = list->next;
137  }
138 
140  *jdata = js_tenant_list;
141  return TM_ECODE_OK;
142 
143 err4:
144  json_object_clear(js_tenant);
145  json_decref(js_tenant);
146 
147 err3:
148  json_object_clear(js_tenant_list);
149  json_decref(js_tenant_list);
150 
151 err2:
153 
154 err1:
155  json_object_set_new(*jdata, "message", json_string("Unable to get info"));
156  return TM_ECODE_FAILED;
157 }
158 
160  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_LAST_RELOAD);
161 }
162 
164  return OutputEngineStats2Json(jdata, OUTPUT_ENGINE_RULESET);
165 }
166 
167 static json_t *OutputStats2Json(json_t *js, const char *key)
168 {
169  void *iter;
170 
171  const char *dot = strchr(key, '.');
172  if (dot == NULL)
173  return NULL;
174  if (strlen(dot) > 2) {
175  if (*(dot + 1) == '.' && *(dot + 2) != '\0')
176  dot = strchr(dot + 2, '.');
177  }
178 
179  size_t predot_len = (dot - key) + 1;
180  char s[predot_len];
181  strlcpy(s, key, predot_len);
182 
183  iter = json_object_iter_at(js, s);
184  const char *s2 = strchr(dot+1, '.');
185 
186  json_t *value = json_object_iter_value(iter);
187  if (value == NULL) {
188  value = json_object();
189 
190  if (!strncmp(s, "detect", 6)) {
191  json_t *js_engine = NULL;
192 
193  TmEcode ret = OutputEngineStats2Json(&js_engine, OUTPUT_ENGINE_ALL);
194  if (ret == TM_ECODE_OK && js_engine) {
195  json_object_set_new(value, "engines", js_engine);
196  }
197  }
198  json_object_set_new(js, s, value);
199  }
200  if (s2 != NULL) {
201  return OutputStats2Json(value, &key[dot-key+1]);
202  }
203  return value;
204 }
205 
206 /** \brief turn StatsTable into a json object
207  * \param flags JSON_STATS_* flags for controlling output
208  */
209 json_t *StatsToJSON(const StatsTable *st, uint8_t flags)
210 {
211  const char delta_suffix[] = "_delta";
212  struct timeval tval;
213  gettimeofday(&tval, NULL);
214 
215  json_t *js_stats = json_object();
216  if (unlikely(js_stats == NULL)) {
217  return NULL;
218  }
219 
220  /* Uptime, in seconds. */
221  double up_time_d = difftime(tval.tv_sec, st->start_time);
222  json_object_set_new(js_stats, "uptime",
223  json_integer((int)up_time_d));
224 
225  uint32_t u = 0;
226  if (flags & JSON_STATS_TOTALS) {
227  for (u = 0; u < st->nstats; u++) {
228  if (st->stats[u].name == NULL)
229  continue;
230  const char *name = st->stats[u].name;
231  const char *shortname = name;
232  if (strrchr(name, '.') != NULL) {
233  shortname = &name[strrchr(name, '.') - name + 1];
234  }
235  json_t *js_type = OutputStats2Json(js_stats, name);
236  if (js_type != NULL) {
237  json_object_set_new(js_type, shortname,
238  json_integer(st->stats[u].value));
239 
240  if (flags & JSON_STATS_DELTAS) {
241  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
242  snprintf(deltaname, sizeof(deltaname), "%s%s", shortname,
243  delta_suffix);
244  json_object_set_new(js_type, deltaname,
245  json_integer(st->stats[u].value - st->stats[u].pvalue));
246  }
247  }
248  }
249  }
250 
251  /* per thread stats - stored in a "threads" object. */
252  if (st->tstats != NULL && (flags & JSON_STATS_THREADS)) {
253  /* for each thread (store) */
254  json_t *threads = json_object();
255  if (unlikely(threads == NULL)) {
256  json_decref(js_stats);
257  return NULL;
258  }
259  uint32_t x;
260  for (x = 0; x < st->ntstats; x++) {
261  uint32_t offset = x * st->nstats;
262 
263  /* for each counter */
264  for (u = offset; u < (offset + st->nstats); u++) {
265  if (st->tstats[u].name == NULL)
266  continue;
267 
268  char str[256];
269  snprintf(str, sizeof(str), "%s.%s", st->tstats[u].tm_name, st->tstats[u].name);
270  char *shortname = &str[strrchr(str, '.') - str + 1];
271  json_t *js_type = OutputStats2Json(threads, str);
272 
273  if (js_type != NULL) {
274  json_object_set_new(js_type, shortname, json_integer(st->tstats[u].value));
275 
276  if (flags & JSON_STATS_DELTAS) {
277  char deltaname[strlen(shortname) + strlen(delta_suffix) + 1];
278  snprintf(deltaname, sizeof(deltaname), "%s%s",
279  shortname, delta_suffix);
280  json_object_set_new(js_type, deltaname,
281  json_integer(st->tstats[u].value - st->tstats[u].pvalue));
282  }
283  }
284  }
285  }
286  json_object_set_new(js_stats, "threads", threads);
287  }
288  return js_stats;
289 }
290 
291 static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
292 {
293  SCEnter();
294  JsonStatsLogThread *aft = (JsonStatsLogThread *)thread_data;
295 
296  struct timeval tval;
297  gettimeofday(&tval, NULL);
298 
299  json_t *js = json_object();
300  if (unlikely(js == NULL))
301  return 0;
302  char timebuf[64];
303  CreateIsoTimeString(SCTIME_FROM_TIMEVAL(&tval), timebuf, sizeof(timebuf));
304  json_object_set_new(js, "timestamp", json_string(timebuf));
305  json_object_set_new(js, "event_type", json_string("stats"));
306 
307  json_t *js_stats = StatsToJSON(st, aft->statslog_ctx->flags);
308  if (js_stats == NULL) {
309  json_decref(js);
310  return 0;
311  }
312 
313  json_object_set_new(js, "stats", js_stats);
314 
315  OutputJSONBuffer(js, aft->file_ctx, &aft->buffer);
316  MemBufferReset(aft->buffer);
317 
318  json_object_clear(js_stats);
319  json_object_del(js, "stats");
320  json_object_clear(js);
321  json_decref(js);
322 
323  SCReturnInt(0);
324 }
325 
326 static TmEcode JsonStatsLogThreadInit(ThreadVars *t, const void *initdata, void **data)
327 {
329  if (unlikely(aft == NULL))
330  return TM_ECODE_FAILED;
331 
332  if(initdata == NULL)
333  {
334  SCLogDebug("Error getting context for EveLogStats. \"initdata\" argument NULL");
335  goto error_exit;
336  }
337 
339  if (aft->buffer == NULL) {
340  goto error_exit;
341  }
342 
343  /* Use the Output Context (file pointer and mutex) */
344  aft->statslog_ctx = ((OutputCtx *)initdata)->data;
345 
347  if (!aft->file_ctx) {
348  goto error_exit;
349  }
350 
351  *data = (void *)aft;
352  return TM_ECODE_OK;
353 
354 error_exit:
355  if (aft->buffer != NULL) {
356  MemBufferFree(aft->buffer);
357  }
358  SCFree(aft);
359  return TM_ECODE_FAILED;
360 }
361 
362 static TmEcode JsonStatsLogThreadDeinit(ThreadVars *t, void *data)
363 {
364  JsonStatsLogThread *aft = (JsonStatsLogThread *)data;
365  if (aft == NULL) {
366  return TM_ECODE_OK;
367  }
368 
369  MemBufferFree(aft->buffer);
370 
371  /* clear memory */
372  memset(aft, 0, sizeof(JsonStatsLogThread));
373 
374  SCFree(aft);
375  return TM_ECODE_OK;
376 }
377 
378 static void OutputStatsLogDeinitSub(OutputCtx *output_ctx)
379 {
380  OutputStatsCtx *stats_ctx = output_ctx->data;
381  SCFree(stats_ctx);
382  SCFree(output_ctx);
383 }
384 
385 static OutputInitResult OutputStatsLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
386 {
387  OutputInitResult result = { NULL, false };
388  OutputJsonCtx *ajt = parent_ctx->data;
389 
390  if (!StatsEnabled()) {
391  SCLogError("eve.stats: stats are disabled globally: set stats.enabled to true. "
392  "See %s/configuration/suricata-yaml.html#stats",
393  GetDocURL());
394  return result;
395  }
396 
397  OutputStatsCtx *stats_ctx = SCMalloc(sizeof(OutputStatsCtx));
398  if (unlikely(stats_ctx == NULL))
399  return result;
400 
401  if (stats_decoder_events &&
402  strcmp(stats_decoder_events_prefix, "decoder") == 0) {
403  SCLogWarning("eve.stats will not display "
404  "all decoder events correctly. See ticket #2225. Set a prefix in "
405  "stats.decoder-events-prefix.");
406  }
407 
408  stats_ctx->flags = JSON_STATS_TOTALS;
409 
410  if (conf != NULL) {
411  const char *totals = ConfNodeLookupChildValue(conf, "totals");
412  const char *threads = ConfNodeLookupChildValue(conf, "threads");
413  const char *deltas = ConfNodeLookupChildValue(conf, "deltas");
414  SCLogDebug("totals %s threads %s deltas %s", totals, threads, deltas);
415 
416  if ((totals != NULL && ConfValIsFalse(totals)) &&
417  (threads != NULL && ConfValIsFalse(threads))) {
418  SCFree(stats_ctx);
419  SCLogError("Cannot disable both totals and threads in stats logging");
420  return result;
421  }
422 
423  if (totals != NULL && ConfValIsFalse(totals)) {
424  stats_ctx->flags &= ~JSON_STATS_TOTALS;
425  }
426  if (threads != NULL && ConfValIsTrue(threads)) {
427  stats_ctx->flags |= JSON_STATS_THREADS;
428  }
429  if (deltas != NULL && ConfValIsTrue(deltas)) {
430  stats_ctx->flags |= JSON_STATS_DELTAS;
431  }
432  SCLogDebug("stats_ctx->flags %08x", stats_ctx->flags);
433  }
434 
435  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
436  if (unlikely(output_ctx == NULL)) {
437  SCFree(stats_ctx);
438  return result;
439  }
440 
441  SCLogDebug("Preparing file context for stats submodule logger");
442  /* Share output slot with thread 1 */
443  stats_ctx->file_ctx = LogFileEnsureExists(ajt->file_ctx);
444  if (!stats_ctx->file_ctx) {
445  SCFree(stats_ctx);
446  SCFree(output_ctx);
447  return result;
448  }
449 
450  output_ctx->data = stats_ctx;
451  output_ctx->DeInit = OutputStatsLogDeinitSub;
452 
453  result.ctx = output_ctx;
454  result.ok = true;
455  return result;
456 }
457 
459  /* register as child of eve-log */
461  "eve-log.stats", OutputStatsLogInitSub, JsonStatsLogger,
462  JsonStatsLogThreadInit, JsonStatsLogThreadDeinit, NULL);
463 }
tm-threads.h
DetectEngineCtx_::tenant_id
int tenant_id
Definition: detect.h:789
StatsTable_::ntstats
uint32_t ntstats
Definition: output-stats.h:42
detect-engine.h
OutputRegisterStatsSubModule
void OutputRegisterStatsSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, StatsLogger StatsLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a stats data output sub-module.
Definition: output.c:746
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
DetectEngineDeReference
void DetectEngineDeReference(DetectEngineCtx **de_ctx)
Definition: detect-engine.c:4307
CreateIsoTimeString
void CreateIsoTimeString(const SCTime_t ts, char *str, size_t size)
Definition: util-time.c:209
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
LOGGER_JSON_STATS
@ LOGGER_JSON_STATS
Definition: suricata-common.h:477
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
JSON_OUTPUT_BUFFER_SIZE
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:63
threads.h
JsonStatsLogThread
struct JsonStatsLogThread_ JsonStatsLogThread
OutputJsonCtx_
Definition: output-json.h:81
LogFileCtx_
Definition: util-logopenfile.h:73
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:785
OutputStatsCtx_::flags
uint8_t flags
Definition: output-json-stats.c:67
DetectEngineGetCurrent
DetectEngineCtx * DetectEngineGetCurrent(void)
Definition: detect-engine.c:3610
util-privs.h
JSON_STATS_TOTALS
#define JSON_STATS_TOTALS
Definition: output-json-stats.h:29
StatsRecord_::pvalue
int64_t pvalue
Definition: output-stats.h:35
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
JsonStatsLogRegister
void JsonStatsLogRegister(void)
Definition: output-json-stats.c:458
util-unittest.h
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:531
OutputCtx_::data
void * data
Definition: tm-modules.h:81
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
OutputCtx_
Definition: tm-modules.h:78
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
OutputEngineInfo_
OutputEngineInfo_
Definition: output-json-stats.c:59
stats_decoder_events_prefix
const char * stats_decoder_events_prefix
Definition: counters.c:102
util-debug.h
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:47
output-json.h
DetectEngineCtx_::last_reload
struct timeval last_reload
Definition: detect.h:965
JsonStatsLogThread_::buffer
MemBuffer * buffer
Definition: output-json-stats.c:73
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
StatsRecord_::name
const char * name
Definition: output-stats.h:32
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:723
SCTIME_FROM_TIMEVAL
#define SCTIME_FROM_TIMEVAL(tv)
Definition: util-time.h:65
pkt-var.h
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:48
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
SigFileLoaderStat_::bad_sigs_total
int bad_sigs_total
Definition: detect.h:752
OutputEngineStatsReloadTime
TmEcode OutputEngineStatsReloadTime(json_t **jdata)
Definition: output-json-stats.c:159
StatsTable_
Definition: output-stats.h:38
StatsRecord_::value
int64_t value
Definition: output-stats.h:34
StatsToJSON
json_t * StatsToJSON(const StatsTable *st, uint8_t flags)
turn StatsTable into a json object
Definition: output-json-stats.c:209
conf.h
SCTime_t
Definition: util-time.h:40
TmEcode
TmEcode
Definition: tm-threads-common.h:83
StatsTable_::start_time
time_t start_time
Definition: output-stats.h:43
MemBuffer_
Definition: util-buffer.h:27
JsonStatsLogThread_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-stats.c:72
StatsTable_::stats
StatsRecord * stats
Definition: output-stats.h:39
MemBufferReset
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
JsonStatsLogThread_
Definition: output-json-stats.c:70
OutputInitResult_
Definition: output.h:46
DetectEngineCtx_::sig_stat
SigFileLoaderStat sig_stat
Definition: detect.h:968
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
output-json-stats.h
OUTPUT_ENGINE_LAST_RELOAD
@ OUTPUT_ENGINE_LAST_RELOAD
Definition: output-json-stats.c:60
DetectEngineCtx_::next
struct DetectEngineCtx_ * next
Definition: detect.h:924
OUTPUT_ENGINE_ALL
@ OUTPUT_ENGINE_ALL
Definition: output-json-stats.c:62
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:87
StatsEnabled
bool StatsEnabled(void)
Definition: counters.c:118
JSON_STATS_DELTAS
#define JSON_STATS_DELTAS
Definition: output-json-stats.h:31
StatsRecord_::tm_name
const char * tm_name
Definition: output-stats.h:33
JSON_STATS_THREADS
#define JSON_STATS_THREADS
Definition: output-json-stats.h:30
StatsTable_::tstats
StatsRecord * tstats
Definition: output-stats.h:40
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
threadvars.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
str
#define str(s)
Definition: suricata-common.h:280
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
SigFileLoaderStat_::good_sigs_total
int good_sigs_total
Definition: detect.h:751
ConfNode_
Definition: conf.h:32
util-logopenfile.h
util-buffer.h
ConfValIsFalse
int ConfValIsFalse(const char *val)
Check if a value is false.
Definition: conf.c:556
OutputStatsCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-stats.c:66
OutputJsonCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json.h:82
OutputJSONBuffer
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:905
OutputEngineInfo
enum OutputEngineInfo_ OutputEngineInfo
GetDocURL
const char * GetDocURL(void)
Definition: suricata.c:1105
OUTPUT_ENGINE_RULESET
@ OUTPUT_ENGINE_RULESET
Definition: output-json-stats.c:61
OutputEngineStatsRuleset
TmEcode OutputEngineStatsRuleset(json_t **jdata)
Definition: output-json-stats.c:163
JsonStatsLogThread_::statslog_ctx
OutputStatsCtx * statslog_ctx
Definition: output-json-stats.c:71
stats_decoder_events
bool stats_decoder_events
Definition: counters.c:101
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
MODULE_NAME
#define MODULE_NAME
Definition: output-json-stats.c:50
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
OutputStatsCtx
struct OutputStatsCtx_ OutputStatsCtx
SigFileLoaderStat_
Signature loader statistics.
Definition: detect.h:747
StatsTable_::nstats
uint32_t nstats
Definition: output-stats.h:41
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
output.h
OutputStatsCtx_
Definition: output-json-stats.c:65
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:808