suricata
util-profiling-rulegroups.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * An API for rule profiling operations.
25  */
26 
27 #include "suricata-common.h"
28 #include "decode.h"
29 #include "detect.h"
30 #include "detect-engine.h"
31 #include "conf.h"
32 
33 #include "tm-threads.h"
34 
35 #include "util-unittest.h"
36 #include "util-byte.h"
37 #include "util-profiling.h"
38 #include "util-profiling-locks.h"
39 
40 #ifdef PROFILING
41 
42 /**
43  * Extra data for rule profiling.
44  */
45 typedef struct SCProfileSghData_ {
46  uint64_t checks;
47 
48  uint64_t non_mpm_generic;
49  uint64_t non_mpm_syn;
50 
53 
56 
58 
59 typedef struct SCProfileSghDetectCtx_ {
60  uint32_t cnt;
62  pthread_mutex_t data_m;
64 
65 static int profiling_sghs_output_to_file = 0;
67 static char profiling_file_name[PATH_MAX];
68 static const char *profiling_file_mode = "a";
69 static int profiling_rulegroup_json = 0;
70 
72 {
73  ConfNode *conf;
74 
75  conf = ConfGetNode("profiling.rulegroups");
76  if (conf != NULL) {
77  if (ConfNodeChildValueIsTrue(conf, "enabled")) {
79  const char *filename = ConfNodeLookupChildValue(conf, "filename");
80  if (filename != NULL) {
81  const char *log_dir;
82  log_dir = ConfigGetLogDirectory();
83 
84  snprintf(profiling_file_name, sizeof(profiling_file_name),
85  "%s/%s", log_dir, filename);
86 
87  const char *v = ConfNodeLookupChildValue(conf, "append");
88  if (v == NULL || ConfValIsTrue(v)) {
89  profiling_file_mode = "a";
90  } else {
91  profiling_file_mode = "w";
92  }
93 
94  profiling_sghs_output_to_file = 1;
95  }
96  if (ConfNodeChildValueIsTrue(conf, "json")) {
97  profiling_rulegroup_json = 1;
98  }
99  }
100  }
101 }
102 
103 static void DoDumpJSON(SCProfileSghDetectCtx *rules_ctx, FILE *fp, const char *name)
104 {
105  char timebuf[64];
106  uint32_t i;
107  struct timeval tval;
108 
109  json_t *js = json_object();
110  if (js == NULL)
111  return;
112  json_t *jsa = json_array();
113  if (jsa == NULL) {
114  json_decref(js);
115  return;
116  }
117 
118  gettimeofday(&tval, NULL);
119  CreateIsoTimeString(&tval, timebuf, sizeof(timebuf));
120  json_object_set_new(js, "timestamp", json_string(timebuf));
121 
122  for (i = 0; i < rules_ctx->cnt; i++) {
123  SCProfileSghData *d = &rules_ctx->data[i];
124  if (d == NULL || d->checks == 0)
125  continue;
126 
127  double avgsigs = 0;
128  double avgmpms = 0;
129 
130  if (d->post_prefilter_sigs_total && d->checks) {
131  avgsigs = (double)((double)d->post_prefilter_sigs_total / (double)d->checks);
132  }
133  if (d->mpm_match_cnt_total && d->checks) {
134  avgmpms = (double)((double)d->mpm_match_cnt_total / (double)d->checks);
135  }
136 
137  json_t *jsm = json_object();
138  if (jsm) {
139  json_object_set_new(jsm, "id", json_integer(i));
140  json_object_set_new(jsm, "checks", json_integer(d->checks));
141  json_object_set_new(jsm, "non_mpm_generic", json_integer(d->non_mpm_generic));
142  json_object_set_new(jsm, "non_mpm_syn", json_integer(d->non_mpm_syn));
143  json_object_set_new(jsm, "avgmpms", json_real(avgmpms));
144  json_object_set_new(jsm, "mpm_match_cnt_max", json_integer(d->mpm_match_cnt_max));
145  json_object_set_new(jsm, "avgsigs", json_real(avgsigs));
146  json_object_set_new(jsm, "post_prefilter_sigs_max", json_integer(d->post_prefilter_sigs_max));
147  json_array_append_new(jsa, jsm);
148  }
149  }
150  json_object_set_new(js, "rule_groups", jsa);
151 
152  char *js_s = json_dumps(js,
153  JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
155  if (likely(js_s != NULL)) {
156  fprintf(fp, "%s", js_s);
157  free(js_s);
158  }
159  json_decref(js);
160 }
161 
162 static void DoDump(SCProfileSghDetectCtx *rules_ctx, FILE *fp, const char *name)
163 {
164  uint32_t i;
165  struct timeval tval;
166  struct tm *tms;
167  struct tm local_tm;
168 
169  gettimeofday(&tval, NULL);
170  tms = SCLocalTime(tval.tv_sec, &local_tm);
171 
172  fprintf(fp, " ----------------------------------------------"
173  "------------------------------------------------------"
174  "----------------------------\n");
175  fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- "
176  "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
177  tms->tm_hour,tms->tm_min, tms->tm_sec);
178 
179  fprintf(fp, " ----------------------------------------------"
180  "------------------------------------------------------"
181  "----------------------------\n");
182  fprintf(fp, " Stats for: %s %u\n", name, rules_ctx->cnt);
183  fprintf(fp, " ----------------------------------------------"
184  "------------------------------------------------------"
185  "----------------------------\n");
186  fprintf(fp, " %-16s %-15s %-15s %-15s %-15s %-15s %-15s %-15s\n", "Sgh", "Checks", "Non-MPM(gen)", "Non-Mpm(syn)", "MPM Matches", "MPM Match Max", "Post-Filter", "Post-Filter Max");
187  fprintf(fp, " ---------------- "
188  "--------------- "
189  "--------------- "
190  "--------------- "
191  "--------------- "
192  "--------------- "
193  "--------------- "
194  "--------------- "
195  "\n");
196  for (i = 0; i < rules_ctx->cnt; i++) {
197  SCProfileSghData *d = &rules_ctx->data[i];
198  if (d == NULL || d->checks == 0)
199  continue;
200 
201  double avgsigs = 0;
202  double avgmpms = 0;
203 
204  if (d->post_prefilter_sigs_total && d->checks) {
205  avgsigs = (double)((double)d->post_prefilter_sigs_total / (double)d->checks);
206  }
207  if (d->mpm_match_cnt_total && d->checks) {
208  avgmpms = (double)((double)d->mpm_match_cnt_total / (double)d->checks);
209  }
210 
211  fprintf(fp,
212  " %-16u %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f %-15"PRIu64" %-15.2f %-15"PRIu64"\n",
213  i,
214  d->checks,
215  d->non_mpm_generic,
216  d->non_mpm_syn,
217  avgmpms,
219  avgsigs,
221  }
222  fprintf(fp,"\n");
223 }
224 
225 static void
226 SCProfilingSghDump(DetectEngineCtx *de_ctx)
227 {
228  FILE *fp;
229 
230  if (profiling_sghs_enabled == 0)
231  return;
232 
233  if (profiling_sghs_output_to_file == 1) {
234  SCLogDebug("file %s mode %s", profiling_file_name, profiling_file_mode);
235 
236  fp = fopen(profiling_file_name, profiling_file_mode);
237 
238  if (fp == NULL) {
239  SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", profiling_file_name,
240  strerror(errno));
241  return;
242  }
243  } else {
244  fp = stdout;
245  }
246 
247  if (profiling_rulegroup_json) {
248  DoDumpJSON(de_ctx->profile_sgh_ctx, fp, "rule groups");
249  } else {
250  DoDump(de_ctx->profile_sgh_ctx, fp, "rule groups");
251  }
252 
253  if (fp != stdout)
254  fclose(fp);
255 
256  SCLogPerf("Done dumping rulegroup profiling data.");
257 }
258 
259 /**
260  * \brief Update a rule counter.
261  *
262  * \param id The ID of this counter.
263  * \param ticks Number of CPU ticks for this rule.
264  * \param match Did the rule match?
265  */
266 void
268 {
269  if (det_ctx != NULL && det_ctx->sgh_perf_data != NULL && sgh->id < det_ctx->de_ctx->sgh_array_cnt) {
270  SCProfileSghData *p = &det_ctx->sgh_perf_data[sgh->id];
271  p->checks++;
272 
273  if (det_ctx->non_pf_store_cnt > 0) {
274  if (det_ctx->non_pf_store_ptr == sgh->non_pf_syn_store_array)
275  p->non_mpm_syn++;
276  else
277  p->non_mpm_generic++;
278  }
280  if (det_ctx->match_array_cnt > p->post_prefilter_sigs_max)
283  if (det_ctx->pmq.rule_id_array_cnt > p->mpm_match_cnt_max)
285  }
286 }
287 
288 static SCProfileSghDetectCtx *SCProfilingSghInitCtx(void)
289 {
291  if (ctx != NULL) {
292  if (pthread_mutex_init(&ctx->data_m, NULL) != 0) {
293  FatalError(SC_ERR_FATAL, "Failed to initialize mutex.");
294  }
295  }
296 
297  return ctx;
298 }
299 
300 static void DetroyCtx(SCProfileSghDetectCtx *ctx)
301 {
302  if (ctx) {
303  if (ctx->data != NULL)
304  SCFree(ctx->data);
305  pthread_mutex_destroy(&ctx->data_m);
306  SCFree(ctx);
307  }
308 }
309 
311 {
312  if (de_ctx != NULL) {
313  SCProfilingSghDump(de_ctx);
314 
315  DetroyCtx(de_ctx->profile_sgh_ctx);
316  }
317 }
318 
320 {
321  if (ctx == NULL)
322  return;
323 
324  uint32_t array_size = det_ctx->de_ctx->sgh_array_cnt;
325 
326  SCProfileSghData *a = SCCalloc(array_size, sizeof(SCProfileSghData));
327  if (a != NULL) {
328  det_ctx->sgh_perf_data = a;
329  }
330 }
331 
332 static void SCProfilingSghThreadMerge(DetectEngineCtx *de_ctx, const DetectEngineThreadCtx *det_ctx)
333 {
334  if (de_ctx == NULL || de_ctx->profile_sgh_ctx == NULL ||
335  de_ctx->profile_sgh_ctx->data == NULL || det_ctx == NULL ||
336  det_ctx->sgh_perf_data == NULL)
337  return;
338 
339 #define ADD(name) de_ctx->profile_sgh_ctx->data[i].name += det_ctx->sgh_perf_data[i].name
340  uint32_t i;
341  for (i = 0; i < de_ctx->sgh_array_cnt; i++) {
342  ADD(checks);
343  ADD(non_mpm_generic);
344  ADD(non_mpm_syn);
345  ADD(post_prefilter_sigs_total);
346  ADD(mpm_match_cnt_total);
347 
352  }
353 #undef ADD
354 }
355 
357 {
358  if (det_ctx == NULL || det_ctx->de_ctx == NULL || det_ctx->sgh_perf_data == NULL)
359  return;
360 
361  pthread_mutex_lock(&det_ctx->de_ctx->profile_sgh_ctx->data_m);
362  SCProfilingSghThreadMerge(det_ctx->de_ctx, det_ctx);
363  pthread_mutex_unlock(&det_ctx->de_ctx->profile_sgh_ctx->data_m);
364 
365  SCFree(det_ctx->sgh_perf_data);
366  det_ctx->sgh_perf_data = NULL;
367 }
368 
369 /**
370  * \brief Register the keyword profiling counters.
371  *
372  * \param de_ctx The active DetectEngineCtx, used to get at the loaded rules.
373  */
374 void
376 {
377  if (profiling_sghs_enabled == 0)
378  return;
379 
380  de_ctx->profile_sgh_ctx = SCProfilingSghInitCtx();
381  BUG_ON(de_ctx->profile_sgh_ctx == NULL);
382 
384  BUG_ON(de_ctx->profile_sgh_ctx->data == NULL);
385 
387 
388  SCLogPerf("Registered %"PRIu32" rulegroup profiling counters.", de_ctx->sgh_array_cnt);
389 }
390 
391 #endif /* PROFILING */
SCProfilingSghThreadCleanup
void SCProfilingSghThreadCleanup(DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-rulegroups.c:356
DetectEngineThreadCtx_::non_pf_store_ptr
SignatureNonPrefilterStore * non_pf_store_ptr
Definition: detect.h:1102
util-byte.h
tm-threads.h
detect-engine.h
ConfNodeChildValueIsTrue
int ConfNodeChildValueIsTrue(const ConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:887
DetectEngineThreadCtx_::match_array_cnt
SigIntId match_array_cnt
Definition: detect.h:1097
SCProfilingSghThreadSetup
void SCProfilingSghThreadSetup(SCProfileSghDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-rulegroups.c:319
CreateIsoTimeString
void CreateIsoTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:213
PrefilterRuleStore_::rule_id_array_cnt
uint32_t rule_id_array_cnt
Definition: util-prefilter.h:38
SigGroupHead_
Container for matching data for a signature group.
Definition: detect.h:1347
DetectEngineThreadCtx_::sgh_perf_data
struct SCProfileSghData_ * sgh_perf_data
Definition: detect.h:1168
SCProfileSghDetectCtx_
Definition: util-profiling-rulegroups.c:59
SCProfileSghDetectCtx_::data_m
pthread_mutex_t data_m
Definition: util-profiling-rulegroups.c:62
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:175
DetectEngineThreadCtx_::pmq
PrefilterRuleStore pmq
Definition: detect.h:1111
SCProfilingSghInitCounters
void SCProfilingSghInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
Definition: util-profiling-rulegroups.c:375
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:767
DetectEngineCtx_::profile_sgh_ctx
struct SCProfileSghDetectCtx_ * profile_sgh_ctx
Definition: detect.h:893
JSON_ESCAPE_SLASH
#define JSON_ESCAPE_SLASH
Definition: suricata-common.h:258
SCProfileSghData_::mpm_match_cnt_total
uint64_t mpm_match_cnt_total
Definition: util-profiling-rulegroups.c:54
util-unittest.h
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:565
SCProfileSghData_::checks
uint64_t checks
Definition: util-profiling-rulegroups.c:46
SCProfileSghData_::post_prefilter_sigs_max
uint64_t post_prefilter_sigs_max
Definition: util-profiling-rulegroups.c:52
decode.h
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
profiling_sghs_enabled
int profiling_sghs_enabled
Definition: util-profiling-rulegroups.c:66
DetectEngineThreadCtx_
Definition: detect.h:1010
SCProfileSghData_
Definition: util-profiling-rulegroups.c:45
detect.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:282
util-profiling.h
SC_ERR_FOPEN
@ SC_ERR_FOPEN
Definition: util-error.h:74
SCLocalTime
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:270
conf.h
DetectEngineCtx_::sgh_array_cnt
uint32_t sgh_array_cnt
Definition: detect.h:849
SCProfileSghData
struct SCProfileSghData_ SCProfileSghData
SCProfileSghData_::non_mpm_generic
uint64_t non_mpm_generic
Definition: util-profiling-rulegroups.c:48
suricata-common.h
SCProfileSghDetectCtx
struct SCProfileSghDetectCtx_ SCProfileSghDetectCtx
SCLogPerf
#define SCLogPerf(...)
Definition: util-debug.h:224
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
FatalError
#define FatalError(x,...)
Definition: util-debug.h:532
ConfigGetLogDirectory
const char * ConfigGetLogDirectory()
Definition: util-conf.c:35
DetectEngineThreadCtx_::non_pf_store_cnt
uint32_t non_pf_store_cnt
Definition: detect.h:1103
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ADD
#define ADD(name)
ConfNode_
Definition: conf.h:32
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
SCProfilingSghDestroyCtx
void SCProfilingSghDestroyCtx(DetectEngineCtx *de_ctx)
Definition: util-profiling-rulegroups.c:310
SCProfileSghData_::mpm_match_cnt_max
uint64_t mpm_match_cnt_max
Definition: util-profiling-rulegroups.c:55
DetectEngineThreadCtx_::de_ctx
DetectEngineCtx * de_ctx
Definition: detect.h:1136
SCProfileSghDetectCtx_::cnt
uint32_t cnt
Definition: util-profiling-rulegroups.c:60
SigGroupHead_::non_pf_syn_store_array
SignatureNonPrefilterStore * non_pf_syn_store_array
Definition: detect.h:1359
SCProfileSghData_::non_mpm_syn
uint64_t non_mpm_syn
Definition: util-profiling-rulegroups.c:49
likely
#define likely(expr)
Definition: util-optimize.h:32
SCProfilingSghUpdateCounter
void SCProfilingSghUpdateCounter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh)
Update a rule counter.
Definition: util-profiling-rulegroups.c:267
SCProfilingSghsGlobalInit
void SCProfilingSghsGlobalInit(void)
Definition: util-profiling-rulegroups.c:71
util-profiling-locks.h
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SigGroupHead_::id
uint32_t id
Definition: detect.h:1365
SCProfileSghDetectCtx_::data
SCProfileSghData * data
Definition: util-profiling-rulegroups.c:61
SCProfileSghData_::post_prefilter_sigs_total
uint64_t post_prefilter_sigs_total
Definition: util-profiling-rulegroups.c:51
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842