suricata
util-profiling-rulegroups.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * An API for rule profiling operations.
25  */
26 
27 #include "suricata-common.h"
28 #include "util-profiling.h"
29 
30 #ifdef PROFILING
31 #include "util-conf.h"
32 #include "util-time.h"
33 
34 /**
35  * Extra data for rule profiling.
36  */
37 typedef struct SCProfileSghData_ {
38  uint64_t checks;
39 
40  uint64_t non_mpm_generic;
41  uint64_t non_mpm_syn;
42 
45 
48 
50 
51 typedef struct SCProfileSghDetectCtx_ {
52  uint32_t cnt;
54  pthread_mutex_t data_m;
56 
57 static int profiling_sghs_output_to_file = 0;
59 static char profiling_file_name[PATH_MAX];
60 static const char *profiling_file_mode = "a";
61 static int profiling_rulegroup_json = 0;
62 
64 {
65  ConfNode *conf;
66 
67  conf = ConfGetNode("profiling.rulegroups");
68  if (conf != NULL) {
69  if (ConfNodeChildValueIsTrue(conf, "enabled")) {
71  const char *filename = ConfNodeLookupChildValue(conf, "filename");
72  if (filename != NULL) {
73  const char *log_dir;
74  log_dir = ConfigGetLogDirectory();
75 
76  snprintf(profiling_file_name, sizeof(profiling_file_name),
77  "%s/%s", log_dir, filename);
78 
79  const char *v = ConfNodeLookupChildValue(conf, "append");
80  if (v == NULL || ConfValIsTrue(v)) {
81  profiling_file_mode = "a";
82  } else {
83  profiling_file_mode = "w";
84  }
85 
86  profiling_sghs_output_to_file = 1;
87  }
88  if (ConfNodeChildValueIsTrue(conf, "json")) {
89  profiling_rulegroup_json = 1;
90  }
91  }
92  }
93 }
94 
95 static void DoDumpJSON(SCProfileSghDetectCtx *rules_ctx, FILE *fp, const char *name)
96 {
97  char timebuf[64];
98  uint32_t i;
99  struct timeval tval;
100 
101  json_t *js = json_object();
102  if (js == NULL)
103  return;
104  json_t *jsa = json_array();
105  if (jsa == NULL) {
106  json_decref(js);
107  return;
108  }
109 
110  gettimeofday(&tval, NULL);
111  CreateIsoTimeString(SCTIME_FROM_TIMEVAL(&tval), timebuf, sizeof(timebuf));
112  json_object_set_new(js, "timestamp", json_string(timebuf));
113 
114  for (i = 0; i < rules_ctx->cnt; i++) {
115  SCProfileSghData *d = &rules_ctx->data[i];
116  if (d == NULL || d->checks == 0)
117  continue;
118 
119  double avgsigs = 0;
120  double avgmpms = 0;
121 
122  if (d->post_prefilter_sigs_total && d->checks) {
123  avgsigs = (double)((double)d->post_prefilter_sigs_total / (double)d->checks);
124  }
125  if (d->mpm_match_cnt_total && d->checks) {
126  avgmpms = (double)((double)d->mpm_match_cnt_total / (double)d->checks);
127  }
128 
129  json_t *jsm = json_object();
130  if (jsm) {
131  json_object_set_new(jsm, "id", json_integer(i));
132  json_object_set_new(jsm, "checks", json_integer(d->checks));
133  json_object_set_new(jsm, "non_mpm_generic", json_integer(d->non_mpm_generic));
134  json_object_set_new(jsm, "non_mpm_syn", json_integer(d->non_mpm_syn));
135  json_object_set_new(jsm, "avgmpms", json_real(avgmpms));
136  json_object_set_new(jsm, "mpm_match_cnt_max", json_integer(d->mpm_match_cnt_max));
137  json_object_set_new(jsm, "avgsigs", json_real(avgsigs));
138  json_object_set_new(jsm, "post_prefilter_sigs_max", json_integer(d->post_prefilter_sigs_max));
139  json_array_append_new(jsa, jsm);
140  }
141  }
142  json_object_set_new(js, "rule_groups", jsa);
143 
144  char *js_s = json_dumps(js,
145  JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
147  if (likely(js_s != NULL)) {
148  fprintf(fp, "%s", js_s);
149  free(js_s);
150  }
151  json_decref(js);
152 }
153 
154 static void DoDump(SCProfileSghDetectCtx *rules_ctx, FILE *fp, const char *name)
155 {
156  uint32_t i;
157  struct timeval tval;
158  struct tm *tms;
159  struct tm local_tm;
160 
161  gettimeofday(&tval, NULL);
162  tms = SCLocalTime(tval.tv_sec, &local_tm);
163 
164  fprintf(fp, " ----------------------------------------------"
165  "------------------------------------------------------"
166  "----------------------------\n");
167  fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- "
168  "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
169  tms->tm_hour,tms->tm_min, tms->tm_sec);
170 
171  fprintf(fp, " ----------------------------------------------"
172  "------------------------------------------------------"
173  "----------------------------\n");
174  fprintf(fp, " Stats for: %s %u\n", name, rules_ctx->cnt);
175  fprintf(fp, " ----------------------------------------------"
176  "------------------------------------------------------"
177  "----------------------------\n");
178  fprintf(fp, " %-16s %-15s %-15s %-15s %-15s %-15s %-15s %-15s\n", "Sgh", "Checks", "Non-MPM(gen)", "Non-Mpm(syn)", "MPM Matches", "MPM Match Max", "Post-Filter", "Post-Filter Max");
179  fprintf(fp, " ---------------- "
180  "--------------- "
181  "--------------- "
182  "--------------- "
183  "--------------- "
184  "--------------- "
185  "--------------- "
186  "--------------- "
187  "\n");
188  for (i = 0; i < rules_ctx->cnt; i++) {
189  SCProfileSghData *d = &rules_ctx->data[i];
190  if (d == NULL || d->checks == 0)
191  continue;
192 
193  double avgsigs = 0;
194  double avgmpms = 0;
195 
196  if (d->post_prefilter_sigs_total && d->checks) {
197  avgsigs = (double)((double)d->post_prefilter_sigs_total / (double)d->checks);
198  }
199  if (d->mpm_match_cnt_total && d->checks) {
200  avgmpms = (double)((double)d->mpm_match_cnt_total / (double)d->checks);
201  }
202 
203  fprintf(fp,
204  " %-16u %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f %-15"PRIu64" %-15.2f %-15"PRIu64"\n",
205  i,
206  d->checks,
207  d->non_mpm_generic,
208  d->non_mpm_syn,
209  avgmpms,
211  avgsigs,
213  }
214  fprintf(fp,"\n");
215 }
216 
217 static void
218 SCProfilingSghDump(DetectEngineCtx *de_ctx)
219 {
220  FILE *fp;
221 
222  if (profiling_sghs_enabled == 0)
223  return;
224 
225  if (profiling_sghs_output_to_file == 1) {
226  SCLogDebug("file %s mode %s", profiling_file_name, profiling_file_mode);
227 
228  fp = fopen(profiling_file_name, profiling_file_mode);
229 
230  if (fp == NULL) {
231  SCLogError("failed to open %s: %s", profiling_file_name, strerror(errno));
232  return;
233  }
234  } else {
235  fp = stdout;
236  }
237 
238  if (profiling_rulegroup_json) {
239  DoDumpJSON(de_ctx->profile_sgh_ctx, fp, "rule groups");
240  } else {
241  DoDump(de_ctx->profile_sgh_ctx, fp, "rule groups");
242  }
243 
244  if (fp != stdout)
245  fclose(fp);
246 
247  SCLogPerf("Done dumping rulegroup profiling data.");
248 }
249 
250 /**
251  * \brief Update a rule counter.
252  *
253  * \param id The ID of this counter.
254  * \param ticks Number of CPU ticks for this rule.
255  * \param match Did the rule match?
256  */
257 void
259 {
260  if (det_ctx != NULL && det_ctx->sgh_perf_data != NULL && sgh->id < det_ctx->de_ctx->sgh_array_cnt) {
261  SCProfileSghData *p = &det_ctx->sgh_perf_data[sgh->id];
262  p->checks++;
263 
264  if (det_ctx->non_pf_store_cnt > 0) {
265  if (det_ctx->non_pf_store_ptr == sgh->non_pf_syn_store_array)
266  p->non_mpm_syn++;
267  else
268  p->non_mpm_generic++;
269  }
271  if (det_ctx->match_array_cnt > p->post_prefilter_sigs_max)
274  if (det_ctx->pmq.rule_id_array_cnt > p->mpm_match_cnt_max)
276  }
277 }
278 
279 static SCProfileSghDetectCtx *SCProfilingSghInitCtx(void)
280 {
282  if (ctx != NULL) {
283  if (pthread_mutex_init(&ctx->data_m, NULL) != 0) {
284  FatalError("Failed to initialize mutex.");
285  }
286  }
287 
288  return ctx;
289 }
290 
291 static void DetroyCtx(SCProfileSghDetectCtx *ctx)
292 {
293  if (ctx) {
294  if (ctx->data != NULL)
295  SCFree(ctx->data);
296  pthread_mutex_destroy(&ctx->data_m);
297  SCFree(ctx);
298  }
299 }
300 
302 {
303  if (de_ctx != NULL) {
304  SCProfilingSghDump(de_ctx);
305 
306  DetroyCtx(de_ctx->profile_sgh_ctx);
307  }
308 }
309 
311 {
312  if (ctx == NULL)
313  return;
314 
315  uint32_t array_size = det_ctx->de_ctx->sgh_array_cnt;
316 
317  SCProfileSghData *a = SCCalloc(array_size, sizeof(SCProfileSghData));
318  if (a != NULL) {
319  det_ctx->sgh_perf_data = a;
320  }
321 }
322 
323 static void SCProfilingSghThreadMerge(DetectEngineCtx *de_ctx, const DetectEngineThreadCtx *det_ctx)
324 {
325  if (de_ctx == NULL || de_ctx->profile_sgh_ctx == NULL ||
326  de_ctx->profile_sgh_ctx->data == NULL || det_ctx == NULL ||
327  det_ctx->sgh_perf_data == NULL)
328  return;
329 
330 #define ADD(name) de_ctx->profile_sgh_ctx->data[i].name += det_ctx->sgh_perf_data[i].name
331  uint32_t i;
332  for (i = 0; i < de_ctx->sgh_array_cnt; i++) {
333  ADD(checks);
334  ADD(non_mpm_generic);
335  ADD(non_mpm_syn);
336  ADD(post_prefilter_sigs_total);
337  ADD(mpm_match_cnt_total);
338 
343  }
344 #undef ADD
345 }
346 
348 {
349  if (det_ctx == NULL || det_ctx->de_ctx == NULL || det_ctx->sgh_perf_data == NULL)
350  return;
351 
352  pthread_mutex_lock(&det_ctx->de_ctx->profile_sgh_ctx->data_m);
353  SCProfilingSghThreadMerge(det_ctx->de_ctx, det_ctx);
354  pthread_mutex_unlock(&det_ctx->de_ctx->profile_sgh_ctx->data_m);
355 
356  SCFree(det_ctx->sgh_perf_data);
357  det_ctx->sgh_perf_data = NULL;
358 }
359 
360 /**
361  * \brief Register the keyword profiling counters.
362  *
363  * \param de_ctx The active DetectEngineCtx, used to get at the loaded rules.
364  */
365 void
367 {
368  if (profiling_sghs_enabled == 0)
369  return;
370 
371  de_ctx->profile_sgh_ctx = SCProfilingSghInitCtx();
372  BUG_ON(de_ctx->profile_sgh_ctx == NULL);
373 
375  BUG_ON(de_ctx->profile_sgh_ctx->data == NULL);
376 
378 
379  SCLogPerf("Registered %"PRIu32" rulegroup profiling counters.", de_ctx->sgh_array_cnt);
380 }
381 
382 #endif /* PROFILING */
SCProfilingSghThreadCleanup
void SCProfilingSghThreadCleanup(DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-rulegroups.c:347
DetectEngineThreadCtx_::non_pf_store_ptr
SignatureNonPrefilterStore * non_pf_store_ptr
Definition: detect.h:1125
ConfNodeChildValueIsTrue
int ConfNodeChildValueIsTrue(const ConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:853
DetectEngineThreadCtx_::match_array_cnt
SigIntId match_array_cnt
Definition: detect.h:1120
SCProfilingSghThreadSetup
void SCProfilingSghThreadSetup(SCProfileSghDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-rulegroups.c:310
CreateIsoTimeString
void CreateIsoTimeString(const SCTime_t ts, char *str, size_t size)
Definition: util-time.c:209
PrefilterRuleStore_::rule_id_array_cnt
uint32_t rule_id_array_cnt
Definition: util-prefilter.h:40
SigGroupHead_
Container for matching data for a signature group.
Definition: detect.h:1395
DetectEngineThreadCtx_::sgh_perf_data
struct SCProfileSghData_ * sgh_perf_data
Definition: detect.h:1191
SCProfileSghDetectCtx_
Definition: util-profiling-rulegroups.c:51
SCProfileSghDetectCtx_::data_m
pthread_mutex_t data_m
Definition: util-profiling-rulegroups.c:54
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:181
DetectEngineThreadCtx_::pmq
PrefilterRuleStore pmq
Definition: detect.h:1134
SCProfilingSghInitCounters
void SCProfilingSghInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
Definition: util-profiling-rulegroups.c:366
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:785
DetectEngineCtx_::profile_sgh_ctx
struct SCProfileSghDetectCtx_ * profile_sgh_ctx
Definition: detect.h:912
JSON_ESCAPE_SLASH
#define JSON_ESCAPE_SLASH
Definition: suricata-common.h:270
SCProfileSghData_::mpm_match_cnt_total
uint64_t mpm_match_cnt_total
Definition: util-profiling-rulegroups.c:46
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:531
SCProfileSghData_::checks
uint64_t checks
Definition: util-profiling-rulegroups.c:38
SCProfileSghData_::post_prefilter_sigs_max
uint64_t post_prefilter_sigs_max
Definition: util-profiling-rulegroups.c:44
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
profiling_sghs_enabled
int profiling_sghs_enabled
Definition: util-profiling-rulegroups.c:58
DetectEngineThreadCtx_
Definition: detect.h:1025
SCProfileSghData_
Definition: util-profiling-rulegroups.c:37
SCTIME_FROM_TIMEVAL
#define SCTIME_FROM_TIMEVAL(tv)
Definition: util-time.h:65
util-time.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
util-profiling.h
SCLocalTime
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:267
DetectEngineCtx_::sgh_array_cnt
uint32_t sgh_array_cnt
Definition: detect.h:868
SCProfileSghData
struct SCProfileSghData_ SCProfileSghData
SCProfileSghData_::non_mpm_generic
uint64_t non_mpm_generic
Definition: util-profiling-rulegroups.c:40
util-conf.h
suricata-common.h
SCProfileSghDetectCtx
struct SCProfileSghDetectCtx_ SCProfileSghDetectCtx
SCLogPerf
#define SCLogPerf(...)
Definition: util-debug.h:230
FatalError
#define FatalError(...)
Definition: util-debug.h:502
ConfigGetLogDirectory
const char * ConfigGetLogDirectory(void)
Definition: util-conf.c:37
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
DetectEngineThreadCtx_::non_pf_store_cnt
uint32_t non_pf_store_cnt
Definition: detect.h:1126
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ADD
#define ADD(name)
ConfNode_
Definition: conf.h:32
SCProfilingSghDestroyCtx
void SCProfilingSghDestroyCtx(DetectEngineCtx *de_ctx)
Definition: util-profiling-rulegroups.c:301
SCProfileSghData_::mpm_match_cnt_max
uint64_t mpm_match_cnt_max
Definition: util-profiling-rulegroups.c:47
DetectEngineThreadCtx_::de_ctx
DetectEngineCtx * de_ctx
Definition: detect.h:1159
SCProfileSghDetectCtx_::cnt
uint32_t cnt
Definition: util-profiling-rulegroups.c:52
SigGroupHead_::non_pf_syn_store_array
SignatureNonPrefilterStore * non_pf_syn_store_array
Definition: detect.h:1404
SCProfileSghData_::non_mpm_syn
uint64_t non_mpm_syn
Definition: util-profiling-rulegroups.c:41
likely
#define likely(expr)
Definition: util-optimize.h:32
SCProfilingSghUpdateCounter
void SCProfilingSghUpdateCounter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh)
Update a rule counter.
Definition: util-profiling-rulegroups.c:258
SCProfilingSghsGlobalInit
void SCProfilingSghsGlobalInit(void)
Definition: util-profiling-rulegroups.c:63
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SigGroupHead_::id
uint32_t id
Definition: detect.h:1410
SCProfileSghDetectCtx_::data
SCProfileSghData * data
Definition: util-profiling-rulegroups.c:53
SCProfileSghData_::post_prefilter_sigs_total
uint64_t post_prefilter_sigs_total
Definition: util-profiling-rulegroups.c:43
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:808