suricata
util-profiling-keywords.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * An API for rule profiling operations.
25  */
26 
27 #include "suricata-common.h"
28 #include "decode.h"
29 #include "detect.h"
30 #include "detect-engine.h"
31 #include "conf.h"
32 
33 #include "tm-threads.h"
34 
35 #include "util-unittest.h"
36 #include "util-byte.h"
37 #include "util-profiling.h"
38 #include "util-profiling-locks.h"
39 
40 #ifdef PROFILING
41 
42 #ifndef MIN
43 #define MIN(a, b) (((a) < (b)) ? (a) : (b))
44 #endif
45 
46 /**
47  * Extra data for rule profiling.
48  */
49 typedef struct SCProfileKeywordData_ {
50  uint64_t checks;
51  uint64_t matches;
52  uint64_t max;
53  uint64_t ticks_match;
54  uint64_t ticks_no_match;
56 
58  uint32_t id;
60  pthread_mutex_t data_m;
62 
63 static int profiling_keywords_output_to_file = 0;
65 __thread int profiling_keyword_entered = 0;
66 static char profiling_file_name[PATH_MAX];
67 static const char *profiling_file_mode = "a";
68 
70 {
71  ConfNode *conf;
72 
73  conf = ConfGetNode("profiling.keywords");
74  if (conf != NULL) {
75  if (ConfNodeChildValueIsTrue(conf, "enabled")) {
77  const char *filename = ConfNodeLookupChildValue(conf, "filename");
78  if (filename != NULL) {
79  const char *log_dir;
80  log_dir = ConfigGetLogDirectory();
81 
82  snprintf(profiling_file_name, sizeof(profiling_file_name), "%s/%s",
83  log_dir, filename);
84 
85  const char *v = ConfNodeLookupChildValue(conf, "append");
86  if (v == NULL || ConfValIsTrue(v)) {
87  profiling_file_mode = "a";
88  } else {
89  profiling_file_mode = "w";
90  }
91 
92  profiling_keywords_output_to_file = 1;
93  }
94  }
95  }
96 }
97 
98 static void DoDump(SCProfileKeywordDetectCtx *rules_ctx, FILE *fp, const char *name)
99 {
100  int i;
101  fprintf(fp, " ----------------------------------------------"
102  "------------------------------------------------------"
103  "----------------------------\n");
104  fprintf(fp, " Stats for: %s\n", name);
105  fprintf(fp, " ----------------------------------------------"
106  "------------------------------------------------------"
107  "----------------------------\n");
108  fprintf(fp, " %-16s %-15s %-15s %-15s %-15s %-15s %-15s %-15s\n", "Keyword", "Ticks", "Checks", "Matches", "Max Ticks", "Avg", "Avg Match", "Avg No Match");
109  fprintf(fp, " ---------------- "
110  "--------------- "
111  "--------------- "
112  "--------------- "
113  "--------------- "
114  "--------------- "
115  "--------------- "
116  "--------------- "
117  "\n");
118  for (i = 0; i < DETECT_TBLSIZE; i++) {
119  SCProfileKeywordData *d = &rules_ctx->data[i];
120  if (d == NULL || d->checks == 0)
121  continue;
122 
123  uint64_t ticks = d->ticks_match + d->ticks_no_match;
124  double avgticks = 0;
125  double avgticks_match = 0;
126  double avgticks_no_match = 0;
127  if (ticks && d->checks) {
128  avgticks = (ticks / d->checks);
129 
130  if (d->ticks_match && d->matches)
131  avgticks_match = (d->ticks_match / d->matches);
132  if (d->ticks_no_match && (d->checks - d->matches) != 0)
133  avgticks_no_match = (d->ticks_no_match / (d->checks - d->matches));
134  }
135 
136  fprintf(fp,
137  " %-16s %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f %-15.2f %-15.2f\n",
138  sigmatch_table[i].name,
139  ticks,
140  d->checks,
141  d->matches,
142  d->max,
143  avgticks,
144  avgticks_match,
145  avgticks_no_match);
146  }
147 }
148 
149 static void
150 SCProfilingKeywordDump(DetectEngineCtx *de_ctx)
151 {
152  int i;
153  FILE *fp;
154  struct timeval tval;
155  struct tm *tms;
156  struct tm local_tm;
157 
158  if (profiling_keyword_enabled == 0)
159  return;
160 
161  const int nlists = de_ctx->buffer_type_id;
162  gettimeofday(&tval, NULL);
163  tms = SCLocalTime(tval.tv_sec, &local_tm);
164 
165  if (profiling_keywords_output_to_file == 1) {
166  SCLogDebug("file %s mode %s", profiling_file_name, profiling_file_mode);
167 
168  fp = fopen(profiling_file_name, profiling_file_mode);
169 
170  if (fp == NULL) {
171  SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", profiling_file_name,
172  strerror(errno));
173  return;
174  }
175  } else {
176  fp = stdout;
177  }
178 
179  fprintf(fp, " ----------------------------------------------"
180  "------------------------------------------------------"
181  "----------------------------\n");
182  fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- "
183  "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
184  tms->tm_hour,tms->tm_min, tms->tm_sec);
185 
186  /* global stats first */
187  DoDump(de_ctx->profile_keyword_ctx, fp, "total");
188  /* per buffer stats next, but only if there are stats to print */
189  for (i = 0; i < nlists; i++) {
190  int j;
191  uint64_t checks = 0;
192  for (j = 0; j < DETECT_TBLSIZE; j++) {
193  checks += de_ctx->profile_keyword_ctx_per_list[i]->data[j].checks;
194  }
195 
196  if (checks) {
197  const char *name = NULL;
200  } else {
201  name = DetectBufferTypeGetNameById(de_ctx, i);
202  }
203 
204  DoDump(de_ctx->profile_keyword_ctx_per_list[i], fp, name);
205  }
206  }
207 
208  fprintf(fp,"\n");
209  if (fp != stdout)
210  fclose(fp);
211 
212  SCLogPerf("Done dumping keyword profiling data.");
213 }
214 
215 /**
216  * \brief Update a rule counter.
217  *
218  * \param id The ID of this counter.
219  * \param ticks Number of CPU ticks for this rule.
220  * \param match Did the rule match?
221  */
222 void
223 SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
224 {
225  if (det_ctx != NULL && det_ctx->keyword_perf_data != NULL && id < DETECT_TBLSIZE) {
226  SCProfileKeywordData *p = &det_ctx->keyword_perf_data[id];
227 
228  p->checks++;
229  p->matches += match;
230  if (ticks > p->max)
231  p->max = ticks;
232  if (match == 1)
233  p->ticks_match += ticks;
234  else
235  p->ticks_no_match += ticks;
236 
237  /* store per list (buffer type) as well */
238  if (det_ctx->keyword_perf_list >= 0) {// && det_ctx->keyword_perf_list < DETECT_SM_LIST_MAX) {
239  p = &det_ctx->keyword_perf_data_per_list[det_ctx->keyword_perf_list][id];
240  p->checks++;
241  p->matches += match;
242  if (ticks > p->max)
243  p->max = ticks;
244  if (match == 1)
245  p->ticks_match += ticks;
246  else
247  p->ticks_no_match += ticks;
248  }
249  }
250 }
251 
252 static SCProfileKeywordDetectCtx *SCProfilingKeywordInitCtx(void)
253 {
255  if (ctx != NULL) {
256  memset(ctx, 0x00, sizeof(SCProfileKeywordDetectCtx));
257 
258  if (pthread_mutex_init(&ctx->data_m, NULL) != 0) {
260  "Failed to initialize hash table mutex.");
261  exit(EXIT_FAILURE);
262  }
263  }
264 
265  return ctx;
266 }
267 
268 static void DetroyCtx(SCProfileKeywordDetectCtx *ctx)
269 {
270  if (ctx) {
271  if (ctx->data != NULL)
272  SCFree(ctx->data);
273  pthread_mutex_destroy(&ctx->data_m);
274  SCFree(ctx);
275  }
276 }
277 
279 {
280  if (de_ctx != NULL) {
281  SCProfilingKeywordDump(de_ctx);
282 
283  DetroyCtx(de_ctx->profile_keyword_ctx);
284 
285  const int nlists = de_ctx->buffer_type_id;
286  int i;
287  for (i = 0; i < nlists; i++) {
288  DetroyCtx(de_ctx->profile_keyword_ctx_per_list[i]);
289  }
291  }
292 }
293 
295 {
296  if (ctx == NULL)
297  return;
298 
300  if (a != NULL) {
301  memset(a, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
302  det_ctx->keyword_perf_data = a;
303  }
304 
305  const int nlists = det_ctx->de_ctx->buffer_type_id;
306  det_ctx->keyword_perf_data_per_list = SCCalloc(nlists, sizeof(SCProfileKeywordData *));
307  BUG_ON(det_ctx->keyword_perf_data_per_list == NULL);
308 
309  int i;
310  for (i = 0; i < nlists; i++) {
311  SCProfileKeywordData *b = SCMalloc(sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
312  if (b != NULL) {
313  memset(b, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
314  det_ctx->keyword_perf_data_per_list[i] = b;
315  }
316 
317  }
318 }
319 
320 static void SCProfilingKeywordThreadMerge(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
321 {
322  if (de_ctx == NULL || de_ctx->profile_keyword_ctx == NULL ||
323  de_ctx->profile_keyword_ctx->data == NULL || det_ctx == NULL ||
324  det_ctx->keyword_perf_data == NULL)
325  return;
326 
327  int i;
328  for (i = 0; i < DETECT_TBLSIZE; i++) {
329  de_ctx->profile_keyword_ctx->data[i].checks += det_ctx->keyword_perf_data[i].checks;
330  de_ctx->profile_keyword_ctx->data[i].matches += det_ctx->keyword_perf_data[i].matches;
333  if (det_ctx->keyword_perf_data[i].max > de_ctx->profile_keyword_ctx->data[i].max)
334  de_ctx->profile_keyword_ctx->data[i].max = det_ctx->keyword_perf_data[i].max;
335  }
336 
337  const int nlists = det_ctx->de_ctx->buffer_type_id;
338  int j;
339  for (j = 0; j < nlists; j++) {
340  for (i = 0; i < DETECT_TBLSIZE; i++) {
345  if (det_ctx->keyword_perf_data_per_list[j][i].max > de_ctx->profile_keyword_ctx_per_list[j]->data[i].max)
346  de_ctx->profile_keyword_ctx_per_list[j]->data[i].max = det_ctx->keyword_perf_data_per_list[j][i].max;
347  }
348  }
349 }
350 
352 {
353  if (det_ctx == NULL || det_ctx->de_ctx == NULL || det_ctx->keyword_perf_data == NULL)
354  return;
355 
356  pthread_mutex_lock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
357  SCProfilingKeywordThreadMerge(det_ctx->de_ctx, det_ctx);
358  pthread_mutex_unlock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
359 
360  SCFree(det_ctx->keyword_perf_data);
361  det_ctx->keyword_perf_data = NULL;
362 
363  const int nlists = det_ctx->de_ctx->buffer_type_id;
364  int i;
365  for (i = 0; i < nlists; i++) {
366  SCFree(det_ctx->keyword_perf_data_per_list[i]);
367  det_ctx->keyword_perf_data_per_list[i] = NULL;
368  }
370 }
371 
372 /**
373  * \brief Register the keyword profiling counters.
374  *
375  * \param de_ctx The active DetectEngineCtx, used to get at the loaded rules.
376  */
377 void
379 {
380  if (profiling_keyword_enabled == 0)
381  return;
382 
383  const int nlists = de_ctx->buffer_type_id;
384 
385  de_ctx->profile_keyword_ctx = SCProfilingKeywordInitCtx();
386  BUG_ON(de_ctx->profile_keyword_ctx == NULL);
387 
389  BUG_ON(de_ctx->profile_keyword_ctx->data == NULL);
390  memset(de_ctx->profile_keyword_ctx->data, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
391 
393  BUG_ON(de_ctx->profile_keyword_ctx_per_list == NULL);
394 
395  int i;
396  for (i = 0; i < nlists; i++) {
397  de_ctx->profile_keyword_ctx_per_list[i] = SCProfilingKeywordInitCtx();
398  BUG_ON(de_ctx->profile_keyword_ctx_per_list[i] == NULL);
399  de_ctx->profile_keyword_ctx_per_list[i]->data = SCMalloc(sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
400  BUG_ON(de_ctx->profile_keyword_ctx_per_list[i]->data == NULL);
401  memset(de_ctx->profile_keyword_ctx_per_list[i]->data, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
402  }
403 
404  SCLogPerf("Registered %"PRIu32" keyword profiling counters.", DETECT_TBLSIZE);
405 }
406 
407 #endif /* PROFILING */
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1431
#define SCLogDebug(...)
Definition: util-debug.h:335
#define BUG_ON(x)
void SCProfilingKeywordDestroyCtx(DetectEngineCtx *de_ctx)
struct SCProfileKeywordDetectCtx_ SCProfileKeywordDetectCtx
int buffer_type_id
Definition: detect.h:905
struct SCProfileKeywordData_ * keyword_perf_data
Definition: detect.h:1139
int ConfNodeChildValueIsTrue(const ConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:888
main detection engine ctx
Definition: detect.h:743
#define SCCalloc(nm, a)
Definition: util-mem.h:197
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:240
void SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
Update a rule counter.
int profiling_keyword_enabled
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *det_ctx)
const char * DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
Definition: conf.h:32
struct SCProfileKeywordData_ SCProfileKeywordData
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:166
DetectEngineCtx * de_ctx
Definition: detect.h:1110
struct SCProfileKeywordDetectCtx_ * profile_keyword_ctx
Definition: detect.h:865
#define SCFree(a)
Definition: util-mem.h:228
struct SCProfileKeywordDetectCtx_ ** profile_keyword_ctx_per_list
Definition: detect.h:867
#define SCLogPerf(...)
Definition: util-debug.h:261
void SCProfilingKeywordsGlobalInit(void)
struct SCProfileKeywordData_ ** keyword_perf_data_per_list
Definition: detect.h:1140
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
void SCProfilingKeywordThreadSetup(SCProfileKeywordDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
const char * DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
__thread int profiling_keyword_entered
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.