suricata
util-profiling-keywords.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * An API for rule profiling operations.
25  */
26 
27 #include "suricata-common.h"
28 #include "util-profiling.h"
29 #include "util-profiling-locks.h"
30 
31 #ifdef PROFILING
32 #include "detect-engine.h"
33 #include "tm-threads.h"
34 #include "util-conf.h"
35 #include "util-time.h"
36 
37 /**
38  * Extra data for rule profiling.
39  */
40 typedef struct SCProfileKeywordData_ {
41  uint64_t checks;
42  uint64_t matches;
43  uint64_t max;
44  uint64_t ticks_match;
45  uint64_t ticks_no_match;
47 
49  uint32_t id;
51  pthread_mutex_t data_m;
53 
54 static int profiling_keywords_output_to_file = 0;
56 thread_local int profiling_keyword_entered = 0;
57 static char profiling_file_name[PATH_MAX];
58 static const char *profiling_file_mode = "a";
59 
61 {
62  ConfNode *conf;
63 
64  conf = ConfGetNode("profiling.keywords");
65  if (conf != NULL) {
66  if (ConfNodeChildValueIsTrue(conf, "enabled")) {
68  const char *filename = ConfNodeLookupChildValue(conf, "filename");
69  if (filename != NULL) {
70  const char *log_dir;
71  log_dir = ConfigGetLogDirectory();
72 
73  snprintf(profiling_file_name, sizeof(profiling_file_name), "%s/%s",
74  log_dir, filename);
75 
76  const char *v = ConfNodeLookupChildValue(conf, "append");
77  if (v == NULL || ConfValIsTrue(v)) {
78  profiling_file_mode = "a";
79  } else {
80  profiling_file_mode = "w";
81  }
82 
83  profiling_keywords_output_to_file = 1;
84  }
85  }
86  }
87 }
88 
89 static void DoDump(SCProfileKeywordDetectCtx *rules_ctx, FILE *fp, const char *name)
90 {
91  int i;
92  fprintf(fp, " ----------------------------------------------"
93  "------------------------------------------------------"
94  "----------------------------\n");
95  fprintf(fp, " Stats for: %s\n", name);
96  fprintf(fp, " ----------------------------------------------"
97  "------------------------------------------------------"
98  "----------------------------\n");
99  fprintf(fp, " %-16s %-15s %-15s %-15s %-15s %-15s %-15s %-15s\n", "Keyword", "Ticks", "Checks", "Matches", "Max Ticks", "Avg", "Avg Match", "Avg No Match");
100  fprintf(fp, " ---------------- "
101  "--------------- "
102  "--------------- "
103  "--------------- "
104  "--------------- "
105  "--------------- "
106  "--------------- "
107  "--------------- "
108  "\n");
109  for (i = 0; i < DETECT_TBLSIZE; i++) {
110  SCProfileKeywordData *d = &rules_ctx->data[i];
111  if (d == NULL || d->checks == 0)
112  continue;
113 
114  uint64_t ticks = d->ticks_match + d->ticks_no_match;
115  double avgticks = 0;
116  double avgticks_match = 0;
117  double avgticks_no_match = 0;
118  if (ticks && d->checks) {
119  avgticks = (double)(ticks / d->checks);
120 
121  if (d->ticks_match && d->matches)
122  avgticks_match = (double)(d->ticks_match / d->matches);
123  if (d->ticks_no_match && (d->checks - d->matches) != 0)
124  avgticks_no_match = (double)(d->ticks_no_match / (d->checks - d->matches));
125  }
126 
127  fprintf(fp,
128  " %-16s %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f %-15.2f %-15.2f\n",
129  sigmatch_table[i].name,
130  ticks,
131  d->checks,
132  d->matches,
133  d->max,
134  avgticks,
135  avgticks_match,
136  avgticks_no_match);
137  }
138 }
139 
140 static void
141 SCProfilingKeywordDump(DetectEngineCtx *de_ctx)
142 {
143  int i;
144  FILE *fp;
145  struct timeval tval;
146  struct tm *tms;
147  struct tm local_tm;
148 
149  if (profiling_keyword_enabled == 0)
150  return;
151 
152  const int nlists = de_ctx->buffer_type_id;
153  gettimeofday(&tval, NULL);
154  tms = SCLocalTime(tval.tv_sec, &local_tm);
155 
156  if (profiling_keywords_output_to_file == 1) {
157  SCLogDebug("file %s mode %s", profiling_file_name, profiling_file_mode);
158 
159  fp = fopen(profiling_file_name, profiling_file_mode);
160 
161  if (fp == NULL) {
162  SCLogError("failed to open %s: %s", profiling_file_name, strerror(errno));
163  return;
164  }
165  } else {
166  fp = stdout;
167  }
168 
169  fprintf(fp, " ----------------------------------------------"
170  "------------------------------------------------------"
171  "----------------------------\n");
172  fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- "
173  "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
174  tms->tm_hour,tms->tm_min, tms->tm_sec);
175 
176  /* global stats first */
177  DoDump(de_ctx->profile_keyword_ctx, fp, "total");
178  /* per buffer stats next, but only if there are stats to print */
179  for (i = 0; i < nlists; i++) {
180  int j;
181  uint64_t checks = 0;
182  for (j = 0; j < DETECT_TBLSIZE; j++) {
184  }
185 
186  if (checks) {
187  const char *name = NULL;
190  } else {
192  }
193 
194  DoDump(de_ctx->profile_keyword_ctx_per_list[i], fp, name);
195  }
196  }
197 
198  fprintf(fp,"\n");
199  if (fp != stdout)
200  fclose(fp);
201 
202  SCLogPerf("Done dumping keyword profiling data.");
203 }
204 
205 /**
206  * \brief Update a rule counter.
207  *
208  * \param id The ID of this counter.
209  * \param ticks Number of CPU ticks for this rule.
210  * \param match Did the rule match?
211  */
212 void
213 SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
214 {
215  if (det_ctx != NULL && det_ctx->keyword_perf_data != NULL && id < DETECT_TBLSIZE) {
216  SCProfileKeywordData *p = &det_ctx->keyword_perf_data[id];
217 
218  p->checks++;
219  p->matches += match;
220  if (ticks > p->max)
221  p->max = ticks;
222  if (match == 1)
223  p->ticks_match += ticks;
224  else
225  p->ticks_no_match += ticks;
226 
227  /* store per list (buffer type) as well */
228  if (det_ctx->keyword_perf_list >= 0) {// && det_ctx->keyword_perf_list < DETECT_SM_LIST_MAX) {
229  p = &det_ctx->keyword_perf_data_per_list[det_ctx->keyword_perf_list][id];
230  p->checks++;
231  p->matches += match;
232  if (ticks > p->max)
233  p->max = ticks;
234  if (match == 1)
235  p->ticks_match += ticks;
236  else
237  p->ticks_no_match += ticks;
238  }
239  }
240 }
241 
242 static SCProfileKeywordDetectCtx *SCProfilingKeywordInitCtx(void)
243 {
245  if (ctx != NULL) {
246  memset(ctx, 0x00, sizeof(SCProfileKeywordDetectCtx));
247 
248  if (pthread_mutex_init(&ctx->data_m, NULL) != 0) {
249  FatalError("Failed to initialize hash table mutex.");
250  }
251  }
252 
253  return ctx;
254 }
255 
256 static void DetroyCtx(SCProfileKeywordDetectCtx *ctx)
257 {
258  if (ctx) {
259  if (ctx->data != NULL)
260  SCFree(ctx->data);
261  pthread_mutex_destroy(&ctx->data_m);
262  SCFree(ctx);
263  }
264 }
265 
267 {
268  if (de_ctx != NULL) {
269  SCProfilingKeywordDump(de_ctx);
270 
271  DetroyCtx(de_ctx->profile_keyword_ctx);
272 
273  const int nlists = de_ctx->buffer_type_id;
274  int i;
275  for (i = 0; i < nlists; i++) {
276  DetroyCtx(de_ctx->profile_keyword_ctx_per_list[i]);
277  }
279  }
280 }
281 
283 {
284  if (ctx == NULL)
285  return;
286 
288  if (a != NULL) {
289  memset(a, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
290  det_ctx->keyword_perf_data = a;
291  }
292 
293  const int nlists = det_ctx->de_ctx->buffer_type_id;
294  det_ctx->keyword_perf_data_per_list = SCCalloc(nlists, sizeof(SCProfileKeywordData *));
295  BUG_ON(det_ctx->keyword_perf_data_per_list == NULL);
296 
297  int i;
298  for (i = 0; i < nlists; i++) {
300  if (b != NULL) {
301  memset(b, 0x00, sizeof(SCProfileKeywordData) * DETECT_TBLSIZE);
302  det_ctx->keyword_perf_data_per_list[i] = b;
303  }
304 
305  }
306 }
307 
308 static void SCProfilingKeywordThreadMerge(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
309 {
310  if (de_ctx == NULL || de_ctx->profile_keyword_ctx == NULL ||
311  de_ctx->profile_keyword_ctx->data == NULL || det_ctx == NULL ||
312  det_ctx->keyword_perf_data == NULL)
313  return;
314 
315  int i;
316  for (i = 0; i < DETECT_TBLSIZE; i++) {
321  if (det_ctx->keyword_perf_data[i].max > de_ctx->profile_keyword_ctx->data[i].max)
323  }
324 
325  const int nlists = det_ctx->de_ctx->buffer_type_id;
326  int j;
327  for (j = 0; j < nlists; j++) {
328  for (i = 0; i < DETECT_TBLSIZE; i++) {
335  }
336  }
337 }
338 
340 {
341  if (det_ctx == NULL || det_ctx->de_ctx == NULL || det_ctx->keyword_perf_data == NULL)
342  return;
343 
344  pthread_mutex_lock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
345  SCProfilingKeywordThreadMerge(det_ctx->de_ctx, det_ctx);
346  pthread_mutex_unlock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
347 
348  SCFree(det_ctx->keyword_perf_data);
349  det_ctx->keyword_perf_data = NULL;
350 
351  const int nlists = det_ctx->de_ctx->buffer_type_id;
352  int i;
353  for (i = 0; i < nlists; i++) {
354  SCFree(det_ctx->keyword_perf_data_per_list[i]);
355  det_ctx->keyword_perf_data_per_list[i] = NULL;
356  }
358 }
359 
360 /**
361  * \brief Register the keyword profiling counters.
362  *
363  * \param de_ctx The active DetectEngineCtx, used to get at the loaded rules.
364  */
365 void
367 {
368  if (profiling_keyword_enabled == 0)
369  return;
370 
371  const int nlists = de_ctx->buffer_type_id;
372 
373  de_ctx->profile_keyword_ctx = SCProfilingKeywordInitCtx();
375 
379 
382 
383  int i;
384  for (i = 0; i < nlists; i++) {
385  de_ctx->profile_keyword_ctx_per_list[i] = SCProfilingKeywordInitCtx();
390  }
391 
392  SCLogPerf("Registered %"PRIu32" keyword profiling counters.", DETECT_TBLSIZE);
393 }
394 
395 #endif /* PROFILING */
tm-threads.h
DetectEngineThreadCtx_::keyword_perf_data_per_list
struct SCProfileKeywordData_ ** keyword_perf_data_per_list
Definition: detect.h:1189
SCProfileKeywordData_
Definition: util-profiling-keywords.c:40
detect-engine.h
SCProfilingKeywordDestroyCtx
void SCProfilingKeywordDestroyCtx(DetectEngineCtx *de_ctx)
Definition: util-profiling-keywords.c:266
SCProfilingKeywordThreadCleanup
void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-keywords.c:339
DetectEngineThreadCtx_::keyword_perf_data
struct SCProfileKeywordData_ * keyword_perf_data
Definition: detect.h:1188
ConfNodeChildValueIsTrue
int ConfNodeChildValueIsTrue(const ConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:853
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
DETECT_SM_LIST_DYNAMIC_START
@ DETECT_SM_LIST_DYNAMIC_START
Definition: detect.h:98
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:181
DETECT_TBLSIZE
@ DETECT_TBLSIZE
Definition: detect-engine-register.h:335
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:785
DetectEngineThreadCtx_::keyword_perf_list
int keyword_perf_list
Definition: detect.h:1190
DetectEngineBufferTypeGetNameById
const char * DetectEngineBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
Definition: detect-engine.c:1109
profiling_keyword_entered
thread_local int profiling_keyword_entered
Definition: util-profiling-keywords.c:56
SCProfileKeywordData_::ticks_match
uint64_t ticks_match
Definition: util-profiling-keywords.c:44
SCProfileKeywordData_::checks
uint64_t checks
Definition: util-profiling-keywords.c:41
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:531
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
DetectEngineThreadCtx_
Definition: detect.h:1025
SCProfileKeywordDetectCtx_
Definition: util-profiling-keywords.c:48
SCProfileKeywordDetectCtx
struct SCProfileKeywordDetectCtx_ SCProfileKeywordDetectCtx
SCProfileKeywordDetectCtx_::data_m
pthread_mutex_t data_m
Definition: util-profiling-keywords.c:51
util-time.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
util-profiling.h
SCLocalTime
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:267
DetectEngineCtx_::profile_keyword_ctx
struct SCProfileKeywordDetectCtx_ * profile_keyword_ctx
Definition: detect.h:909
SCProfilingKeywordUpdateCounter
void SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
Update a rule counter.
Definition: util-profiling-keywords.c:213
DetectSigmatchListEnumToString
const char * DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
Definition: detect-engine.c:4618
util-conf.h
suricata-common.h
sigmatch_table
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect-parse.c:76
SCLogPerf
#define SCLogPerf(...)
Definition: util-debug.h:230
SCProfileKeywordData_::matches
uint64_t matches
Definition: util-profiling-keywords.c:42
FatalError
#define FatalError(...)
Definition: util-debug.h:502
DetectEngineCtx_::profile_keyword_ctx_per_list
struct SCProfileKeywordDetectCtx_ ** profile_keyword_ctx_per_list
Definition: detect.h:911
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
ConfigGetLogDirectory
const char * ConfigGetLogDirectory(void)
Definition: util-conf.c:37
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
profiling_keyword_enabled
int profiling_keyword_enabled
Definition: util-profiling-keywords.c:55
ConfNode_
Definition: conf.h:32
SCProfileKeywordData_::max
uint64_t max
Definition: util-profiling-keywords.c:43
SCProfileKeywordDetectCtx_::id
uint32_t id
Definition: util-profiling-keywords.c:49
SCProfileKeywordDetectCtx_::data
SCProfileKeywordData * data
Definition: util-profiling-keywords.c:50
SCProfilingKeywordInitCounters
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
Definition: util-profiling-keywords.c:366
DetectEngineThreadCtx_::de_ctx
DetectEngineCtx * de_ctx
Definition: detect.h:1159
SCProfilingKeywordsGlobalInit
void SCProfilingKeywordsGlobalInit(void)
Definition: util-profiling-keywords.c:60
DetectEngineCtx_::buffer_type_id
uint32_t buffer_type_id
Definition: detect.h:947
util-profiling-locks.h
SCProfileKeywordData
struct SCProfileKeywordData_ SCProfileKeywordData
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCProfileKeywordData_::ticks_no_match
uint64_t ticks_no_match
Definition: util-profiling-keywords.c:45
SCProfilingKeywordThreadSetup
void SCProfilingKeywordThreadSetup(SCProfileKeywordDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-keywords.c:282
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:808