suricata
util-profiling-keywords.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * An API for rule profiling operations.
25  */
26 
27 #include "suricata-common.h"
28 #include "util-profiling.h"
29 #include "util-profiling-locks.h"
30 
31 #ifdef PROFILING
32 #include "detect-engine.h"
33 #include "tm-threads.h"
34 #include "util-conf.h"
35 #include "util-path.h"
36 #include "util-time.h"
37 
38 /**
39  * Extra data for rule profiling.
40  */
41 typedef struct SCProfileKeywordData_ {
42  uint64_t checks;
43  uint64_t matches;
44  uint64_t max;
45  uint64_t ticks_match;
46  uint64_t ticks_no_match;
48 
50  uint32_t id;
52  pthread_mutex_t data_m;
54 
55 static int profiling_keywords_output_to_file = 0;
57 thread_local int profiling_keyword_entered = 0;
58 static char profiling_file_name[PATH_MAX];
59 static const char *profiling_file_mode = "a";
60 
62 {
63  ConfNode *conf;
64 
65  conf = ConfGetNode("profiling.keywords");
66  if (conf != NULL) {
67  if (ConfNodeChildValueIsTrue(conf, "enabled")) {
69  const char *filename = ConfNodeLookupChildValue(conf, "filename");
70  if (filename != NULL) {
71  if (PathIsAbsolute(filename)) {
72  strlcpy(profiling_file_name, filename, sizeof(profiling_file_name));
73  } else {
74  const char *log_dir = ConfigGetLogDirectory();
75  snprintf(profiling_file_name, sizeof(profiling_file_name), "%s/%s", log_dir,
76  filename);
77  }
78 
79  const char *v = ConfNodeLookupChildValue(conf, "append");
80  if (v == NULL || ConfValIsTrue(v)) {
81  profiling_file_mode = "a";
82  } else {
83  profiling_file_mode = "w";
84  }
85 
86  profiling_keywords_output_to_file = 1;
87  }
88  }
89  }
90 }
91 
92 static void DoDump(SCProfileKeywordDetectCtx *rules_ctx, FILE *fp, const char *name)
93 {
94  int i;
95  fprintf(fp, " ----------------------------------------------"
96  "------------------------------------------------------"
97  "----------------------------\n");
98  fprintf(fp, " Stats for: %s\n", name);
99  fprintf(fp, " ----------------------------------------------"
100  "------------------------------------------------------"
101  "----------------------------\n");
102  fprintf(fp, " %-16s %-15s %-15s %-15s %-15s %-15s %-15s %-15s\n", "Keyword", "Ticks", "Checks", "Matches", "Max Ticks", "Avg", "Avg Match", "Avg No Match");
103  fprintf(fp, " ---------------- "
104  "--------------- "
105  "--------------- "
106  "--------------- "
107  "--------------- "
108  "--------------- "
109  "--------------- "
110  "--------------- "
111  "\n");
112  for (i = 0; i < DETECT_TBLSIZE; i++) {
113  SCProfileKeywordData *d = &rules_ctx->data[i];
114  if (d == NULL || d->checks == 0)
115  continue;
116 
117  uint64_t ticks = d->ticks_match + d->ticks_no_match;
118  double avgticks = 0;
119  double avgticks_match = 0;
120  double avgticks_no_match = 0;
121  if (ticks && d->checks) {
122  avgticks = (double)(ticks / d->checks);
123 
124  if (d->ticks_match && d->matches)
125  avgticks_match = (double)(d->ticks_match / d->matches);
126  if (d->ticks_no_match && (d->checks - d->matches) != 0)
127  avgticks_no_match = (double)(d->ticks_no_match / (d->checks - d->matches));
128  }
129 
130  fprintf(fp,
131  " %-16s %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f %-15.2f %-15.2f\n",
132  sigmatch_table[i].name,
133  ticks,
134  d->checks,
135  d->matches,
136  d->max,
137  avgticks,
138  avgticks_match,
139  avgticks_no_match);
140  }
141 }
142 
143 static void
144 SCProfilingKeywordDump(DetectEngineCtx *de_ctx)
145 {
146  int i;
147  FILE *fp;
148  struct timeval tval;
149  struct tm *tms;
150  struct tm local_tm;
151 
152  if (profiling_keyword_enabled == 0)
153  return;
154 
155  const int nlists = de_ctx->buffer_type_id;
156  gettimeofday(&tval, NULL);
157  tms = SCLocalTime(tval.tv_sec, &local_tm);
158 
159  if (profiling_keywords_output_to_file == 1) {
160  SCLogDebug("file %s mode %s", profiling_file_name, profiling_file_mode);
161 
162  fp = fopen(profiling_file_name, profiling_file_mode);
163 
164  if (fp == NULL) {
165  SCLogError("failed to open %s: %s", profiling_file_name, strerror(errno));
166  return;
167  }
168  } else {
169  fp = stdout;
170  }
171 
172  fprintf(fp, " ----------------------------------------------"
173  "------------------------------------------------------"
174  "----------------------------\n");
175  fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- "
176  "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
177  tms->tm_hour,tms->tm_min, tms->tm_sec);
178 
179  /* global stats first */
180  DoDump(de_ctx->profile_keyword_ctx, fp, "total");
181  /* per buffer stats next, but only if there are stats to print */
182  for (i = 0; i < nlists; i++) {
183  int j;
184  uint64_t checks = 0;
185  for (j = 0; j < DETECT_TBLSIZE; j++) {
187  }
188 
189  if (checks) {
190  const char *name = NULL;
193  } else {
195  }
196 
197  DoDump(de_ctx->profile_keyword_ctx_per_list[i], fp, name);
198  }
199  }
200 
201  fprintf(fp,"\n");
202  if (fp != stdout)
203  fclose(fp);
204 
205  SCLogPerf("Done dumping keyword profiling data.");
206 }
207 
208 /**
209  * \brief Update a rule counter.
210  *
211  * \param id The ID of this counter.
212  * \param ticks Number of CPU ticks for this rule.
213  * \param match Did the rule match?
214  */
215 void
216 SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
217 {
218  if (det_ctx != NULL && det_ctx->keyword_perf_data != NULL && id < DETECT_TBLSIZE) {
219  SCProfileKeywordData *p = &det_ctx->keyword_perf_data[id];
220 
221  p->checks++;
222  p->matches += match;
223  if (ticks > p->max)
224  p->max = ticks;
225  if (match == 1)
226  p->ticks_match += ticks;
227  else
228  p->ticks_no_match += ticks;
229 
230  /* store per list (buffer type) as well */
231  if (det_ctx->keyword_perf_list >= 0) {// && det_ctx->keyword_perf_list < DETECT_SM_LIST_MAX) {
232  p = &det_ctx->keyword_perf_data_per_list[det_ctx->keyword_perf_list][id];
233  p->checks++;
234  p->matches += match;
235  if (ticks > p->max)
236  p->max = ticks;
237  if (match == 1)
238  p->ticks_match += ticks;
239  else
240  p->ticks_no_match += ticks;
241  }
242  }
243 }
244 
245 static SCProfileKeywordDetectCtx *SCProfilingKeywordInitCtx(void)
246 {
248  if (ctx != NULL) {
249 
250  if (pthread_mutex_init(&ctx->data_m, NULL) != 0) {
251  FatalError("Failed to initialize hash table mutex.");
252  }
253  }
254 
255  return ctx;
256 }
257 
258 static void DetroyCtx(SCProfileKeywordDetectCtx *ctx)
259 {
260  if (ctx) {
261  if (ctx->data != NULL)
262  SCFree(ctx->data);
263  pthread_mutex_destroy(&ctx->data_m);
264  SCFree(ctx);
265  }
266 }
267 
269 {
270  if (de_ctx != NULL) {
271  SCProfilingKeywordDump(de_ctx);
272 
273  DetroyCtx(de_ctx->profile_keyword_ctx);
274 
275  const int nlists = de_ctx->buffer_type_id;
276  int i;
277  for (i = 0; i < nlists; i++) {
278  DetroyCtx(de_ctx->profile_keyword_ctx_per_list[i]);
279  }
281  }
282 }
283 
285 {
286  if (ctx == NULL)
287  return;
288 
290  if (a != NULL) {
291  det_ctx->keyword_perf_data = a;
292  }
293 
294  const int nlists = det_ctx->de_ctx->buffer_type_id;
295  det_ctx->keyword_perf_data_per_list = SCCalloc(nlists, sizeof(SCProfileKeywordData *));
296  BUG_ON(det_ctx->keyword_perf_data_per_list == NULL);
297 
298  int i;
299  for (i = 0; i < nlists; i++) {
301  if (b != NULL) {
302  det_ctx->keyword_perf_data_per_list[i] = b;
303  }
304  }
305 }
306 
307 static void SCProfilingKeywordThreadMerge(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
308 {
309  if (de_ctx == NULL || de_ctx->profile_keyword_ctx == NULL ||
310  de_ctx->profile_keyword_ctx->data == NULL || det_ctx == NULL ||
311  det_ctx->keyword_perf_data == NULL)
312  return;
313 
314  int i;
315  for (i = 0; i < DETECT_TBLSIZE; i++) {
320  if (det_ctx->keyword_perf_data[i].max > de_ctx->profile_keyword_ctx->data[i].max)
322  }
323 
324  const int nlists = det_ctx->de_ctx->buffer_type_id;
325  int j;
326  for (j = 0; j < nlists; j++) {
327  for (i = 0; i < DETECT_TBLSIZE; i++) {
334  }
335  }
336 }
337 
339 {
340  if (det_ctx == NULL || det_ctx->de_ctx == NULL || det_ctx->keyword_perf_data == NULL)
341  return;
342 
343  pthread_mutex_lock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
344  SCProfilingKeywordThreadMerge(det_ctx->de_ctx, det_ctx);
345  pthread_mutex_unlock(&det_ctx->de_ctx->profile_keyword_ctx->data_m);
346 
347  SCFree(det_ctx->keyword_perf_data);
348  det_ctx->keyword_perf_data = NULL;
349 
350  const int nlists = det_ctx->de_ctx->buffer_type_id;
351  int i;
352  for (i = 0; i < nlists; i++) {
353  SCFree(det_ctx->keyword_perf_data_per_list[i]);
354  det_ctx->keyword_perf_data_per_list[i] = NULL;
355  }
357 }
358 
359 /**
360  * \brief Register the keyword profiling counters.
361  *
362  * \param de_ctx The active DetectEngineCtx, used to get at the loaded rules.
363  */
364 void
366 {
367  if (profiling_keyword_enabled == 0)
368  return;
369 
370  const int nlists = de_ctx->buffer_type_id;
371 
372  de_ctx->profile_keyword_ctx = SCProfilingKeywordInitCtx();
374 
377 
380 
381  int i;
382  for (i = 0; i < nlists; i++) {
383  de_ctx->profile_keyword_ctx_per_list[i] = SCProfilingKeywordInitCtx();
388  }
389 
390  SCLogPerf("Registered %"PRIu32" keyword profiling counters.", DETECT_TBLSIZE);
391 }
392 
393 #endif /* PROFILING */
tm-threads.h
DetectEngineThreadCtx_::keyword_perf_data_per_list
struct SCProfileKeywordData_ ** keyword_perf_data_per_list
Definition: detect.h:1253
SCProfileKeywordData_
Definition: util-profiling-keywords.c:41
detect-engine.h
SCProfilingKeywordDestroyCtx
void SCProfilingKeywordDestroyCtx(DetectEngineCtx *de_ctx)
Definition: util-profiling-keywords.c:268
SCProfilingKeywordThreadCleanup
void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-keywords.c:338
DetectEngineThreadCtx_::keyword_perf_data
struct SCProfileKeywordData_ * keyword_perf_data
Definition: detect.h:1252
ConfNodeChildValueIsTrue
int ConfNodeChildValueIsTrue(const ConfNode *node, const char *key)
Test if a configuration node has a true value.
Definition: conf.c:854
sigmatch_table
SigTableElmt * sigmatch_table
Definition: detect-parse.c:127
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
DETECT_SM_LIST_DYNAMIC_START
@ DETECT_SM_LIST_DYNAMIC_START
Definition: detect.h:135
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:181
ctx
struct Thresholds ctx
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:841
DetectEngineThreadCtx_::keyword_perf_list
int keyword_perf_list
Definition: detect.h:1254
DetectEngineBufferTypeGetNameById
const char * DetectEngineBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
Definition: detect-engine.c:1116
profiling_keyword_entered
thread_local int profiling_keyword_entered
Definition: util-profiling-keywords.c:57
SCProfileKeywordData_::ticks_match
uint64_t ticks_match
Definition: util-profiling-keywords.c:45
SCProfileKeywordData_::checks
uint64_t checks
Definition: util-profiling-keywords.c:42
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:536
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
DetectEngineThreadCtx_
Definition: detect.h:1090
SCProfileKeywordDetectCtx_
Definition: util-profiling-keywords.c:49
SCProfileKeywordDetectCtx
struct SCProfileKeywordDetectCtx_ SCProfileKeywordDetectCtx
SCProfileKeywordDetectCtx_::data_m
pthread_mutex_t data_m
Definition: util-profiling-keywords.c:52
util-time.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
util-profiling.h
SCLocalTime
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:267
DetectEngineCtx_::profile_keyword_ctx
struct SCProfileKeywordDetectCtx_ * profile_keyword_ctx
Definition: detect.h:951
SCProfilingKeywordUpdateCounter
void SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
Update a rule counter.
Definition: util-profiling-keywords.c:216
DetectSigmatchListEnumToString
const char * DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
Definition: detect-engine.c:4901
util-conf.h
suricata-common.h
util-path.h
SCLogPerf
#define SCLogPerf(...)
Definition: util-debug.h:230
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
SCProfileKeywordData_::matches
uint64_t matches
Definition: util-profiling-keywords.c:43
FatalError
#define FatalError(...)
Definition: util-debug.h:502
DetectEngineCtx_::profile_keyword_ctx_per_list
struct SCProfileKeywordDetectCtx_ ** profile_keyword_ctx_per_list
Definition: detect.h:953
ConfigGetLogDirectory
const char * ConfigGetLogDirectory(void)
Definition: util-conf.c:38
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
DETECT_TBLSIZE
int DETECT_TBLSIZE
Definition: detect-engine-register.c:318
SCFree
#define SCFree(p)
Definition: util-mem.h:61
profiling_keyword_enabled
int profiling_keyword_enabled
Definition: util-profiling-keywords.c:56
ConfNode_
Definition: conf.h:32
SCProfileKeywordData_::max
uint64_t max
Definition: util-profiling-keywords.c:44
SCProfileKeywordDetectCtx_::id
uint32_t id
Definition: util-profiling-keywords.c:50
SCProfileKeywordDetectCtx_::data
SCProfileKeywordData * data
Definition: util-profiling-keywords.c:51
SCProfilingKeywordInitCounters
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
Definition: util-profiling-keywords.c:365
DetectEngineThreadCtx_::de_ctx
DetectEngineCtx * de_ctx
Definition: detect.h:1212
SCProfilingKeywordsGlobalInit
void SCProfilingKeywordsGlobalInit(void)
Definition: util-profiling-keywords.c:61
DetectEngineCtx_::buffer_type_id
uint32_t buffer_type_id
Definition: detect.h:987
util-profiling-locks.h
SCProfileKeywordData
struct SCProfileKeywordData_ SCProfileKeywordData
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCProfileKeywordData_::ticks_no_match
uint64_t ticks_no_match
Definition: util-profiling-keywords.c:46
SCProfilingKeywordThreadSetup
void SCProfilingKeywordThreadSetup(SCProfileKeywordDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
Definition: util-profiling-keywords.c:284
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:809