suricata
util-profiling.h
Go to the documentation of this file.
1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  * \author Victor Julien <victor@inliniac.net>
23  */
24 
25 #ifndef SURICATA_UTIL_PROFILE_H
26 #define SURICATA_UTIL_PROFILE_H
27 
28 #include "util-cpu.h"
29 
30 #include "detect.h"
31 
32 #ifdef PROFILING
33 
34 #include "util-cpu.h"
35 #include "util-profiling-locks.h"
36 
37 extern int profiling_rules_enabled;
38 extern int profiling_packets_enabled;
39 extern int profiling_sghs_enabled;
40 
43 extern int profiling_keyword_enabled;
44 extern thread_local int profiling_keyword_entered;
45 
46 #define KEYWORD_PROFILING_SET_LIST(ctx, list) { \
47  (ctx)->keyword_perf_list = (list); \
48 }
49 
50 #define KEYWORD_PROFILING_START \
51  uint64_t profile_keyword_start_ = 0; \
52  uint64_t profile_keyword_end_ = 0; \
53  if (profiling_keyword_enabled) { \
54  if (profiling_keyword_entered > 0) { \
55  SCLogError("Re-entered profiling, exiting."); \
56  abort(); \
57  } \
58  profiling_keyword_entered++; \
59  profile_keyword_start_ = UtilCpuGetTicks(); \
60  }
61 
62 /* we allow this macro to be called if profiling_keyword_entered == 0,
63  * so that we don't have to refactor some of the detection code. */
64 #define KEYWORD_PROFILING_END(ctx, type, m) \
65  if (profiling_keyword_enabled && profiling_keyword_entered) { \
66  profile_keyword_end_ = UtilCpuGetTicks(); \
67  SCProfilingKeywordUpdateCounter((ctx),(type),(profile_keyword_end_ - profile_keyword_start_),(m)); \
68  profiling_keyword_entered--; \
69  }
70 
72 
73 #define PACKET_PROFILING_START(p) \
74  if (profiling_packets_enabled) { \
75  (p)->profile = SCProfilePacketStart(); \
76  if ((p)->profile != NULL) \
77  (p)->profile->ticks_start = UtilCpuGetTicks(); \
78  }
79 
80 #define PACKET_PROFILING_RESTART(p) \
81  if (profiling_packets_enabled) { \
82  if ((p)->profile != NULL) \
83  (p)->profile->ticks_start = UtilCpuGetTicks(); \
84  }
85 
86 #define PACKET_PROFILING_END(p) \
87  if (profiling_packets_enabled && (p)->profile != NULL) { \
88  (p)->profile->ticks_end = UtilCpuGetTicks(); \
89  SCProfilingAddPacket((p)); \
90  }
91 
92 #ifdef PROFILE_LOCKING
93 #define PACKET_PROFILING_RESET_LOCKS do { \
94  mutex_lock_cnt = 0; \
95  mutex_lock_wait_ticks = 0; \
96  mutex_lock_contention = 0; \
97  spin_lock_cnt = 0; \
98  spin_lock_wait_ticks = 0; \
99  spin_lock_contention = 0; \
100  rww_lock_cnt = 0; \
101  rww_lock_wait_ticks = 0; \
102  rww_lock_contention = 0; \
103  rwr_lock_cnt = 0; \
104  rwr_lock_wait_ticks = 0; \
105  rwr_lock_contention = 0; \
106  locks_idx = 0; \
107  record_locks = 1;\
108  } while (0)
109 
110 #define PACKET_PROFILING_COPY_LOCKS(p, id) do { \
111  (p)->profile->tmm[(id)].mutex_lock_cnt = mutex_lock_cnt; \
112  (p)->profile->tmm[(id)].mutex_lock_wait_ticks = mutex_lock_wait_ticks; \
113  (p)->profile->tmm[(id)].mutex_lock_contention = mutex_lock_contention; \
114  (p)->profile->tmm[(id)].spin_lock_cnt = spin_lock_cnt; \
115  (p)->profile->tmm[(id)].spin_lock_wait_ticks = spin_lock_wait_ticks; \
116  (p)->profile->tmm[(id)].spin_lock_contention = spin_lock_contention; \
117  (p)->profile->tmm[(id)].rww_lock_cnt = rww_lock_cnt; \
118  (p)->profile->tmm[(id)].rww_lock_wait_ticks = rww_lock_wait_ticks; \
119  (p)->profile->tmm[(id)].rww_lock_contention = rww_lock_contention; \
120  (p)->profile->tmm[(id)].rwr_lock_cnt = rwr_lock_cnt; \
121  (p)->profile->tmm[(id)].rwr_lock_wait_ticks = rwr_lock_wait_ticks; \
122  (p)->profile->tmm[(id)].rwr_lock_contention = rwr_lock_contention; \
123  record_locks = 0; \
124  SCProfilingAddPacketLocks((p)); \
125  } while(0)
126 #else
127 #define PACKET_PROFILING_RESET_LOCKS
128 #define PACKET_PROFILING_COPY_LOCKS(p, id)
129 #endif
130 
131 #define PACKET_PROFILING_TMM_START(p, id) \
132  if (profiling_packets_enabled && (p)->profile != NULL) { \
133  if ((id) < TMM_SIZE) { \
134  (p)->profile->tmm[(id)].ticks_start = UtilCpuGetTicks();\
135  PACKET_PROFILING_RESET_LOCKS; \
136  } \
137  }
138 
139 #define PACKET_PROFILING_TMM_END(p, id) \
140  if (profiling_packets_enabled && (p)->profile != NULL) { \
141  if ((id) < TMM_SIZE) { \
142  PACKET_PROFILING_COPY_LOCKS((p), (id)); \
143  (p)->profile->tmm[(id)].ticks_end = UtilCpuGetTicks(); \
144  } \
145  }
146 
147 #define FLOWWORKER_PROFILING_START(p, id) \
148  if (profiling_packets_enabled && (p)->profile != NULL) { \
149  if ((id) < PROFILE_FLOWWORKER_SIZE) { \
150  (p)->profile->flowworker[(id)].ticks_start = UtilCpuGetTicks();\
151  } \
152  }
153 
154 #define FLOWWORKER_PROFILING_END(p, id) \
155  if (profiling_packets_enabled && (p)->profile != NULL) { \
156  if ((id) < PROFILE_FLOWWORKER_SIZE) { \
157  (p)->profile->flowworker[(id)].ticks_end = UtilCpuGetTicks(); \
158  } \
159  }
160 
161 #define PACKET_PROFILING_RESET(p) \
162  if (profiling_packets_enabled && (p)->profile != NULL) { \
163  SCFree((p)->profile); \
164  (p)->profile = NULL; \
165  }
166 
167 #define PACKET_PROFILING_APP_START(dp, id) \
168  if (profiling_packets_enabled) { \
169  (dp)->ticks_start = UtilCpuGetTicks(); \
170  (dp)->alproto = (id); \
171  }
172 
173 #define PACKET_PROFILING_APP_END(dp) \
174  if (profiling_packets_enabled) { \
175  (dp)->ticks_end = UtilCpuGetTicks(); \
176  if ((dp)->ticks_start != 0 && (dp)->ticks_start < ((dp)->ticks_end)) { \
177  (dp)->ticks_spent = ((dp)->ticks_end - (dp)->ticks_start); \
178  } \
179  }
180 
181 #define PACKET_PROFILING_APP_PD_START(dp) \
182  if (profiling_packets_enabled) { \
183  (dp)->proto_detect_ticks_start = UtilCpuGetTicks(); \
184  }
185 
186 #define PACKET_PROFILING_APP_PD_END(dp) \
187  if (profiling_packets_enabled) { \
188  (dp)->proto_detect_ticks_end = UtilCpuGetTicks(); \
189  if ((dp)->proto_detect_ticks_start != 0 && (dp)->proto_detect_ticks_start < ((dp)->proto_detect_ticks_end)) { \
190  (dp)->proto_detect_ticks_spent = \
191  ((dp)->proto_detect_ticks_end - (dp)->proto_detect_ticks_start); \
192  } \
193  }
194 
195 #define PACKET_PROFILING_APP_RESET(dp) \
196  if (profiling_packets_enabled) { \
197  (dp)->ticks_start = 0; \
198  (dp)->ticks_end = 0; \
199  (dp)->ticks_spent = 0; \
200  (dp)->alproto = 0; \
201  (dp)->proto_detect_ticks_start = 0; \
202  (dp)->proto_detect_ticks_end = 0; \
203  (dp)->proto_detect_ticks_spent = 0; \
204  }
205 
206 #define PACKET_PROFILING_APP_STORE(dp, p) \
207  if (profiling_packets_enabled && (p)->profile != NULL) { \
208  if ((dp)->alproto < ALPROTO_MAX) { \
209  (p)->profile->app[(dp)->alproto].ticks_spent += (dp)->ticks_spent; \
210  (p)->profile->proto_detect += (dp)->proto_detect_ticks_spent; \
211  } \
212  }
213 
214 #define PACKET_PROFILING_DETECT_START(p, id) \
215  if (profiling_packets_enabled && (p)->profile != NULL) { \
216  if ((id) < PROF_DETECT_SIZE) { \
217  (p)->profile->detect[(id)].ticks_start = UtilCpuGetTicks(); \
218  } \
219  }
220 
221 #define PACKET_PROFILING_DETECT_END(p, id) \
222  if (profiling_packets_enabled && (p)->profile != NULL) { \
223  if ((id) < PROF_DETECT_SIZE) { \
224  (p)->profile->detect[(id)].ticks_end = UtilCpuGetTicks();\
225  if ((p)->profile->detect[(id)].ticks_start != 0 && \
226  (p)->profile->detect[(id)].ticks_start < (p)->profile->detect[(id)].ticks_end) { \
227  (p)->profile->detect[(id)].ticks_spent += \
228  ((p)->profile->detect[(id)].ticks_end - (p)->profile->detect[(id)].ticks_start); \
229  } \
230  } \
231  }
232 
233 #define PACKET_PROFILING_LOGGER_START(p, id) \
234  if (profiling_packets_enabled && (p)->profile != NULL) { \
235  if ((id) < LOGGER_SIZE) { \
236  (p)->profile->logger[(id)].ticks_start = UtilCpuGetTicks(); \
237  } \
238  }
239 
240 #define PACKET_PROFILING_LOGGER_END(p, id) \
241  if (profiling_packets_enabled && (p)->profile != NULL) { \
242  if ((id) < LOGGER_SIZE) { \
243  (p)->profile->logger[(id)].ticks_end = UtilCpuGetTicks();\
244  if ((p)->profile->logger[(id)].ticks_start != 0 && \
245  (p)->profile->logger[(id)].ticks_start < (p)->profile->logger[(id)].ticks_end) { \
246  (p)->profile->logger[(id)].ticks_spent += \
247  ((p)->profile->logger[(id)].ticks_end - (p)->profile->logger[(id)].ticks_start); \
248  } \
249  } \
250  }
251 
252 #define SGH_PROFILING_RECORD(det_ctx, sgh) \
253  if (profiling_sghs_enabled) { \
254  SCProfilingSghUpdateCounter((det_ctx), (sgh)); \
255  }
256 
257 extern int profiling_prefilter_enabled;
258 extern thread_local int profiling_prefilter_entered;
259 
260 #define PREFILTER_PROFILING_START(det_ctx) \
261  (det_ctx)->prefilter_bytes = 0; \
262  (det_ctx)->prefilter_bytes_called = 0; \
263  uint64_t profile_prefilter_start_ = 0; \
264  uint64_t profile_prefilter_end_ = 0; \
265  if (profiling_prefilter_enabled) { \
266  if (profiling_prefilter_entered > 0) { \
267  SCLogError("Re-entered profiling, exiting."); \
268  abort(); \
269  } \
270  profiling_prefilter_entered++; \
271  profile_prefilter_start_ = UtilCpuGetTicks(); \
272  }
273 
274 /* we allow this macro to be called if profiling_prefilter_entered == 0,
275  * so that we don't have to refactor some of the detection code. */
276 #define PREFILTER_PROFILING_END(ctx, profile_id) \
277  if (profiling_prefilter_enabled && profiling_prefilter_entered) { \
278  profile_prefilter_end_ = UtilCpuGetTicks(); \
279  if (profile_prefilter_end_ > profile_prefilter_start_) \
280  SCProfilingPrefilterUpdateCounter((ctx), (profile_id), \
281  (profile_prefilter_end_ - profile_prefilter_start_), (ctx)->prefilter_bytes, \
282  (ctx)->prefilter_bytes_called); \
283  profiling_prefilter_entered--; \
284  }
285 
286 #define PREFILTER_PROFILING_ADD_BYTES(det_ctx, bytes) \
287  (det_ctx)->prefilter_bytes += (bytes); \
288  (det_ctx)->prefilter_bytes_called++
289 
291 void SCProfilingKeywordDestroyCtx(DetectEngineCtx *);//struct SCProfileKeywordDetectCtx_ *);
293 void SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match);
296 
301 void SCProfilingPrefilterUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks,
302  uint64_t bytes, uint64_t bytes_called);
305 
306 void SCProfilingSghsGlobalInit(void);
312 
313 void SCProfilingInit(void);
314 void SCProfilingDestroy(void);
315 void SCProfilingRegisterTests(void);
316 void SCProfilingDump(void);
317 
318 #else
319 
320 #define KEYWORD_PROFILING_SET_LIST(a,b)
321 #define KEYWORD_PROFILING_START
322 #define KEYWORD_PROFILING_END(a,b,c)
323 
324 #define PACKET_PROFILING_START(p)
325 #define PACKET_PROFILING_RESTART(p)
326 #define PACKET_PROFILING_END(p)
327 
328 #define PACKET_PROFILING_TMM_START(p, id)
329 #define PACKET_PROFILING_TMM_END(p, id)
330 
331 #define PACKET_PROFILING_RESET(p)
332 
333 #define PACKET_PROFILING_APP_START(dp, id)
334 #define PACKET_PROFILING_APP_END(d)
335 #define PACKET_PROFILING_APP_RESET(dp)
336 #define PACKET_PROFILING_APP_STORE(dp, p)
337 
338 #define PACKET_PROFILING_APP_PD_START(dp)
339 #define PACKET_PROFILING_APP_PD_END(dp)
340 
341 #define PACKET_PROFILING_DETECT_START(p, id)
342 #define PACKET_PROFILING_DETECT_END(p, id)
343 
344 #define PACKET_PROFILING_LOGGER_START(p, id)
345 #define PACKET_PROFILING_LOGGER_END(p, id)
346 
347 #define SGH_PROFILING_RECORD(det_ctx, sgh)
348 
349 #define FLOWWORKER_PROFILING_START(p, id)
350 #define FLOWWORKER_PROFILING_END(p, id)
351 
352 #define PREFILTER_PROFILING_START(ctx)
353 #define PREFILTER_PROFILING_END(ctx, profile_id)
354 #define PREFILTER_PROFILING_ADD_BYTES(det_ctx, bytes)
355 
356 #endif /* PROFILING */
357 
358 #ifdef PROFILE_RULES
359 
360 extern int profiling_rules_enabled;
361 extern thread_local int profiling_rules_entered;
362 
363 #ifndef PROFILING
364 void SCProfilingInit(void);
365 #endif
366 /**
367  * Extra data for rule profiling.
368  */
369 typedef struct SCProfileData_ {
370  uint32_t sid;
371  uint32_t gid;
372  uint32_t rev;
373  uint64_t checks;
374  uint64_t matches;
375  uint64_t max;
376  uint64_t ticks_match;
377  uint64_t ticks_no_match;
378 } SCProfileData;
379 
380 typedef struct SCProfileDetectCtx_ {
381  uint32_t size;
382  uint32_t id;
383  SCProfileData *data;
384  pthread_mutex_t data_m;
385 } SCProfileDetectCtx;
386 
387 void SCProfilingRulesGlobalInit(void);
388 void SCProfilingRuleDestroyCtx(struct SCProfileDetectCtx_ *);
389 void SCProfilingRuleInitCounters(DetectEngineCtx *);
390 void SCProfilingRuleUpdateCounter(DetectEngineThreadCtx *, uint16_t, uint64_t, int);
391 void SCProfilingRuleThreadSetup(struct SCProfileDetectCtx_ *, DetectEngineThreadCtx *);
392 void SCProfilingRuleThreadCleanup(DetectEngineThreadCtx *);
393 int SCProfileRuleStart(Packet *p);
394 json_t *SCProfileRuleTriggerDump(DetectEngineCtx *de_ctx);
396 void SCProfileRuleStopCollection(void);
397 void SCProfilingRuleThreatAggregate(DetectEngineThreadCtx *det_ctx);
398 
399 #define RULE_PROFILING_START(p) \
400  uint64_t profile_rule_start_ = 0; \
401  uint64_t profile_rule_end_ = 0; \
402  if (profiling_rules_enabled && SCProfileRuleStart((p))) { \
403  if (profiling_rules_entered > 0) { \
404  FatalError("Re-entered profiling, exiting."); \
405  } \
406  profiling_rules_entered++; \
407  profile_rule_start_ = UtilCpuGetTicks(); \
408  }
409 
410 #define RULE_PROFILING_END(ctx, r, m, p) \
411  if (profiling_rules_enabled && profiling_rules_entered) { \
412  profile_rule_end_ = UtilCpuGetTicks(); \
413  SCProfilingRuleUpdateCounter( \
414  ctx, r->profiling_id, profile_rule_end_ - profile_rule_start_, m); \
415  profiling_rules_entered--; \
416  BUG_ON(profiling_rules_entered < 0); \
417  }
418 
419 #else /* PROFILE_RULES */
420 
421 #define RULE_PROFILING_START(p)
422 #define RULE_PROFILING_END(a, b, c, p)
423 
424 #endif /* PROFILE_RULES */
425 
426 #endif /* ! SURICATA_UTIL_PROFILE_H */
SCProfilingSghDestroyCtx
void SCProfilingSghDestroyCtx(DetectEngineCtx *)
Definition: util-profiling-rulegroups.c:304
SCProfilingRegisterTests
void SCProfilingRegisterTests(void)
Definition: util-profiling.c:1393
SCProfilingInit
void SCProfilingInit(void)
Initialize profiling.
Definition: util-profiling.c:133
PktProfiling_
Per pkt stats storage.
Definition: decode.h:345
SigGroupHead_
Container for matching data for a signature group.
Definition: detect.h:1465
SCProfileSghDetectCtx_
Definition: util-profiling-rulegroups.c:52
SCProfileRuleStopCollection
void SCProfileRuleStopCollection(void)
Definition: util-profiling.c:1404
SCProfilingPrefilterGlobalInit
void SCProfilingPrefilterGlobalInit(void)
Definition: util-profiling-prefilter.c:61
profiling_prefilter_entered
thread_local int profiling_prefilter_entered
Definition: util-profiling-prefilter.c:57
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:843
SCProfilePrefilterDetectCtx_
Definition: util-profiling-prefilter.c:48
SCProfilePacketStart
PktProfiling * SCProfilePacketStart(void)
Definition: util-profiling.c:1183
SCProfilingPrefilterThreadCleanup
void SCProfilingPrefilterThreadCleanup(DetectEngineThreadCtx *)
Definition: util-profiling-prefilter.c:281
SCProfileRuleStart
int SCProfileRuleStart(Packet *p)
Definition: util-profiling.c:1192
SCProfilingKeywordUpdateCounter
void SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, int match)
Update a rule counter.
Definition: util-profiling-keywords.c:216
profiling_packets_enabled
int profiling_packets_enabled
Definition: util-profiling.c:94
SCProfilingDump
void SCProfilingDump(void)
Definition: util-profiling.c:296
SCProfilingKeywordThreadSetup
void SCProfilingKeywordThreadSetup(struct SCProfileKeywordDetectCtx_ *, DetectEngineThreadCtx *)
Definition: util-profiling-keywords.c:284
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
DetectEngineThreadCtx_
Definition: detect.h:1098
SCProfileRuleStartCollection
void SCProfileRuleStartCollection(void)
Definition: util-profiling.c:1400
util-cpu.h
SCProfilingKeywordInitCounters
void SCProfilingKeywordInitCounters(DetectEngineCtx *)
Register the keyword profiling counters.
Definition: util-profiling-keywords.c:365
SCProfileKeywordDetectCtx_
Definition: util-profiling-keywords.c:49
SCProfilingPrintPacketProfile
void SCProfilingPrintPacketProfile(Packet *)
Definition: util-profiling.c:765
detect.h
profiling_keyword_entered
thread_local int profiling_keyword_entered
Definition: util-profiling-keywords.c:57
SCProfilingPrefilterUpdateCounter
void SCProfilingPrefilterUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks, uint64_t bytes, uint64_t bytes_called)
Update a rule counter.
Definition: util-profiling-prefilter.c:194
profiling_keyword_enabled
int profiling_keyword_enabled
Definition: util-profiling-keywords.c:56
SCProfilingPrefilterInitCounters
void SCProfilingPrefilterInitCounters(DetectEngineCtx *)
Register the prefilter profiling counters.
Definition: util-profiling-prefilter.c:300
Packet_
Definition: decode.h:476
SCProfilingAddPacket
void SCProfilingAddPacket(Packet *)
Definition: util-profiling.c:1093
SCProfilingKeywordThreadCleanup
void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *)
Definition: util-profiling-keywords.c:338
SCProfilingSghUpdateCounter
void SCProfilingSghUpdateCounter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh)
Update a rule counter.
Definition: util-profiling-rulegroups.c:261
SCProfilingSghsGlobalInit
void SCProfilingSghsGlobalInit(void)
Definition: util-profiling-rulegroups.c:64
profiling_sghs_enabled
int profiling_sghs_enabled
Definition: util-profiling-rulegroups.c:59
SCProfilingDestroy
void SCProfilingDestroy(void)
Free resources used by profiling.
Definition: util-profiling.c:270
SCProfilingKeywordDestroyCtx
void SCProfilingKeywordDestroyCtx(DetectEngineCtx *)
Definition: util-profiling-keywords.c:268
SCProfilingPrefilterDestroyCtx
void SCProfilingPrefilterDestroyCtx(DetectEngineCtx *)
Definition: util-profiling-prefilter.c:236
SCProfilingSghInitCounters
void SCProfilingSghInitCounters(DetectEngineCtx *)
Register the keyword profiling counters.
Definition: util-profiling-rulegroups.c:369
SCProfilingSghThreadSetup
void SCProfilingSghThreadSetup(struct SCProfileSghDetectCtx_ *, DetectEngineThreadCtx *)
Definition: util-profiling-rulegroups.c:313
SCProfilingSghThreadCleanup
void SCProfilingSghThreadCleanup(DetectEngineThreadCtx *)
Definition: util-profiling-rulegroups.c:350
SCProfilingKeywordsGlobalInit
void SCProfilingKeywordsGlobalInit(void)
Definition: util-profiling-keywords.c:61
profiling_rules_enabled
int profiling_rules_enabled
util-profiling-locks.h
profiling_rules_entered
thread_local int profiling_rules_entered
Definition: util-profiling.c:110
SCProfilingPrefilterThreadSetup
void SCProfilingPrefilterThreadSetup(struct SCProfilePrefilterDetectCtx_ *, DetectEngineThreadCtx *)
Definition: util-profiling-prefilter.c:245
profiling_prefilter_enabled
int profiling_prefilter_enabled
Definition: util-profiling-prefilter.c:56