suricata
detect-engine-build.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2025 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "detect.h"
20 #include "detect-engine.h"
21 #include "detect-parse.h"
22 #include "detect-content.h"
23 
24 #include "detect-engine-build.h"
25 #include "detect-engine-address.h"
26 #include "detect-engine-analyzer.h"
27 #include "detect-engine-iponly.h"
28 #include "detect-engine-mpm.h"
29 #include "detect-engine-siggroup.h"
30 #include "detect-engine-port.h"
32 #include "detect-engine-proto.h"
34 
35 #include "detect-dsize.h"
36 #include "detect-tcp-flags.h"
37 #include "detect-flow.h"
38 #include "detect-config.h"
39 #include "detect-flowbits.h"
40 #include "app-layer-events.h"
41 
43 #include "util-profiling.h"
44 #include "util-validate.h"
45 #include "util-var-name.h"
46 #include "util-conf.h"
47 
48 /* Magic numbers to make the rules of a certain order fall in the same group */
49 #define DETECT_PGSCORE_RULE_PORT_PRIORITIZED 111 /* Rule port group contains a priority port */
50 #define DETECT_PGSCORE_RULE_MPM_FAST_PATTERN 99 /* Rule contains an MPM fast pattern */
51 #define DETECT_PGSCORE_RULE_MPM_NEGATED 77 /* Rule contains a negated MPM */
52 #define DETECT_PGSCORE_RULE_NO_MPM 55 /* Rule does not contain MPM */
53 #define DETECT_PGSCORE_RULE_SYN_ONLY 33 /* Rule needs SYN check */
54 
56 {
57  if (de_ctx == NULL)
58  return;
59 
60  for (Signature *s = de_ctx->sig_list; s != NULL;) {
61  Signature *ns = s->next;
62  SigFree(de_ctx, s);
63  s = ns;
64  }
65  de_ctx->sig_list = NULL;
66 
68  de_ctx->sig_list = NULL;
69 }
70 
71 /** \brief Find a specific signature by sid and gid
72  * \param de_ctx detection engine ctx
73  * \param sid the signature id
74  * \param gid the signature group id
75  *
76  * \retval s sig found
77  * \retval NULL sig not found
78  */
80 {
81  if (de_ctx == NULL)
82  return NULL;
83 
84  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
85  if (s->id == sid && s->gid == gid)
86  return s;
87  }
88 
89  return NULL;
90 }
91 
92 /**
93  * \brief Check if a signature contains the filestore keyword.
94  *
95  * \param s signature
96  *
97  * \retval 0 no
98  * \retval 1 yes
99  */
101 {
102  if (s == NULL)
103  return 0;
104 
105  if (s->flags & SIG_FLAG_FILESTORE)
106  return 1;
107 
108  return 0;
109 }
110 
111 /**
112  * \brief Check if a signature contains the filemagic keyword.
113  *
114  * \param s signature
115  *
116  * \retval 0 no
117  * \retval 1 yes
118  */
120 {
121  if (s == NULL)
122  return 0;
123 
125  return 1;
126 
127  return 0;
128 }
129 
130 /**
131  * \brief Check if a signature contains the filemd5 keyword.
132  *
133  * \param s signature
134  *
135  * \retval 0 no
136  * \retval 1 yes
137  */
139 {
140  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5))
141  return 1;
142 
143  return 0;
144 }
145 
146 /**
147  * \brief Check if a signature contains the filesha1 keyword.
148  *
149  * \param s signature
150  *
151  * \retval 0 no
152  * \retval 1 yes
153  */
155 {
156  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1))
157  return 1;
158 
159  return 0;
160 }
161 
162 /**
163  * \brief Check if a signature contains the filesha256 keyword.
164  *
165  * \param s signature
166  *
167  * \retval 0 no
168  * \retval 1 yes
169  */
171 {
172  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256))
173  return 1;
174 
175  return 0;
176 }
177 
178 /**
179  * \brief Check if a signature contains the filesize keyword.
180  *
181  * \param s signature
182  *
183  * \retval 0 no
184  * \retval 1 yes
185  */
187 {
188  if (s == NULL)
189  return 0;
190 
192  return 1;
193 
194  return 0;
195 }
196 
197 static bool SignatureInspectsBuffers(const Signature *s)
198 {
199  return (s->init_data->buffer_index > 0);
200 }
201 
202 /** \brief Test is a initialized signature is IP only
203  * \param de_ctx detection engine ctx
204  * \param s the signature
205  * \retval 1 sig is ip only
206  * \retval 2 sig is like ip only
207  * \retval 0 sig is not ip only
208  */
210 {
211  /* explicit hook means no IP-only */
213  return 0;
214 
215  if (s->alproto != ALPROTO_UNKNOWN)
216  return 0;
217 
218  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
219  return 0;
220 
221  // may happen for 'config' keyword, postmatch
222  if (s->flags & SIG_FLAG_APPLAYER)
223  return 0;
224 
225  /* if flow dir is set we can't process it in ip-only */
226  if (!(((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) ||
229  return 0;
230 
231  /* for now assume that all registered buffer types are incompatible */
232  if (SignatureInspectsBuffers(s)) {
233  SCReturnInt(0);
234  }
235 
236  /* TMATCH list can be ignored, it contains TAGs and
237  * tags are compatible to IP-only. */
238 
240  for (; sm != NULL; sm = sm->next) {
242  return 0;
243  /* we have enabled flowbits to be compatible with ip only sigs, as long
244  * as the sig only has a "set" flowbits */
245  if (sm->type == DETECT_FLOWBITS &&
246  (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET)) {
247  return 0;
248  }
249  }
251  for ( ; sm != NULL; sm = sm->next) {
253  return 0;
254  /* we have enabled flowbits to be compatible with ip only sigs, as long
255  * as the sig only has a "set" flowbits */
256  if (sm->type == DETECT_FLOWBITS &&
257  (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
258  return 0;
259  }
260  }
261 
263  /* Rule is IP only, but contains negated addresses. */
264  return 2;
265  }
266  if (!(de_ctx->flags & DE_QUIET)) {
267  SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
268  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
269  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
270  }
271  return 1;
272 }
273 
274 /** \internal
275  * \brief Test is a initialized signature is inspecting protocol detection only
276  * \param de_ctx detection engine ctx
277  * \param s the signature
278  * \retval 1 sig is dp only
279  * \retval 0 sig is not dp only
280  */
281 static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s)
282 {
283  /* explicit hook means no PD-only */
285  return 0;
286 
287  if (s->alproto != ALPROTO_UNKNOWN)
288  return 0;
289 
290  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
291  return 0;
292 
293  /* for now assume that all registered buffer types are incompatible */
294  if (SignatureInspectsBuffers(s)) {
295  SCReturnInt(0);
296  }
297 
298  /* TMATCH list can be ignored, it contains TAGs and
299  * tags are compatible to DP-only. */
300 
301  /* match list matches may be compatible to DP only. We follow the same
302  * logic as IP-only so we can use that flag */
303 
305  if (sm == NULL)
306  return 0;
307 
308  int pd = 0;
309  for ( ; sm != NULL; sm = sm->next) {
310  if (sm->type == DETECT_APP_LAYER_PROTOCOL) {
311  pd = 1;
312  } else {
313  /* flowbits are supported for dp only sigs, as long
314  * as the sig only has a "set" flowbits */
315  if (sm->type == DETECT_FLOWBITS) {
316  if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
317  SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id);
318  return 0;
319  }
320  } else if (sm->type == DETECT_FLOW) {
322  SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id);
323  return 0;
324  }
325  } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) {
326  SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name);
327  return 0;
328  }
329  }
330  }
331 
332  if (pd) {
333  SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id);
334  }
335  return pd;
336 }
337 
338 /**
339  * \internal
340  * \brief Check if the initialized signature is inspecting the packet payload
341  * \param de_ctx detection engine ctx
342  * \param s the signature
343  * \retval 1 sig is inspecting the payload
344  * \retval 0 sig is not inspecting the payload
345  */
346 static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s)
347 {
348 
349  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
350  return 1;
351  }
352  return 0;
353 }
354 
355 /**
356  * \internal
357  * \brief check if a signature is decoder event matching only
358  * \param de_ctx detection engine
359  * \param s the signature to test
360  * \retval 0 not a DEOnly sig
361  * \retval 1 DEOnly sig
362  */
363 static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
364 {
365  /* explicit hook means no DE-only */
367  SCReturnInt(0);
368 
369  if (s->alproto != ALPROTO_UNKNOWN) {
370  SCReturnInt(0);
371  }
372 
373  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
374  {
375  SCReturnInt(0);
376  }
377 
378  /* for now assume that all registered buffer types are incompatible */
379  if (SignatureInspectsBuffers(s)) {
380  SCReturnInt(0);
381  }
382 
383  /* check for conflicting keywords */
385  for ( ;sm != NULL; sm = sm->next) {
387  SCReturnInt(0);
388  }
389 
390  /* need at least one decode event keyword to be considered decode event. */
392  for ( ;sm != NULL; sm = sm->next) {
393  if (sm->type == DETECT_DECODE_EVENT)
394  goto deonly;
395  if (sm->type == DETECT_ENGINE_EVENT)
396  goto deonly;
397  if (sm->type == DETECT_STREAM_EVENT)
398  goto deonly;
399  }
400 
401  SCReturnInt(0);
402 
403 deonly:
404  if (!(de_ctx->flags & DE_QUIET)) {
405  SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
406  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
407  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
408  }
409 
410  SCReturnInt(1);
411 }
412 
413 #define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN)
414 #define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR)
415 
416 /* Create mask for this packet + it's flow if it has one
417  */
418 void
420  bool app_decoder_events)
421 {
422  if (!(PKT_IS_PSEUDOPKT(p))) {
423  (*mask) |= SIG_MASK_REQUIRE_REAL_PKT;
424  }
425  if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
426  SCLogDebug("packet has payload");
427  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
428  } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
429  SCLogDebug("stream data available");
430  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
431  } else {
432  SCLogDebug("packet has no payload");
433  (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
434  }
435 
436  if (p->events.cnt > 0 || app_decoder_events != 0 ||
437  (p->app_layer_events != NULL && p->app_layer_events->cnt)) {
438  SCLogDebug("packet/flow has events set");
440  }
441 
442  if (!(PKT_IS_PSEUDOPKT(p)) && PacketIsTCP(p)) {
443  const TCPHdr *tcph = PacketGetTCP(p);
444  if ((tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) {
446  }
447  if ((tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) {
449  }
450  }
451 
452  if (p->flags & PKT_HAS_FLOW) {
453  SCLogDebug("packet has flow");
454  (*mask) |= SIG_MASK_REQUIRE_FLOW;
455  }
456 }
457 
458 static int SignatureCreateMask(Signature *s)
459 {
460  SCEnter();
461 
465  }
466  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
468  SCLogDebug("sig requires payload");
469  }
470 
471  SigMatch *sm;
472  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
473  switch(sm->type) {
474  case DETECT_FLOWBITS:
475  {
476  /* figure out what flowbit action */
478  if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
479  /* not a mask flag, but still set it here */
481 
482  SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has "
483  "flowbit isset option.");
484  }
485 
486  /* flow is required for any flowbit manipulation */
488  SCLogDebug("sig requires flow to be able to manipulate "
489  "flowbit(s)");
490  break;
491  }
492  case DETECT_FLOWINT:
493  /* flow is required for any flowint manipulation */
495  SCLogDebug("sig requires flow to be able to manipulate "
496  "flowint(s)");
497  break;
498  case DETECT_FLAGS:
499  {
500  DetectFlagsData *fl = (DetectFlagsData *)sm->ctx;
501 
502  if (fl->flags & MASK_TCP_INITDEINIT_FLAGS) {
504  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
505  }
506  if (fl->flags & MASK_TCP_UNUSUAL_FLAGS) {
508  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
509  }
510  break;
511  }
512  case DETECT_DSIZE:
513  {
514  DetectU16Data *ds = (DetectU16Data *)sm->ctx;
515  /* LT will include 0, so no payload.
516  * if GT is used in the same rule the
517  * flag will be set anyway. */
518  if (ds->mode == DETECT_UINT_RA || ds->mode == DETECT_UINT_GT ||
519  ds->mode == DETECT_UINT_NE || ds->mode == DETECT_UINT_GTE) {
520 
522  SCLogDebug("sig requires payload");
523 
524  } else if (ds->mode == DETECT_UINT_EQ) {
525  if (ds->arg1 > 0) {
527  SCLogDebug("sig requires payload");
528  } else {
530  SCLogDebug("sig requires no payload");
531  }
532  }
533  break;
534  }
535  case DETECT_DECODE_EVENT:
536  // fallthrough
537  case DETECT_STREAM_EVENT:
538  // fallthrough
540  // fallthrough
541  case DETECT_ENGINE_EVENT:
543  break;
544  }
545  }
546 
547  for (sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL; sm = sm->next) {
548  switch (sm->type) {
549  case DETECT_CONFIG: {
551  if (fd->scope == CONFIG_SCOPE_FLOW) {
553  }
554  break;
555  }
556  }
557  }
558 
561  SCLogDebug("sig requires flow");
562  }
563 
564  if (s->flags & SIG_FLAG_APPLAYER) {
566  SCLogDebug("sig requires flow");
567  }
568 
569  SCLogDebug("mask %02X", s->mask);
570  SCReturnInt(0);
571 }
572 
573 static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
574 {
576 }
577 
578 /** \brief Pure-PCRE or bytetest rule */
579 static bool RuleInspectsPayloadHasNoMpm(const Signature *s)
580 {
581  if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
582  return true;
583  return false;
584 }
585 
586 static int RuleGetMpmPatternSize(const Signature *s)
587 {
588  if (s->init_data->mpm_sm == NULL)
589  return -1;
590  int mpm_list = s->init_data->mpm_sm_list;
591  if (mpm_list < 0)
592  return -1;
593  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
594  if (cd == NULL)
595  return -1;
596  return (int)cd->content_len;
597 }
598 
599 static bool RuleMpmIsNegated(const Signature *s)
600 {
601  if (s->flags & SIG_FLAG_MPM_NEG)
602  return true;
603  if (s->init_data->mpm_sm == NULL)
604  return false;
605  int mpm_list = s->init_data->mpm_sm_list;
606  if (mpm_list < 0)
607  return false;
608  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
609  if (cd == NULL)
610  return false;
611  return (cd->flags & DETECT_CONTENT_NEGATED) ? true : false;
612 }
613 
614 static SCJsonBuilder *RulesGroupPrintSghStats(const DetectEngineCtx *de_ctx,
615  const SigGroupHead *sgh, const int add_rules, const int add_mpm_stats)
616 {
617  uint32_t prefilter_cnt = 0;
618  uint32_t mpm_cnt = 0;
619  uint32_t nonmpm_cnt = 0;
620  uint32_t mpm_depth_cnt = 0;
621  uint32_t mpm_endswith_cnt = 0;
622  uint32_t negmpm_cnt = 0;
623  uint32_t any5_cnt = 0;
624  uint32_t payload_no_mpm_cnt = 0;
625  uint32_t syn_cnt = 0;
626 
627  uint32_t mpms_min = 0;
628  uint32_t mpms_max = 0;
629 
630  int max_buffer_type_id = de_ctx->buffer_type_id;
631 
632  struct {
633  uint32_t total;
634  uint32_t cnt;
635  uint32_t min;
636  uint32_t max;
637  } mpm_stats[max_buffer_type_id];
638  memset(mpm_stats, 0x00, sizeof(mpm_stats));
639 
640  uint32_t alstats[g_alproto_max];
641  memset(alstats, 0, g_alproto_max * sizeof(uint32_t));
642  uint32_t mpm_sizes[max_buffer_type_id][256];
643  memset(mpm_sizes, 0, sizeof(mpm_sizes));
644  uint32_t alproto_mpm_bufs[g_alproto_max][max_buffer_type_id];
645  memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs));
646 
647  DEBUG_VALIDATE_BUG_ON(sgh->init == NULL);
648  if (sgh->init == NULL)
649  return NULL;
650 
651  SCJsonBuilder *js = SCJbNewObject();
652  if (unlikely(js == NULL))
653  return NULL;
654 
655  SCJbSetUint(js, "id", sgh->id);
656 
657  SCJbOpenArray(js, "rules");
658  for (uint32_t x = 0; x < sgh->init->sig_cnt; x++) {
659  const Signature *s = sgh->init->match_array[x];
660  if (s == NULL)
661  continue;
662 
663  int any = 0;
664  if (s->proto.flags & DETECT_PROTO_ANY) {
665  any++;
666  }
667  if (s->flags & SIG_FLAG_DST_ANY) {
668  any++;
669  }
670  if (s->flags & SIG_FLAG_SRC_ANY) {
671  any++;
672  }
673  if (s->flags & SIG_FLAG_DP_ANY) {
674  any++;
675  }
676  if (s->flags & SIG_FLAG_SP_ANY) {
677  any++;
678  }
679  if (any == 5) {
680  any5_cnt++;
681  }
682 
683  prefilter_cnt += (s->init_data->prefilter_sm != NULL);
684  if (s->init_data->mpm_sm == NULL) {
685  nonmpm_cnt++;
686 
687  if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) {
688  SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id);
689  }
690 
691  DetectPort *sp = s->sp;
692  DetectPort *dp = s->dp;
693 
694  if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) {
695  SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id);
696  }
697  if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) {
698  SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id);
699  }
700 
702  syn_cnt++;
703  }
704 
705  } else {
706  int mpm_list = s->init_data->mpm_sm_list;
707  BUG_ON(mpm_list < 0);
708  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
709  uint32_t size = cd->content_len < 256 ? cd->content_len : 255;
710 
711  mpm_sizes[mpm_list][size]++;
712  alproto_mpm_bufs[s->alproto][mpm_list]++;
713 
714  if (mpm_list == DETECT_SM_LIST_PMATCH) {
715  if (size == 1) {
716  DetectPort *sp = s->sp;
717  DetectPort *dp = s->dp;
718  if (s->flags & SIG_FLAG_TOSERVER) {
719  if (dp->port == 0 && dp->port2 == 65535) {
720  SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
721  } else {
722  SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id);
723  }
724  }
725  if (s->flags & SIG_FLAG_TOCLIENT) {
726  if (sp->port == 0 && sp->port2 == 65535) {
727  SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
728  } else {
729  SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id);
730  }
731  }
732  }
733  }
734 
735  uint32_t w = PatternStrength(cd->content, cd->content_len);
736  if (mpms_min == 0)
737  mpms_min = w;
738  if (w < mpms_min)
739  mpms_min = w;
740  if (w > mpms_max)
741  mpms_max = w;
742 
743  BUG_ON(mpm_list >= max_buffer_type_id);
744  mpm_stats[mpm_list].total += w;
745  mpm_stats[mpm_list].cnt++;
746  if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min)
747  mpm_stats[mpm_list].min = w;
748  if (w > mpm_stats[mpm_list].max)
749  mpm_stats[mpm_list].max = w;
750 
751  mpm_cnt++;
752 
753  if (w < 10) {
754  SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id);
755  }
756  if (w < 10 && any == 5) {
757  SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id);
758  }
759 
760  if (cd->flags & DETECT_CONTENT_NEGATED) {
761  SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id);
762  negmpm_cnt++;
763  }
764  if (cd->flags & DETECT_CONTENT_ENDS_WITH) {
765  mpm_endswith_cnt++;
766  }
767  if (cd->flags & DETECT_CONTENT_DEPTH) {
768  mpm_depth_cnt++;
769  }
770  }
771 
772  if (RuleInspectsPayloadHasNoMpm(s)) {
773  SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id);
774  payload_no_mpm_cnt++;
775  }
776 
777  alstats[s->alproto]++;
778 
779  if (add_rules) {
780  SCJsonBuilder *e = SCJbNewObject();
781  if (e != NULL) {
782  SCJbSetUint(e, "sig_id", s->id);
783  SCJbClose(e);
784  SCJbAppendObject(js, e);
785  SCJbFree(e);
786  }
787  }
788  }
789  SCJbClose(js);
790 
791  SCJbOpenObject(js, "stats");
792  SCJbSetUint(js, "total", sgh->init->sig_cnt);
793 
794  SCJbOpenObject(js, "types");
795  SCJbSetUint(js, "mpm", mpm_cnt);
796  SCJbSetUint(js, "non_mpm", nonmpm_cnt);
797  SCJbSetUint(js, "mpm_depth", mpm_depth_cnt);
798  SCJbSetUint(js, "mpm_endswith", mpm_endswith_cnt);
799  SCJbSetUint(js, "negated_mpm", negmpm_cnt);
800  SCJbSetUint(js, "payload_but_no_mpm", payload_no_mpm_cnt);
801  SCJbSetUint(js, "prefilter", prefilter_cnt);
802  SCJbSetUint(js, "syn", syn_cnt);
803  SCJbSetUint(js, "any5", any5_cnt);
804  SCJbClose(js);
805 
806  for (AppProto i = 0; i < g_alproto_max; i++) {
807  if (alstats[i] > 0) {
808  const char *proto_name = (i == ALPROTO_UNKNOWN) ? "payload" : AppProtoToString(i);
809  SCJbOpenObject(js, proto_name);
810  SCJbSetUint(js, "total", alstats[i]);
811 
812  for (int y = 0; y < max_buffer_type_id; y++) {
813  if (alproto_mpm_bufs[i][y] == 0)
814  continue;
815 
816  const char *name;
819  else
821 
822  SCJbSetUint(js, name, alproto_mpm_bufs[i][y]);
823  }
824  SCJbClose(js);
825  }
826  }
827 
828  if (add_mpm_stats) {
829  SCJbOpenObject(js, "mpm");
830 
831  for (int i = 0; i < max_buffer_type_id; i++) {
832  if (mpm_stats[i].cnt > 0) {
833  const char *name;
836  else
838 
839  SCJbOpenArray(js, name);
840 
841  for (int y = 0; y < 256; y++) {
842  if (mpm_sizes[i][y] == 0)
843  continue;
844 
845  SCJsonBuilder *e = SCJbNewObject();
846  if (e != NULL) {
847  SCJbSetUint(e, "size", y);
848  SCJbSetUint(e, "count", mpm_sizes[i][y]);
849  SCJbClose(e);
850  SCJbAppendObject(js, e);
851  SCJbFree(e);
852  }
853  }
854 
855  SCJsonBuilder *e = SCJbNewObject();
856  if (e != NULL) {
857  SCJbSetUint(e, "total", mpm_stats[i].cnt);
858  SCJbSetUint(e, "avg_strength", mpm_stats[i].total / mpm_stats[i].cnt);
859  SCJbSetUint(e, "min_strength", mpm_stats[i].min);
860  SCJbSetUint(e, "max_strength", mpm_stats[i].max);
861  SCJbClose(e);
862  SCJbAppendObject(js, e);
863  SCJbFree(e);
864  }
865 
866  SCJbClose(js);
867  }
868  }
869  SCJbClose(js);
870  }
871  SCJbClose(js);
872 
873  SCJbSetUint(js, "score", sgh->init->score);
874  SCJbClose(js);
875 
876  return js;
877 }
878 
879 static void RulesDumpGrouping(const DetectEngineCtx *de_ctx,
880  const int add_rules, const int add_mpm_stats)
881 {
882  SCJsonBuilder *js = SCJbNewObject();
883  if (unlikely(js == NULL))
884  return;
885 
886  for (int p = 0; p < 256; p++) {
887  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
888  const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp";
889 
890  SCJbOpenObject(js, name);
891  SCJbOpenArray(js, "toserver");
892  const DetectPort *list =
893  (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp : de_ctx->flow_gh[1].udp;
894  while (list != NULL) {
895  SCJsonBuilder *port = SCJbNewObject();
896  SCJbSetUint(port, "port", list->port);
897  SCJbSetUint(port, "port2", list->port2);
898 
899  SCJsonBuilder *stats =
900  RulesGroupPrintSghStats(de_ctx, list->sh, add_rules, add_mpm_stats);
901  SCJbSetObject(port, "rulegroup", stats);
902  SCJbFree(stats);
903  SCJbClose(port);
904  SCJbAppendObject(js, port);
905  SCJbFree(port);
906 
907  list = list->next;
908  }
909  SCJbClose(js); // toserver array
910 
911  SCJbOpenArray(js, "toclient");
912  list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp :
913  de_ctx->flow_gh[0].udp;
914  while (list != NULL) {
915  SCJsonBuilder *port = SCJbNewObject();
916  SCJbSetUint(port, "port", list->port);
917  SCJbSetUint(port, "port2", list->port2);
918 
919  SCJsonBuilder *stats =
920  RulesGroupPrintSghStats(de_ctx, list->sh, add_rules, add_mpm_stats);
921  SCJbSetObject(port, "rulegroup", stats);
922  SCJbFree(stats);
923  SCJbClose(port);
924  SCJbAppendObject(js, port);
925  SCJbFree(port);
926 
927  list = list->next;
928  }
929  SCJbClose(js); // toclient array
930  SCJbClose(js);
931  } else if (p == IPPROTO_ICMP || p == IPPROTO_ICMPV6) {
932  const char *name = (p == IPPROTO_ICMP) ? "icmpv4" : "icmpv6";
933  SCJbOpenObject(js, name);
934  if (de_ctx->flow_gh[1].sgh[p]) {
935  SCJbOpenObject(js, "toserver");
936  SCJsonBuilder *stats = RulesGroupPrintSghStats(
937  de_ctx, de_ctx->flow_gh[1].sgh[p], add_rules, add_mpm_stats);
938  SCJbSetObject(js, "rulegroup", stats);
939  SCJbFree(stats);
940  SCJbClose(js);
941  }
942  if (de_ctx->flow_gh[0].sgh[p]) {
943  SCJbOpenObject(js, "toclient");
944  SCJsonBuilder *stats = RulesGroupPrintSghStats(
945  de_ctx, de_ctx->flow_gh[0].sgh[p], add_rules, add_mpm_stats);
946  SCJbSetObject(js, "rulegroup", stats);
947  SCJbFree(stats);
948  SCJbClose(js);
949  }
950  SCJbClose(js);
951  }
952  }
953  SCJbClose(js);
954 
955  const char *filename = "rule_group.json";
956  const char *log_dir = SCConfigGetLogDirectory();
957  char log_path[PATH_MAX] = "";
958  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
959 
960  FILE *fp = fopen(log_path, "w");
961  if (fp != NULL) {
962  fwrite(SCJbPtr(js), SCJbLen(js), 1, fp);
963  (void)fclose(fp);
964  }
965  SCJbFree(js);
966 }
967 
968 static int RulesGroupByIPProto(DetectEngineCtx *de_ctx)
969 {
970  Signature *s = de_ctx->sig_list;
971 
972  SigGroupHead *sgh_ts[256] = {NULL};
973  SigGroupHead *sgh_tc[256] = {NULL};
974 
975  for ( ; s != NULL; s = s->next) {
976  if (s->type == SIG_TYPE_IPONLY)
977  continue;
978 
979  /* traverse over IP protocol list from libc */
980  for (int p = 0; p < 256; p++) {
981  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
982  continue;
983  }
984  if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) {
985  continue;
986  }
987 
988  /* Signatures that are ICMP, SCTP, not IP only are handled here */
989  if (s->flags & SIG_FLAG_TOCLIENT) {
990  SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s);
991  }
992  if (s->flags & SIG_FLAG_TOSERVER) {
993  SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s);
994  }
995  }
996  }
997 
998  /* lets look at deduplicating this list */
1001 
1002  uint32_t cnt = 0;
1003  uint32_t own = 0;
1004  uint32_t ref = 0;
1005  int p;
1006  for (p = 0; p < 256; p++) {
1007  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1008  continue;
1009  if (sgh_ts[p] == NULL)
1010  continue;
1011 
1012  cnt++;
1013 
1014  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]);
1015  if (lookup_sgh == NULL) {
1016  SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]);
1017 
1018  SigGroupHeadSetSigCnt(sgh_ts[p], 0);
1019  SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], 0);
1020 
1021  SigGroupHeadHashAdd(de_ctx, sgh_ts[p]);
1022  SigGroupHeadStore(de_ctx, sgh_ts[p]);
1023  own++;
1024  } else {
1025  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]);
1026 
1027  SigGroupHeadFree(de_ctx, sgh_ts[p]);
1028  sgh_ts[p] = lookup_sgh;
1029  ref++;
1030  }
1031  }
1032  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1033  "toserver", cnt, own, ref);
1034 
1035  cnt = 0;
1036  own = 0;
1037  ref = 0;
1038  for (p = 0; p < 256; p++) {
1039  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1040  continue;
1041  if (sgh_tc[p] == NULL)
1042  continue;
1043 
1044  cnt++;
1045 
1046  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]);
1047  if (lookup_sgh == NULL) {
1048  SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]);
1049 
1050  SigGroupHeadSetSigCnt(sgh_tc[p], 0);
1051  SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], 0);
1052 
1053  SigGroupHeadHashAdd(de_ctx, sgh_tc[p]);
1054  SigGroupHeadStore(de_ctx, sgh_tc[p]);
1055  own++;
1056 
1057  } else {
1058  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]);
1059 
1060  SigGroupHeadFree(de_ctx, sgh_tc[p]);
1061  sgh_tc[p] = lookup_sgh;
1062  ref++;
1063  }
1064  }
1065  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1066  "toclient", cnt, own, ref);
1067 
1068  for (p = 0; p < 256; p++) {
1069  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1070  continue;
1071 
1072  de_ctx->flow_gh[0].sgh[p] = sgh_tc[p];
1073  de_ctx->flow_gh[1].sgh[p] = sgh_ts[p];
1074  }
1075 
1076  return 0;
1077 }
1078 
1079 static int PortIsPriority(const DetectEngineCtx *de_ctx, const DetectPort *a, int ipproto)
1080 {
1082  if (ipproto == IPPROTO_UDP)
1084 
1085  while (w) {
1086  /* Make sure the priority port falls in the port range of a */
1087  DEBUG_VALIDATE_BUG_ON(a->port > a->port2);
1088  if (a->port == w->port && w->port2 == a->port2) {
1089  return 1;
1090  }
1091  w = w->next;
1092  }
1093 
1094  return 0;
1095 }
1096 
1097 static int RuleSetScore(Signature *s)
1098 {
1099  DetectPort *p = NULL;
1100  if (s->flags & SIG_FLAG_TOSERVER)
1101  p = s->dp;
1102  else if (s->flags & SIG_FLAG_TOCLIENT)
1103  p = s->sp;
1104  else
1105  return 0;
1106 
1107  /* for sigs that don't use 'any' as port, see if we want to
1108  * prioritize poor sigs */
1109  int wl = 0;
1110  if (!(p->port == 0 && p->port2 == 65535)) {
1111  /* pure pcre, bytetest, etc rules */
1112  if (RuleInspectsPayloadHasNoMpm(s)) {
1113  SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Prioritizing SGH's.", s->id);
1115 
1116  } else if (RuleMpmIsNegated(s)) {
1117  SCLogDebug("Rule %u MPM is negated. Prioritizing SGH's.", s->id);
1119 
1120  /* one byte pattern in packet/stream payloads */
1121  } else if (s->init_data->mpm_sm != NULL &&
1123  RuleGetMpmPatternSize(s) == 1) {
1124  SCLogDebug("Rule %u No MPM. Payload inspecting. Prioritizing SGH's.", s->id);
1126 
1128  SCLogDebug("Rule %u Needs SYN, so inspected often. Prioritizing SGH's.", s->id);
1130  }
1131  }
1132 
1133  s->init_data->score = wl;
1134  return wl;
1135 }
1136 
1137 static int SortCompare(const void *a, const void *b)
1138 {
1139  const DetectPort *pa = *(const DetectPort **)a;
1140  const DetectPort *pb = *(const DetectPort **)b;
1141 
1142  if (pa->sh->init->score < pb->sh->init->score) {
1143  return 1;
1144  } else if (pa->sh->init->score > pb->sh->init->score) {
1145  return -1;
1146  }
1147 
1148  if (pa->sh->init->sig_cnt < pb->sh->init->sig_cnt) {
1149  return 1;
1150  } else if (pa->sh->init->sig_cnt > pb->sh->init->sig_cnt) {
1151  return -1;
1152  }
1153 
1154  /* Hack to make the qsort output deterministic across platforms.
1155  * This had to be done because the order of equal elements sorted
1156  * by qsort is undeterministic and showed different output on BSD,
1157  * MacOS and Windows. Sorting based on id makes it deterministic. */
1158  if (pa->sh->id < pb->sh->id)
1159  return -1;
1160 
1161  return 1;
1162 }
1163 
1164 static inline void SortGroupList(
1165  uint32_t *groups, DetectPort **list, int (*CompareFunc)(const void *, const void *))
1166 {
1167  int cnt = 0;
1168  for (DetectPort *x = *list; x != NULL; x = x->next) {
1169  DEBUG_VALIDATE_BUG_ON(x->port > x->port2);
1170  cnt++;
1171  }
1172  if (cnt <= 1)
1173  return;
1174 
1175  /* build temporary array to sort with qsort */
1176  DetectPort **array = (DetectPort **)SCCalloc(cnt, sizeof(DetectPort *));
1177  if (array == NULL)
1178  return;
1179 
1180  int idx = 0;
1181  for (DetectPort *x = *list; x != NULL;) {
1182  /* assign a temporary id to resolve otherwise equal groups */
1183  x->sh->id = idx + 1;
1184  SigGroupHeadSetSigCnt(x->sh, 0);
1185  DetectPort *next = x->next;
1186  x->next = x->prev = x->last = NULL;
1187  DEBUG_VALIDATE_BUG_ON(x->port > x->port2);
1188  array[idx++] = x;
1189  x = next;
1190  }
1191  DEBUG_VALIDATE_BUG_ON(cnt != idx);
1192 
1193  qsort(array, idx, sizeof(DetectPort *), SortCompare);
1194 
1195  /* rebuild the list based on the qsort-ed array */
1196  DetectPort *new_list = NULL, *tail = NULL;
1197  for (int i = 0; i < idx; i++) {
1198  DetectPort *p = array[i];
1199  /* unset temporary group id */
1200  p->sh->id = 0;
1201 
1202  if (new_list == NULL) {
1203  new_list = p;
1204  }
1205  if (tail != NULL) {
1206  tail->next = p;
1207  }
1208  p->prev = tail;
1209  tail = p;
1210  }
1211 
1212  *list = new_list;
1213  *groups = idx;
1214 
1215 #if DEBUG
1216  int dbgcnt = 0;
1217  SCLogDebug("SORTED LIST:");
1218  for (DetectPort *tmp = *list; tmp != NULL; tmp = tmp->next) {
1219  SCLogDebug("item:= [%u:%u]; score: %d; sig_cnt: %d", tmp->port, tmp->port2,
1220  tmp->sh->init->score, tmp->sh->init->sig_cnt);
1221  dbgcnt++;
1222  BUG_ON(dbgcnt > cnt);
1223  }
1224 #endif
1225  SCFree(array);
1226 }
1227 /** \internal
1228  * \brief Create a list of DetectPort objects sorted based on CompareFunc's
1229  * logic.
1230  *
1231  * List can limit the number of groups. In this case an extra "join" group
1232  * is created that contains the sigs belonging to that. It's *appended* to
1233  * the list, meaning that if the list is walked linearly it's found last.
1234  * The joingr is meant to be a catch all.
1235  *
1236  */
1237 static int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list,
1238  DetectPort **newhead, uint32_t unique_groups,
1239  int (*CompareFunc)(const void *, const void *))
1240 {
1241  DetectPort *tmplist = NULL, *joingr = NULL;
1242  uint32_t groups = 0;
1243 
1244  /* insert the ports into the tmplist, where it will
1245  * be sorted descending on 'cnt' and on whether a group
1246  * is prioritized. */
1247  tmplist = port_list;
1248  SortGroupList(&groups, &tmplist, SortCompare);
1249  uint32_t left = unique_groups;
1250  if (left == 0)
1251  left = groups;
1252 
1253  /* create another list: take the port groups from above
1254  * and add them to the 2nd list until we have met our
1255  * count. The rest is added to the 'join' group. */
1256  DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL;
1257  DetectPort *gr, *next_gr;
1258  for (gr = tmplist; gr != NULL;) {
1259  next_gr = gr->next;
1260 
1261  SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2);
1262  DetectPortPrint(gr);
1263 
1264  /* if we've set up all the unique groups, add the rest to the
1265  * catch-all joingr */
1266  if (left == 0) {
1267  if (joingr == NULL) {
1268  DetectPortParse(de_ctx, &joingr, "0:65535");
1269  if (joingr == NULL) {
1270  goto error;
1271  }
1272  SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2);
1273  joingr->next = NULL;
1274  }
1275  SigGroupHeadCopySigs(de_ctx, gr->sh, &joingr->sh);
1276 
1277  /* when a group's sigs are added to the joingr, we can free it */
1278  gr->next = NULL;
1279  DetectPortFree(de_ctx, gr);
1280  /* append */
1281  } else {
1282  gr->next = NULL;
1283 
1284  if (tmplist2 == NULL) {
1285  tmplist2 = gr;
1286  tmplist2_tail = gr;
1287  } else {
1288  tmplist2_tail->next = gr;
1289  tmplist2_tail = gr;
1290  }
1291  }
1292 
1293  if (left > 0)
1294  left--;
1295 
1296  gr = next_gr;
1297  }
1298 
1299  /* if present, append the joingr that covers the rest */
1300  if (joingr != NULL) {
1301  SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2);
1302 
1303  if (tmplist2 == NULL) {
1304  tmplist2 = joingr;
1305  // tmplist2_tail = joingr;
1306  } else {
1307  tmplist2_tail->next = joingr;
1308  // tmplist2_tail = joingr;
1309  }
1310  } else {
1311  SCLogDebug("no joingr");
1312  }
1313 
1314  /* pass back our new list to the caller */
1315  *newhead = tmplist2;
1316  DetectPortPrintList(*newhead);
1317 
1318  return 0;
1319 error:
1320  return -1;
1321 }
1322 
1323 #define UNDEFINED_PORT 0
1324 #define RANGE_PORT 1
1325 #define SINGLE_PORT 2
1327 typedef struct UniquePortPoint_ {
1328  uint16_t port; /* value of the port */
1329  bool single; /* is the port single or part of a range */
1331 
1332 /**
1333  * \brief Function to set unique port points. Consider all the ports
1334  * flattened out on one line, set the points that correspond
1335  * to a valid port. Also store whether the port point stored
1336  * was a single port or part of a range.
1337  *
1338  * \param p Port object to be set
1339  * \param unique_list List of unique port points to be updated
1340  * \param size_list Current size of the list
1341  *
1342  * \return Updated size of the list
1343  */
1344 static inline uint32_t SetUniquePortPoints(
1345  const DetectPort *p, uint8_t *unique_list, uint32_t size_list)
1346 {
1347  if (unique_list[p->port] == UNDEFINED_PORT) {
1348  if (p->port == p->port2) {
1349  unique_list[p->port] = SINGLE_PORT;
1350  } else {
1351  unique_list[p->port] = RANGE_PORT;
1352  }
1353  size_list++;
1354  } else if (((unique_list[p->port] == SINGLE_PORT) && (p->port != p->port2)) ||
1355  ((unique_list[p->port] == RANGE_PORT) && (p->port == p->port2))) {
1356  if ((p->port != UINT16_MAX) && (unique_list[p->port + 1] == UNDEFINED_PORT)) {
1357  unique_list[p->port + 1] = RANGE_PORT;
1358  size_list++;
1359  }
1360  }
1361 
1362  /* Treat right boundary as single point to avoid creating unneeded
1363  * ranges later on */
1364  if (unique_list[p->port2] == UNDEFINED_PORT) {
1365  size_list++;
1366  }
1367  unique_list[p->port2] = SINGLE_PORT;
1368  return size_list;
1369 }
1370 
1371 /**
1372  * \brief Function to set the *final* unique port points and save them
1373  * for later use. The points are already sorted because of the way
1374  * they have been retrieved and saved earlier for use at this point.
1375  *
1376  * \param unique_list List of the unique port points to be used
1377  * \param size_unique_arr Number of unique port points
1378  * \param final_arr List of the final unique port points to be created
1379  */
1380 static inline void SetFinalUniquePortPoints(
1381  const uint8_t *unique_list, const uint32_t size_unique_arr, UniquePortPoint *final_arr)
1382 {
1383  for (uint32_t i = 0, j = 0; i < (UINT16_MAX + 1); i++) {
1384  DEBUG_VALIDATE_BUG_ON(j > size_unique_arr);
1385  if (unique_list[i] == RANGE_PORT) {
1386  final_arr[j].port = (uint16_t)i;
1387  final_arr[j++].single = false;
1388  } else if (unique_list[i] == SINGLE_PORT) {
1389  final_arr[j].port = (uint16_t)i;
1390  final_arr[j++].single = true;
1391  }
1392  }
1393 }
1394 
1395 /**
1396  * \brief Function to create the list of ports with the smallest ranges
1397  * by resolving overlaps and end point conditions. These contain the
1398  * correct SGHs as well after going over the interval tree to find
1399  * any range overlaps.
1400  *
1401  * \param de_ctx Detection Engine Context
1402  * \param unique_list Final list of unique port points
1403  * \param size_list Size of the unique_list
1404  * \param it Pointer to the interval tree
1405  * \param list Pointer to the list where final ports will be stored
1406  *
1407  * \return 0 on success, -1 otherwise
1408  */
1409 static inline int CreatePortList(DetectEngineCtx *de_ctx, const uint8_t *unique_list,
1410  const uint32_t size_list, SCPortIntervalTree *it, DetectPort **list)
1411 {
1412  /* Only do the operations if there is at least one unique port */
1413  if (size_list == 0)
1414  return 0;
1415  UniquePortPoint *final_unique_points =
1416  (UniquePortPoint *)SCCalloc(size_list, sizeof(UniquePortPoint));
1417  if (final_unique_points == NULL)
1418  return -1;
1419  SetFinalUniquePortPoints(unique_list, size_list, final_unique_points);
1420  /* Handle edge case when there is just one unique port */
1421  if (size_list == 1) {
1423  de_ctx, final_unique_points[0].port, final_unique_points[0].port, &it->tree, list);
1424  } else {
1425  UniquePortPoint *p1 = &final_unique_points[0];
1426  UniquePortPoint *p2 = &final_unique_points[1];
1427  uint16_t port = p1 ? p1->port : 0; // just for cppcheck
1428  uint16_t port2 = p2->port;
1429  for (uint32_t i = 1; i < size_list; i++) {
1430  DEBUG_VALIDATE_BUG_ON(port > port2);
1431  if ((p1 && p1->single) && p2->single) {
1432  SCPortIntervalFindOverlappingRanges(de_ctx, port, port, &it->tree, list);
1433  SCPortIntervalFindOverlappingRanges(de_ctx, port2, port2, &it->tree, list);
1434  port = port2 + 1;
1435  } else if (p1 && p1->single) {
1436  SCPortIntervalFindOverlappingRanges(de_ctx, port, port, &it->tree, list);
1437  if ((port2 > port + 1)) {
1439  de_ctx, port + 1, port2 - 1, &it->tree, list);
1440  port = port2;
1441  } else {
1442  port = port + 1;
1443  }
1444  } else if (p2->single) {
1445  /* If port2 is boundary and less or equal to port + 1, create a range
1446  * keeping the boundary away as it is single port */
1447  if ((port2 >= port + 1)) {
1448  SCPortIntervalFindOverlappingRanges(de_ctx, port, port2 - 1, &it->tree, list);
1449  }
1450  /* Deal with port2 as it is a single port */
1451  SCPortIntervalFindOverlappingRanges(de_ctx, port2, port2, &it->tree, list);
1452  port = port2 + 1;
1453  } else {
1454  if ((port2 > port + 1)) {
1455  SCPortIntervalFindOverlappingRanges(de_ctx, port, port2 - 1, &it->tree, list);
1456  port = port2;
1457  } else {
1458  SCPortIntervalFindOverlappingRanges(de_ctx, port, port2, &it->tree, list);
1459  port = port2 + 1;
1460  }
1461  }
1462  /* if the current port matches the p2->port, assign it to p1 so that
1463  * there is a UniquePortPoint object to check other info like whether
1464  * the port with this value is single */
1465  if (port == p2->port) {
1466  p1 = p2;
1467  } else {
1468  p1 = NULL;
1469  }
1470  if (i + 1 < size_list) {
1471  p2 = &final_unique_points[i + 1];
1472  port2 = p2->port;
1473  }
1474  }
1475  }
1476  /* final_unique_points array is no longer needed */
1477  SCFree(final_unique_points);
1478  return 0;
1479 }
1480 
1481 static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, uint32_t direction)
1482 {
1483  /* step 1: create a hash of 'DetectPort' objects based on all the
1484  * rules. Each object will have a SGH with the sigs added
1485  * that belong to the SGH. */
1487 
1488  uint32_t size_unique_port_arr = 0;
1489  const Signature *s = de_ctx->sig_list;
1490  DetectPort *list = NULL;
1491 
1492  uint8_t *unique_port_points = (uint8_t *)SCCalloc(UINT16_MAX + 1, sizeof(uint8_t));
1493  if (unique_port_points == NULL)
1494  return NULL;
1495 
1496  while (s) {
1497  /* IP Only rules are handled separately */
1498  if (s->type == SIG_TYPE_IPONLY)
1499  goto next;
1500  /* Protocol does not match the Signature protocol and is neither IP or pkthdr */
1501  if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY)))
1502  goto next;
1503  /* Direction does not match Signature direction */
1504  if (direction == SIG_FLAG_TOSERVER) {
1505  if (!(s->flags & SIG_FLAG_TOSERVER))
1506  goto next;
1507  } else if (direction == SIG_FLAG_TOCLIENT) {
1508  if (!(s->flags & SIG_FLAG_TOCLIENT))
1509  goto next;
1510  }
1511 
1512  /* see if we want to exclude directionless sigs that really care only for
1513  * to_server syn scans/floods */
1517  (!(s->dp->port == 0 && s->dp->port2 == 65535))) {
1518  SCLogWarning("rule %u: SYN-only to port(s) %u:%u "
1519  "w/o direction specified, disabling for toclient direction",
1520  s->id, s->dp->port, s->dp->port2);
1521  goto next;
1522  }
1523 
1524  DetectPort *p = NULL;
1525  if (direction == SIG_FLAG_TOSERVER)
1526  p = s->dp;
1527  else if (direction == SIG_FLAG_TOCLIENT)
1528  p = s->sp;
1529  else
1530  BUG_ON(1);
1531 
1532  int wl = s->init_data->score;
1533  while (p) {
1534  int pwl = PortIsPriority(de_ctx, p, ipproto) ? DETECT_PGSCORE_RULE_PORT_PRIORITIZED : 0;
1535  pwl = MAX(wl,pwl);
1536 
1537  DetectPort *lookup = DetectPortHashLookup(de_ctx, p);
1538  if (lookup) {
1539  SigGroupHeadAppendSig(de_ctx, &lookup->sh, s);
1540  lookup->sh->init->score = MAX(lookup->sh->init->score, pwl);
1541  } else {
1543  BUG_ON(tmp2 == NULL);
1544  SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s);
1545  tmp2->sh->init->score = pwl;
1546  DetectPortHashAdd(de_ctx, tmp2);
1547  size_unique_port_arr =
1548  SetUniquePortPoints(tmp2, unique_port_points, size_unique_port_arr);
1549  }
1550 
1551  p = p->next;
1552  }
1553  next:
1554  s = s->next;
1555  }
1556 
1557  /* step 2: create a list of the smallest port ranges with
1558  * appropriate SGHs */
1559 
1560  /* Create an interval tree of all the given ports to make the search
1561  * for overlaps later on easier */
1563  if (it == NULL)
1564  goto error;
1565 
1566  HashListTableBucket *htb = NULL;
1567  for (htb = HashListTableGetListHead(de_ctx->dport_hash_table); htb != NULL;
1568  htb = HashListTableGetListNext(htb)) {
1570  if (SCPortIntervalInsert(de_ctx, it, p) != SC_OK) {
1571  SCLogDebug("Port was not inserted in the tree");
1572  goto error;
1573  }
1574  }
1575 
1576  /* Create a sorted list of ports in ascending order after resolving overlaps
1577  * and corresponding SGHs */
1578  if (CreatePortList(de_ctx, unique_port_points, size_unique_port_arr, it, &list) < 0)
1579  goto error;
1580 
1581  /* unique_port_points array is no longer needed */
1582  SCFree(unique_port_points);
1583 
1584  /* Port hashes are no longer needed */
1586 
1587  SCLogDebug("rules analyzed");
1588 
1589  /* step 3: group the list and shrink it if necessary */
1590  DetectPort *newlist = NULL;
1591  uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups :
1593  CreateGroupedPortList(de_ctx, list, &newlist, groupmax, SortCompare);
1594  list = newlist;
1595 
1596  /* step 4: deduplicate the SGH's */
1599 
1600  uint32_t cnt = 0;
1601  uint32_t own = 0;
1602  uint32_t ref = 0;
1603  DetectPort *iter;
1604  for (iter = list ; iter != NULL; iter = iter->next) {
1605  BUG_ON (iter->sh == NULL);
1606  DEBUG_VALIDATE_BUG_ON(own + ref != cnt);
1607  cnt++;
1608 
1609  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh);
1610  if (lookup_sgh == NULL) {
1611  SCLogDebug("port group %p sgh %p is the original", iter, iter->sh);
1612 
1613  SigGroupHeadSetSigCnt(iter->sh, 0);
1615  SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction);
1616  SigGroupHeadHashAdd(de_ctx, iter->sh);
1617  SigGroupHeadStore(de_ctx, iter->sh);
1618  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1619  own++;
1620  } else {
1621  SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh);
1622 
1623  SigGroupHeadFree(de_ctx, iter->sh);
1624  iter->sh = lookup_sgh;
1625  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1626  ref++;
1627  }
1628  }
1629 #if 0
1630  for (iter = list ; iter != NULL; iter = iter->next) {
1631  SCLogInfo("PORT %u-%u %p (sgh=%s, prioritized=%s/%d)",
1632  iter->port, iter->port2, iter->sh,
1633  iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own",
1634  iter->sh->init->score ? "true" : "false",
1635  iter->sh->init->score);
1636  }
1637 #endif
1638  SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies",
1639  ipproto == 6 ? "TCP" : "UDP",
1640  direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1641  cnt, own, ref);
1643  return list;
1644 
1645 error:
1646  if (unique_port_points != NULL)
1647  SCFree(unique_port_points);
1648  if (it != NULL)
1650 
1651  return NULL;
1652 }
1653 
1655 {
1656  BUG_ON(s->type != SIG_TYPE_NOT_SET);
1657  int iponly = 0;
1658 
1660  s->type = SIG_TYPE_APP_TX;
1661  SCLogDebug("%u: set to app_tx due to hook type app", s->id);
1662  SCReturn;
1663  }
1664 
1665  /* see if the sig is dp only */
1666  if (SignatureIsPDOnly(de_ctx, s) == 1) {
1667  s->type = SIG_TYPE_PDONLY;
1668 
1669  /* see if the sig is ip only */
1670  } else if ((iponly = SignatureIsIPOnly(de_ctx, s)) > 0) {
1671  if (iponly == 1) {
1672  s->type = SIG_TYPE_IPONLY;
1673  } else if (iponly == 2) {
1675  }
1676  } else if (SignatureIsDEOnly(de_ctx, s) == 1) {
1677  s->type = SIG_TYPE_DEONLY;
1678 
1679  } else {
1680  const bool has_match = s->init_data->smlists[DETECT_SM_LIST_MATCH] != NULL;
1681  const bool has_pmatch = s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL;
1682  bool has_buffer_frame_engine = false;
1683  bool has_buffer_packet_engine = false;
1684  bool has_buffer_app_engine = false;
1685 
1686  for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
1687  const uint32_t id = s->init_data->buffers[x].id;
1688 
1690  has_buffer_packet_engine = true;
1692  has_buffer_frame_engine = true;
1693  } else {
1694  has_buffer_app_engine = true;
1695  }
1696  }
1697 
1698  if (has_buffer_packet_engine) {
1699  s->type = SIG_TYPE_PKT;
1700  } else if (has_buffer_frame_engine || has_buffer_app_engine) {
1701  s->type = SIG_TYPE_APP_TX;
1702  } else if (has_pmatch) {
1705  s->type = SIG_TYPE_PKT;
1706  } else if ((s->flags & (SIG_FLAG_REQUIRE_PACKET | SIG_FLAG_REQUIRE_STREAM)) ==
1708  s->type = SIG_TYPE_STREAM;
1709  } else {
1711  }
1712  } else if (has_match) {
1713  s->type = SIG_TYPE_PKT;
1714 
1715  /* app-layer but no inspect engines */
1716  } else if (s->flags & SIG_FLAG_APPLAYER) {
1717  s->type = SIG_TYPE_APPLAYER;
1718  } else {
1719  s->type = SIG_TYPE_PKT;
1720  }
1721  }
1722 }
1723 
1724 /**
1725  * \brief Preprocess signature, classify ip-only, etc, build sig array
1726  *
1727  * \param de_ctx Pointer to the Detection Engine Context
1728  *
1729  * \retval 0 on success
1730  * \retval -1 on failure
1731  */
1733 {
1734  uint32_t cnt_iponly = 0;
1735  uint32_t cnt_payload = 0;
1736  uint32_t cnt_applayer = 0;
1737  uint32_t cnt_deonly = 0;
1738 
1739  if (!(de_ctx->flags & DE_QUIET)) {
1740  SCLogDebug("building signature grouping structure, stage 1: "
1741  "preprocessing rules...");
1742  }
1743 
1746  if (de_ctx->sig_array == NULL)
1747  goto error;
1748 
1749  /* now for every rule add the source group */
1750  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1751  de_ctx->sig_array[s->num] = s;
1752 
1753  SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", s->id, s->num, s, de_ctx->sig_array[s->num]);
1754 
1755  if (s->type == SIG_TYPE_PDONLY) {
1756  SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", s->id);
1757  } else if (s->type == SIG_TYPE_IPONLY) {
1758  SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", s->id);
1759  cnt_iponly++;
1760  } else if (SignatureIsInspectingPayload(de_ctx, s) == 1) {
1761  SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", s->id);
1762  cnt_payload++;
1763  } else if (s->type == SIG_TYPE_DEONLY) {
1764  SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", s->id);
1765  cnt_deonly++;
1766  } else if (s->flags & SIG_FLAG_APPLAYER) {
1767  SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", s->id);
1768  cnt_applayer++;
1769  }
1770 
1771 #ifdef DEBUG
1772  if (SCLogDebugEnabled()) {
1773  uint16_t colen = 0;
1774  char copresent = 0;
1775  SigMatch *sm;
1776  DetectContentData *co;
1777  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) {
1778  if (sm->type != DETECT_CONTENT)
1779  continue;
1780 
1781  copresent = 1;
1782  co = (DetectContentData *)sm->ctx;
1783  if (co->content_len > colen)
1784  colen = co->content_len;
1785  }
1786 
1787  if (copresent && colen == 1) {
1788  SCLogDebug("signature %8u content maxlen 1", s->id);
1789  for (int proto = 0; proto < 256; proto++) {
1790  if (s->proto.proto[(proto/8)] & (1<<(proto%8)))
1791  SCLogDebug("=> proto %" PRId32 "", proto);
1792  }
1793  }
1794  }
1795 #endif /* DEBUG */
1796 
1797  if (RuleMpmIsNegated(s)) {
1798  s->flags |= SIG_FLAG_MPM_NEG;
1799  }
1800 
1801  SignatureCreateMask(s);
1804 
1805  RuleSetScore(s);
1806 
1807  /* run buffer type callbacks if any */
1808  for (int x = 0; x < DETECT_SM_LIST_MAX; x++) {
1809  if (s->init_data->smlists[x])
1811  }
1812  for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
1814  }
1815 
1816  de_ctx->sig_cnt++;
1817  }
1818 
1819  if (!(de_ctx->flags & DE_QUIET)) {
1820  if (strlen(de_ctx->config_prefix) > 0)
1821  SCLogInfo("tenant id %d: %" PRIu32 " signatures processed. %" PRIu32 " are IP-only "
1822  "rules, %" PRIu32 " are inspecting packet payload, %" PRIu32
1823  " inspect application layer, %" PRIu32 " are decoder event only",
1824  de_ctx->tenant_id, de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer,
1825  cnt_deonly);
1826  else
1827  SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only "
1828  "rules, %" PRIu32 " are inspecting packet payload, %" PRIu32
1829  " inspect application layer, %" PRIu32 " are decoder event only",
1830  de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer, cnt_deonly);
1831 
1832  SCLogConfig("building signature grouping structure, stage 1: "
1833  "preprocessing rules... complete");
1834  }
1835 
1836  if (DetectFlowbitsAnalyze(de_ctx) != 0)
1837  goto error;
1838 
1839  return 0;
1840 
1841 error:
1842  return -1;
1843 }
1844 
1845 /**
1846  * \internal
1847  * \brief add a decoder event signature to the detection engine ctx
1848  */
1849 static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s)
1850 {
1851  SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id);
1853 }
1854 
1855 /**
1856  * \brief Fill the global src group head, with the sigs included
1857  *
1858  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1859  * to be processed
1860  *
1861  * \retval 0 On success
1862  * \retval -1 On failure
1863  */
1865 {
1866  SCLogDebug("building signature grouping structure, stage 2: "
1867  "building source address lists...");
1868 
1870 
1871  de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER);
1872  de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT);
1873  de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER);
1874  de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT);
1875 
1876  /* Setup the other IP Protocols (so not TCP/UDP) */
1877  RulesGroupByIPProto(de_ctx);
1878 
1879  /* now for every rule add the source group to our temp lists */
1880  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1881  SCLogDebug("s->id %"PRIu32, s->id);
1882  if (s->type == SIG_TYPE_IPONLY) {
1884  } else if (s->type == SIG_TYPE_DEONLY) {
1885  DetectEngineAddDecoderEventSig(de_ctx, s);
1886  }
1887  }
1888 
1891  return 0;
1892 }
1893 
1894 static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx)
1895 {
1896  if (de_ctx->decoder_event_sgh == NULL)
1897  return;
1898 
1899  uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx);
1902 }
1903 
1905 {
1906  /* prepare the decoder event sgh */
1907  DetectEngineBuildDecoderEventSgh(de_ctx);
1908  return 0;
1909 }
1910 
1912 {
1913  BUG_ON(de_ctx == NULL);
1914 
1915  SCLogDebug("cleaning up signature grouping structure...");
1916 
1919  de_ctx->decoder_event_sgh = NULL;
1920 
1921  for (int f = 0; f < FLOW_STATES; f++) {
1922  for (int p = 0; p < 256; p++) {
1923  de_ctx->flow_gh[f].sgh[p] = NULL;
1924  }
1925 
1926  /* free lookup lists */
1928  de_ctx->flow_gh[f].tcp = NULL;
1930  de_ctx->flow_gh[f].udp = NULL;
1931  }
1932 
1933  for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1934  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1935  if (sgh == NULL)
1936  continue;
1937 
1938  SCLogDebug("sgh %p", sgh);
1939  SigGroupHeadFree(de_ctx, sgh);
1940  }
1942  de_ctx->sgh_array = NULL;
1943  de_ctx->sgh_array_cnt = 0;
1944  de_ctx->sgh_array_size = 0;
1945 
1947 
1948  SCLogDebug("cleaning up signature grouping structure... complete");
1949  return 0;
1950 }
1951 
1952 #if 0
1953 static void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1954 {
1955  if (sgh == NULL) {
1956  printf("\n");
1957  return;
1958  }
1959 
1960  uint32_t sig;
1961  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1962  printf("%" PRIu32 " ", sgh->match_array[sig]->id);
1963  }
1964  printf("\n");
1965 }
1966 
1967 static void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1968 {
1969  if (sgh == NULL || sgh->init == NULL) {
1970  printf("\n");
1971  return;
1972  }
1973 
1974  uint32_t sig;
1975  for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
1976  if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) {
1977  printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id);
1978  }
1979  }
1980  printf("\n");
1981 }
1982 #endif
1983 
1984 /** \brief finalize preparing sgh's */
1986 {
1987  SCEnter();
1988 
1989  //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt);
1990 
1991  uint32_t cnt = 0;
1992  for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1993  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1994  if (sgh == NULL)
1995  continue;
1996 
1997  SCLogDebug("sgh %p", sgh);
1998 
2000  SCLogDebug("filestore count %u", sgh->filestore_cnt);
2001 
2003 
2004  sgh->id = idx;
2005  cnt++;
2006  }
2007  SCLogPerf("Unique rule groups: %u", cnt);
2008 
2010 
2011  if (de_ctx->decoder_event_sgh != NULL) {
2012  /* no need to set filestore count here as that would make a
2013  * signature not decode event only. */
2015  }
2016 
2017  int dump_grouping = 0;
2018  (void)SCConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping);
2019 
2020  if (dump_grouping) {
2021  int add_rules = 0;
2022  (void)SCConfGetBool("detect.profiling.grouping.include-rules", &add_rules);
2023  int add_mpm_stats = 0;
2024  (void)SCConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_mpm_stats);
2025 
2026  RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats);
2027  }
2028 
2029  for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
2030  SigGroupHead *sgh = de_ctx->sgh_array[idx];
2031  if (sgh == NULL)
2032  continue;
2034  sgh->init = NULL;
2035  }
2036  /* cleanup the hashes now since we won't need them
2037  * after the initialization phase. */
2039 
2040 #ifdef PROFILING
2042 #endif
2043  SCReturnInt(0);
2044 }
2045 
2046 extern bool rule_engine_analysis_set;
2047 /** \internal
2048  * \brief perform final per signature setup tasks
2049  *
2050  * - Create SigMatchData arrays from the init only SigMatch lists
2051  * - Setup per signature inspect engines
2052  * - remove signature init data.
2053  */
2054 static int SigMatchPrepare(DetectEngineCtx *de_ctx)
2055 {
2056  SCEnter();
2057 
2058  Signature *s = de_ctx->sig_list;
2059  for (; s != NULL; s = s->next) {
2060  /* set up inspect engines */
2062 
2063  /* built-ins */
2064  for (int type = 0; type < DETECT_SM_LIST_MAX; type++) {
2065  /* skip PMATCH if it is used in a stream 'app engine' instead */
2067  continue;
2068  SigMatch *sm = s->init_data->smlists[type];
2070  }
2071  /* set up the pkt inspection engines */
2073 
2077  }
2078  /* free lists. Ctx' are xferred to sm_arrays so won't get freed */
2079  for (uint32_t i = 0; i < DETECT_SM_LIST_MAX; i++) {
2080  SigMatch *sm = s->init_data->smlists[i];
2081  while (sm != NULL) {
2082  SigMatch *nsm = sm->next;
2083  SigMatchFree(de_ctx, sm);
2084  sm = nsm;
2085  }
2086  }
2087  for (uint32_t i = 0; i < (uint32_t)s->init_data->transforms.cnt; i++) {
2088  if (s->init_data->transforms.transforms[i].options) {
2089  int transform = s->init_data->transforms.transforms[i].transform;
2090  sigmatch_table[transform].Free(
2092  s->init_data->transforms.transforms[i].options = NULL;
2093  }
2094  }
2095  for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
2096  SigMatch *sm = s->init_data->buffers[x].head;
2097  while (sm != NULL) {
2098  SigMatch *nsm = sm->next;
2099  SigMatchFree(de_ctx, sm);
2100  sm = nsm;
2101  }
2102  }
2103  if (s->init_data->cidr_dst != NULL)
2105 
2106  if (s->init_data->cidr_src != NULL)
2108 
2109  SCFree(s->init_data->buffers);
2112  SCFree(s->init_data);
2113  s->init_data = NULL;
2114  }
2115 
2117  SCReturnInt(0);
2118 }
2119 
2120 /**
2121  * \brief Convert the signature list into the runtime match structure.
2122  *
2123  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
2124  * to be processed
2125  *
2126  * \retval 0 On Success.
2127  * \retval -1 On failure.
2128  */
2130 {
2131  Signature *s = de_ctx->sig_list;
2132 
2133  /* Assign the unique order id of signatures after sorting,
2134  * so the IP Only engine process them in order too. Also
2135  * reset the old signums and assign new signums. We would
2136  * have experienced Sig reordering by now, hence the new
2137  * signums. */
2138  de_ctx->signum = 0;
2139  while (s != NULL) {
2140  s->num = de_ctx->signum++;
2141 
2142  s = s->next;
2143  }
2144 
2146  return -1;
2147 
2148  SigInitStandardMpmFactoryContexts(de_ctx);
2149 
2150  if (SigPrepareStage1(de_ctx) != 0) {
2151  FatalError("initializing the detection engine failed");
2152  }
2153 
2154  if (SigPrepareStage2(de_ctx) != 0) {
2155  FatalError("initializing the detection engine failed");
2156  }
2157 
2158  if (SigPrepareStage3(de_ctx) != 0) {
2159  FatalError("initializing the detection engine failed");
2160  }
2161  if (SigPrepareStage4(de_ctx) != 0) {
2162  FatalError("initializing the detection engine failed");
2163  }
2164 
2169  if (r != 0) {
2170  FatalError("initializing the detection engine failed");
2171  }
2172 
2173  if (SigMatchPrepare(de_ctx) != 0) {
2174  FatalError("initializing the detection engine failed");
2175  }
2176 
2177 #ifdef PROFILING
2180  de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled
2181 
2182  intmax_t v = 0;
2183  if (SCConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1)
2184  de_ctx->profile_match_logging_threshold = (uint32_t)v;
2185 #endif
2186 #ifdef PROFILE_RULES
2187  SCProfilingRuleInitCounters(de_ctx);
2188 #endif
2189 
2192  }
2193 
2194  if (de_ctx->flags & DE_HAS_FIREWALL) {
2196  }
2197  return 0;
2198 }
2199 
2201 {
2203 
2204  return 0;
2205 }
detect-tcp-flags.h
HashListTableGetListData
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:56
SIG_TYPE_STREAM
@ SIG_TYPE_STREAM
Definition: detect.h:73
DETECT_FLOW_FLAG_TOCLIENT
#define DETECT_FLOW_FLAG_TOCLIENT
Definition: detect-flow.h:28
SignatureInitDataBuffer_::head
SigMatch * head
Definition: detect.h:547
detect-content.h
DETECT_APP_LAYER_EVENT
@ DETECT_APP_LAYER_EVENT
Definition: detect-engine-register.h:203
util-port-interval-tree.h
detect-engine.h
DetectEngineResetMaxSigId
void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
Definition: detect-engine.c:3211
DetectEngineBufferTypeSupportsPacketGetById
bool DetectEngineBufferTypeSupportsPacketGetById(const DetectEngineCtx *de_ctx, const int id)
Definition: detect-engine.c:1324
DETECT_SM_LIST_PMATCH
@ DETECT_SM_LIST_PMATCH
Definition: detect.h:118
SIG_MASK_REQUIRE_REAL_PKT
#define SIG_MASK_REQUIRE_REAL_PKT
Definition: detect.h:315
DE_HAS_FIREWALL
#define DE_HAS_FIREWALL
Definition: detect.h:330
MASK_TCP_UNUSUAL_FLAGS
#define MASK_TCP_UNUSUAL_FLAGS
Definition: detect-engine-build.c:414
DetectFlagsSignatureNeedsSynOnlyPackets
int DetectFlagsSignatureNeedsSynOnlyPackets(const Signature *s)
Definition: detect-tcp-flags.c:536
SignatureInitData_::smlists
struct SigMatch_ * smlists[DETECT_SM_LIST_MAX]
Definition: detect.h:643
detect-engine-proto.h
detect-dsize.h
sigmatch_table
SigTableElmt * sigmatch_table
Definition: detect-parse.c:155
PKT_HAS_FLOW
#define PKT_HAS_FLOW
Definition: decode.h:1242
UniquePortPoint_
Definition: detect-engine-build.c:1327
SIG_TYPE_APP_TX
@ SIG_TYPE_APP_TX
Definition: detect.h:76
SigMatchFree
void SigMatchFree(DetectEngineCtx *de_ctx, SigMatch *sm)
free a SigMatch
Definition: detect-parse.c:364
SigTableElmt_::Free
void(* Free)(DetectEngineCtx *, void *)
Definition: detect.h:1392
DetectEngineCtx_::decoder_event_sgh
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:1005
DetectPortHashInit
int DetectPortHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the DetectPort hash.
Definition: detect-engine-port.c:1370
DetectEngineCtx_::flow_gh
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:947
FILE_SIG_NEED_SHA1
#define FILE_SIG_NEED_SHA1
Definition: detect.h:324
SigPrepareStage4
int SigPrepareStage4(DetectEngineCtx *de_ctx)
finalize preparing sgh's
Definition: detect-engine-build.c:1985
detect-engine-siggroup.h
DetectFlowData_
Definition: detect-flow.h:37
SigTableElmt_::name
const char * name
Definition: detect.h:1402
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1297
Signature_::num
SigIntId num
Definition: detect.h:681
DetectEngineBufferRunSetupCallback
void DetectEngineBufferRunSetupCallback(const DetectEngineCtx *de_ctx, const int id, Signature *s)
Definition: detect-engine.c:1361
SigGroupHead_
Container for matching data for a signature group.
Definition: detect.h:1570
DetectListToHumanString
const char * DetectListToHumanString(int list)
Definition: detect-parse.c:188
SIG_FLAG_INIT_FLOW
#define SIG_FLAG_INIT_FLOW
Definition: detect.h:291
SigFree
void SigFree(DetectEngineCtx *, Signature *)
Definition: detect-parse.c:2021
DumpPatterns
void DumpPatterns(DetectEngineCtx *de_ctx)
Definition: detect-engine-analyzer.c:1418
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
DetectFlagsData_::flags
uint8_t flags
Definition: detect-tcp-flags.h:38
SigGroupHeadInitData_::sig_array
uint8_t * sig_array
Definition: detect.h:1544
DetectPortFree
void DetectPortFree(const DetectEngineCtx *de_ctx, DetectPort *dp)
Free a DetectPort and its members.
Definition: detect-engine-port.c:80
DETECT_CONTENT
@ DETECT_CONTENT
Definition: detect-engine-register.h:69
DetectEngineCtx_::max_uniq_toclient_groups
uint16_t max_uniq_toclient_groups
Definition: detect.h:978
MpmStoreReportStats
void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
Definition: detect-engine-mpm.c:1473
SignatureInitData_::prefilter_sm
SigMatch * prefilter_sm
Definition: detect.h:626
SignatureInitData_::src_contains_negation
bool src_contains_negation
Definition: detect.h:602
DETECT_FLOW
@ DETECT_FLOW
Definition: detect-engine-register.h:54
Signature_::alproto
AppProto alproto
Definition: detect.h:674
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
DETECT_UINT_NE
#define DETECT_UINT_NE
Definition: detect-engine-uint.h:36
IPOnlyDeinit
void IPOnlyDeinit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Deinitialize the IP Only detection engine context.
Definition: detect-engine-iponly.c:952
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
DetectPort_::port
uint16_t port
Definition: detect.h:220
SCPortIntervalFindOverlappingRanges
void SCPortIntervalFindOverlappingRanges(DetectEngineCtx *de_ctx, const uint16_t port, const uint16_t port2, const struct PI *head, DetectPort **list)
Callee function to find all overlapping port ranges as asked by the detection engine during Stage 2 o...
Definition: util-port-interval-tree.c:315
PORT_SIGGROUPHEAD_COPY
#define PORT_SIGGROUPHEAD_COPY
Definition: detect.h:216
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:85
DETECT_DECODE_EVENT
@ DETECT_DECODE_EVENT
Definition: detect-engine-register.h:121
DETECT_SM_LIST_DYNAMIC_START
@ DETECT_SM_LIST_DYNAMIC_START
Definition: detect.h:137
Packet_::flags
uint32_t flags
Definition: decode.h:527
UNDEFINED_PORT
#define UNDEFINED_PORT
Definition: detect-engine-build.c:1323
PatternStrength
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
Predict a strength value for patterns.
Definition: detect-engine-mpm.c:930
DetectFlowbitsAnalyze
int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
Definition: detect-flowbits.c:564
DetectSetFastPatternAndItsId
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
Figure out the FP and their respective content ids for all the sigs in the engine.
Definition: detect-engine-mpm.c:2461
AppProtoToString
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
Definition: app-layer-protos.c:40
SigTableElmt_::flags
uint16_t flags
Definition: detect.h:1396
SCProfilingSghInitCounters
void SCProfilingSghInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
Definition: util-profiling-rulegroups.c:348
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:920
DETECT_PROTO_ANY
#define DETECT_PROTO_ANY
Definition: detect-engine-proto.h:28
SIG_TYPE_PKT_STREAM
@ SIG_TYPE_PKT_STREAM
Definition: detect.h:72
DetectFlagsSignatureNeedsSynPackets
int DetectFlagsSignatureNeedsSynPackets(const Signature *s)
Definition: detect-tcp-flags.c:517
TransformData_::options
void * options
Definition: detect.h:412
DetectFlowbitsData_::cmd
uint8_t cmd
Definition: detect-flowbits.h:37
HashListTableGetListHead
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
Definition: util-hashlist.c:287
SigGroupHeadSetProtoAndDirection
void SigGroupHeadSetProtoAndDirection(SigGroupHead *sgh, uint8_t ipproto, int dir)
Definition: detect-engine-siggroup.c:486
SIGMATCH_DEONLY_COMPAT
#define SIGMATCH_DEONLY_COMPAT
Definition: detect.h:1596
DETECT_UINT_EQ
#define DETECT_UINT_EQ
Definition: detect-engine-uint.h:35
DetectEngineBufferTypeGetNameById
const char * DetectEngineBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
Definition: detect-engine.c:1187
DetectMpmInitializeBuiltinMpms
void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
Definition: detect-engine-mpm.c:726
SIG_FLAG_DST_ANY
#define SIG_FLAG_DST_ANY
Definition: detect.h:241
util-var-name.h
SIG_FLAG_REQUIRE_STREAM
#define SIG_FLAG_REQUIRE_STREAM
Definition: detect.h:254
rule_engine_analysis_set
bool rule_engine_analysis_set
Definition: detect-engine-loader.c:56
SCConfGetBool
int SCConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:497
DE_QUIET
#define DE_QUIET
Definition: detect.h:329
DetectPort_::next
struct DetectPort_ * next
Definition: detect.h:233
DetectEngineCtx_::tcp_priorityports
DetectPort * tcp_priorityports
Definition: detect.h:1056
DetectEngineCtx_::dport_hash_table
HashListTable * dport_hash_table
Definition: detect.h:1054
Signature_::sm_arrays
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:732
SignatureInitData_::init_flags
uint32_t init_flags
Definition: detect.h:609
proto
uint8_t proto
Definition: decode-template.h:0
DetectPort_::sh
struct SigGroupHead_ * sh
Definition: detect.h:230
DetectEngineCtx_::udp_priorityports
DetectPort * udp_priorityports
Definition: detect.h:1057
DetectContentData_
Definition: detect-content.h:93
DETECT_FLOWBITS_CMD_ISSET
#define DETECT_FLOWBITS_CMD_ISSET
Definition: detect-flowbits.h:32
PKT_NOPAYLOAD_INSPECTION
#define PKT_NOPAYLOAD_INSPECTION
Definition: decode.h:1231
SigCleanSignatures
void SigCleanSignatures(DetectEngineCtx *de_ctx)
Definition: detect-engine-build.c:55
DetectPortPrintList
void DetectPortPrintList(DetectPort *head)
Helper function used to print the list of ports present in this DetectPort list.
Definition: detect-engine-port.c:100
DETECT_UINT_GT
#define DETECT_UINT_GT
Definition: detect-engine-uint.h:32
SigPrepareStage1
int SigPrepareStage1(DetectEngineCtx *de_ctx)
Preprocess signature, classify ip-only, etc, build sig array.
Definition: detect-engine-build.c:1732
DETECT_PGSCORE_RULE_SYN_ONLY
#define DETECT_PGSCORE_RULE_SYN_ONLY
Definition: detect-engine-build.c:53
SIG_FLAG_TOCLIENT
#define SIG_FLAG_TOCLIENT
Definition: detect.h:271
MAX
#define MAX(x, y)
Definition: suricata-common.h:404
SIG_FLAG_SRC_ANY
#define SIG_FLAG_SRC_ANY
Definition: detect.h:240
SigGroupHeadSetupFiles
void SigGroupHeadSetupFiles(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need hash flag in the sgh.
Definition: detect-engine-siggroup.c:573
SIG_TYPE_APPLAYER
@ SIG_TYPE_APPLAYER
Definition: detect.h:75
SignatureIsFilesizeInspecting
int SignatureIsFilesizeInspecting(const Signature *s)
Check if a signature contains the filesize keyword.
Definition: detect-engine-build.c:186
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:589
DetectPort_::port2
uint16_t port2
Definition: detect.h:221
detect-engine-prefilter.h
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:615
Packet_::events
PacketEngineEvents events
Definition: decode.h:613
FLOW_STATES
#define FLOW_STATES
Definition: detect.h:912
EngineAnalysisAddAllRulePatterns
void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature *s)
add all patterns on our stats hash Used to fill the hash later used by DumpPatterns()
Definition: detect-engine-mpm.c:2518
DetectPort_::flags
uint8_t flags
Definition: detect.h:223
DetectPortPrint
void DetectPortPrint(DetectPort *dp)
Helper function that print the DetectPort info.
Definition: detect-engine-port.c:590
FirewallAnalyzer
int FirewallAnalyzer(const DetectEngineCtx *de_ctx)
Definition: detect-engine-analyzer.c:2001
PacketCreateMask
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
Definition: detect-engine-build.c:419
SIG_FLAG_APPLAYER
#define SIG_FLAG_APPLAYER
Definition: detect.h:248
SigGroupHeadHashInit
int SigGroupHeadHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the SigGroupHeads.
Definition: detect-engine-siggroup.c:244
FILE_SIG_NEED_MD5
#define FILE_SIG_NEED_MD5
Definition: detect.h:323
DetectEngineCtx_::sgh_array_size
uint32_t sgh_array_size
Definition: detect.h:990
SIGNATURE_HOOK_TYPE_APP
@ SIGNATURE_HOOK_TYPE_APP
Definition: detect.h:560
SignatureIsFileSha256Inspecting
int SignatureIsFileSha256Inspecting(const Signature *s)
Check if a signature contains the filesha256 keyword.
Definition: detect-engine-build.c:170
Signature_::next
struct Signature_ * next
Definition: detect.h:751
DetectFlowbitsData_
Definition: detect-flowbits.h:35
DetectEngineMultiTenantEnabled
int DetectEngineMultiTenantEnabled(void)
Definition: detect-engine.c:4019
HashListTableGetListNext
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:55
DETECT_APP_LAYER_PROTOCOL
@ DETECT_APP_LAYER_PROTOCOL
Definition: detect-engine-register.h:35
DETECT_SM_LIST_POSTMATCH
@ DETECT_SM_LIST_POSTMATCH
Definition: detect.h:126
SigGroupHeadInitData_::score
int score
Definition: detect.h:1549
DetectPortCopySingle
DetectPort * DetectPortCopySingle(DetectEngineCtx *de_ctx, DetectPort *src)
Function that return a copy of DetectPort src sigs.
Definition: detect-engine-port.c:550
SIG_FLAG_TOSERVER
#define SIG_FLAG_TOSERVER
Definition: detect.h:270
DetectPortParse
int DetectPortParse(const DetectEngineCtx *de_ctx, DetectPort **head, const char *str)
Function for parsing port strings.
Definition: detect-engine-port.c:1182
SigAddressCleanupStage1
int SigAddressCleanupStage1(DetectEngineCtx *de_ctx)
Definition: detect-engine-build.c:1911
SCPortIntervalInsert
int SCPortIntervalInsert(DetectEngineCtx *de_ctx, SCPortIntervalTree *it, const DetectPort *p)
Function to insert a node in the interval tree.
Definition: util-port-interval-tree.c:109
SIG_TYPE_PKT
@ SIG_TYPE_PKT
Definition: detect.h:71
TCPHdr_::th_flags
uint8_t th_flags
Definition: decode-tcp.h:155
SigGroupHeadBuildMatchArray
int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t max_idx)
Create an array with all the internal ids of the sigs that this sig group head will check for.
Definition: detect-engine-siggroup.c:535
SigGroupHeadInitData_::sig_cnt
SigIntId sig_cnt
Definition: detect.h:1563
DETECT_CONTENT_ENDS_WITH
#define DETECT_CONTENT_ENDS_WITH
Definition: detect-content.h:42
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:18
g_alproto_max
AppProto g_alproto_max
Definition: app-layer-protos.c:29
DETECT_FLOWINT
@ DETECT_FLOWINT
Definition: detect-engine-register.h:62
IPOnlyInit
void IPOnlyInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Setup the IP Only detection engine context.
Definition: detect-engine-iponly.c:913
SIG_MASK_REQUIRE_ENGINE_EVENT
#define SIG_MASK_REQUIRE_ENGINE_EVENT
Definition: detect.h:317
SIG_TYPE_IPONLY
@ SIG_TYPE_IPONLY
Definition: detect.h:65
SignatureInitData_::mpm_sm
SigMatch * mpm_sm
Definition: detect.h:624
SigFindSignatureBySidGid
Signature * SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
Find a specific signature by sid and gid.
Definition: detect-engine-build.c:79
DetectConfigData_::scope
enum ConfigScope scope
Definition: detect-config.h:32
DetectEngineGetMaxSigId
#define DetectEngineGetMaxSigId(de_ctx)
Definition: detect-engine.h:110
DETECT_FLOW_FLAG_TOSERVER
#define DETECT_FLOW_FLAG_TOSERVER
Definition: detect-flow.h:27
FILE_SIG_NEED_MAGIC
#define FILE_SIG_NEED_MAGIC
Definition: detect.h:321
SignatureInitData_::mpm_sm_list
int mpm_sm_list
Definition: detect.h:622
SIG_MASK_REQUIRE_FLOW
#define SIG_MASK_REQUIRE_FLOW
Definition: detect.h:311
SCConfGetInt
int SCConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:414
SignatureInitData_::cidr_dst
IPOnlyCIDRItem * cidr_dst
Definition: detect.h:619
DETECT_CONTENT_DEPTH
#define DETECT_CONTENT_DEPTH
Definition: detect-content.h:33
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
detect-engine-mpm.h
DetectEngineLookupFlow_::sgh
struct SigGroupHead_ * sgh[256]
Definition: detect.h:864
detect.h
SIG_MASK_REQUIRE_FLAGS_INITDEINIT
#define SIG_MASK_REQUIRE_FLAGS_INITDEINIT
Definition: detect.h:312
SCProfilingPrefilterInitCounters
void SCProfilingPrefilterInitCounters(DetectEngineCtx *de_ctx)
Register the prefilter profiling counters.
Definition: util-profiling-prefilter.c:300
SigMatchList2DataArray
SigMatchData * SigMatchList2DataArray(SigMatch *head)
convert SigMatch list to SigMatchData array
Definition: detect-parse.c:2354
PKT_DETECT_HAS_STREAMDATA
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1281
SigMatch_::next
struct SigMatch_ * next
Definition: detect.h:360
DETECT_CONTENT_NEGATED
#define DETECT_CONTENT_NEGATED
Definition: detect-content.h:40
detect-engine-port.h
DETECT_ENGINE_EVENT
@ DETECT_ENGINE_EVENT
Definition: detect-engine-register.h:225
DETECT_SM_LIST_MATCH
@ DETECT_SM_LIST_MATCH
Definition: detect.h:116
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
DetectContentPropagateLimits
void DetectContentPropagateLimits(Signature *s)
Definition: detect-content.c:717
DetectPort_
Port structure for detection engine.
Definition: detect.h:219
SigGroupHead_::init
SigGroupHeadInitData * init
Definition: detect.h:1587
DetectEngineCtx_::sig_cnt
uint32_t sig_cnt
Definition: detect.h:930
DETECT_PGSCORE_RULE_NO_MPM
#define DETECT_PGSCORE_RULE_NO_MPM
Definition: detect-engine-build.c:52
UniquePortPoint_::single
bool single
Definition: detect-engine-build.c:1329
SignatureInitData_::cidr_src
IPOnlyCIDRItem * cidr_src
Definition: detect.h:619
SigMatch_::ctx
SigMatchCtx * ctx
Definition: detect.h:359
SignatureInitData_::hook
SignatureHook hook
Definition: detect.h:591
SIGNATURE_HOOK_TYPE_NOT_SET
@ SIGNATURE_HOOK_TYPE_NOT_SET
Definition: detect.h:558
DetectProto_::proto
uint8_t proto[256/8]
Definition: detect-engine-proto.h:36
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:309
AppLayerDecoderEvents_::cnt
uint8_t cnt
Definition: app-layer-events.h:40
IPOnlyPrepare
void IPOnlyPrepare(DetectEngineCtx *de_ctx)
Build the radix trees from the lists of parsed addresses in CIDR format the result should be 4 radix ...
Definition: detect-engine-iponly.c:1138
SigGroupCleanup
int SigGroupCleanup(DetectEngineCtx *de_ctx)
Definition: detect-engine-build.c:2200
DETECT_PGSCORE_RULE_MPM_FAST_PATTERN
#define DETECT_PGSCORE_RULE_MPM_FAST_PATTERN
Definition: detect-engine-build.c:50
SIG_FLAG_REQUIRE_FLOWVAR
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:266
util-profiling.h
FILE_SIG_NEED_SHA256
#define FILE_SIG_NEED_SHA256
Definition: detect.h:325
DetectEngineLookupFlow_::udp
DetectPort * udp
Definition: detect.h:863
SCReturn
#define SCReturn
Definition: util-debug.h:273
Signature_::flags
uint32_t flags
Definition: detect.h:670
Packet_
Definition: decode.h:484
detect-engine-build.h
type
uint16_t type
Definition: decode-vlan.c:106
DETECT_UINT_GTE
#define DETECT_UINT_GTE
Definition: detect-engine-uint.h:33
SCConfigGetLogDirectory
const char * SCConfigGetLogDirectory(void)
Definition: util-conf.c:38
IPOnlyPrint
void IPOnlyPrint(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Print stats of the IP Only engine.
Definition: detect-engine-iponly.c:941
DetectConfigData_
Definition: detect-config.h:29
DetectContentData_::flags
uint32_t flags
Definition: detect-content.h:104
DetectEngineCtx_::max_uniq_toserver_groups
uint16_t max_uniq_toserver_groups
Definition: detect.h:979
IPOnlyAddSignature
void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx, Signature *s)
Add a signature to the lists of Addresses in CIDR format (sorted) this step is necessary to build the...
Definition: detect-engine-iponly.c:1410
SC_OK
@ SC_OK
Definition: util-error.h:27
DETECT_CONFIG
@ DETECT_CONFIG
Definition: detect-engine-register.h:228
name
const char * name
Definition: tm-threads.c:2135
Signature_::init_data
SignatureInitData * init_data
Definition: detect.h:748
SignatureInitData_::rule_state_dependant_sids_array
uint32_t * rule_state_dependant_sids_array
Definition: detect.h:658
DetectEngineCtx_::sgh_array_cnt
uint32_t sgh_array_cnt
Definition: detect.h:989
SigGroupHeadAppendSig
int SigGroupHeadAppendSig(const DetectEngineCtx *de_ctx, SigGroupHead **sgh, const Signature *s)
Add a Signature to a SigGroupHead.
Definition: detect-engine-siggroup.c:319
DetectEngineCtx_::sgh_array
struct SigGroupHead_ ** sgh_array
Definition: detect.h:988
UniquePortPoint_::port
uint16_t port
Definition: detect-engine-build.c:1328
SignatureInitData_::dst_contains_negation
bool dst_contains_negation
Definition: detect.h:603
SigGroupHeadFree
void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Free a SigGroupHead and its members.
Definition: detect-engine-siggroup.c:163
DetectEngineTransforms::transforms
TransformData transforms[DETECT_TRANSFORMS_MAX]
Definition: detect.h:416
SIG_TYPE_DEONLY
@ SIG_TYPE_DEONLY
Definition: detect.h:70
detect-flowbits.h
SIG_MASK_REQUIRE_PAYLOAD
#define SIG_MASK_REQUIRE_PAYLOAD
Definition: detect.h:310
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
DETECT_DSIZE
@ DETECT_DSIZE
Definition: detect-engine-register.h:52
Signature_::sp
DetectPort * sp
Definition: detect.h:720
SIG_TYPE_NOT_SET
@ SIG_TYPE_NOT_SET
Definition: detect.h:64
SignatureIsFileSha1Inspecting
int SignatureIsFileSha1Inspecting(const Signature *s)
Check if a signature contains the filesha1 keyword.
Definition: detect-engine-build.c:154
SCPortIntervalTreeInit
SCPortIntervalTree * SCPortIntervalTreeInit(void)
Function to initialize the interval tree.
Definition: util-port-interval-tree.c:58
IPOnlyCIDRListFree
void IPOnlyCIDRListFree(IPOnlyCIDRItem *tmphead)
This function free a IPOnlyCIDRItem list.
Definition: detect-engine-iponly.c:482
SigGroupBuild
int SigGroupBuild(DetectEngineCtx *de_ctx)
Convert the signature list into the runtime match structure.
Definition: detect-engine-build.c:2129
DetectEngineCtx_::config_prefix
char config_prefix[64]
Definition: detect.h:1039
SIG_FLAG_MPM_NEG
#define SIG_FLAG_MPM_NEG
Definition: detect.h:256
SCPortIntervalTree_::tree
struct PI tree
Definition: util-port-interval-tree.h:30
detect-engine-analyzer.h
SIG_FLAG_INIT_STATE_MATCH
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:295
DetectEngineBufferTypeSupportsFramesGetById
bool DetectEngineBufferTypeSupportsFramesGetById(const DetectEngineCtx *de_ctx, const int id)
Definition: detect-engine.c:1342
EngineAnalysisRules2
void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
Definition: detect-engine-analyzer.c:971
SigGroupHeadSetSigCnt
void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx)
Updates the SigGroupHead->sig_cnt with the total count of all the Signatures present in this SigGroup...
Definition: detect-engine-siggroup.c:446
DetectProto_::flags
uint8_t flags
Definition: detect-engine-proto.h:37
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
util-conf.h
DetectMpmPrepareFrameMpms
int DetectMpmPrepareFrameMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
Definition: detect-engine-mpm.c:513
SINGLE_PORT
#define SINGLE_PORT
Definition: detect-engine-build.c:1325
DetectEnginePktInspectionSetup
int DetectEnginePktInspectionSetup(Signature *s)
Definition: detect-engine.c:2115
tail
Host * tail
Definition: host.h:2
Signature_::proto
DetectProto proto
Definition: detect.h:688
suricata-common.h
SIG_MASK_REQUIRE_NO_PAYLOAD
#define SIG_MASK_REQUIRE_NO_PAYLOAD
Definition: detect.h:314
SIG_FLAG_SP_ANY
#define SIG_FLAG_SP_ANY
Definition: detect.h:242
SigGroupHeadStore
void SigGroupHeadStore(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Definition: detect-engine-siggroup.c:109
SigMatch_::type
uint16_t type
Definition: detect.h:357
SigGroupHeadInitData_::match_array
Signature ** match_array
Definition: detect.h:1566
Signature_::file_flags
uint8_t file_flags
Definition: detect.h:685
SigPrepareStage3
int SigPrepareStage3(DetectEngineCtx *de_ctx)
Definition: detect-engine-build.c:1904
SignatureHook_::type
enum SignatureHookType type
Definition: detect.h:573
DetectPort_::prev
struct DetectPort_ * prev
Definition: detect.h:232
SCLogPerf
#define SCLogPerf(...)
Definition: util-debug.h:230
DetectContentData_::content
uint8_t * content
Definition: detect-content.h:94
MASK_TCP_INITDEINIT_FLAGS
#define MASK_TCP_INITDEINIT_FLAGS
Definition: detect-engine-build.c:413
DetectListToString
const char * DetectListToString(int list)
Definition: detect-parse.c:206
DetectEngineCtx_::profile_match_logging_threshold
uint32_t profile_match_logging_threshold
Definition: detect.h:1037
FatalError
#define FatalError(...)
Definition: util-debug.h:502
DetectEngineCtx_::sig_list
Signature * sig_list
Definition: detect.h:929
DetectMpmPreparePktMpms
int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
Definition: detect-engine-mpm.c:680
SigGroupHeadHashFree
void SigGroupHeadHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by SigGroupHeadHashInit() function.
Definition: detect-engine-siggroup.c:299
SIG_MASK_REQUIRE_FLAGS_UNUSUAL
#define SIG_MASK_REQUIRE_FLAGS_UNUSUAL
Definition: detect.h:313
TransformData_::transform
int transform
Definition: detect.h:411
DETECT_FLOWBITS
@ DETECT_FLOWBITS
Definition: detect-engine-register.h:59
app-layer-events.h
SigGroupHeadHashAdd
int SigGroupHeadHashAdd(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Adds a SigGroupHead to the detection engine context SigGroupHead hash table.
Definition: detect-engine-siggroup.c:266
util-validate.h
detect-flow.h
SCPortIntervalTree_
Definition: util-port-interval-tree.h:29
SigPrepareStage2
int SigPrepareStage2(DetectEngineCtx *de_ctx)
Fill the global src group head, with the sigs included.
Definition: detect-engine-build.c:1864
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
SignatureInitData_::buffers
SignatureInitDataBuffer * buffers
Definition: detect.h:648
HtpBodyChunk_::next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:123
SigGroupHeadHashLookup
SigGroupHead * SigGroupHeadHashLookup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Used to lookup a SigGroupHead hash from the detection engine context SigGroupHead hash table.
Definition: detect-engine-siggroup.c:283
Signature_::dp
DetectPort * dp
Definition: detect.h:720
DETECT_STREAM_EVENT
@ DETECT_STREAM_EVENT
Definition: detect-engine-register.h:226
DETECT_PGSCORE_RULE_PORT_PRIORITIZED
#define DETECT_PGSCORE_RULE_PORT_PRIORITIZED
Definition: detect-engine-build.c:49
SignatureIsFilestoring
int SignatureIsFilestoring(const Signature *s)
Check if a signature contains the filestore keyword.
Definition: detect-engine-build.c:100
SCFree
#define SCFree(p)
Definition: util-mem.h:61
SignatureIsIPOnly
int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
Test is a initialized signature is IP only.
Definition: detect-engine-build.c:209
DetectMpmPrepareBuiltinMpms
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for builtin buffers that are in "single or "shared" mode.
Definition: detect-engine-mpm.c:739
Signature_::id
uint32_t id
Definition: detect.h:714
HashListTableBucket_
Definition: util-hashlist.h:28
detect-engine-iponly.h
SignatureInitData_::score
int score
Definition: detect.h:637
detect-parse.h
SignatureInitDataBuffer_::id
uint32_t id
Definition: detect.h:538
Signature_
Signature container.
Definition: detect.h:669
SigMatch_
a single match condition for a signature
Definition: detect.h:356
DETECT_SM_LIST_MAX
@ DETECT_SM_LIST_MAX
Definition: detect.h:134
SigGroupHeadInitDataFree
void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid)
Definition: detect-engine-siggroup.c:60
SignatureIsFileMd5Inspecting
int SignatureIsFileMd5Inspecting(const Signature *s)
Check if a signature contains the filemd5 keyword.
Definition: detect-engine-build.c:138
DETECT_PGSCORE_RULE_MPM_NEGATED
#define DETECT_PGSCORE_RULE_MPM_NEGATED
Definition: detect-engine-build.c:51
DetectFlagsData_
Definition: detect-tcp-flags.h:37
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
SignatureSetType
void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
Definition: detect-engine-build.c:1654
RANGE_PORT
#define RANGE_PORT
Definition: detect-engine-build.c:1324
PrefilterSetupRuleGroup
int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Definition: detect-engine-prefilter.c:1176
SCProfilingKeywordInitCounters
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
Definition: util-profiling-keywords.c:365
DetectEngineTransforms::cnt
int cnt
Definition: detect.h:417
DetectPortHashAdd
int DetectPortHashAdd(DetectEngineCtx *de_ctx, DetectPort *dp)
Adds a DetectPort to the detection engine context DetectPort hash table.
Definition: detect-engine-port.c:1390
DetectEngineCtx_::sig_array
Signature ** sig_array
Definition: detect.h:938
CONFIG_SCOPE_FLOW
@ CONFIG_SCOPE_FLOW
Definition: util-config.h:48
DetectContentData_::content_len
uint16_t content_len
Definition: detect-content.h:95
DetectEngineCtx_::buffer_type_id
uint32_t buffer_type_id
Definition: detect.h:1069
SigParseApplyDsizeToContent
void SigParseApplyDsizeToContent(Signature *s)
Apply dsize as depth to content matches in the rule.
Definition: detect-dsize.c:325
SignatureInitData_::rule_state_flowbits_ids_array
uint32_t * rule_state_flowbits_ids_array
Definition: detect.h:661
DetectEngineCtx_::flags
uint8_t flags
Definition: detect.h:922
UniquePortPoint
struct UniquePortPoint_ UniquePortPoint
SignatureInitData_::transforms
DetectEngineTransforms transforms
Definition: detect.h:632
DetectEngineCtx_::io_ctx
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:958
DetectU16Data
DetectUintData_u16 DetectU16Data
Definition: detect-engine-uint.h:42
DetectPortHashFree
void DetectPortHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by DetectPortInit() function.
Definition: detect-engine-port.c:1421
DetectPortHashLookup
DetectPort * DetectPortHashLookup(DetectEngineCtx *de_ctx, DetectPort *dp)
Used to lookup a DetectPort hash from the detection engine context DetectPort hash table.
Definition: detect-engine-port.c:1406
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
DetectEngineCtx_::sig_array_len
uint32_t sig_array_len
Definition: detect.h:939
detect-config.h
DETECT_FLAGS
@ DETECT_FLAGS
Definition: detect-engine-register.h:42
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
Signature_::type
enum SignatureType type
Definition: detect.h:672
SignatureIsFilemagicInspecting
int SignatureIsFilemagicInspecting(const Signature *s)
Check if a signature contains the filemagic keyword.
Definition: detect-engine-build.c:119
DetectEngineCtx_::signum
uint32_t signum
Definition: detect.h:941
DetectEngineCtx_::tenant_id
uint32_t tenant_id
Definition: detect.h:927
SigGroupHead_::filestore_cnt
uint16_t filestore_cnt
Definition: detect.h:1576
SignatureInitData_::buffer_index
uint32_t buffer_index
Definition: detect.h:649
DetectMpmPrepareAppMpms
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
Definition: detect-engine-mpm.c:286
SIGMATCH_IPONLY_COMPAT
#define SIGMATCH_IPONLY_COMPAT
Definition: detect.h:1594
DetectEngineLookupFlow_::tcp
DetectPort * tcp
Definition: detect.h:862
DetectPortCleanupList
void DetectPortCleanupList(const DetectEngineCtx *de_ctx, DetectPort *head)
Free a DetectPort list and each of its members.
Definition: detect-engine-port.c:124
SIG_TYPE_PDONLY
@ SIG_TYPE_PDONLY
Definition: detect.h:69
SCLogDebugEnabled
int SCLogDebugEnabled(void)
Returns whether debug messages are enabled to be logged or not.
Definition: util-debug.c:767
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
DetectEngineAppInspectionEngine2Signature
int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s)
Definition: detect-engine.c:749
SCPortIntervalTreeFree
void SCPortIntervalTreeFree(DetectEngineCtx *de_ctx, SCPortIntervalTree *it)
Function to free an entire interval tree.
Definition: util-port-interval-tree.c:92
TCPHdr_
Definition: decode-tcp.h:149
detect-engine-address.h
SigGroupHead_::id
uint32_t id
Definition: detect.h:1578
VarNameStoreActivate
int VarNameStoreActivate(void)
Definition: util-var-name.c:214
SIG_FLAG_FILESTORE
#define SIG_FLAG_FILESTORE
Definition: detect.h:268
DETECT_UINT_RA
#define DETECT_UINT_RA
Definition: detect-engine-uint.h:34
FILE_SIG_NEED_SIZE
#define FILE_SIG_NEED_SIZE
Definition: detect.h:326
SIG_FLAG_DP_ANY
#define SIG_FLAG_DP_ANY
Definition: detect.h:243
DETECT_FLOWBITS_CMD_SET
#define DETECT_FLOWBITS_CMD_SET
Definition: detect-flowbits.h:28
detect-engine-threshold.h
SignatureMask
#define SignatureMask
Definition: decode.h:99
SigGroupHeadCopySigs
int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHead **dst)
Copies the bitarray holding the sids from the source SigGroupHead to the destination SigGroupHead.
Definition: detect-engine-siggroup.c:390
Signature_::mask
SignatureMask mask
Definition: detect.h:680
SIG_TYPE_LIKE_IPONLY
@ SIG_TYPE_LIKE_IPONLY
Definition: detect.h:66
PacketEngineEvents_::cnt
uint8_t cnt
Definition: decode.h:292
SIG_FLAG_REQUIRE_PACKET
#define SIG_FLAG_REQUIRE_PACKET
Definition: detect.h:253