suricata
detect-engine-build.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2018 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "detect.h"
20 #include "detect-engine.h"
21 #include "detect-parse.h"
22 
23 #include "detect-engine-address.h"
24 #include "detect-engine-analyzer.h"
25 #include "detect-engine-iponly.h"
26 #include "detect-engine-mpm.h"
27 #include "detect-engine-siggroup.h"
28 #include "detect-engine-port.h"
30 #include "detect-engine-proto.h"
31 
32 #include "detect-dsize.h"
33 #include "detect-flags.h"
34 #include "detect-flow.h"
35 #include "detect-flowbits.h"
36 
37 #include "util-profiling.h"
38 
40 {
41  if (de_ctx == NULL)
42  return;
43 
44  for (Signature *s = de_ctx->sig_list; s != NULL;) {
45  Signature *ns = s->next;
46  SigFree(s);
47  s = ns;
48  }
49  de_ctx->sig_list = NULL;
50 
52  de_ctx->sig_list = NULL;
53 }
54 
55 /** \brief Find a specific signature by sid and gid
56  * \param de_ctx detection engine ctx
57  * \param sid the signature id
58  * \param gid the signature group id
59  *
60  * \retval s sig found
61  * \retval NULL sig not found
62  */
63 Signature *SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
64 {
65  if (de_ctx == NULL)
66  return NULL;
67 
68  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
69  if (s->id == sid && s->gid == gid)
70  return s;
71  }
72 
73  return NULL;
74 }
75 
76 /**
77  * \brief Check if a signature contains the filestore keyword.
78  *
79  * \param s signature
80  *
81  * \retval 0 no
82  * \retval 1 yes
83  */
85 {
86  if (s == NULL)
87  return 0;
88 
89  if (s->flags & SIG_FLAG_FILESTORE)
90  return 1;
91 
92  return 0;
93 }
94 
95 /**
96  * \brief Check if a signature contains the filemagic keyword.
97  *
98  * \param s signature
99  *
100  * \retval 0 no
101  * \retval 1 yes
102  */
104 {
105  if (s == NULL)
106  return 0;
107 
109  return 1;
110 
111  return 0;
112 }
113 
114 /**
115  * \brief Check if a signature contains the filemd5 keyword.
116  *
117  * \param s signature
118  *
119  * \retval 0 no
120  * \retval 1 yes
121  */
123 {
124  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5))
125  return 1;
126 
127  return 0;
128 }
129 
130 /**
131  * \brief Check if a signature contains the filesha1 keyword.
132  *
133  * \param s signature
134  *
135  * \retval 0 no
136  * \retval 1 yes
137  */
139 {
140  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1))
141  return 1;
142 
143  return 0;
144 }
145 
146 /**
147  * \brief Check if a signature contains the filesha256 keyword.
148  *
149  * \param s signature
150  *
151  * \retval 0 no
152  * \retval 1 yes
153  */
155 {
156  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256))
157  return 1;
158 
159  return 0;
160 }
161 
162 /**
163  * \brief Check if a signature contains the filesize keyword.
164  *
165  * \param s signature
166  *
167  * \retval 0 no
168  * \retval 1 yes
169  */
171 {
172  if (s == NULL)
173  return 0;
174 
176  return 1;
177 
178  return 0;
179 }
180 
181 /** \brief Test is a initialized signature is IP only
182  * \param de_ctx detection engine ctx
183  * \param s the signature
184  * \retval 1 sig is ip only
185  * \retval 0 sig is not ip only
186  */
188 {
189  if (s->alproto != ALPROTO_UNKNOWN)
190  return 0;
191 
192  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
193  return 0;
194 
195  /* for now assume that all registered buffer types are incompatible */
196  const int nlists = s->init_data->smlists_array_size;
197  for (int i = 0; i < nlists; i++) {
198  if (s->init_data->smlists[i] == NULL)
199  continue;
200  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
201  continue;
202 
203  SCReturnInt(0);
204  }
205 
206  /* TMATCH list can be ignored, it contains TAGs and
207  * tags are compatible to IP-only. */
208 
209  IPOnlyCIDRItem *cidr_item;
210  cidr_item = s->CidrSrc;
211  while (cidr_item != NULL) {
212  if (cidr_item->negated)
213  return 0;
214 
215  cidr_item = cidr_item->next;
216  }
217  cidr_item = s->CidrDst;
218  while (cidr_item != NULL) {
219  if (cidr_item->negated)
220  return 0;
221 
222  cidr_item = cidr_item->next;
223  }
224 
226  if (sm == NULL)
227  goto iponly;
228 
229  for ( ; sm != NULL; sm = sm->next) {
231  return 0;
232  /* we have enabled flowbits to be compatible with ip only sigs, as long
233  * as the sig only has a "set" flowbits */
234  if (sm->type == DETECT_FLOWBITS &&
235  (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
236  return 0;
237  }
238  }
239 
240 iponly:
241  if (!(de_ctx->flags & DE_QUIET)) {
242  SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
243  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
244  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
245  }
246  return 1;
247 }
248 
249 /** \internal
250  * \brief Test is a initialized signature is inspecting protocol detection only
251  * \param de_ctx detection engine ctx
252  * \param s the signature
253  * \retval 1 sig is dp only
254  * \retval 0 sig is not dp only
255  */
256 static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s)
257 {
258  if (s->alproto != ALPROTO_UNKNOWN)
259  return 0;
260 
261  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
262  return 0;
263 
264  /* for now assume that all registered buffer types are incompatible */
265  const int nlists = s->init_data->smlists_array_size;
266  for (int i = 0; i < nlists; i++) {
267  if (s->init_data->smlists[i] == NULL)
268  continue;
269  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
270  continue;
271 
272  SCReturnInt(0);
273  }
274 
275  /* TMATCH list can be ignored, it contains TAGs and
276  * tags are compatible to DP-only. */
277 
278  /* match list matches may be compatible to DP only. We follow the same
279  * logic as IP-only so we can use that flag */
280 
282  if (sm == NULL)
283  return 0;
284 
285  int pd = 0;
286  for ( ; sm != NULL; sm = sm->next) {
287  if (sm->type == DETECT_AL_APP_LAYER_PROTOCOL) {
288  pd = 1;
289  } else {
290  /* flowbits are supported for dp only sigs, as long
291  * as the sig only has a "set" flowbits */
292  if (sm->type == DETECT_FLOWBITS) {
293  if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
294  SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id);
295  return 0;
296  }
297  } else if (sm->type == DETECT_FLOW) {
299  SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id);
300  return 0;
301  }
302  } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) {
303  SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name);
304  return 0;
305  }
306  }
307  }
308 
309  if (pd) {
310  SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id);
311  }
312  return pd;
313 }
314 
315 /**
316  * \internal
317  * \brief Check if the initialized signature is inspecting the packet payload
318  * \param de_ctx detection engine ctx
319  * \param s the signature
320  * \retval 1 sig is inspecting the payload
321  * \retval 0 sig is not inspecting the payload
322  */
323 static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s)
324 {
325 
326  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
327  return 1;
328  }
329  return 0;
330 }
331 
332 /**
333  * \internal
334  * \brief check if a signature is decoder event matching only
335  * \param de_ctx detection engine
336  * \param s the signature to test
337  * \retval 0 not a DEOnly sig
338  * \retval 1 DEOnly sig
339  */
340 static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
341 {
342  if (s->alproto != ALPROTO_UNKNOWN) {
343  SCReturnInt(0);
344  }
345 
346  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
347  {
348  SCReturnInt(0);
349  }
350 
351  /* for now assume that all registered buffer types are incompatible */
352  const int nlists = s->init_data->smlists_array_size;
353  for (int i = 0; i < nlists; i++) {
354  if (s->init_data->smlists[i] == NULL)
355  continue;
356  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
357  continue;
358 
359  SCReturnInt(0);
360  }
361 
362  /* check for conflicting keywords */
364  for ( ;sm != NULL; sm = sm->next) {
366  SCReturnInt(0);
367  }
368 
369  /* need at least one decode event keyword to be considered decode event. */
371  for ( ;sm != NULL; sm = sm->next) {
372  if (sm->type == DETECT_DECODE_EVENT)
373  goto deonly;
374  if (sm->type == DETECT_ENGINE_EVENT)
375  goto deonly;
376  if (sm->type == DETECT_STREAM_EVENT)
377  goto deonly;
378  }
379 
380  SCReturnInt(0);
381 
382 deonly:
383  if (!(de_ctx->flags & DE_QUIET)) {
384  SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
385  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
386  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
387  }
388 
389  SCReturnInt(1);
390 }
391 
392 #define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN)
393 #define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR)
394 
395 /* Create mask for this packet + it's flow if it has one
396  */
397 void
399  bool app_decoder_events)
400 {
401  if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
402  SCLogDebug("packet has payload");
403  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
404  } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
405  SCLogDebug("stream data available");
406  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
407  } else {
408  SCLogDebug("packet has no payload");
409  (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
410  }
411 
412  if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) {
413  SCLogDebug("packet/flow has events set");
415  }
416 
417  if (!(PKT_IS_PSEUDOPKT(p)) && PKT_IS_TCP(p)) {
418  if ((p->tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) {
420  }
421  if ((p->tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) {
423  }
424  }
425 
426  if (p->flags & PKT_HAS_FLOW) {
427  SCLogDebug("packet has flow");
428  (*mask) |= SIG_MASK_REQUIRE_FLOW;
429  }
430 
431  if (alproto == ALPROTO_SMB || alproto == ALPROTO_DCERPC) {
432  SCLogDebug("packet will be inspected for DCERPC");
433  (*mask) |= SIG_MASK_REQUIRE_DCERPC;
434  }
435 }
436 
437 static int g_dce_generic_list_id = -1;
438 static int g_dce_stub_data_buffer_id = -1;
439 
440 static bool SignatureNeedsDCERPCMask(const Signature *s)
441 {
442  if (g_dce_generic_list_id == -1) {
443  g_dce_generic_list_id = DetectBufferTypeGetByName("dce_generic");
444  SCLogDebug("g_dce_generic_list_id %d", g_dce_generic_list_id);
445  }
446  if (g_dce_stub_data_buffer_id == -1) {
447  g_dce_stub_data_buffer_id = DetectBufferTypeGetByName("dce_stub_data");
448  SCLogDebug("g_dce_stub_data_buffer_id %d", g_dce_stub_data_buffer_id);
449  }
450 
451  if (g_dce_generic_list_id >= 0 &&
452  s->init_data->smlists[g_dce_generic_list_id] != NULL)
453  {
454  return true;
455  }
456  if (g_dce_stub_data_buffer_id >= 0 &&
457  s->init_data->smlists[g_dce_stub_data_buffer_id] != NULL)
458  {
459  return true;
460  }
461  return false;
462 }
463 
464 static int SignatureCreateMask(Signature *s)
465 {
466  SCEnter();
467 
468  if (SignatureNeedsDCERPCMask(s)) {
470  SCLogDebug("sig requires DCERPC");
471  }
472 
473  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
475  SCLogDebug("sig requires payload");
476  }
477 
478  SigMatch *sm;
479  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
480  switch(sm->type) {
481  case DETECT_FLOWBITS:
482  {
483  /* figure out what flowbit action */
485  if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
486  /* not a mask flag, but still set it here */
488 
489  SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has "
490  "flowbit isset option.");
491  }
492 
493  /* flow is required for any flowbit manipulation */
495  SCLogDebug("sig requires flow to be able to manipulate "
496  "flowbit(s)");
497  break;
498  }
499  case DETECT_FLOWINT:
500  /* flow is required for any flowint manipulation */
502  SCLogDebug("sig requires flow to be able to manipulate "
503  "flowint(s)");
504  break;
505  case DETECT_FLAGS:
506  {
507  DetectFlagsData *fl = (DetectFlagsData *)sm->ctx;
508 
509  if (fl->flags & TH_SYN) {
511  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
512  }
513  if (fl->flags & TH_RST) {
515  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
516  }
517  if (fl->flags & TH_FIN) {
519  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
520  }
521  if (fl->flags & TH_URG) {
523  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
524  }
525  if (fl->flags & TH_ECN) {
527  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
528  }
529  if (fl->flags & TH_CWR) {
531  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
532  }
533  break;
534  }
535  case DETECT_DSIZE:
536  {
537  DetectDsizeData *ds = (DetectDsizeData *)sm->ctx;
538  switch (ds->mode) {
539  case DETECTDSIZE_LT:
540  /* LT will include 0, so no payload.
541  * if GT is used in the same rule the
542  * flag will be set anyway. */
543  break;
544  case DETECTDSIZE_RA:
545  case DETECTDSIZE_GT:
547  SCLogDebug("sig requires payload");
548  break;
549  case DETECTDSIZE_EQ:
550  if (ds->dsize > 0) {
552  SCLogDebug("sig requires payload");
553  } else if (ds->dsize == 0) {
555  SCLogDebug("sig requires no payload");
556  }
557  break;
558  }
559  break;
560  }
563  break;
564  case DETECT_ENGINE_EVENT:
566  break;
567  }
568  }
569 
572  SCLogDebug("sig requires flow");
573  }
574 
575  if (s->flags & SIG_FLAG_APPLAYER) {
577  SCLogDebug("sig requires flow");
578  }
579 
580  SCLogDebug("mask %02X", s->mask);
581  SCReturnInt(0);
582 }
583 
584 static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
585 {
587  DetectMpmSetupAppMpms(de_ctx);
588 
589  return;
590 }
591 
592 /** \brief Pure-PCRE or bytetest rule */
593 static int RuleInspectsPayloadHasNoMpm(const Signature *s)
594 {
595  if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
596  return 1;
597  return 0;
598 }
599 
600 static int RuleGetMpmPatternSize(const Signature *s)
601 {
602  if (s->init_data->mpm_sm == NULL)
603  return -1;
604  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
605  if (mpm_list < 0)
606  return -1;
607  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
608  if (cd == NULL)
609  return -1;
610  return (int)cd->content_len;
611 }
612 
613 static int RuleMpmIsNegated(const Signature *s)
614 {
615  if (s->init_data->mpm_sm == NULL)
616  return 0;
617  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
618  if (mpm_list < 0)
619  return 0;
620  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
621  if (cd == NULL)
622  return 0;
623  return (cd->flags & DETECT_CONTENT_NEGATED);
624 }
625 
626 #ifdef HAVE_LIBJANSSON
627 static json_t *RulesGroupPrintSghStats(const SigGroupHead *sgh,
628  const int add_rules, const int add_mpm_stats)
629 {
630  uint32_t mpm_cnt = 0;
631  uint32_t nonmpm_cnt = 0;
632  uint32_t negmpm_cnt = 0;
633  uint32_t any5_cnt = 0;
634  uint32_t payload_no_mpm_cnt = 0;
635  uint32_t syn_cnt = 0;
636 
637  uint32_t mpms_total = 0;
638  uint32_t mpms_min = 0;
639  uint32_t mpms_max = 0;
640 
641  struct {
642  uint32_t total;
643  uint32_t cnt;
644  uint32_t min;
645  uint32_t max;
646  } mpm_stats[DETECT_SM_LIST_MAX];
647  memset(mpm_stats, 0x00, sizeof(mpm_stats));
648 
649  uint32_t alstats[ALPROTO_MAX] = {0};
650  uint32_t mpm_sizes[DETECT_SM_LIST_MAX][256];
651  memset(mpm_sizes, 0, sizeof(mpm_sizes));
652  uint32_t alproto_mpm_bufs[ALPROTO_MAX][DETECT_SM_LIST_MAX];
653  memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs));
654 
655  json_t *js = json_object();
656  if (unlikely(js == NULL))
657  return NULL;
658 
659  json_object_set_new(js, "id", json_integer(sgh->id));
660 
661  json_t *js_array = json_array();
662 
663  const Signature *s;
664  uint32_t x;
665  for (x = 0; x < sgh->sig_cnt; x++) {
666  s = sgh->match_array[x];
667  if (s == NULL)
668  continue;
669 
670  int any = 0;
671  if (s->proto.flags & DETECT_PROTO_ANY) {
672  any++;
673  }
674  if (s->flags & SIG_FLAG_DST_ANY) {
675  any++;
676  }
677  if (s->flags & SIG_FLAG_SRC_ANY) {
678  any++;
679  }
680  if (s->flags & SIG_FLAG_DP_ANY) {
681  any++;
682  }
683  if (s->flags & SIG_FLAG_SP_ANY) {
684  any++;
685  }
686  if (any == 5) {
687  any5_cnt++;
688  }
689 
690  if (s->init_data->mpm_sm == NULL) {
691  nonmpm_cnt++;
692 
693  if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) {
694  SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id);
695  }
696 
697  DetectPort *sp = s->sp;
698  DetectPort *dp = s->dp;
699 
700  if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) {
701  SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id);
702  }
703  if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) {
704  SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id);
705  }
706 
708  syn_cnt++;
709  }
710 
711  } else {
712  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
713  BUG_ON(mpm_list < 0);
714  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
715  uint32_t size = cd->content_len < 256 ? cd->content_len : 255;
716 
717  mpm_sizes[mpm_list][size]++;
718  if (s->alproto != ALPROTO_UNKNOWN) {
719  alproto_mpm_bufs[s->alproto][mpm_list]++;
720  }
721 
722  if (mpm_list == DETECT_SM_LIST_PMATCH) {
723  if (size == 1) {
724  DetectPort *sp = s->sp;
725  DetectPort *dp = s->dp;
726  if (s->flags & SIG_FLAG_TOSERVER) {
727  if (dp->port == 0 && dp->port2 == 65535) {
728  SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
729  } else {
730  SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id);
731  }
732  }
733  if (s->flags & SIG_FLAG_TOCLIENT) {
734  if (sp->port == 0 && sp->port2 == 65535) {
735  SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
736  } else {
737  SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id);
738  }
739  }
740  }
741  }
742 
743  uint32_t w = PatternStrength(cd->content, cd->content_len);
744  mpms_total += w;
745  if (mpms_min == 0)
746  mpms_min = w;
747  if (w < mpms_min)
748  mpms_min = w;
749  if (w > mpms_max)
750  mpms_max = w;
751 
752  mpm_stats[mpm_list].total += w;
753  mpm_stats[mpm_list].cnt++;
754  if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min)
755  mpm_stats[mpm_list].min = w;
756  if (w > mpm_stats[mpm_list].max)
757  mpm_stats[mpm_list].max = w;
758 
759  mpm_cnt++;
760 
761  if (w < 10) {
762  SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id);
763  }
764  if (w < 10 && any == 5) {
765  SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id);
766  }
767 
768  if (cd->flags & DETECT_CONTENT_NEGATED) {
769  SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id);
770  negmpm_cnt++;
771  }
772  }
773 
774  if (RuleInspectsPayloadHasNoMpm(s)) {
775  SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id);
776  payload_no_mpm_cnt++;
777  }
778 
779  if (s->alproto != ALPROTO_UNKNOWN) {
780  alstats[s->alproto]++;
781  }
782 
783  if (add_rules) {
784  json_t *js_sig = json_object();
785  if (unlikely(js == NULL))
786  continue;
787  json_object_set_new(js_sig, "sig_id", json_integer(s->id));
788  json_array_append_new(js_array, js_sig);
789  }
790  }
791 
792  json_object_set_new(js, "rules", js_array);
793 
794  json_t *stats = json_object();
795  json_object_set_new(stats, "total", json_integer(sgh->sig_cnt));
796 
797  json_t *types = json_object();
798  json_object_set_new(types, "mpm", json_integer(mpm_cnt));
799  json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt));
800  json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt));
801  json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt));
802  json_object_set_new(types, "syn", json_integer(syn_cnt));
803  json_object_set_new(types, "any5", json_integer(any5_cnt));
804  json_object_set_new(stats, "types", types);
805 
806  int i;
807  for (i = 0; i < ALPROTO_MAX; i++) {
808  if (alstats[i] > 0) {
809  json_t *app = json_object();
810  json_object_set_new(app, "total", json_integer(alstats[i]));
811 
812  for (x = 0; x < DETECT_SM_LIST_MAX; x++) {
813  if (alproto_mpm_bufs[i][x] == 0)
814  continue;
815  json_object_set_new(app, DetectListToHumanString(x), json_integer(alproto_mpm_bufs[i][x]));
816  }
817 
818  json_object_set_new(stats, AppProtoToString(i), app);
819  }
820  }
821 
822  if (add_mpm_stats) {
823  json_t *mpm_js = json_object();
824 
825  for (i = 0; i < DETECT_SM_LIST_MAX; i++) {
826  if (mpm_stats[i].cnt > 0) {
827 
828  json_t *mpm_sizes_array = json_array();
829  for (x = 0; x < 256; x++) {
830  if (mpm_sizes[i][x] == 0)
831  continue;
832 
833  json_t *e = json_object();
834  json_object_set_new(e, "size", json_integer(x));
835  json_object_set_new(e, "count", json_integer(mpm_sizes[i][x]));
836  json_array_append_new(mpm_sizes_array, e);
837  }
838 
839  json_t *buf = json_object();
840  json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt));
841  json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt));
842  json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min));
843  json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max));
844 
845  json_object_set_new(buf, "sizes", mpm_sizes_array);
846 
847  json_object_set_new(mpm_js, DetectListToHumanString(i), buf);
848  }
849  }
850 
851  json_object_set_new(stats, "mpm", mpm_js);
852  }
853  json_object_set_new(js, "stats", stats);
854 
855  json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist));
856 
857  return js;
858 }
859 #endif /* HAVE_LIBJANSSON */
860 
861 static void RulesDumpGrouping(const DetectEngineCtx *de_ctx,
862  const int add_rules, const int add_mpm_stats)
863 {
864 #ifdef HAVE_LIBJANSSON
865  json_t *js = json_object();
866  if (unlikely(js == NULL))
867  return;
868 
869  int p;
870  for (p = 0; p < 256; p++) {
871  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
872  const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp";
873 
874  json_t *tcp = json_object();
875 
876  json_t *ts_array = json_array();
877  DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp :
878  de_ctx->flow_gh[1].udp;
879  while (list != NULL) {
880  json_t *port = json_object();
881  json_object_set_new(port, "port", json_integer(list->port));
882  json_object_set_new(port, "port2", json_integer(list->port2));
883 
884  json_t *tcp_ts = RulesGroupPrintSghStats(list->sh,
885  add_rules, add_mpm_stats);
886  json_object_set_new(port, "rulegroup", tcp_ts);
887  json_array_append_new(ts_array, port);
888 
889  list = list->next;
890  }
891  json_object_set_new(tcp, "toserver", ts_array);
892 
893  json_t *tc_array = json_array();
894  list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp :
895  de_ctx->flow_gh[0].udp;
896  while (list != NULL) {
897  json_t *port = json_object();
898  json_object_set_new(port, "port", json_integer(list->port));
899  json_object_set_new(port, "port2", json_integer(list->port2));
900 
901  json_t *tcp_tc = RulesGroupPrintSghStats(list->sh,
902  add_rules, add_mpm_stats);
903  json_object_set_new(port, "rulegroup", tcp_tc);
904  json_array_append_new(tc_array, port);
905 
906  list = list->next;
907  }
908  json_object_set_new(tcp, "toclient", tc_array);
909 
910  json_object_set_new(js, name, tcp);
911  }
912 
913  }
914 
915  const char *filename = "rule_group.json";
916  const char *log_dir = ConfigGetLogDirectory();
917  char log_path[PATH_MAX] = "";
918 
919  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
920 
921  FILE *fp = fopen(log_path, "w");
922  if (fp == NULL) {
923  return;
924  }
925 
926  char *js_s = json_dumps(js,
927  JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH);
928  if (unlikely(js_s == NULL)) {
929  fclose(fp);
930  return;
931  }
932 
933  json_object_clear(js);
934  json_decref(js);
935 
936  fprintf(fp, "%s\n", js_s);
937  free(js_s);
938  fclose(fp);
939 #endif
940  return;
941 }
942 
943 static int RulesGroupByProto(DetectEngineCtx *de_ctx)
944 {
945  Signature *s = de_ctx->sig_list;
946 
947  uint32_t max_idx = 0;
948  SigGroupHead *sgh_ts[256] = {NULL};
949  SigGroupHead *sgh_tc[256] = {NULL};
950 
951  for ( ; s != NULL; s = s->next) {
952  if (s->flags & SIG_FLAG_IPONLY)
953  continue;
954 
955  int p;
956  for (p = 0; p < 256; p++) {
957  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
958  continue;
959  }
960  if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) {
961  continue;
962  }
963 
964  if (s->flags & SIG_FLAG_TOCLIENT) {
965  SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s);
966  max_idx = s->num;
967  }
968  if (s->flags & SIG_FLAG_TOSERVER) {
969  SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s);
970  max_idx = s->num;
971  }
972  }
973  }
974  SCLogDebug("max_idx %u", max_idx);
975 
976  /* lets look at deduplicating this list */
977  SigGroupHeadHashFree(de_ctx);
978  SigGroupHeadHashInit(de_ctx);
979 
980  uint32_t cnt = 0;
981  uint32_t own = 0;
982  uint32_t ref = 0;
983  int p;
984  for (p = 0; p < 256; p++) {
985  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
986  continue;
987  if (sgh_ts[p] == NULL)
988  continue;
989 
990  cnt++;
991 
992  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]);
993  if (lookup_sgh == NULL) {
994  SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]);
995 
996  SigGroupHeadSetSigCnt(sgh_ts[p], max_idx);
997  SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx);
998 
999  SigGroupHeadHashAdd(de_ctx, sgh_ts[p]);
1000  SigGroupHeadStore(de_ctx, sgh_ts[p]);
1001 
1002  de_ctx->gh_unique++;
1003  own++;
1004  } else {
1005  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]);
1006 
1007  SigGroupHeadFree(de_ctx, sgh_ts[p]);
1008  sgh_ts[p] = lookup_sgh;
1009 
1010  de_ctx->gh_reuse++;
1011  ref++;
1012  }
1013  }
1014  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1015  "toserver", cnt, own, ref);
1016 
1017  cnt = 0;
1018  own = 0;
1019  ref = 0;
1020  for (p = 0; p < 256; p++) {
1021  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1022  continue;
1023  if (sgh_tc[p] == NULL)
1024  continue;
1025 
1026  cnt++;
1027 
1028  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]);
1029  if (lookup_sgh == NULL) {
1030  SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]);
1031 
1032  SigGroupHeadSetSigCnt(sgh_tc[p], max_idx);
1033  SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx);
1034 
1035  SigGroupHeadHashAdd(de_ctx, sgh_tc[p]);
1036  SigGroupHeadStore(de_ctx, sgh_tc[p]);
1037 
1038  de_ctx->gh_unique++;
1039  own++;
1040 
1041  } else {
1042  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]);
1043 
1044  SigGroupHeadFree(de_ctx, sgh_tc[p]);
1045  sgh_tc[p] = lookup_sgh;
1046 
1047  de_ctx->gh_reuse++;
1048  ref++;
1049  }
1050  }
1051  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1052  "toclient", cnt, own, ref);
1053 
1054  for (p = 0; p < 256; p++) {
1055  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1056  continue;
1057 
1058  de_ctx->flow_gh[0].sgh[p] = sgh_tc[p];
1059  de_ctx->flow_gh[1].sgh[p] = sgh_ts[p];
1060  }
1061 
1062  return 0;
1063 }
1064 
1065 static int PortIsWhitelisted(const DetectEngineCtx *de_ctx,
1066  const DetectPort *a, int ipproto)
1067 {
1068  DetectPort *w = de_ctx->tcp_whitelist;
1069  if (ipproto == IPPROTO_UDP)
1070  w = de_ctx->udp_whitelist;
1071 
1072  while (w) {
1073  if (a->port >= w->port && a->port2 <= w->port) {
1074  SCLogDebug("port group %u:%u whitelisted -> %d", a->port, a->port2, w->port);
1075  return 1;
1076  }
1077  w = w->next;
1078  }
1079 
1080  return 0;
1081 }
1082 
1083 static int RuleSetWhitelist(Signature *s)
1084 {
1085  DetectPort *p = NULL;
1086  if (s->flags & SIG_FLAG_TOSERVER)
1087  p = s->dp;
1088  else if (s->flags & SIG_FLAG_TOCLIENT)
1089  p = s->sp;
1090  else
1091  return 0;
1092 
1093  /* for sigs that don't use 'any' as port, see if we want to
1094  * whitelist poor sigs */
1095  int wl = 0;
1096  if (!(p->port == 0 && p->port2 == 65535)) {
1097  /* pure pcre, bytetest, etc rules */
1098  if (RuleInspectsPayloadHasNoMpm(s)) {
1099  SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id);
1100  wl = 99;
1101 
1102  } else if (RuleMpmIsNegated(s)) {
1103  SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id);
1104  wl = 77;
1105 
1106  /* one byte pattern in packet/stream payloads */
1107  } else if (s->init_data->mpm_sm != NULL &&
1109  RuleGetMpmPatternSize(s) == 1)
1110  {
1111  SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id);
1112  wl = 55;
1113 
1114  } else if (DetectFlagsSignatureNeedsSynPackets(s) &&
1116  {
1117  SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id);
1118  wl = 33;
1119  }
1120  }
1121 
1122  s->init_data->whitelist = wl;
1123  return wl;
1124 }
1125 
1126 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx);
1128 
1129 static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint32_t direction) {
1130  /* step 1: create a hash of 'DetectPort' objects based on all the
1131  * rules. Each object will have a SGH with the sigs added
1132  * that belong to the SGH. */
1133  DetectPortHashInit(de_ctx);
1134 
1135  uint32_t max_idx = 0;
1136  const Signature *s = de_ctx->sig_list;
1137  DetectPort *list = NULL;
1138  while (s) {
1139  /* IP Only rules are handled separately */
1140  if (s->flags & SIG_FLAG_IPONLY)
1141  goto next;
1142  if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY)))
1143  goto next;
1144  if (direction == SIG_FLAG_TOSERVER) {
1145  if (!(s->flags & SIG_FLAG_TOSERVER))
1146  goto next;
1147  } else if (direction == SIG_FLAG_TOCLIENT) {
1148  if (!(s->flags & SIG_FLAG_TOCLIENT))
1149  goto next;
1150  }
1151 
1152  DetectPort *p = NULL;
1153  if (direction == SIG_FLAG_TOSERVER)
1154  p = s->dp;
1155  else if (direction == SIG_FLAG_TOCLIENT)
1156  p = s->sp;
1157  else
1158  BUG_ON(1);
1159 
1160  /* see if we want to exclude directionless sigs that really care only for
1161  * to_server syn scans/floods */
1162  if ((direction == SIG_FLAG_TOCLIENT) &&
1166  (!(s->dp->port == 0 && s->dp->port2 == 65535)))
1167  {
1168  SCLogWarning(SC_WARN_POOR_RULE, "rule %u: SYN-only to port(s) %u:%u "
1169  "w/o direction specified, disabling for toclient direction",
1170  s->id, s->dp->port, s->dp->port2);
1171  goto next;
1172  }
1173 
1174  int wl = s->init_data->whitelist;
1175  while (p) {
1176  int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? 111 : 0;
1177  pwl = MAX(wl,pwl);
1178 
1179  DetectPort *lookup = DetectPortHashLookup(de_ctx, p);
1180  if (lookup) {
1181  SigGroupHeadAppendSig(de_ctx, &lookup->sh, s);
1182  lookup->sh->init->whitelist = MAX(lookup->sh->init->whitelist, pwl);
1183  } else {
1184  DetectPort *tmp2 = DetectPortCopySingle(de_ctx, p);
1185  BUG_ON(tmp2 == NULL);
1186  SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s);
1187  tmp2->sh->init->whitelist = pwl;
1188  DetectPortHashAdd(de_ctx, tmp2);
1189  }
1190 
1191  p = p->next;
1192  }
1193  max_idx = s->num;
1194  next:
1195  s = s->next;
1196  }
1197 
1198  /* step 2: create a list of DetectPort objects */
1199  HashListTableBucket *htb = NULL;
1200  for (htb = HashListTableGetListHead(de_ctx->dport_hash_table);
1201  htb != NULL;
1202  htb = HashListTableGetListNext(htb))
1203  {
1205  DetectPort *tmp = DetectPortCopySingle(de_ctx, p);
1206  BUG_ON(tmp == NULL);
1207  int r = DetectPortInsert(de_ctx, &list , tmp);
1208  BUG_ON(r == -1);
1209  }
1210  DetectPortHashFree(de_ctx);
1211  de_ctx->dport_hash_table = NULL;
1212 
1213  SCLogDebug("rules analyzed");
1214 
1215  /* step 3: group the list and shrink it if necessary */
1216  DetectPort *newlist = NULL;
1217  uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups :
1218  de_ctx->max_uniq_toserver_groups;
1219  CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx);
1220  list = newlist;
1221 
1222  /* step 4: deduplicate the SGH's */
1223  SigGroupHeadHashFree(de_ctx);
1224  SigGroupHeadHashInit(de_ctx);
1225 
1226  uint32_t cnt = 0;
1227  uint32_t own = 0;
1228  uint32_t ref = 0;
1229  DetectPort *iter;
1230  for (iter = list ; iter != NULL; iter = iter->next) {
1231  BUG_ON (iter->sh == NULL);
1232  cnt++;
1233 
1234  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh);
1235  if (lookup_sgh == NULL) {
1236  SCLogDebug("port group %p sgh %p is the original", iter, iter->sh);
1237 
1238  SigGroupHeadSetSigCnt(iter->sh, max_idx);
1239  SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx);
1240  SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction);
1241  SigGroupHeadHashAdd(de_ctx, iter->sh);
1242  SigGroupHeadStore(de_ctx, iter->sh);
1243  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1244  de_ctx->gh_unique++;
1245  own++;
1246  } else {
1247  SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh);
1248 
1249  SigGroupHeadFree(de_ctx, iter->sh);
1250  iter->sh = lookup_sgh;
1251  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1252 
1253  de_ctx->gh_reuse++;
1254  ref++;
1255  }
1256  }
1257 #if 0
1258  for (iter = list ; iter != NULL; iter = iter->next) {
1259  SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)",
1260  iter->port, iter->port2, iter->sh,
1261  iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own",
1262  iter->sh->init->whitelist ? "true" : "false",
1263  iter->sh->init->whitelist);
1264  }
1265 #endif
1266  SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies",
1267  ipproto == 6 ? "TCP" : "UDP",
1268  direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1269  cnt, own, ref);
1270  return list;
1271 }
1272 
1273 /**
1274  * \brief Preprocess signature, classify ip-only, etc, build sig array
1275  *
1276  * \param de_ctx Pointer to the Detection Engine Context
1277  *
1278  * \retval 0 on success
1279  * \retval -1 on failure
1280  */
1282 {
1283  Signature *tmp_s = NULL;
1284  uint32_t cnt_iponly = 0;
1285  uint32_t cnt_payload = 0;
1286  uint32_t cnt_applayer = 0;
1287  uint32_t cnt_deonly = 0;
1288 
1289  if (!(de_ctx->flags & DE_QUIET)) {
1290  SCLogDebug("building signature grouping structure, stage 1: "
1291  "preprocessing rules...");
1292  }
1293 
1294  de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx);
1295  de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *));
1296  de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size);
1297  if (de_ctx->sig_array == NULL)
1298  goto error;
1299  memset(de_ctx->sig_array,0,de_ctx->sig_array_size);
1300 
1301  SCLogDebug("signature lookup array: %" PRIu32 " sigs, %" PRIu32 " bytes",
1302  de_ctx->sig_array_len, de_ctx->sig_array_size);
1303 
1304  /* now for every rule add the source group */
1305  for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) {
1306  de_ctx->sig_array[tmp_s->num] = tmp_s;
1307 
1308  SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]);
1309 
1310  /* see if the sig is dp only */
1311  if (SignatureIsPDOnly(de_ctx, tmp_s) == 1) {
1312  tmp_s->flags |= SIG_FLAG_PDONLY;
1313  SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", tmp_s->id);
1314 
1315  /* see if the sig is ip only */
1316  } else if (SignatureIsIPOnly(de_ctx, tmp_s) == 1) {
1317  tmp_s->flags |= SIG_FLAG_IPONLY;
1318  cnt_iponly++;
1319 
1320  SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", tmp_s->id);
1321 
1322  /* see if any sig is inspecting the packet payload */
1323  } else if (SignatureIsInspectingPayload(de_ctx, tmp_s) == 1) {
1324  cnt_payload++;
1325 
1326  SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", tmp_s->id);
1327  } else if (SignatureIsDEOnly(de_ctx, tmp_s) == 1) {
1329  SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", tmp_s->id);
1330  cnt_deonly++;
1331  }
1332 
1333  if (tmp_s->flags & SIG_FLAG_APPLAYER) {
1334  SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", tmp_s->id);
1335  cnt_applayer++;
1336  }
1337 
1338 #ifdef DEBUG
1339  if (SCLogDebugEnabled()) {
1340  uint16_t colen = 0;
1341  char copresent = 0;
1342  SigMatch *sm;
1343  DetectContentData *co;
1344  for (sm = tmp_s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) {
1345  if (sm->type != DETECT_CONTENT)
1346  continue;
1347 
1348  copresent = 1;
1349  co = (DetectContentData *)sm->ctx;
1350  if (co->content_len > colen)
1351  colen = co->content_len;
1352  }
1353 
1354  if (copresent && colen == 1) {
1355  SCLogDebug("signature %8u content maxlen 1", tmp_s->id);
1356  int proto;
1357  for (proto = 0; proto < 256; proto++) {
1358  if (tmp_s->proto.proto[(proto/8)] & (1<<(proto%8)))
1359  SCLogDebug("=> proto %" PRId32 "", proto);
1360  }
1361  }
1362  }
1363 #endif /* DEBUG */
1364 
1365  if (RuleMpmIsNegated(tmp_s)) {
1366  tmp_s->flags |= SIG_FLAG_MPM_NEG;
1367  }
1368 
1369  SignatureCreateMask(tmp_s);
1372 
1373  RuleSetWhitelist(tmp_s);
1374 
1375  /* if keyword engines are enabled in the config, handle them here */
1376  if (de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
1377  !(tmp_s->flags & SIG_FLAG_PREFILTER))
1378  {
1379  int i;
1380  int prefilter_list = DETECT_TBLSIZE;
1381 
1382  /* get the keyword supporting prefilter with the lowest type */
1383  for (i = 0; i < (int)tmp_s->init_data->smlists_array_size; i++) {
1384  SigMatch *sm = tmp_s->init_data->smlists[i];
1385  while (sm != NULL) {
1386  if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
1387  if (sigmatch_table[sm->type].SupportsPrefilter(tmp_s) == TRUE) {
1388  prefilter_list = MIN(prefilter_list, sm->type);
1389  }
1390  }
1391  sm = sm->next;
1392  }
1393  }
1394 
1395  /* apply that keyword as prefilter */
1396  if (prefilter_list != DETECT_TBLSIZE) {
1397  for (i = 0; i < (int)tmp_s->init_data->smlists_array_size; i++) {
1398  SigMatch *sm = tmp_s->init_data->smlists[i];
1399  while (sm != NULL) {
1400  if (sm->type == prefilter_list) {
1401  tmp_s->init_data->prefilter_sm = sm;
1402  tmp_s->flags |= SIG_FLAG_PREFILTER;
1403  SCLogConfig("sid %u: prefilter is on \"%s\"", tmp_s->id, sigmatch_table[sm->type].name);
1404  break;
1405  }
1406  sm = sm->next;
1407  }
1408  }
1409  }
1410  }
1411 
1412  /* run buffer type callbacks if any */
1413  int x;
1414  for (x = 0; x < (int)tmp_s->init_data->smlists_array_size; x++) {
1415  if (tmp_s->init_data->smlists[x])
1416  DetectBufferRunSetupCallback(de_ctx, x, tmp_s);
1417  }
1418 
1419  de_ctx->sig_cnt++;
1420  }
1421 
1422  if (!(de_ctx->flags & DE_QUIET)) {
1423  SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only "
1424  "rules, %" PRIu32 " are inspecting packet payload, %"PRIu32
1425  " inspect application layer, %"PRIu32" are decoder event only",
1426  de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer,
1427  cnt_deonly);
1428 
1429  SCLogConfig("building signature grouping structure, stage 1: "
1430  "preprocessing rules... complete");
1431  }
1432  DetectFlowbitsAnalyze(de_ctx);
1433 
1434  return 0;
1435 
1436 error:
1437  return -1;
1438 }
1439 
1440 static int PortGroupWhitelist(const DetectPort *a)
1441 {
1442  return a->sh->init->whitelist;
1443 }
1444 
1446 {
1447  if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) {
1448  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1449  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1450  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1451  return 1;
1452  } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) {
1453  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1454  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1455  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1456  return 0;
1457  } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) {
1458  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1459  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1460  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1461  return 1;
1462  } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) {
1463  if (a->sh->sig_cnt > b->sh->sig_cnt) {
1464  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1465  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1466  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1467  return 1;
1468  }
1469  }
1470 
1471  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1472  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1473  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1474  return 0;
1475 }
1476 
1477 /** \internal
1478  * \brief Create a list of DetectPort objects sorted based on CompareFunc's
1479  * logic.
1480  *
1481  * List can limit the number of groups. In this case an extra "join" group
1482  * is created that contains the sigs belonging to that. It's *appended* to
1483  * the list, meaning that if the list is walked linearly it's found last.
1484  * The joingr is meant to be a catch all.
1485  *
1486  */
1487 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
1488 {
1489  DetectPort *tmplist = NULL, *joingr = NULL;
1490  char insert = 0;
1491  uint32_t groups = 0;
1492  DetectPort *list;
1493 
1494  /* insert the addresses into the tmplist, where it will
1495  * be sorted descending on 'cnt' and on wehther a group
1496  * is whitelisted. */
1497 
1498  DetectPort *oldhead = port_list;
1499  while (oldhead) {
1500  /* take the top of the list */
1501  list = oldhead;
1502  oldhead = oldhead->next;
1503  list->next = NULL;
1504 
1505  groups++;
1506 
1507  SigGroupHeadSetSigCnt(list->sh, max_idx);
1508 
1509  /* insert it */
1510  DetectPort *tmpgr = tmplist, *prevtmpgr = NULL;
1511  if (tmplist == NULL) {
1512  /* empty list, set head */
1513  tmplist = list;
1514  } else {
1515  /* look for the place to insert */
1516  for ( ; tmpgr != NULL && !insert; tmpgr = tmpgr->next) {
1517  if (CompareFunc(list, tmpgr) == 1) {
1518  if (tmpgr == tmplist) {
1519  list->next = tmplist;
1520  tmplist = list;
1521  SCLogDebug("new list top: %u:%u", tmplist->port, tmplist->port2);
1522  } else {
1523  list->next = prevtmpgr->next;
1524  prevtmpgr->next = list;
1525  }
1526  insert = 1;
1527  break;
1528  }
1529  prevtmpgr = tmpgr;
1530  }
1531  if (insert == 0) {
1532  list->next = NULL;
1533  prevtmpgr->next = list;
1534  }
1535  insert = 0;
1536  }
1537  }
1538 
1539  uint32_t left = unique_groups;
1540  if (left == 0)
1541  left = groups;
1542 
1543  /* create another list: take the port groups from above
1544  * and add them to the 2nd list until we have met our
1545  * count. The rest is added to the 'join' group. */
1546  DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL;
1547  DetectPort *gr, *next_gr;
1548  for (gr = tmplist; gr != NULL; ) {
1549  next_gr = gr->next;
1550 
1551  SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2);
1552  DetectPortPrint(gr);
1553 
1554  /* if we've set up all the unique groups, add the rest to the
1555  * catch-all joingr */
1556  if (left == 0) {
1557  if (joingr == NULL) {
1558  DetectPortParse(de_ctx, &joingr, "0:65535");
1559  if (joingr == NULL) {
1560  goto error;
1561  }
1562  SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2);
1563  joingr->next = NULL;
1564  }
1565  SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh);
1566 
1567  /* when a group's sigs are added to the joingr, we can free it */
1568  gr->next = NULL;
1569  DetectPortFree(de_ctx, gr);
1570  gr = NULL;
1571 
1572  /* append */
1573  } else {
1574  gr->next = NULL;
1575 
1576  if (tmplist2 == NULL) {
1577  tmplist2 = gr;
1578  tmplist2_tail = gr;
1579  } else {
1580  tmplist2_tail->next = gr;
1581  tmplist2_tail = gr;
1582  }
1583  }
1584 
1585  if (left > 0)
1586  left--;
1587 
1588  gr = next_gr;
1589  }
1590 
1591  /* if present, append the joingr that covers the rest */
1592  if (joingr != NULL) {
1593  SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2);
1594 
1595  if (tmplist2 == NULL) {
1596  tmplist2 = joingr;
1597  //tmplist2_tail = joingr;
1598  } else {
1599  tmplist2_tail->next = joingr;
1600  //tmplist2_tail = joingr;
1601  }
1602  } else {
1603  SCLogDebug("no joingr");
1604  }
1605 
1606  /* pass back our new list to the caller */
1607  *newhead = tmplist2;
1608  DetectPortPrintList(*newhead);
1609 
1610  return 0;
1611 error:
1612  return -1;
1613 }
1614 
1615 /**
1616  * \internal
1617  * \brief add a decoder event signature to the detection engine ctx
1618  */
1619 static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s)
1620 {
1621  SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id);
1622  SigGroupHeadAppendSig(de_ctx, &de_ctx->decoder_event_sgh, s);
1623 }
1624 
1625 /**
1626  * \brief Fill the global src group head, with the sigs included
1627  *
1628  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1629  * to be processed
1630  *
1631  * \retval 0 On success
1632  * \retval -1 On failure
1633  */
1635 {
1636  Signature *tmp_s = NULL;
1637  uint32_t sigs = 0;
1638 
1639  if (!(de_ctx->flags & DE_QUIET)) {
1640  SCLogDebug("building signature grouping structure, stage 2: "
1641  "building source address lists...");
1642  }
1643 
1644  IPOnlyInit(de_ctx, &de_ctx->io_ctx);
1645 
1646  de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER);
1647  de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT);
1648  de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER);
1649  de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT);
1650 
1651  /* Setup the other IP Protocols (so not TCP/UDP) */
1652  RulesGroupByProto(de_ctx);
1653 
1654  /* now for every rule add the source group to our temp lists */
1655  for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) {
1656  SCLogDebug("tmp_s->id %"PRIu32, tmp_s->id);
1657  if (tmp_s->flags & SIG_FLAG_IPONLY) {
1658  IPOnlyAddSignature(de_ctx, &de_ctx->io_ctx, tmp_s);
1659  }
1660 
1661  if (tmp_s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) {
1662  DetectEngineAddDecoderEventSig(de_ctx, tmp_s);
1663  }
1664 
1665  sigs++;
1666  }
1667 
1668  IPOnlyPrepare(de_ctx);
1669  IPOnlyPrint(de_ctx, &de_ctx->io_ctx);
1670 
1671  return 0;
1672 }
1673 
1674 static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx)
1675 {
1676  if (de_ctx->decoder_event_sgh == NULL)
1677  return;
1678 
1679  uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx);
1680  SigGroupHeadSetSigCnt(de_ctx->decoder_event_sgh, max_idx);
1681  SigGroupHeadBuildMatchArray(de_ctx, de_ctx->decoder_event_sgh, max_idx);
1682 }
1683 
1685 {
1686  /* prepare the decoder event sgh */
1687  DetectEngineBuildDecoderEventSgh(de_ctx);
1688  return 0;
1689 }
1690 
1692 {
1693  BUG_ON(de_ctx == NULL);
1694 
1695  if (!(de_ctx->flags & DE_QUIET)) {
1696  SCLogDebug("cleaning up signature grouping structure...");
1697  }
1698  if (de_ctx->decoder_event_sgh)
1699  SigGroupHeadFree(de_ctx, de_ctx->decoder_event_sgh);
1700  de_ctx->decoder_event_sgh = NULL;
1701 
1702  int f;
1703  for (f = 0; f < FLOW_STATES; f++) {
1704  int p;
1705  for (p = 0; p < 256; p++) {
1706  de_ctx->flow_gh[f].sgh[p] = NULL;
1707  }
1708 
1709  /* free lookup lists */
1710  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].tcp);
1711  de_ctx->flow_gh[f].tcp = NULL;
1712  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].udp);
1713  de_ctx->flow_gh[f].udp = NULL;
1714  }
1715 
1716  uint32_t idx;
1717  for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1718  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1719  if (sgh == NULL)
1720  continue;
1721 
1722  SCLogDebug("sgh %p", sgh);
1723  SigGroupHeadFree(de_ctx, sgh);
1724  }
1725  SCFree(de_ctx->sgh_array);
1726  de_ctx->sgh_array = NULL;
1727  de_ctx->sgh_array_cnt = 0;
1728  de_ctx->sgh_array_size = 0;
1729 
1730  IPOnlyDeinit(de_ctx, &de_ctx->io_ctx);
1731 
1732  if (!(de_ctx->flags & DE_QUIET)) {
1733  SCLogInfo("cleaning up signature grouping structure... complete");
1734  }
1735  return 0;
1736 }
1737 
1738 #if 0
1739 static void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1740 {
1741  if (sgh == NULL) {
1742  printf("\n");
1743  return;
1744  }
1745 
1746  uint32_t sig;
1747  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1748  printf("%" PRIu32 " ", sgh->match_array[sig]->id);
1749  }
1750  printf("\n");
1751 }
1752 
1753 static void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1754 {
1755  if (sgh == NULL || sgh->init == NULL) {
1756  printf("\n");
1757  return;
1758  }
1759 
1760  uint32_t sig;
1761  for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
1762  if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) {
1763  printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id);
1764  }
1765  }
1766  printf("\n");
1767 }
1768 #endif
1769 
1770 /** \brief finalize preparing sgh's */
1772 {
1773  SCEnter();
1774 
1775  //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt);
1776 
1777  uint32_t cnt = 0;
1778  uint32_t idx = 0;
1779  for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1780  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1781  if (sgh == NULL)
1782  continue;
1783 
1784  SCLogDebug("sgh %p", sgh);
1785 
1786  SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
1787  SigGroupHeadSetFileHashFlag(de_ctx, sgh);
1788  SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
1789  SigGroupHeadSetFilestoreCount(de_ctx, sgh);
1790  SCLogDebug("filestore count %u", sgh->filestore_cnt);
1791 
1792  PrefilterSetupRuleGroup(de_ctx, sgh);
1793 
1795 
1797  sgh->init = NULL;
1798 
1799  sgh->id = idx;
1800  cnt++;
1801  }
1802  SCLogPerf("Unique rule groups: %u", cnt);
1803 
1804  MpmStoreReportStats(de_ctx);
1805 
1806  if (de_ctx->decoder_event_sgh != NULL) {
1807  /* no need to set filestore count here as that would make a
1808  * signature not decode event only. */
1809  }
1810 
1811  /* cleanup the hashes now since we won't need them
1812  * after the initialization phase. */
1813  SigGroupHeadHashFree(de_ctx);
1814 
1815  int dump_grouping = 0;
1816  (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping);
1817 
1818  if (dump_grouping) {
1819  int add_rules = 0;
1820  (void)ConfGetBool("detect.profiling.grouping.include-rules", &add_rules);
1821  int add_mpm_stats = 0;
1822  (void)ConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_rules);
1823 
1824  RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats);
1825  }
1826 
1827 #ifdef PROFILING
1829 #endif
1830  SCReturnInt(0);
1831 }
1832 
1833 extern int rule_engine_analysis_set;
1834 /** \internal
1835  * \brief perform final per signature setup tasks
1836  *
1837  * - Create SigMatchData arrays from the init only SigMatch lists
1838  * - Setup per signature inspect engines
1839  * - remove signature init data.
1840  */
1841 static int SigMatchPrepare(DetectEngineCtx *de_ctx)
1842 {
1843  SCEnter();
1844 
1845  Signature *s = de_ctx->sig_list;
1846  for (; s != NULL; s = s->next) {
1847  /* set up inspect engines */
1849 
1850  /* built-ins */
1851  int type;
1852  for (type = 0; type < DETECT_SM_LIST_MAX; type++) {
1853  /* skip PMATCH if it is used in a stream 'app engine' instead */
1855  continue;
1856  SigMatch *sm = s->init_data->smlists[type];
1858  }
1859  if (rule_engine_analysis_set) {
1860 #ifdef HAVE_LIBJANSSON
1861  EngineAnalysisRules2(de_ctx, s);
1862 #endif
1863  }
1864  /* free lists. Ctx' are xferred to sm_arrays so won't get freed */
1865  uint32_t i;
1866  for (i = 0; i < s->init_data->smlists_array_size; i++) {
1867  SigMatch *sm = s->init_data->smlists[i];
1868  while (sm != NULL) {
1869  SigMatch *nsm = sm->next;
1870  SigMatchFree(sm);
1871  sm = nsm;
1872  }
1873  }
1874  SCFree(s->init_data->smlists);
1876  SCFree(s->init_data);
1877  s->init_data = NULL;
1878  }
1879 
1880 
1881  SCReturnInt(0);
1882 }
1883 
1884 /**
1885  * \brief Convert the signature list into the runtime match structure.
1886  *
1887  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1888  * to be processed
1889  *
1890  * \retval 0 On Success.
1891  * \retval -1 On failure.
1892  */
1894 {
1895  Signature *s = de_ctx->sig_list;
1896 
1897  /* Assign the unique order id of signatures after sorting,
1898  * so the IP Only engine process them in order too. Also
1899  * reset the old signums and assign new signums. We would
1900  * have experienced Sig reordering by now, hence the new
1901  * signums. */
1902  de_ctx->signum = 0;
1903  while (s != NULL) {
1904  s->num = de_ctx->signum++;
1905 
1906  s = s->next;
1907  }
1908 
1909  if (DetectSetFastPatternAndItsId(de_ctx) < 0)
1910  return -1;
1911 
1912  SigInitStandardMpmFactoryContexts(de_ctx);
1913 
1914  if (SigAddressPrepareStage1(de_ctx) != 0) {
1915  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1916  exit(EXIT_FAILURE);
1917  }
1918 
1919  if (SigAddressPrepareStage2(de_ctx) != 0) {
1920  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1921  exit(EXIT_FAILURE);
1922  }
1923 
1924  if (SigAddressPrepareStage3(de_ctx) != 0) {
1925  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1926  exit(EXIT_FAILURE);
1927  }
1928  if (SigAddressPrepareStage4(de_ctx) != 0) {
1929  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1930  exit(EXIT_FAILURE);
1931  }
1932 
1933  int r = DetectMpmPrepareBuiltinMpms(de_ctx);
1934  r |= DetectMpmPrepareAppMpms(de_ctx);
1935  if (r != 0) {
1936  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1937  exit(EXIT_FAILURE);
1938  }
1939 
1940  if (SigMatchPrepare(de_ctx) != 0) {
1941  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1942  exit(EXIT_FAILURE);
1943  }
1944 
1945 #ifdef PROFILING
1948  de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled
1949 
1950  intmax_t v = 0;
1951  if (ConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1)
1952  de_ctx->profile_match_logging_threshold = (uint32_t)v;
1953 
1955 #endif
1956  SCFree(de_ctx->app_mpms);
1957  de_ctx->app_mpms = NULL;
1958 
1961  }
1962  return 0;
1963 }
1964 
1966 {
1967  SigAddressCleanupStage1(de_ctx);
1968 
1969  return 0;
1970 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:241
void DetectMpmSetupAppMpms(DetectEngineCtx *de_ctx)
uint16_t filestore_cnt
Definition: detect.h:1311
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1403
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
Predict a strength value for patterns.
DetectProto proto
Definition: detect.h:509
#define DETECTDSIZE_LT
Definition: detect-dsize.h:27
SigIntId sig_cnt
Definition: detect.h:1300
SignatureInitData * init_data
Definition: detect.h:560
int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
Preprocess signature, classify ip-only, etc, build sig array.
#define SIG_FLAG_PDONLY
Definition: detect.h:254
#define SCLogDebug(...)
Definition: util-debug.h:335
#define MAX(x, y)
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Free a SigGroupHead and its members.
Signature ** sig_array
Definition: detect.h:735
void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx, Signature *s)
Add a signature to the lists of Adrresses in CIDR format (sorted) this step is necesary to build the ...
#define SIG_FLAG_PREFILTER
Definition: detect.h:250
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
Figured out the FP and their respective content ids for all the sigs in the engine.
#define DETECTDSIZE_GT
Definition: detect-dsize.h:29
#define SignatureMask
Definition: detect.h:284
uint32_t sig_array_len
Definition: detect.h:737
struct HtpBodyChunk_ * next
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:572
#define SIGMATCH_DEONLY_COMPAT
Definition: detect.h:1332
void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
SigMatch * prefilter_sm
Definition: detect.h:465
#define BUG_ON(x)
uint32_t flags
Definition: detect.h:493
_Bool(* SupportsPrefilter)(const Signature *s)
Definition: detect.h:1148
uint32_t id
Definition: detect.h:525
#define DETECT_PROTO_ANY
struct SigGroupHead_ * sh
Definition: detect.h:209
Signature ** match_array
Definition: detect.h:1320
#define DETECTDSIZE_RA
Definition: detect-dsize.h:30
#define unlikely(expr)
Definition: util-optimize.h:35
int SigAddressPrepareStage2(DetectEngineCtx *de_ctx)
Fill the global src group head, with the sigs included.
DetectPort * tcp_whitelist
Definition: detect.h:866
void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:59
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
DetectPort * udp_whitelist
Definition: detect.h:867
DetectMpmAppLayerKeyword * app_mpms
Definition: detect.h:895
void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx)
Updates the SigGroupHead->sig_cnt with the total count of all the Signatures present in this SigGroup...
int rule_engine_analysis_set
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
#define TH_RST
Definition: decode-tcp.h:37
Signature * sig_list
Definition: detect.h:726
#define TH_FIN
Definition: decode-tcp.h:35
uint32_t sig_array_size
Definition: detect.h:736
#define MIN(x, y)
IPOnlyCIDRItem * CidrSrc
Definition: detect.h:538
int SigAddressPrepareStage3(DetectEngineCtx *de_ctx)
void SCProfilingSghInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
uint8_t proto[256/8]
IPOnlyCIDRItem * CidrDst
Definition: detect.h:538
void SigCleanSignatures(DetectEngineCtx *de_ctx)
int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
finalize preparing sgh&#39;s
Container for matching data for a signature group.
Definition: detect.h:1295
#define DETECT_FLOWBITS_CMD_ISSET
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:239
const char * name
Definition: detect.h:1160
uint16_t AppProto
TCPHdr * tcph
Definition: decode.h:525
Signature container.
Definition: detect.h:492
#define SIG_MASK_REQUIRE_ENGINE_EVENT
Definition: detect.h:281
#define TRUE
uint32_t id
Definition: detect.h:1313
uint32_t signum
Definition: detect.h:739
#define SIG_FLAG_APPLAYER
Definition: detect.h:225
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:818
void IPOnlyPrepare(DetectEngineCtx *de_ctx)
Build the radix trees from the lists of parsed adresses in CIDR format the result should be 4 radix t...
int DetectEngineMultiTenantEnabled(void)
void VarNameStoreActivateStaging(void)
int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t max_idx)
Create an array with all the internal ids of the sigs that this sig group head will check for...
SigIntId num
Definition: detect.h:502
struct SigMatch_ * next
Definition: detect.h:328
main detection engine ctx
Definition: detect.h:720
int SCLogDebugEnabled(void)
Returns whether debug messages are enabled to be logged or not.
Definition: util-debug.c:631
#define SIG_MASK_REQUIRE_FLAGS_INITDEINIT
Definition: detect.h:276
void IPOnlyDeinit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Deinitialize the IP Only detection engine context.
DetectPort * udp
Definition: detect.h:659
#define SIG_FLAG_DST_ANY
Definition: detect.h:219
int SignatureIsFilesizeInspecting(const Signature *s)
Check if a signature contains the filesize keyword.
#define SIG_FLAG_MPM_NEG
Definition: detect.h:233
uint32_t sgh_array_cnt
Definition: detect.h:802
#define DE_QUIET
Definition: detect.h:298
DetectPort * DetectPortHashLookup(DetectEngineCtx *de_ctx, DetectPort *dp)
Used to lookup a DetectPort hash from the detection engine context DetectPort hash table...
#define DetectEngineGetMaxSigId(de_ctx)
Definition: detect-engine.h:81
int DetectBufferTypeGetByName(const char *name)
void SigGroupHeadStore(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
#define PORT_SIGGROUPHEAD_COPY
Definition: detect.h:195
uint16_t max_uniq_toserver_groups
Definition: detect.h:779
int SignatureIsFileSha256Inspecting(const Signature *s)
Check if a signature contains the filesha256 keyword.
#define SIG_FLAG_TOCLIENT
Definition: detect.h:244
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:58
DetectPort * dp
Definition: detect.h:531
void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need size flag in the sgh.
#define SIG_FLAG_SP_ANY
Definition: detect.h:220
uint16_t type
DetectPort * tcp
Definition: detect.h:658
uint8_t flags
Definition: detect.h:721
struct SigGroupHead_ * sgh[256]
Definition: detect.h:660
#define SIG_FLAG_IPONLY
Definition: detect.h:226
#define SIG_MASK_REQUIRE_FLOW
Definition: detect.h:275
int SigAddressCleanupStage1(DetectEngineCtx *de_ctx)
void SigGroupHeadHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by SigGroupHeadHashInit() function...
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SIG_FLAG_INIT_DEONLY
Definition: detect.h:263
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1131
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void SigFree(Signature *)
int SignatureIsFileSha1Inspecting(const Signature *s)
Check if a signature contains the filesha1 keyword.
int SigGroupBuild(DetectEngineCtx *de_ctx)
Convert the signature list into the runtime match structure.
#define SIG_FLAG_INIT_FLOW
Definition: detect.h:265
#define FILE_SIG_NEED_MAGIC
Definition: detect.h:290
Signature * SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
Find a specific signature by sid and gid.
#define TH_URG
Definition: decode-tcp.h:40
#define SIG_MASK_REQUIRE_FLAGS_UNUSUAL
Definition: detect.h:277
#define SIG_FLAG_TOSERVER
Definition: detect.h:243
#define SCEnter(...)
Definition: util-debug.h:337
int DetectPortHashAdd(DetectEngineCtx *de_ctx, DetectPort *dp)
Adds a DetectPort to the detection engine context DetectPort hash table.
int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
build an array of rule id&#39;s for sigs with no prefilter Also updated de_ctx::non_pf_store_cnt_max to t...
void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid)
uint32_t profile_match_logging_threshold
Definition: detect.h:845
PacketEngineEvents events
Definition: decode.h:570
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:544
void SigParseApplyDsizeToContent(Signature *s)
Apply dsize as depth to content matches in the rule.
Definition: detect-dsize.c:440
void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need magic flag in the sgh.
AppProto alproto
Definition: detect.h:496
struct SigMatch_ ** smlists_tail
Definition: detect.h:488
#define TH_ECN
Definition: decode-tcp.h:42
uint16_t max_uniq_toclient_groups
Definition: detect.h:778
#define DETECTDSIZE_EQ
Definition: detect-dsize.h:28
int SigGroupCleanup(DetectEngineCtx *de_ctx)
void DetectPortHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by DetectPortInit() function...
#define DETECT_FLOW_FLAG_TOCLIENT
Definition: detect-flow.h:28
struct Signature_ * next
Definition: detect.h:563
uint8_t negated
Definition: detect.h:306
uint8_t type
Definition: detect.h:325
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
const char * DetectListToString(int list)
Definition: detect-parse.c:128
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:269
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:766
#define FILE_SIG_NEED_MD5
Definition: detect.h:292
SigGroupHead * SigGroupHeadHashLookup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Used to lookup a SigGroupHead hash from the detection engine context SigGroupHead hash table...
int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHead **dst)
Copies the bitarray holding the sids from the source SigGroupHead to the destination SigGroupHead...
void DetectPortCleanupList(const DetectEngineCtx *de_ctx, DetectPort *head)
Free a DetectPort list and each of its members.
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
uint32_t gh_unique
Definition: detect.h:756
struct SigMatch_ ** smlists
Definition: detect.h:486
int DetectPortHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the DetectPort hash...
uint32_t sgh_array_size
Definition: detect.h:803
#define SIG_MASK_REQUIRE_PAYLOAD
Definition: detect.h:274
int SignatureIsFilestoring(const Signature *s)
Check if a signature contains the filestore keyword.
uint32_t smlists_array_size
Definition: detect.h:484
void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
enum DetectEnginePrefilterSetting prefilter_setting
Definition: detect.h:862
SigMatchCtx * ctx
Definition: detect.h:327
#define MASK_TCP_INITDEINIT_FLAGS
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
int SigGroupHeadHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the SigGroupHeads.
int SigGroupHeadAppendSig(const DetectEngineCtx *de_ctx, SigGroupHead **sgh, const Signature *s)
Add a Signature to a SigGroupHead.
#define SCMalloc(a)
Definition: util-mem.h:174
void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
uint8_t flags
Definition: detect.h:202
int SignatureIsFileMd5Inspecting(const Signature *s)
Check if a signature contains the filemd5 keyword.
struct IPOnlyCIDRItem_ * next
Definition: detect.h:312
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define DETECT_CONTENT_NEGATED
#define TH_SYN
Definition: decode-tcp.h:36
#define SCFree(a)
Definition: util-mem.h:236
struct SigGroupHead_ ** sgh_array
Definition: detect.h:801
void IPOnlyPrint(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Print stats of the IP Only engine.
DetectPort * sp
Definition: detect.h:531
#define PKT_IS_TCP(p)
Definition: decode.h:252
uint16_t port
Definition: detect.h:199
void SigGroupHeadSetFileHashFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need hash flag in the sgh.
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for builtin buffers that are in "single or "shared" mode. ...
SigMatchData * SigMatchList2DataArray(SigMatch *head)
convert SigMatch list to SigMatchData array
uint32_t init_flags
Definition: detect.h:456
#define FILE_SIG_NEED_SHA1
Definition: detect.h:293
int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
Test is a initialized signature is IP only.
#define DETECT_FLOWBITS_CMD_SET
int DetectPortInsert(DetectEngineCtx *de_ctx, DetectPort **head, DetectPort *new)
function for inserting a port group object. This also makes sure SigGroupContainer lists are handled ...
void SigMatchFree(SigMatch *sm)
free a SigMatch
Definition: detect-parse.c:241
void DetectPortFree(const DetectEngineCtx *de_ctx, DetectPort *dp)
Free a DetectPort and its members.
void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
#define SCLogPerf(...)
Definition: util-debug.h:261
int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int(*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
struct DetectPort_ * next
Definition: detect.h:212
#define FILE_SIG_NEED_SHA256
Definition: detect.h:294
int SignatureIsFilemagicInspecting(const Signature *s)
Check if a signature contains the filemagic keyword.
void SCProfilingPrefilterInitCounters(DetectEngineCtx *de_ctx)
Register the prefilter profiling counters.
#define PKT_HAS_FLOW
Definition: decode.h:1101
uint32_t gh_reuse
Definition: detect.h:756
int SigGroupHeadHashAdd(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Adds a SigGroupHead to the detection engine context SigGroupHead hash table.
void DetectBufferRunSetupCallback(const DetectEngineCtx *de_ctx, const int id, Signature *s)
#define SIGMATCH_IPONLY_COMPAT
Definition: detect.h:1330
const char * DetectListToHumanString(int list)
Definition: detect-parse.c:111
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1140
#define SIG_FLAG_DP_ANY
Definition: detect.h:221
void SCProfilingRuleInitCounters(DetectEngineCtx *de_ctx)
Register the rule profiling counters.
void IPOnlyInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Setup the IP Only detection engine context.
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:754
#define DETECT_FLOW_FLAG_TOSERVER
Definition: detect-flow.h:27
SignatureMask mask
Definition: detect.h:501
uint8_t file_flags
Definition: detect.h:506
int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s)
#define PKT_NOPAYLOAD_INSPECTION
Definition: decode.h:1095
uint32_t flags
Definition: decode.h:442
uint16_t port2
Definition: detect.h:200
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:555
uint16_t payload_len
Definition: decode.h:546
uint32_t sig_cnt
Definition: detect.h:727
int DetectFlagsSignatureNeedsSynOnlyPackets(const Signature *s)
Definition: detect-flags.c:526
void SigGroupHeadSetProtoAndDirection(SigGroupHead *sgh, uint8_t ipproto, int dir)
void DetectPortPrintList(DetectPort *head)
Helper function used to print the list of ports present in this DetectPort list.
#define SIG_FLAG_SRC_ANY
Definition: detect.h:218
DetectPort * DetectPortCopySingle(DetectEngineCtx *de_ctx, DetectPort *src)
Function that return a copy of DetectPort src sigs.
uint16_t flags
Definition: detect.h:1154
#define SIG_MASK_REQUIRE_DCERPC
Definition: detect.h:279
#define TH_CWR
Definition: decode-tcp.h:44
int DetectPortParse(const DetectEngineCtx *de_ctx, DetectPort **head, const char *str)
Function for parsing port strings.
const char * DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
void SigGroupHeadSetFilestoreCount(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the filestore_cnt in the sgh.
#define SIG_MASK_REQUIRE_NO_PAYLOAD
Definition: detect.h:278
SigMatch * mpm_sm
Definition: detect.h:463
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
HashListTable * dport_hash_table
Definition: detect.h:864
uint8_t * sig_array
Definition: detect.h:1277
#define FLOW_STATES
Definition: detect.h:668
int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
int DetectFlagsSignatureNeedsSynPackets(const Signature *s)
Definition: detect-flags.c:507
Port structure for detection engine.
Definition: detect.h:198
a single match condition for a signature
Definition: detect.h:324
void DetectPortPrint(DetectPort *dp)
Helper function that print the DetectPort info.
void DetectContentPropagateLimits(Signature *s)
apply depth/offset and distance/within to content matches
#define FILE_SIG_NEED_SIZE
Definition: detect.h:295
#define MASK_TCP_UNUSUAL_FLAGS
SigGroupHeadInitData * init
Definition: detect.h:1323