suricata
detect-engine-build.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2018 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "detect.h"
20 #include "detect-engine.h"
21 #include "detect-parse.h"
22 
23 #include "detect-engine-address.h"
24 #include "detect-engine-analyzer.h"
25 #include "detect-engine-iponly.h"
26 #include "detect-engine-mpm.h"
27 #include "detect-engine-siggroup.h"
28 #include "detect-engine-port.h"
30 #include "detect-engine-proto.h"
31 
32 #include "detect-dsize.h"
33 #include "detect-tcp-flags.h"
34 #include "detect-flow.h"
35 #include "detect-flowbits.h"
36 
37 #include "util-profiling.h"
38 
40 {
41  if (de_ctx == NULL)
42  return;
43 
44  for (Signature *s = de_ctx->sig_list; s != NULL;) {
45  Signature *ns = s->next;
46  SigFree(s);
47  s = ns;
48  }
49  de_ctx->sig_list = NULL;
50 
52  de_ctx->sig_list = NULL;
53 }
54 
55 /** \brief Find a specific signature by sid and gid
56  * \param de_ctx detection engine ctx
57  * \param sid the signature id
58  * \param gid the signature group id
59  *
60  * \retval s sig found
61  * \retval NULL sig not found
62  */
63 Signature *SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
64 {
65  if (de_ctx == NULL)
66  return NULL;
67 
68  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
69  if (s->id == sid && s->gid == gid)
70  return s;
71  }
72 
73  return NULL;
74 }
75 
76 /**
77  * \brief Check if a signature contains the filestore keyword.
78  *
79  * \param s signature
80  *
81  * \retval 0 no
82  * \retval 1 yes
83  */
85 {
86  if (s == NULL)
87  return 0;
88 
89  if (s->flags & SIG_FLAG_FILESTORE)
90  return 1;
91 
92  return 0;
93 }
94 
95 /**
96  * \brief Check if a signature contains the filemagic keyword.
97  *
98  * \param s signature
99  *
100  * \retval 0 no
101  * \retval 1 yes
102  */
104 {
105  if (s == NULL)
106  return 0;
107 
109  return 1;
110 
111  return 0;
112 }
113 
114 /**
115  * \brief Check if a signature contains the filemd5 keyword.
116  *
117  * \param s signature
118  *
119  * \retval 0 no
120  * \retval 1 yes
121  */
123 {
124  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5))
125  return 1;
126 
127  return 0;
128 }
129 
130 /**
131  * \brief Check if a signature contains the filesha1 keyword.
132  *
133  * \param s signature
134  *
135  * \retval 0 no
136  * \retval 1 yes
137  */
139 {
140  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1))
141  return 1;
142 
143  return 0;
144 }
145 
146 /**
147  * \brief Check if a signature contains the filesha256 keyword.
148  *
149  * \param s signature
150  *
151  * \retval 0 no
152  * \retval 1 yes
153  */
155 {
156  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256))
157  return 1;
158 
159  return 0;
160 }
161 
162 /**
163  * \brief Check if a signature contains the filesize keyword.
164  *
165  * \param s signature
166  *
167  * \retval 0 no
168  * \retval 1 yes
169  */
171 {
172  if (s == NULL)
173  return 0;
174 
176  return 1;
177 
178  return 0;
179 }
180 
181 /** \brief Test is a initialized signature is IP only
182  * \param de_ctx detection engine ctx
183  * \param s the signature
184  * \retval 1 sig is ip only
185  * \retval 0 sig is not ip only
186  */
188 {
189  if (s->alproto != ALPROTO_UNKNOWN)
190  return 0;
191 
192  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
193  return 0;
194 
195  /* if flow dir is set we can't process it in ip-only */
196  if (!(((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) ||
199  return 0;
200 
201  /* for now assume that all registered buffer types are incompatible */
202  const int nlists = s->init_data->smlists_array_size;
203  for (int i = 0; i < nlists; i++) {
204  if (s->init_data->smlists[i] == NULL)
205  continue;
206  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
207  continue;
208 
209  SCReturnInt(0);
210  }
211 
212  /* TMATCH list can be ignored, it contains TAGs and
213  * tags are compatible to IP-only. */
214 
215  /* if any of the addresses uses negation, we don't support
216  * it in ip-only */
218  return 0;
220  return 0;
221 
223  if (sm == NULL)
224  goto iponly;
225 
226  for ( ; sm != NULL; sm = sm->next) {
228  return 0;
229  /* we have enabled flowbits to be compatible with ip only sigs, as long
230  * as the sig only has a "set" flowbits */
231  if (sm->type == DETECT_FLOWBITS &&
232  (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
233  return 0;
234  }
235  }
236 
237 iponly:
238  if (!(de_ctx->flags & DE_QUIET)) {
239  SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
240  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
241  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
242  }
243  return 1;
244 }
245 
246 /** \internal
247  * \brief Test is a initialized signature is inspecting protocol detection only
248  * \param de_ctx detection engine ctx
249  * \param s the signature
250  * \retval 1 sig is dp only
251  * \retval 0 sig is not dp only
252  */
253 static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s)
254 {
255  if (s->alproto != ALPROTO_UNKNOWN)
256  return 0;
257 
258  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
259  return 0;
260 
261  /* for now assume that all registered buffer types are incompatible */
262  const int nlists = s->init_data->smlists_array_size;
263  for (int i = 0; i < nlists; i++) {
264  if (s->init_data->smlists[i] == NULL)
265  continue;
266  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
267  continue;
268 
269  SCReturnInt(0);
270  }
271 
272  /* TMATCH list can be ignored, it contains TAGs and
273  * tags are compatible to DP-only. */
274 
275  /* match list matches may be compatible to DP only. We follow the same
276  * logic as IP-only so we can use that flag */
277 
279  if (sm == NULL)
280  return 0;
281 
282  int pd = 0;
283  for ( ; sm != NULL; sm = sm->next) {
284  if (sm->type == DETECT_AL_APP_LAYER_PROTOCOL) {
285  pd = 1;
286  } else {
287  /* flowbits are supported for dp only sigs, as long
288  * as the sig only has a "set" flowbits */
289  if (sm->type == DETECT_FLOWBITS) {
290  if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
291  SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id);
292  return 0;
293  }
294  } else if (sm->type == DETECT_FLOW) {
296  SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id);
297  return 0;
298  }
299  } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) {
300  SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name);
301  return 0;
302  }
303  }
304  }
305 
306  if (pd) {
307  SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id);
308  }
309  return pd;
310 }
311 
312 /**
313  * \internal
314  * \brief Check if the initialized signature is inspecting the packet payload
315  * \param de_ctx detection engine ctx
316  * \param s the signature
317  * \retval 1 sig is inspecting the payload
318  * \retval 0 sig is not inspecting the payload
319  */
320 static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s)
321 {
322 
323  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
324  return 1;
325  }
326  return 0;
327 }
328 
329 /**
330  * \internal
331  * \brief check if a signature is decoder event matching only
332  * \param de_ctx detection engine
333  * \param s the signature to test
334  * \retval 0 not a DEOnly sig
335  * \retval 1 DEOnly sig
336  */
337 static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
338 {
339  if (s->alproto != ALPROTO_UNKNOWN) {
340  SCReturnInt(0);
341  }
342 
343  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
344  {
345  SCReturnInt(0);
346  }
347 
348  /* for now assume that all registered buffer types are incompatible */
349  const int nlists = s->init_data->smlists_array_size;
350  for (int i = 0; i < nlists; i++) {
351  if (s->init_data->smlists[i] == NULL)
352  continue;
353  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
354  continue;
355 
356  SCReturnInt(0);
357  }
358 
359  /* check for conflicting keywords */
361  for ( ;sm != NULL; sm = sm->next) {
363  SCReturnInt(0);
364  }
365 
366  /* need at least one decode event keyword to be considered decode event. */
368  for ( ;sm != NULL; sm = sm->next) {
369  if (sm->type == DETECT_DECODE_EVENT)
370  goto deonly;
371  if (sm->type == DETECT_ENGINE_EVENT)
372  goto deonly;
373  if (sm->type == DETECT_STREAM_EVENT)
374  goto deonly;
375  }
376 
377  SCReturnInt(0);
378 
379 deonly:
380  if (!(de_ctx->flags & DE_QUIET)) {
381  SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
382  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
383  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
384  }
385 
386  SCReturnInt(1);
387 }
388 
389 #define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN)
390 #define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR)
391 
392 /* Create mask for this packet + it's flow if it has one
393  */
394 void
396  bool app_decoder_events)
397 {
398  if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
399  SCLogDebug("packet has payload");
400  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
401  } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
402  SCLogDebug("stream data available");
403  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
404  } else {
405  SCLogDebug("packet has no payload");
406  (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
407  }
408 
409  if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) {
410  SCLogDebug("packet/flow has events set");
412  }
413 
414  if (!(PKT_IS_PSEUDOPKT(p)) && PKT_IS_TCP(p)) {
415  if ((p->tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) {
417  }
418  if ((p->tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) {
420  }
421  }
422 
423  if (p->flags & PKT_HAS_FLOW) {
424  SCLogDebug("packet has flow");
425  (*mask) |= SIG_MASK_REQUIRE_FLOW;
426  }
427 
428  if (alproto == ALPROTO_SMB || alproto == ALPROTO_DCERPC) {
429  SCLogDebug("packet will be inspected for DCERPC");
430  (*mask) |= SIG_MASK_REQUIRE_DCERPC;
431  }
432 }
433 
434 static int g_dce_generic_list_id = -1;
435 static int g_dce_stub_data_buffer_id = -1;
436 
437 static bool SignatureNeedsDCERPCMask(const Signature *s)
438 {
439  if (g_dce_generic_list_id == -1) {
440  g_dce_generic_list_id = DetectBufferTypeGetByName("dce_generic");
441  SCLogDebug("g_dce_generic_list_id %d", g_dce_generic_list_id);
442  }
443  if (g_dce_stub_data_buffer_id == -1) {
444  g_dce_stub_data_buffer_id = DetectBufferTypeGetByName("dce_stub_data");
445  SCLogDebug("g_dce_stub_data_buffer_id %d", g_dce_stub_data_buffer_id);
446  }
447 
448  if (g_dce_generic_list_id >= 0 &&
449  s->init_data->smlists[g_dce_generic_list_id] != NULL)
450  {
451  return true;
452  }
453  if (g_dce_stub_data_buffer_id >= 0 &&
454  s->init_data->smlists[g_dce_stub_data_buffer_id] != NULL)
455  {
456  return true;
457  }
458  return false;
459 }
460 
461 static int SignatureCreateMask(Signature *s)
462 {
463  SCEnter();
464 
465  if (SignatureNeedsDCERPCMask(s)) {
467  SCLogDebug("sig requires DCERPC");
468  }
469 
470  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
472  SCLogDebug("sig requires payload");
473  }
474 
475  SigMatch *sm;
476  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
477  switch(sm->type) {
478  case DETECT_FLOWBITS:
479  {
480  /* figure out what flowbit action */
482  if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
483  /* not a mask flag, but still set it here */
485 
486  SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has "
487  "flowbit isset option.");
488  }
489 
490  /* flow is required for any flowbit manipulation */
492  SCLogDebug("sig requires flow to be able to manipulate "
493  "flowbit(s)");
494  break;
495  }
496  case DETECT_FLOWINT:
497  /* flow is required for any flowint manipulation */
499  SCLogDebug("sig requires flow to be able to manipulate "
500  "flowint(s)");
501  break;
502  case DETECT_FLAGS:
503  {
504  DetectFlagsData *fl = (DetectFlagsData *)sm->ctx;
505 
506  if (fl->flags & TH_SYN) {
508  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
509  }
510  if (fl->flags & TH_RST) {
512  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
513  }
514  if (fl->flags & TH_FIN) {
516  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
517  }
518  if (fl->flags & TH_URG) {
520  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
521  }
522  if (fl->flags & TH_ECN) {
524  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
525  }
526  if (fl->flags & TH_CWR) {
528  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
529  }
530  break;
531  }
532  case DETECT_DSIZE:
533  {
534  DetectDsizeData *ds = (DetectDsizeData *)sm->ctx;
535  switch (ds->mode) {
536  case DETECTDSIZE_LT:
537  /* LT will include 0, so no payload.
538  * if GT is used in the same rule the
539  * flag will be set anyway. */
540  break;
541  case DETECTDSIZE_RA:
542  case DETECTDSIZE_GT:
544  SCLogDebug("sig requires payload");
545  break;
546  case DETECTDSIZE_EQ:
547  if (ds->dsize > 0) {
549  SCLogDebug("sig requires payload");
550  } else if (ds->dsize == 0) {
552  SCLogDebug("sig requires no payload");
553  }
554  break;
555  }
556  break;
557  }
560  break;
561  case DETECT_ENGINE_EVENT:
563  break;
564  }
565  }
566 
569  SCLogDebug("sig requires flow");
570  }
571 
572  if (s->flags & SIG_FLAG_APPLAYER) {
574  SCLogDebug("sig requires flow");
575  }
576 
577  SCLogDebug("mask %02X", s->mask);
578  SCReturnInt(0);
579 }
580 
581 static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
582 {
584 }
585 
586 /** \brief Pure-PCRE or bytetest rule */
587 static int RuleInspectsPayloadHasNoMpm(const Signature *s)
588 {
589  if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
590  return 1;
591  return 0;
592 }
593 
594 static int RuleGetMpmPatternSize(const Signature *s)
595 {
596  if (s->init_data->mpm_sm == NULL)
597  return -1;
598  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
599  if (mpm_list < 0)
600  return -1;
601  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
602  if (cd == NULL)
603  return -1;
604  return (int)cd->content_len;
605 }
606 
607 static int RuleMpmIsNegated(const Signature *s)
608 {
609  if (s->init_data->mpm_sm == NULL)
610  return 0;
611  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
612  if (mpm_list < 0)
613  return 0;
614  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
615  if (cd == NULL)
616  return 0;
617  return (cd->flags & DETECT_CONTENT_NEGATED);
618 }
619 
620 #ifdef HAVE_LIBJANSSON
621 static json_t *RulesGroupPrintSghStats(const SigGroupHead *sgh,
622  const int add_rules, const int add_mpm_stats)
623 {
624  uint32_t mpm_cnt = 0;
625  uint32_t nonmpm_cnt = 0;
626  uint32_t negmpm_cnt = 0;
627  uint32_t any5_cnt = 0;
628  uint32_t payload_no_mpm_cnt = 0;
629  uint32_t syn_cnt = 0;
630 
631  uint32_t mpms_total = 0;
632  uint32_t mpms_min = 0;
633  uint32_t mpms_max = 0;
634 
635  struct {
636  uint32_t total;
637  uint32_t cnt;
638  uint32_t min;
639  uint32_t max;
640  } mpm_stats[DETECT_SM_LIST_MAX];
641  memset(mpm_stats, 0x00, sizeof(mpm_stats));
642 
643  uint32_t alstats[ALPROTO_MAX] = {0};
644  uint32_t mpm_sizes[DETECT_SM_LIST_MAX][256];
645  memset(mpm_sizes, 0, sizeof(mpm_sizes));
646  uint32_t alproto_mpm_bufs[ALPROTO_MAX][DETECT_SM_LIST_MAX];
647  memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs));
648 
649  json_t *js = json_object();
650  if (unlikely(js == NULL))
651  return NULL;
652 
653  json_object_set_new(js, "id", json_integer(sgh->id));
654 
655  json_t *js_array = json_array();
656 
657  const Signature *s;
658  uint32_t x;
659  for (x = 0; x < sgh->sig_cnt; x++) {
660  s = sgh->match_array[x];
661  if (s == NULL)
662  continue;
663 
664  int any = 0;
665  if (s->proto.flags & DETECT_PROTO_ANY) {
666  any++;
667  }
668  if (s->flags & SIG_FLAG_DST_ANY) {
669  any++;
670  }
671  if (s->flags & SIG_FLAG_SRC_ANY) {
672  any++;
673  }
674  if (s->flags & SIG_FLAG_DP_ANY) {
675  any++;
676  }
677  if (s->flags & SIG_FLAG_SP_ANY) {
678  any++;
679  }
680  if (any == 5) {
681  any5_cnt++;
682  }
683 
684  if (s->init_data->mpm_sm == NULL) {
685  nonmpm_cnt++;
686 
687  if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) {
688  SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id);
689  }
690 
691  DetectPort *sp = s->sp;
692  DetectPort *dp = s->dp;
693 
694  if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) {
695  SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id);
696  }
697  if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) {
698  SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id);
699  }
700 
702  syn_cnt++;
703  }
704 
705  } else {
706  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
707  BUG_ON(mpm_list < 0);
708  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
709  uint32_t size = cd->content_len < 256 ? cd->content_len : 255;
710 
711  mpm_sizes[mpm_list][size]++;
712  if (s->alproto != ALPROTO_UNKNOWN) {
713  alproto_mpm_bufs[s->alproto][mpm_list]++;
714  }
715 
716  if (mpm_list == DETECT_SM_LIST_PMATCH) {
717  if (size == 1) {
718  DetectPort *sp = s->sp;
719  DetectPort *dp = s->dp;
720  if (s->flags & SIG_FLAG_TOSERVER) {
721  if (dp->port == 0 && dp->port2 == 65535) {
722  SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
723  } else {
724  SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id);
725  }
726  }
727  if (s->flags & SIG_FLAG_TOCLIENT) {
728  if (sp->port == 0 && sp->port2 == 65535) {
729  SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
730  } else {
731  SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id);
732  }
733  }
734  }
735  }
736 
737  uint32_t w = PatternStrength(cd->content, cd->content_len);
738  mpms_total += w;
739  if (mpms_min == 0)
740  mpms_min = w;
741  if (w < mpms_min)
742  mpms_min = w;
743  if (w > mpms_max)
744  mpms_max = w;
745 
746  mpm_stats[mpm_list].total += w;
747  mpm_stats[mpm_list].cnt++;
748  if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min)
749  mpm_stats[mpm_list].min = w;
750  if (w > mpm_stats[mpm_list].max)
751  mpm_stats[mpm_list].max = w;
752 
753  mpm_cnt++;
754 
755  if (w < 10) {
756  SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id);
757  }
758  if (w < 10 && any == 5) {
759  SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id);
760  }
761 
762  if (cd->flags & DETECT_CONTENT_NEGATED) {
763  SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id);
764  negmpm_cnt++;
765  }
766  }
767 
768  if (RuleInspectsPayloadHasNoMpm(s)) {
769  SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id);
770  payload_no_mpm_cnt++;
771  }
772 
773  if (s->alproto != ALPROTO_UNKNOWN) {
774  alstats[s->alproto]++;
775  }
776 
777  if (add_rules) {
778  json_t *js_sig = json_object();
779  if (unlikely(js == NULL))
780  continue;
781  json_object_set_new(js_sig, "sig_id", json_integer(s->id));
782  json_array_append_new(js_array, js_sig);
783  }
784  }
785 
786  json_object_set_new(js, "rules", js_array);
787 
788  json_t *stats = json_object();
789  json_object_set_new(stats, "total", json_integer(sgh->sig_cnt));
790 
791  json_t *types = json_object();
792  json_object_set_new(types, "mpm", json_integer(mpm_cnt));
793  json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt));
794  json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt));
795  json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt));
796  json_object_set_new(types, "syn", json_integer(syn_cnt));
797  json_object_set_new(types, "any5", json_integer(any5_cnt));
798  json_object_set_new(stats, "types", types);
799 
800  int i;
801  for (i = 0; i < ALPROTO_MAX; i++) {
802  if (alstats[i] > 0) {
803  json_t *app = json_object();
804  json_object_set_new(app, "total", json_integer(alstats[i]));
805 
806  for (x = 0; x < DETECT_SM_LIST_MAX; x++) {
807  if (alproto_mpm_bufs[i][x] == 0)
808  continue;
809  json_object_set_new(app, DetectListToHumanString(x), json_integer(alproto_mpm_bufs[i][x]));
810  }
811 
812  json_object_set_new(stats, AppProtoToString(i), app);
813  }
814  }
815 
816  if (add_mpm_stats) {
817  json_t *mpm_js = json_object();
818 
819  for (i = 0; i < DETECT_SM_LIST_MAX; i++) {
820  if (mpm_stats[i].cnt > 0) {
821 
822  json_t *mpm_sizes_array = json_array();
823  for (x = 0; x < 256; x++) {
824  if (mpm_sizes[i][x] == 0)
825  continue;
826 
827  json_t *e = json_object();
828  json_object_set_new(e, "size", json_integer(x));
829  json_object_set_new(e, "count", json_integer(mpm_sizes[i][x]));
830  json_array_append_new(mpm_sizes_array, e);
831  }
832 
833  json_t *buf = json_object();
834  json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt));
835  json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt));
836  json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min));
837  json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max));
838 
839  json_object_set_new(buf, "sizes", mpm_sizes_array);
840 
841  json_object_set_new(mpm_js, DetectListToHumanString(i), buf);
842  }
843  }
844 
845  json_object_set_new(stats, "mpm", mpm_js);
846  }
847  json_object_set_new(js, "stats", stats);
848 
849  json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist));
850 
851  return js;
852 }
853 #endif /* HAVE_LIBJANSSON */
854 
855 static void RulesDumpGrouping(const DetectEngineCtx *de_ctx,
856  const int add_rules, const int add_mpm_stats)
857 {
858 #ifdef HAVE_LIBJANSSON
859  json_t *js = json_object();
860  if (unlikely(js == NULL))
861  return;
862 
863  int p;
864  for (p = 0; p < 256; p++) {
865  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
866  const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp";
867 
868  json_t *tcp = json_object();
869 
870  json_t *ts_array = json_array();
871  DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp :
872  de_ctx->flow_gh[1].udp;
873  while (list != NULL) {
874  json_t *port = json_object();
875  json_object_set_new(port, "port", json_integer(list->port));
876  json_object_set_new(port, "port2", json_integer(list->port2));
877 
878  json_t *tcp_ts = RulesGroupPrintSghStats(list->sh,
879  add_rules, add_mpm_stats);
880  json_object_set_new(port, "rulegroup", tcp_ts);
881  json_array_append_new(ts_array, port);
882 
883  list = list->next;
884  }
885  json_object_set_new(tcp, "toserver", ts_array);
886 
887  json_t *tc_array = json_array();
888  list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp :
889  de_ctx->flow_gh[0].udp;
890  while (list != NULL) {
891  json_t *port = json_object();
892  json_object_set_new(port, "port", json_integer(list->port));
893  json_object_set_new(port, "port2", json_integer(list->port2));
894 
895  json_t *tcp_tc = RulesGroupPrintSghStats(list->sh,
896  add_rules, add_mpm_stats);
897  json_object_set_new(port, "rulegroup", tcp_tc);
898  json_array_append_new(tc_array, port);
899 
900  list = list->next;
901  }
902  json_object_set_new(tcp, "toclient", tc_array);
903 
904  json_object_set_new(js, name, tcp);
905  }
906 
907  }
908 
909  const char *filename = "rule_group.json";
910  const char *log_dir = ConfigGetLogDirectory();
911  char log_path[PATH_MAX] = "";
912 
913  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
914 
915  FILE *fp = fopen(log_path, "w");
916  if (fp == NULL) {
917  return;
918  }
919 
920  char *js_s = json_dumps(js,
921  JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH);
922  if (unlikely(js_s == NULL)) {
923  fclose(fp);
924  return;
925  }
926 
927  json_object_clear(js);
928  json_decref(js);
929 
930  fprintf(fp, "%s\n", js_s);
931  free(js_s);
932  fclose(fp);
933 #endif
934  return;
935 }
936 
937 static int RulesGroupByProto(DetectEngineCtx *de_ctx)
938 {
939  Signature *s = de_ctx->sig_list;
940 
941  uint32_t max_idx = 0;
942  SigGroupHead *sgh_ts[256] = {NULL};
943  SigGroupHead *sgh_tc[256] = {NULL};
944 
945  for ( ; s != NULL; s = s->next) {
946  if (s->flags & SIG_FLAG_IPONLY)
947  continue;
948 
949  int p;
950  for (p = 0; p < 256; p++) {
951  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
952  continue;
953  }
954  if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) {
955  continue;
956  }
957 
958  if (s->flags & SIG_FLAG_TOCLIENT) {
959  SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s);
960  max_idx = s->num;
961  }
962  if (s->flags & SIG_FLAG_TOSERVER) {
963  SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s);
964  max_idx = s->num;
965  }
966  }
967  }
968  SCLogDebug("max_idx %u", max_idx);
969 
970  /* lets look at deduplicating this list */
971  SigGroupHeadHashFree(de_ctx);
972  SigGroupHeadHashInit(de_ctx);
973 
974  uint32_t cnt = 0;
975  uint32_t own = 0;
976  uint32_t ref = 0;
977  int p;
978  for (p = 0; p < 256; p++) {
979  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
980  continue;
981  if (sgh_ts[p] == NULL)
982  continue;
983 
984  cnt++;
985 
986  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]);
987  if (lookup_sgh == NULL) {
988  SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]);
989 
990  SigGroupHeadSetSigCnt(sgh_ts[p], max_idx);
991  SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx);
992 
993  SigGroupHeadHashAdd(de_ctx, sgh_ts[p]);
994  SigGroupHeadStore(de_ctx, sgh_ts[p]);
995 
996  de_ctx->gh_unique++;
997  own++;
998  } else {
999  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]);
1000 
1001  SigGroupHeadFree(de_ctx, sgh_ts[p]);
1002  sgh_ts[p] = lookup_sgh;
1003 
1004  de_ctx->gh_reuse++;
1005  ref++;
1006  }
1007  }
1008  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1009  "toserver", cnt, own, ref);
1010 
1011  cnt = 0;
1012  own = 0;
1013  ref = 0;
1014  for (p = 0; p < 256; p++) {
1015  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1016  continue;
1017  if (sgh_tc[p] == NULL)
1018  continue;
1019 
1020  cnt++;
1021 
1022  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]);
1023  if (lookup_sgh == NULL) {
1024  SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]);
1025 
1026  SigGroupHeadSetSigCnt(sgh_tc[p], max_idx);
1027  SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx);
1028 
1029  SigGroupHeadHashAdd(de_ctx, sgh_tc[p]);
1030  SigGroupHeadStore(de_ctx, sgh_tc[p]);
1031 
1032  de_ctx->gh_unique++;
1033  own++;
1034 
1035  } else {
1036  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]);
1037 
1038  SigGroupHeadFree(de_ctx, sgh_tc[p]);
1039  sgh_tc[p] = lookup_sgh;
1040 
1041  de_ctx->gh_reuse++;
1042  ref++;
1043  }
1044  }
1045  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1046  "toclient", cnt, own, ref);
1047 
1048  for (p = 0; p < 256; p++) {
1049  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1050  continue;
1051 
1052  de_ctx->flow_gh[0].sgh[p] = sgh_tc[p];
1053  de_ctx->flow_gh[1].sgh[p] = sgh_ts[p];
1054  }
1055 
1056  return 0;
1057 }
1058 
1059 static int PortIsWhitelisted(const DetectEngineCtx *de_ctx,
1060  const DetectPort *a, int ipproto)
1061 {
1062  DetectPort *w = de_ctx->tcp_whitelist;
1063  if (ipproto == IPPROTO_UDP)
1064  w = de_ctx->udp_whitelist;
1065 
1066  while (w) {
1067  if (a->port >= w->port && a->port2 <= w->port) {
1068  SCLogDebug("port group %u:%u whitelisted -> %d", a->port, a->port2, w->port);
1069  return 1;
1070  }
1071  w = w->next;
1072  }
1073 
1074  return 0;
1075 }
1076 
1077 static int RuleSetWhitelist(Signature *s)
1078 {
1079  DetectPort *p = NULL;
1080  if (s->flags & SIG_FLAG_TOSERVER)
1081  p = s->dp;
1082  else if (s->flags & SIG_FLAG_TOCLIENT)
1083  p = s->sp;
1084  else
1085  return 0;
1086 
1087  /* for sigs that don't use 'any' as port, see if we want to
1088  * whitelist poor sigs */
1089  int wl = 0;
1090  if (!(p->port == 0 && p->port2 == 65535)) {
1091  /* pure pcre, bytetest, etc rules */
1092  if (RuleInspectsPayloadHasNoMpm(s)) {
1093  SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id);
1094  wl = 99;
1095 
1096  } else if (RuleMpmIsNegated(s)) {
1097  SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id);
1098  wl = 77;
1099 
1100  /* one byte pattern in packet/stream payloads */
1101  } else if (s->init_data->mpm_sm != NULL &&
1103  RuleGetMpmPatternSize(s) == 1)
1104  {
1105  SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id);
1106  wl = 55;
1107 
1108  } else if (DetectFlagsSignatureNeedsSynPackets(s) &&
1110  {
1111  SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id);
1112  wl = 33;
1113  }
1114  }
1115 
1116  s->init_data->whitelist = wl;
1117  return wl;
1118 }
1119 
1120 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx);
1122 
1123 static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint32_t direction) {
1124  /* step 1: create a hash of 'DetectPort' objects based on all the
1125  * rules. Each object will have a SGH with the sigs added
1126  * that belong to the SGH. */
1127  DetectPortHashInit(de_ctx);
1128 
1129  uint32_t max_idx = 0;
1130  const Signature *s = de_ctx->sig_list;
1131  DetectPort *list = NULL;
1132  while (s) {
1133  /* IP Only rules are handled separately */
1134  if (s->flags & SIG_FLAG_IPONLY)
1135  goto next;
1136  if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY)))
1137  goto next;
1138  if (direction == SIG_FLAG_TOSERVER) {
1139  if (!(s->flags & SIG_FLAG_TOSERVER))
1140  goto next;
1141  } else if (direction == SIG_FLAG_TOCLIENT) {
1142  if (!(s->flags & SIG_FLAG_TOCLIENT))
1143  goto next;
1144  }
1145 
1146  DetectPort *p = NULL;
1147  if (direction == SIG_FLAG_TOSERVER)
1148  p = s->dp;
1149  else if (direction == SIG_FLAG_TOCLIENT)
1150  p = s->sp;
1151  else
1152  BUG_ON(1);
1153 
1154  /* see if we want to exclude directionless sigs that really care only for
1155  * to_server syn scans/floods */
1156  if ((direction == SIG_FLAG_TOCLIENT) &&
1160  (!(s->dp->port == 0 && s->dp->port2 == 65535)))
1161  {
1162  SCLogWarning(SC_WARN_POOR_RULE, "rule %u: SYN-only to port(s) %u:%u "
1163  "w/o direction specified, disabling for toclient direction",
1164  s->id, s->dp->port, s->dp->port2);
1165  goto next;
1166  }
1167 
1168  int wl = s->init_data->whitelist;
1169  while (p) {
1170  int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? 111 : 0;
1171  pwl = MAX(wl,pwl);
1172 
1173  DetectPort *lookup = DetectPortHashLookup(de_ctx, p);
1174  if (lookup) {
1175  SigGroupHeadAppendSig(de_ctx, &lookup->sh, s);
1176  lookup->sh->init->whitelist = MAX(lookup->sh->init->whitelist, pwl);
1177  } else {
1178  DetectPort *tmp2 = DetectPortCopySingle(de_ctx, p);
1179  BUG_ON(tmp2 == NULL);
1180  SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s);
1181  tmp2->sh->init->whitelist = pwl;
1182  DetectPortHashAdd(de_ctx, tmp2);
1183  }
1184 
1185  p = p->next;
1186  }
1187  max_idx = s->num;
1188  next:
1189  s = s->next;
1190  }
1191 
1192  /* step 2: create a list of DetectPort objects */
1193  HashListTableBucket *htb = NULL;
1194  for (htb = HashListTableGetListHead(de_ctx->dport_hash_table);
1195  htb != NULL;
1196  htb = HashListTableGetListNext(htb))
1197  {
1199  DetectPort *tmp = DetectPortCopySingle(de_ctx, p);
1200  BUG_ON(tmp == NULL);
1201  int r = DetectPortInsert(de_ctx, &list , tmp);
1202  BUG_ON(r == -1);
1203  }
1204  DetectPortHashFree(de_ctx);
1205  de_ctx->dport_hash_table = NULL;
1206 
1207  SCLogDebug("rules analyzed");
1208 
1209  /* step 3: group the list and shrink it if necessary */
1210  DetectPort *newlist = NULL;
1211  uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups :
1212  de_ctx->max_uniq_toserver_groups;
1213  CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx);
1214  list = newlist;
1215 
1216  /* step 4: deduplicate the SGH's */
1217  SigGroupHeadHashFree(de_ctx);
1218  SigGroupHeadHashInit(de_ctx);
1219 
1220  uint32_t cnt = 0;
1221  uint32_t own = 0;
1222  uint32_t ref = 0;
1223  DetectPort *iter;
1224  for (iter = list ; iter != NULL; iter = iter->next) {
1225  BUG_ON (iter->sh == NULL);
1226  cnt++;
1227 
1228  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh);
1229  if (lookup_sgh == NULL) {
1230  SCLogDebug("port group %p sgh %p is the original", iter, iter->sh);
1231 
1232  SigGroupHeadSetSigCnt(iter->sh, max_idx);
1233  SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx);
1234  SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction);
1235  SigGroupHeadHashAdd(de_ctx, iter->sh);
1236  SigGroupHeadStore(de_ctx, iter->sh);
1237  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1238  de_ctx->gh_unique++;
1239  own++;
1240  } else {
1241  SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh);
1242 
1243  SigGroupHeadFree(de_ctx, iter->sh);
1244  iter->sh = lookup_sgh;
1245  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1246 
1247  de_ctx->gh_reuse++;
1248  ref++;
1249  }
1250  }
1251 #if 0
1252  for (iter = list ; iter != NULL; iter = iter->next) {
1253  SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)",
1254  iter->port, iter->port2, iter->sh,
1255  iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own",
1256  iter->sh->init->whitelist ? "true" : "false",
1257  iter->sh->init->whitelist);
1258  }
1259 #endif
1260  SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies",
1261  ipproto == 6 ? "TCP" : "UDP",
1262  direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1263  cnt, own, ref);
1264  return list;
1265 }
1266 
1268 {
1269  /* see if the sig is dp only */
1270  if (SignatureIsPDOnly(de_ctx, s) == 1) {
1271  s->flags |= SIG_FLAG_PDONLY;
1272 
1273  /* see if the sig is ip only */
1274  } else if (SignatureIsIPOnly(de_ctx, s) == 1) {
1275  s->flags |= SIG_FLAG_IPONLY;
1276 
1277  } else if (SignatureIsDEOnly(de_ctx, s) == 1) {
1279  }
1280 }
1281 
1282 /**
1283  * \brief Preprocess signature, classify ip-only, etc, build sig array
1284  *
1285  * \param de_ctx Pointer to the Detection Engine Context
1286  *
1287  * \retval 0 on success
1288  * \retval -1 on failure
1289  */
1291 {
1292  uint32_t cnt_iponly = 0;
1293  uint32_t cnt_payload = 0;
1294  uint32_t cnt_applayer = 0;
1295  uint32_t cnt_deonly = 0;
1296 
1297  if (!(de_ctx->flags & DE_QUIET)) {
1298  SCLogDebug("building signature grouping structure, stage 1: "
1299  "preprocessing rules...");
1300  }
1301 
1302  de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx);
1303  de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *));
1304  de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size);
1305  if (de_ctx->sig_array == NULL)
1306  goto error;
1307  memset(de_ctx->sig_array,0,de_ctx->sig_array_size);
1308 
1309  SCLogDebug("signature lookup array: %" PRIu32 " sigs, %" PRIu32 " bytes",
1310  de_ctx->sig_array_len, de_ctx->sig_array_size);
1311 
1312  /* now for every rule add the source group */
1313  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1314  de_ctx->sig_array[s->num] = s;
1315 
1316  SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", s->id, s->num, s, de_ctx->sig_array[s->num]);
1317 
1318  if (s->flags & SIG_FLAG_PDONLY) {
1319  SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", s->id);
1320  } else if (s->flags & SIG_FLAG_IPONLY) {
1321  SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", s->id);
1322  cnt_iponly++;
1323  } else if (SignatureIsInspectingPayload(de_ctx, s) == 1) {
1324  SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", s->id);
1325  cnt_payload++;
1326  } else if (s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) {
1327  SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", s->id);
1328  cnt_deonly++;
1329  } else if (s->flags & SIG_FLAG_APPLAYER) {
1330  SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", s->id);
1331  cnt_applayer++;
1332  }
1333 
1334 #ifdef DEBUG
1335  if (SCLogDebugEnabled()) {
1336  uint16_t colen = 0;
1337  char copresent = 0;
1338  SigMatch *sm;
1339  DetectContentData *co;
1340  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) {
1341  if (sm->type != DETECT_CONTENT)
1342  continue;
1343 
1344  copresent = 1;
1345  co = (DetectContentData *)sm->ctx;
1346  if (co->content_len > colen)
1347  colen = co->content_len;
1348  }
1349 
1350  if (copresent && colen == 1) {
1351  SCLogDebug("signature %8u content maxlen 1", s->id);
1352  for (int proto = 0; proto < 256; proto++) {
1353  if (s->proto.proto[(proto/8)] & (1<<(proto%8)))
1354  SCLogDebug("=> proto %" PRId32 "", proto);
1355  }
1356  }
1357  }
1358 #endif /* DEBUG */
1359 
1360  if (RuleMpmIsNegated(s)) {
1361  s->flags |= SIG_FLAG_MPM_NEG;
1362  }
1363 
1364  SignatureCreateMask(s);
1367 
1368  RuleSetWhitelist(s);
1369 
1370  /* if keyword engines are enabled in the config, handle them here */
1371  if (de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
1372  !(s->flags & SIG_FLAG_PREFILTER))
1373  {
1374  int prefilter_list = DETECT_TBLSIZE;
1375 
1376  /* get the keyword supporting prefilter with the lowest type */
1377  for (int i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1378  SigMatch *sm = s->init_data->smlists[i];
1379  while (sm != NULL) {
1380  if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
1381  if (sigmatch_table[sm->type].SupportsPrefilter(s) == TRUE) {
1382  prefilter_list = MIN(prefilter_list, sm->type);
1383  }
1384  }
1385  sm = sm->next;
1386  }
1387  }
1388 
1389  /* apply that keyword as prefilter */
1390  if (prefilter_list != DETECT_TBLSIZE) {
1391  for (int i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1392  SigMatch *sm = s->init_data->smlists[i];
1393  while (sm != NULL) {
1394  if (sm->type == prefilter_list) {
1395  s->init_data->prefilter_sm = sm;
1396  s->flags |= SIG_FLAG_PREFILTER;
1397  SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name);
1398  break;
1399  }
1400  sm = sm->next;
1401  }
1402  }
1403  }
1404  }
1405 
1406  /* run buffer type callbacks if any */
1407  for (int x = 0; x < (int)s->init_data->smlists_array_size; x++) {
1408  if (s->init_data->smlists[x])
1409  DetectBufferRunSetupCallback(de_ctx, x, s);
1410  }
1411 
1412  de_ctx->sig_cnt++;
1413  }
1414 
1415  if (!(de_ctx->flags & DE_QUIET)) {
1416  SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only "
1417  "rules, %" PRIu32 " are inspecting packet payload, %"PRIu32
1418  " inspect application layer, %"PRIu32" are decoder event only",
1419  de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer,
1420  cnt_deonly);
1421 
1422  SCLogConfig("building signature grouping structure, stage 1: "
1423  "preprocessing rules... complete");
1424  }
1425  DetectFlowbitsAnalyze(de_ctx);
1426 
1427  return 0;
1428 
1429 error:
1430  return -1;
1431 }
1432 
1433 static int PortGroupWhitelist(const DetectPort *a)
1434 {
1435  return a->sh->init->whitelist;
1436 }
1437 
1439 {
1440  if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) {
1441  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1442  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1443  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1444  return 1;
1445  } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) {
1446  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1447  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1448  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1449  return 0;
1450  } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) {
1451  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1452  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1453  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1454  return 1;
1455  } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) {
1456  if (a->sh->sig_cnt > b->sh->sig_cnt) {
1457  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1458  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1459  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1460  return 1;
1461  }
1462  }
1463 
1464  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1465  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1466  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1467  return 0;
1468 }
1469 
1470 /** \internal
1471  * \brief Create a list of DetectPort objects sorted based on CompareFunc's
1472  * logic.
1473  *
1474  * List can limit the number of groups. In this case an extra "join" group
1475  * is created that contains the sigs belonging to that. It's *appended* to
1476  * the list, meaning that if the list is walked linearly it's found last.
1477  * The joingr is meant to be a catch all.
1478  *
1479  */
1480 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
1481 {
1482  DetectPort *tmplist = NULL, *joingr = NULL;
1483  char insert = 0;
1484  uint32_t groups = 0;
1485  DetectPort *list;
1486 
1487  /* insert the addresses into the tmplist, where it will
1488  * be sorted descending on 'cnt' and on wehther a group
1489  * is whitelisted. */
1490 
1491  DetectPort *oldhead = port_list;
1492  while (oldhead) {
1493  /* take the top of the list */
1494  list = oldhead;
1495  oldhead = oldhead->next;
1496  list->next = NULL;
1497 
1498  groups++;
1499 
1500  SigGroupHeadSetSigCnt(list->sh, max_idx);
1501 
1502  /* insert it */
1503  DetectPort *tmpgr = tmplist, *prevtmpgr = NULL;
1504  if (tmplist == NULL) {
1505  /* empty list, set head */
1506  tmplist = list;
1507  } else {
1508  /* look for the place to insert */
1509  for ( ; tmpgr != NULL && !insert; tmpgr = tmpgr->next) {
1510  if (CompareFunc(list, tmpgr) == 1) {
1511  if (tmpgr == tmplist) {
1512  list->next = tmplist;
1513  tmplist = list;
1514  SCLogDebug("new list top: %u:%u", tmplist->port, tmplist->port2);
1515  } else {
1516  list->next = prevtmpgr->next;
1517  prevtmpgr->next = list;
1518  }
1519  insert = 1;
1520  break;
1521  }
1522  prevtmpgr = tmpgr;
1523  }
1524  if (insert == 0) {
1525  list->next = NULL;
1526  prevtmpgr->next = list;
1527  }
1528  insert = 0;
1529  }
1530  }
1531 
1532  uint32_t left = unique_groups;
1533  if (left == 0)
1534  left = groups;
1535 
1536  /* create another list: take the port groups from above
1537  * and add them to the 2nd list until we have met our
1538  * count. The rest is added to the 'join' group. */
1539  DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL;
1540  DetectPort *gr, *next_gr;
1541  for (gr = tmplist; gr != NULL; ) {
1542  next_gr = gr->next;
1543 
1544  SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2);
1545  DetectPortPrint(gr);
1546 
1547  /* if we've set up all the unique groups, add the rest to the
1548  * catch-all joingr */
1549  if (left == 0) {
1550  if (joingr == NULL) {
1551  DetectPortParse(de_ctx, &joingr, "0:65535");
1552  if (joingr == NULL) {
1553  goto error;
1554  }
1555  SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2);
1556  joingr->next = NULL;
1557  }
1558  SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh);
1559 
1560  /* when a group's sigs are added to the joingr, we can free it */
1561  gr->next = NULL;
1562  DetectPortFree(de_ctx, gr);
1563  gr = NULL;
1564 
1565  /* append */
1566  } else {
1567  gr->next = NULL;
1568 
1569  if (tmplist2 == NULL) {
1570  tmplist2 = gr;
1571  tmplist2_tail = gr;
1572  } else {
1573  tmplist2_tail->next = gr;
1574  tmplist2_tail = gr;
1575  }
1576  }
1577 
1578  if (left > 0)
1579  left--;
1580 
1581  gr = next_gr;
1582  }
1583 
1584  /* if present, append the joingr that covers the rest */
1585  if (joingr != NULL) {
1586  SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2);
1587 
1588  if (tmplist2 == NULL) {
1589  tmplist2 = joingr;
1590  //tmplist2_tail = joingr;
1591  } else {
1592  tmplist2_tail->next = joingr;
1593  //tmplist2_tail = joingr;
1594  }
1595  } else {
1596  SCLogDebug("no joingr");
1597  }
1598 
1599  /* pass back our new list to the caller */
1600  *newhead = tmplist2;
1601  DetectPortPrintList(*newhead);
1602 
1603  return 0;
1604 error:
1605  return -1;
1606 }
1607 
1608 /**
1609  * \internal
1610  * \brief add a decoder event signature to the detection engine ctx
1611  */
1612 static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s)
1613 {
1614  SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id);
1615  SigGroupHeadAppendSig(de_ctx, &de_ctx->decoder_event_sgh, s);
1616 }
1617 
1618 /**
1619  * \brief Fill the global src group head, with the sigs included
1620  *
1621  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1622  * to be processed
1623  *
1624  * \retval 0 On success
1625  * \retval -1 On failure
1626  */
1628 {
1629  uint32_t sigs = 0;
1630 
1631  SCLogDebug("building signature grouping structure, stage 2: "
1632  "building source address lists...");
1633 
1634  IPOnlyInit(de_ctx, &de_ctx->io_ctx);
1635 
1636  de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER);
1637  de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT);
1638  de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER);
1639  de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT);
1640 
1641  /* Setup the other IP Protocols (so not TCP/UDP) */
1642  RulesGroupByProto(de_ctx);
1643 
1644  /* now for every rule add the source group to our temp lists */
1645  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1646  SCLogDebug("s->id %"PRIu32, s->id);
1647  if (s->flags & SIG_FLAG_IPONLY) {
1648  IPOnlyAddSignature(de_ctx, &de_ctx->io_ctx, s);
1649  }
1650 
1652  DetectEngineAddDecoderEventSig(de_ctx, s);
1653  }
1654 
1655  sigs++;
1656  }
1657 
1658  IPOnlyPrepare(de_ctx);
1659  IPOnlyPrint(de_ctx, &de_ctx->io_ctx);
1660 
1661  return 0;
1662 }
1663 
1664 static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx)
1665 {
1666  if (de_ctx->decoder_event_sgh == NULL)
1667  return;
1668 
1669  uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx);
1670  SigGroupHeadSetSigCnt(de_ctx->decoder_event_sgh, max_idx);
1671  SigGroupHeadBuildMatchArray(de_ctx, de_ctx->decoder_event_sgh, max_idx);
1672 }
1673 
1675 {
1676  /* prepare the decoder event sgh */
1677  DetectEngineBuildDecoderEventSgh(de_ctx);
1678  return 0;
1679 }
1680 
1682 {
1683  BUG_ON(de_ctx == NULL);
1684 
1685  if (!(de_ctx->flags & DE_QUIET)) {
1686  SCLogDebug("cleaning up signature grouping structure...");
1687  }
1688  if (de_ctx->decoder_event_sgh)
1689  SigGroupHeadFree(de_ctx, de_ctx->decoder_event_sgh);
1690  de_ctx->decoder_event_sgh = NULL;
1691 
1692  for (int f = 0; f < FLOW_STATES; f++) {
1693  for (int p = 0; p < 256; p++) {
1694  de_ctx->flow_gh[f].sgh[p] = NULL;
1695  }
1696 
1697  /* free lookup lists */
1698  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].tcp);
1699  de_ctx->flow_gh[f].tcp = NULL;
1700  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].udp);
1701  de_ctx->flow_gh[f].udp = NULL;
1702  }
1703 
1704  for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1705  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1706  if (sgh == NULL)
1707  continue;
1708 
1709  SCLogDebug("sgh %p", sgh);
1710  SigGroupHeadFree(de_ctx, sgh);
1711  }
1712  SCFree(de_ctx->sgh_array);
1713  de_ctx->sgh_array = NULL;
1714  de_ctx->sgh_array_cnt = 0;
1715  de_ctx->sgh_array_size = 0;
1716 
1717  IPOnlyDeinit(de_ctx, &de_ctx->io_ctx);
1718 
1719  if (!(de_ctx->flags & DE_QUIET)) {
1720  SCLogInfo("cleaning up signature grouping structure... complete");
1721  }
1722  return 0;
1723 }
1724 
1725 #if 0
1726 static void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1727 {
1728  if (sgh == NULL) {
1729  printf("\n");
1730  return;
1731  }
1732 
1733  uint32_t sig;
1734  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1735  printf("%" PRIu32 " ", sgh->match_array[sig]->id);
1736  }
1737  printf("\n");
1738 }
1739 
1740 static void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1741 {
1742  if (sgh == NULL || sgh->init == NULL) {
1743  printf("\n");
1744  return;
1745  }
1746 
1747  uint32_t sig;
1748  for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
1749  if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) {
1750  printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id);
1751  }
1752  }
1753  printf("\n");
1754 }
1755 #endif
1756 
1757 /** \brief finalize preparing sgh's */
1759 {
1760  SCEnter();
1761 
1762  //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt);
1763 
1764  uint32_t cnt = 0;
1765  for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1766  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1767  if (sgh == NULL)
1768  continue;
1769 
1770  SCLogDebug("sgh %p", sgh);
1771 
1772  SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
1773  SigGroupHeadSetFileHashFlag(de_ctx, sgh);
1774  SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
1775  SigGroupHeadSetFilestoreCount(de_ctx, sgh);
1776  SCLogDebug("filestore count %u", sgh->filestore_cnt);
1777 
1778  PrefilterSetupRuleGroup(de_ctx, sgh);
1779 
1781 
1783  sgh->init = NULL;
1784 
1785  sgh->id = idx;
1786  cnt++;
1787  }
1788  SCLogPerf("Unique rule groups: %u", cnt);
1789 
1790  MpmStoreReportStats(de_ctx);
1791 
1792  if (de_ctx->decoder_event_sgh != NULL) {
1793  /* no need to set filestore count here as that would make a
1794  * signature not decode event only. */
1795  }
1796 
1797  /* cleanup the hashes now since we won't need them
1798  * after the initialization phase. */
1799  SigGroupHeadHashFree(de_ctx);
1800 
1801  int dump_grouping = 0;
1802  (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping);
1803 
1804  if (dump_grouping) {
1805  int add_rules = 0;
1806  (void)ConfGetBool("detect.profiling.grouping.include-rules", &add_rules);
1807  int add_mpm_stats = 0;
1808  (void)ConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_rules);
1809 
1810  RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats);
1811  }
1812 
1813 #ifdef PROFILING
1815 #endif
1816  SCReturnInt(0);
1817 }
1818 
1819 extern int rule_engine_analysis_set;
1820 /** \internal
1821  * \brief perform final per signature setup tasks
1822  *
1823  * - Create SigMatchData arrays from the init only SigMatch lists
1824  * - Setup per signature inspect engines
1825  * - remove signature init data.
1826  */
1827 static int SigMatchPrepare(DetectEngineCtx *de_ctx)
1828 {
1829  SCEnter();
1830 
1831  Signature *s = de_ctx->sig_list;
1832  for (; s != NULL; s = s->next) {
1833  /* set up inspect engines */
1835 
1836  /* built-ins */
1837  int type;
1838  for (type = 0; type < DETECT_SM_LIST_MAX; type++) {
1839  /* skip PMATCH if it is used in a stream 'app engine' instead */
1841  continue;
1842  SigMatch *sm = s->init_data->smlists[type];
1844  }
1845  /* set up the pkt inspection engines */
1847 
1848  if (rule_engine_analysis_set) {
1849 #ifdef HAVE_LIBJANSSON
1850  EngineAnalysisRules2(de_ctx, s);
1851 #endif
1852  }
1853  /* free lists. Ctx' are xferred to sm_arrays so won't get freed */
1854  uint32_t i;
1855  for (i = 0; i < s->init_data->smlists_array_size; i++) {
1856  SigMatch *sm = s->init_data->smlists[i];
1857  while (sm != NULL) {
1858  SigMatch *nsm = sm->next;
1859  SigMatchFree(sm);
1860  sm = nsm;
1861  }
1862  }
1863  SCFree(s->init_data->smlists);
1865  SCFree(s->init_data);
1866  s->init_data = NULL;
1867  }
1868 
1869 
1870  SCReturnInt(0);
1871 }
1872 
1873 /**
1874  * \brief Convert the signature list into the runtime match structure.
1875  *
1876  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1877  * to be processed
1878  *
1879  * \retval 0 On Success.
1880  * \retval -1 On failure.
1881  */
1883 {
1884  Signature *s = de_ctx->sig_list;
1885 
1886  /* Assign the unique order id of signatures after sorting,
1887  * so the IP Only engine process them in order too. Also
1888  * reset the old signums and assign new signums. We would
1889  * have experienced Sig reordering by now, hence the new
1890  * signums. */
1891  de_ctx->signum = 0;
1892  while (s != NULL) {
1893  s->num = de_ctx->signum++;
1894 
1895  s = s->next;
1896  }
1897 
1898  if (DetectSetFastPatternAndItsId(de_ctx) < 0)
1899  return -1;
1900 
1901  SigInitStandardMpmFactoryContexts(de_ctx);
1902 
1903  if (SigAddressPrepareStage1(de_ctx) != 0) {
1904  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1905  exit(EXIT_FAILURE);
1906  }
1907 
1908  if (SigAddressPrepareStage2(de_ctx) != 0) {
1909  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1910  exit(EXIT_FAILURE);
1911  }
1912 
1913  if (SigAddressPrepareStage3(de_ctx) != 0) {
1914  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1915  exit(EXIT_FAILURE);
1916  }
1917  if (SigAddressPrepareStage4(de_ctx) != 0) {
1918  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1919  exit(EXIT_FAILURE);
1920  }
1921 
1922  int r = DetectMpmPrepareBuiltinMpms(de_ctx);
1923  r |= DetectMpmPrepareAppMpms(de_ctx);
1924  r |= DetectMpmPreparePktMpms(de_ctx);
1925  if (r != 0) {
1926  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1927  exit(EXIT_FAILURE);
1928  }
1929 
1930  if (SigMatchPrepare(de_ctx) != 0) {
1931  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1932  exit(EXIT_FAILURE);
1933  }
1934 
1935 #ifdef PROFILING
1938  de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled
1939 
1940  intmax_t v = 0;
1941  if (ConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1)
1942  de_ctx->profile_match_logging_threshold = (uint32_t)v;
1943 
1945 #endif
1946 
1949  }
1950  return 0;
1951 }
1952 
1954 {
1955  SigAddressCleanupStage1(de_ctx);
1956 
1957  return 0;
1958 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:230
uint16_t filestore_cnt
Definition: detect.h:1345
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1439
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
Predict a strength value for patterns.
DetectProto proto
Definition: detect.h:534
#define DETECTDSIZE_LT
Definition: detect-dsize.h:27
SigIntId sig_cnt
Definition: detect.h:1334
SignatureInitData * init_data
Definition: detect.h:586
int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
Preprocess signature, classify ip-only, etc, build sig array.
#define SIG_FLAG_PDONLY
Definition: detect.h:243
#define SCLogDebug(...)
Definition: util-debug.h:335
#define MAX(x, y)
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Free a SigGroupHead and its members.
Signature ** sig_array
Definition: detect.h:771
void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx, Signature *s)
Add a signature to the lists of Adrresses in CIDR format (sorted) this step is necesary to build the ...
#define SIG_FLAG_PREFILTER
Definition: detect.h:239
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
Figured out the FP and their respective content ids for all the sigs in the engine.
#define DETECTDSIZE_GT
Definition: detect-dsize.h:29
#define SignatureMask
Definition: detect.h:273
uint32_t sig_array_len
Definition: detect.h:773
struct HtpBodyChunk_ * next
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
#define SIGMATCH_DEONLY_COMPAT
Definition: detect.h:1366
void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
SigMatch * prefilter_sm
Definition: detect.h:490
#define BUG_ON(x)
uint32_t flags
Definition: detect.h:518
_Bool(* SupportsPrefilter)(const Signature *s)
Definition: detect.h:1181
uint32_t id
Definition: detect.h:550
#define DETECT_PROTO_ANY
struct SigGroupHead_ * sh
Definition: detect.h:198
Signature ** match_array
Definition: detect.h:1354
#define DETECTDSIZE_RA
Definition: detect-dsize.h:30
#define unlikely(expr)
Definition: util-optimize.h:35
int SigAddressPrepareStage2(DetectEngineCtx *de_ctx)
Fill the global src group head, with the sigs included.
DetectPort * tcp_whitelist
Definition: detect.h:902
void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:59
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
DetectPort * udp_whitelist
Definition: detect.h:903
void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx)
Updates the SigGroupHead->sig_cnt with the total count of all the Signatures present in this SigGroup...
int rule_engine_analysis_set
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
#define TH_RST
Definition: decode-tcp.h:37
Signature * sig_list
Definition: detect.h:762
#define TH_FIN
Definition: decode-tcp.h:35
uint32_t sig_array_size
Definition: detect.h:772
#define MIN(x, y)
int SigAddressPrepareStage3(DetectEngineCtx *de_ctx)
void SCProfilingSghInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
uint8_t proto[256/8]
void SigCleanSignatures(DetectEngineCtx *de_ctx)
int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
finalize preparing sgh&#39;s
Container for matching data for a signature group.
Definition: detect.h:1329
#define DETECT_FLOWBITS_CMD_ISSET
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:228
const char * name
Definition: detect.h:1193
uint16_t AppProto
TCPHdr * tcph
Definition: decode.h:522
Signature container.
Definition: detect.h:517
int DetectFlagsSignatureNeedsSynPackets(const Signature *s)
int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
#define SIG_MASK_REQUIRE_ENGINE_EVENT
Definition: detect.h:270
#define TRUE
uint32_t id
Definition: detect.h:1347
uint32_t signum
Definition: detect.h:775
#define SIG_FLAG_APPLAYER
Definition: detect.h:214
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:854
void IPOnlyPrepare(DetectEngineCtx *de_ctx)
Build the radix trees from the lists of parsed adresses in CIDR format the result should be 4 radix t...
int DetectEngineMultiTenantEnabled(void)
void VarNameStoreActivateStaging(void)
int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t max_idx)
Create an array with all the internal ids of the sigs that this sig group head will check for...
SigIntId num
Definition: detect.h:527
struct SigMatch_ * next
Definition: detect.h:317
main detection engine ctx
Definition: detect.h:756
int SCLogDebugEnabled(void)
Returns whether debug messages are enabled to be logged or not.
Definition: util-debug.c:631
#define SIG_MASK_REQUIRE_FLAGS_INITDEINIT
Definition: detect.h:265
void IPOnlyDeinit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Deinitialize the IP Only detection engine context.
DetectPort * udp
Definition: detect.h:694
#define SIG_FLAG_DST_ANY
Definition: detect.h:208
int SignatureIsFilesizeInspecting(const Signature *s)
Check if a signature contains the filesize keyword.
#define SIG_FLAG_MPM_NEG
Definition: detect.h:222
bool src_contains_negation
Definition: detect.h:477
uint32_t sgh_array_cnt
Definition: detect.h:838
#define DE_QUIET
Definition: detect.h:287
DetectPort * DetectPortHashLookup(DetectEngineCtx *de_ctx, DetectPort *dp)
Used to lookup a DetectPort hash from the detection engine context DetectPort hash table...
#define DetectEngineGetMaxSigId(de_ctx)
Definition: detect-engine.h:81
int DetectBufferTypeGetByName(const char *name)
void SigGroupHeadStore(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
#define PORT_SIGGROUPHEAD_COPY
Definition: detect.h:184
uint16_t max_uniq_toserver_groups
Definition: detect.h:815
int SignatureIsFileSha256Inspecting(const Signature *s)
Check if a signature contains the filesha256 keyword.
#define SIG_FLAG_TOCLIENT
Definition: detect.h:233
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:58
DetectPort * dp
Definition: detect.h:556
void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need size flag in the sgh.
#define SIG_FLAG_SP_ANY
Definition: detect.h:209
DetectPort * tcp
Definition: detect.h:693
uint8_t flags
Definition: detect.h:757
struct SigGroupHead_ * sgh[256]
Definition: detect.h:695
#define SIG_FLAG_IPONLY
Definition: detect.h:215
#define SIG_MASK_REQUIRE_FLOW
Definition: detect.h:264
int SigAddressCleanupStage1(DetectEngineCtx *de_ctx)
void SigGroupHeadHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by SigGroupHeadHashInit() function...
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SIG_FLAG_INIT_DEONLY
Definition: detect.h:252
uint8_t type
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1120
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void SigFree(Signature *)
int SignatureIsFileSha1Inspecting(const Signature *s)
Check if a signature contains the filesha1 keyword.
int SigGroupBuild(DetectEngineCtx *de_ctx)
Convert the signature list into the runtime match structure.
#define SIG_FLAG_INIT_FLOW
Definition: detect.h:254
bool dst_contains_negation
Definition: detect.h:478
#define FILE_SIG_NEED_MAGIC
Definition: detect.h:279
Signature * SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
Find a specific signature by sid and gid.
#define TH_URG
Definition: decode-tcp.h:40
#define SIG_MASK_REQUIRE_FLAGS_UNUSUAL
Definition: detect.h:266
#define SIG_FLAG_TOSERVER
Definition: detect.h:232
#define SCEnter(...)
Definition: util-debug.h:337
int DetectPortHashAdd(DetectEngineCtx *de_ctx, DetectPort *dp)
Adds a DetectPort to the detection engine context DetectPort hash table.
int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
build an array of rule id&#39;s for sigs with no prefilter Also updated de_ctx::non_pf_store_cnt_max to t...
void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid)
uint32_t profile_match_logging_threshold
Definition: detect.h:881
PacketEngineEvents events
Definition: decode.h:565
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:570
void SigParseApplyDsizeToContent(Signature *s)
Apply dsize as depth to content matches in the rule.
Definition: detect-dsize.c:440
void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need magic flag in the sgh.
AppProto alproto
Definition: detect.h:521
struct SigMatch_ ** smlists_tail
Definition: detect.h:513
#define TH_ECN
Definition: decode-tcp.h:42
uint16_t max_uniq_toclient_groups
Definition: detect.h:814
#define DETECTDSIZE_EQ
Definition: detect-dsize.h:28
int SigGroupCleanup(DetectEngineCtx *de_ctx)
void DetectPortHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by DetectPortInit() function...
#define DETECT_FLOW_FLAG_TOCLIENT
Definition: detect-flow.h:28
struct Signature_ * next
Definition: detect.h:589
uint8_t type
Definition: detect.h:314
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
const char * DetectListToString(int list)
Definition: detect-parse.c:128
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:258
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:802
#define FILE_SIG_NEED_MD5
Definition: detect.h:281
SigGroupHead * SigGroupHeadHashLookup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Used to lookup a SigGroupHead hash from the detection engine context SigGroupHead hash table...
int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHead **dst)
Copies the bitarray holding the sids from the source SigGroupHead to the destination SigGroupHead...
void DetectPortCleanupList(const DetectEngineCtx *de_ctx, DetectPort *head)
Free a DetectPort list and each of its members.
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
uint32_t gh_unique
Definition: detect.h:792
struct SigMatch_ ** smlists
Definition: detect.h:511
int DetectPortHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the DetectPort hash...
uint32_t sgh_array_size
Definition: detect.h:839
#define SIG_MASK_REQUIRE_PAYLOAD
Definition: detect.h:263
int SignatureIsFilestoring(const Signature *s)
Check if a signature contains the filestore keyword.
uint32_t smlists_array_size
Definition: detect.h:509
void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
enum DetectEnginePrefilterSetting prefilter_setting
Definition: detect.h:898
SigMatchCtx * ctx
Definition: detect.h:316
#define MASK_TCP_INITDEINIT_FLAGS
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
int SigGroupHeadHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the SigGroupHeads.
int SigGroupHeadAppendSig(const DetectEngineCtx *de_ctx, SigGroupHead **sgh, const Signature *s)
Add a Signature to a SigGroupHead.
#define SCMalloc(a)
Definition: util-mem.h:222
void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
uint8_t flags
Definition: detect.h:191
int SignatureIsFileMd5Inspecting(const Signature *s)
Check if a signature contains the filemd5 keyword.
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define DETECT_CONTENT_NEGATED
#define TH_SYN
Definition: decode-tcp.h:36
#define SCFree(a)
Definition: util-mem.h:322
struct SigGroupHead_ ** sgh_array
Definition: detect.h:837
void IPOnlyPrint(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Print stats of the IP Only engine.
DetectPort * sp
Definition: detect.h:556
#define PKT_IS_TCP(p)
Definition: decode.h:253
int DetectEnginePktInspectionSetup(Signature *s)
uint16_t port
Definition: detect.h:188
void SigGroupHeadSetFileHashFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need hash flag in the sgh.
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for builtin buffers that are in "single or "shared" mode. ...
SigMatchData * SigMatchList2DataArray(SigMatch *head)
convert SigMatch list to SigMatchData array
uint32_t init_flags
Definition: detect.h:481
#define FILE_SIG_NEED_SHA1
Definition: detect.h:282
int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
Test is a initialized signature is IP only.
#define DETECT_FLOWBITS_CMD_SET
int DetectPortInsert(DetectEngineCtx *de_ctx, DetectPort **head, DetectPort *new)
function for inserting a port group object. This also makes sure SigGroupContainer lists are handled ...
void SigMatchFree(SigMatch *sm)
free a SigMatch
Definition: detect-parse.c:247
void DetectPortFree(const DetectEngineCtx *de_ctx, DetectPort *dp)
Free a DetectPort and its members.
void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
#define SCLogPerf(...)
Definition: util-debug.h:261
int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int(*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
struct DetectPort_ * next
Definition: detect.h:201
#define FILE_SIG_NEED_SHA256
Definition: detect.h:283
int SignatureIsFilemagicInspecting(const Signature *s)
Check if a signature contains the filemagic keyword.
void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
void SCProfilingPrefilterInitCounters(DetectEngineCtx *de_ctx)
Register the prefilter profiling counters.
#define PKT_HAS_FLOW
Definition: decode.h:1090
uint32_t gh_reuse
Definition: detect.h:792
int SigGroupHeadHashAdd(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Adds a SigGroupHead to the detection engine context SigGroupHead hash table.
void DetectBufferRunSetupCallback(const DetectEngineCtx *de_ctx, const int id, Signature *s)
#define SIGMATCH_IPONLY_COMPAT
Definition: detect.h:1364
const char * DetectListToHumanString(int list)
Definition: detect-parse.c:111
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1129
#define SIG_FLAG_DP_ANY
Definition: detect.h:210
void SCProfilingRuleInitCounters(DetectEngineCtx *de_ctx)
Register the rule profiling counters.
void IPOnlyInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Setup the IP Only detection engine context.
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:790
#define DETECT_FLOW_FLAG_TOSERVER
Definition: detect-flow.h:27
SignatureMask mask
Definition: detect.h:526
uint8_t file_flags
Definition: detect.h:531
int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s)
#define PKT_NOPAYLOAD_INSPECTION
Definition: decode.h:1084
uint32_t flags
Definition: decode.h:443
uint16_t port2
Definition: detect.h:189
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:561
uint16_t payload_len
Definition: decode.h:541
uint32_t sig_cnt
Definition: detect.h:763
void SigGroupHeadSetProtoAndDirection(SigGroupHead *sgh, uint8_t ipproto, int dir)
void DetectPortPrintList(DetectPort *head)
Helper function used to print the list of ports present in this DetectPort list.
#define SIG_FLAG_SRC_ANY
Definition: detect.h:207
DetectPort * DetectPortCopySingle(DetectEngineCtx *de_ctx, DetectPort *src)
Function that return a copy of DetectPort src sigs.
uint16_t flags
Definition: detect.h:1187
#define SIG_MASK_REQUIRE_DCERPC
Definition: detect.h:268
#define TH_CWR
Definition: decode-tcp.h:44
int DetectPortParse(const DetectEngineCtx *de_ctx, DetectPort **head, const char *str)
Function for parsing port strings.
const char * DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
int DetectFlagsSignatureNeedsSynOnlyPackets(const Signature *s)
void SigGroupHeadSetFilestoreCount(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the filestore_cnt in the sgh.
#define SIG_MASK_REQUIRE_NO_PAYLOAD
Definition: detect.h:267
SigMatch * mpm_sm
Definition: detect.h:488
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
HashListTable * dport_hash_table
Definition: detect.h:900
uint8_t * sig_array
Definition: detect.h:1310
#define FLOW_STATES
Definition: detect.h:753
int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
Port structure for detection engine.
Definition: detect.h:187
a single match condition for a signature
Definition: detect.h:313
void DetectPortPrint(DetectPort *dp)
Helper function that print the DetectPort info.
void DetectContentPropagateLimits(Signature *s)
apply depth/offset and distance/within to content matches
#define FILE_SIG_NEED_SIZE
Definition: detect.h:284
#define MASK_TCP_UNUSUAL_FLAGS
SigGroupHeadInitData * init
Definition: detect.h:1357