suricata
detect-engine-build.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2018 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "detect.h"
20 #include "detect-engine.h"
21 #include "detect-parse.h"
22 
23 #include "detect-engine-address.h"
24 #include "detect-engine-analyzer.h"
25 #include "detect-engine-iponly.h"
26 #include "detect-engine-mpm.h"
27 #include "detect-engine-siggroup.h"
28 #include "detect-engine-port.h"
30 #include "detect-engine-proto.h"
31 
32 #include "detect-dsize.h"
33 #include "detect-flags.h"
34 #include "detect-flow.h"
35 #include "detect-flowbits.h"
36 
37 #include "util-profiling.h"
38 
40 {
41  if (de_ctx == NULL)
42  return;
43 
44  for (Signature *s = de_ctx->sig_list; s != NULL;) {
45  Signature *ns = s->next;
46  SigFree(s);
47  s = ns;
48  }
49  de_ctx->sig_list = NULL;
50 
52  de_ctx->sig_list = NULL;
53 }
54 
55 /** \brief Find a specific signature by sid and gid
56  * \param de_ctx detection engine ctx
57  * \param sid the signature id
58  * \param gid the signature group id
59  *
60  * \retval s sig found
61  * \retval NULL sig not found
62  */
63 Signature *SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
64 {
65  if (de_ctx == NULL)
66  return NULL;
67 
68  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
69  if (s->id == sid && s->gid == gid)
70  return s;
71  }
72 
73  return NULL;
74 }
75 
76 /**
77  * \brief Check if a signature contains the filestore keyword.
78  *
79  * \param s signature
80  *
81  * \retval 0 no
82  * \retval 1 yes
83  */
85 {
86  if (s == NULL)
87  return 0;
88 
89  if (s->flags & SIG_FLAG_FILESTORE)
90  return 1;
91 
92  return 0;
93 }
94 
95 /**
96  * \brief Check if a signature contains the filemagic keyword.
97  *
98  * \param s signature
99  *
100  * \retval 0 no
101  * \retval 1 yes
102  */
104 {
105  if (s == NULL)
106  return 0;
107 
109  return 1;
110 
111  return 0;
112 }
113 
114 /**
115  * \brief Check if a signature contains the filemd5 keyword.
116  *
117  * \param s signature
118  *
119  * \retval 0 no
120  * \retval 1 yes
121  */
123 {
124  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5))
125  return 1;
126 
127  return 0;
128 }
129 
130 /**
131  * \brief Check if a signature contains the filesha1 keyword.
132  *
133  * \param s signature
134  *
135  * \retval 0 no
136  * \retval 1 yes
137  */
139 {
140  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1))
141  return 1;
142 
143  return 0;
144 }
145 
146 /**
147  * \brief Check if a signature contains the filesha256 keyword.
148  *
149  * \param s signature
150  *
151  * \retval 0 no
152  * \retval 1 yes
153  */
155 {
156  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256))
157  return 1;
158 
159  return 0;
160 }
161 
162 /**
163  * \brief Check if a signature contains the filesize keyword.
164  *
165  * \param s signature
166  *
167  * \retval 0 no
168  * \retval 1 yes
169  */
171 {
172  if (s == NULL)
173  return 0;
174 
176  return 1;
177 
178  return 0;
179 }
180 
181 /** \brief Test is a initialized signature is IP only
182  * \param de_ctx detection engine ctx
183  * \param s the signature
184  * \retval 1 sig is ip only
185  * \retval 0 sig is not ip only
186  */
188 {
189  if (s->alproto != ALPROTO_UNKNOWN)
190  return 0;
191 
192  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
193  return 0;
194 
195  /* if flow dir is set we can't process it in ip-only */
196  if (!(((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) ||
199  return 0;
200 
201  /* for now assume that all registered buffer types are incompatible */
202  const int nlists = s->init_data->smlists_array_size;
203  for (int i = 0; i < nlists; i++) {
204  if (s->init_data->smlists[i] == NULL)
205  continue;
206  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
207  continue;
208 
209  SCReturnInt(0);
210  }
211 
212  /* TMATCH list can be ignored, it contains TAGs and
213  * tags are compatible to IP-only. */
214 
215  /* if any of the addresses uses negation, we don't support
216  * it in ip-only */
218  return 0;
220  return 0;
221 
223  if (sm == NULL)
224  goto iponly;
225 
226  for ( ; sm != NULL; sm = sm->next) {
228  return 0;
229  /* we have enabled flowbits to be compatible with ip only sigs, as long
230  * as the sig only has a "set" flowbits */
231  if (sm->type == DETECT_FLOWBITS &&
232  (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
233  return 0;
234  }
235  }
236 
237 iponly:
238  if (!(de_ctx->flags & DE_QUIET)) {
239  SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
240  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
241  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
242  }
243  return 1;
244 }
245 
246 /** \internal
247  * \brief Test is a initialized signature is inspecting protocol detection only
248  * \param de_ctx detection engine ctx
249  * \param s the signature
250  * \retval 1 sig is dp only
251  * \retval 0 sig is not dp only
252  */
253 static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s)
254 {
255  if (s->alproto != ALPROTO_UNKNOWN)
256  return 0;
257 
258  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
259  return 0;
260 
261  /* for now assume that all registered buffer types are incompatible */
262  const int nlists = s->init_data->smlists_array_size;
263  for (int i = 0; i < nlists; i++) {
264  if (s->init_data->smlists[i] == NULL)
265  continue;
266  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
267  continue;
268 
269  SCReturnInt(0);
270  }
271 
272  /* TMATCH list can be ignored, it contains TAGs and
273  * tags are compatible to DP-only. */
274 
275  /* match list matches may be compatible to DP only. We follow the same
276  * logic as IP-only so we can use that flag */
277 
279  if (sm == NULL)
280  return 0;
281 
282  int pd = 0;
283  for ( ; sm != NULL; sm = sm->next) {
284  if (sm->type == DETECT_AL_APP_LAYER_PROTOCOL) {
285  pd = 1;
286  } else {
287  /* flowbits are supported for dp only sigs, as long
288  * as the sig only has a "set" flowbits */
289  if (sm->type == DETECT_FLOWBITS) {
290  if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
291  SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id);
292  return 0;
293  }
294  } else if (sm->type == DETECT_FLOW) {
296  SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id);
297  return 0;
298  }
299  } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) {
300  SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name);
301  return 0;
302  }
303  }
304  }
305 
306  if (pd) {
307  SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id);
308  }
309  return pd;
310 }
311 
312 /**
313  * \internal
314  * \brief Check if the initialized signature is inspecting the packet payload
315  * \param de_ctx detection engine ctx
316  * \param s the signature
317  * \retval 1 sig is inspecting the payload
318  * \retval 0 sig is not inspecting the payload
319  */
320 static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s)
321 {
322 
323  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
324  return 1;
325  }
326  return 0;
327 }
328 
329 /**
330  * \internal
331  * \brief check if a signature is decoder event matching only
332  * \param de_ctx detection engine
333  * \param s the signature to test
334  * \retval 0 not a DEOnly sig
335  * \retval 1 DEOnly sig
336  */
337 static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
338 {
339  if (s->alproto != ALPROTO_UNKNOWN) {
340  SCReturnInt(0);
341  }
342 
343  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
344  {
345  SCReturnInt(0);
346  }
347 
348  /* for now assume that all registered buffer types are incompatible */
349  const int nlists = s->init_data->smlists_array_size;
350  for (int i = 0; i < nlists; i++) {
351  if (s->init_data->smlists[i] == NULL)
352  continue;
353  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
354  continue;
355 
356  SCReturnInt(0);
357  }
358 
359  /* check for conflicting keywords */
361  for ( ;sm != NULL; sm = sm->next) {
363  SCReturnInt(0);
364  }
365 
366  /* need at least one decode event keyword to be considered decode event. */
368  for ( ;sm != NULL; sm = sm->next) {
369  if (sm->type == DETECT_DECODE_EVENT)
370  goto deonly;
371  if (sm->type == DETECT_ENGINE_EVENT)
372  goto deonly;
373  if (sm->type == DETECT_STREAM_EVENT)
374  goto deonly;
375  }
376 
377  SCReturnInt(0);
378 
379 deonly:
380  if (!(de_ctx->flags & DE_QUIET)) {
381  SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
382  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
383  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
384  }
385 
386  SCReturnInt(1);
387 }
388 
389 #define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN)
390 #define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR)
391 
392 /* Create mask for this packet + it's flow if it has one
393  */
394 void
396  bool app_decoder_events)
397 {
398  if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
399  SCLogDebug("packet has payload");
400  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
401  } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
402  SCLogDebug("stream data available");
403  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
404  } else {
405  SCLogDebug("packet has no payload");
406  (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
407  }
408 
409  if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) {
410  SCLogDebug("packet/flow has events set");
412  }
413 
414  if (!(PKT_IS_PSEUDOPKT(p)) && PKT_IS_TCP(p)) {
415  if ((p->tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) {
417  }
418  if ((p->tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) {
420  }
421  }
422 
423  if (p->flags & PKT_HAS_FLOW) {
424  SCLogDebug("packet has flow");
425  (*mask) |= SIG_MASK_REQUIRE_FLOW;
426  }
427 
428  if (alproto == ALPROTO_SMB || alproto == ALPROTO_DCERPC) {
429  SCLogDebug("packet will be inspected for DCERPC");
430  (*mask) |= SIG_MASK_REQUIRE_DCERPC;
431  }
432 }
433 
434 static int g_dce_generic_list_id = -1;
435 static int g_dce_stub_data_buffer_id = -1;
436 
437 static bool SignatureNeedsDCERPCMask(const Signature *s)
438 {
439  if (g_dce_generic_list_id == -1) {
440  g_dce_generic_list_id = DetectBufferTypeGetByName("dce_generic");
441  SCLogDebug("g_dce_generic_list_id %d", g_dce_generic_list_id);
442  }
443  if (g_dce_stub_data_buffer_id == -1) {
444  g_dce_stub_data_buffer_id = DetectBufferTypeGetByName("dce_stub_data");
445  SCLogDebug("g_dce_stub_data_buffer_id %d", g_dce_stub_data_buffer_id);
446  }
447 
448  if (g_dce_generic_list_id >= 0 &&
449  s->init_data->smlists[g_dce_generic_list_id] != NULL)
450  {
451  return true;
452  }
453  if (g_dce_stub_data_buffer_id >= 0 &&
454  s->init_data->smlists[g_dce_stub_data_buffer_id] != NULL)
455  {
456  return true;
457  }
458  return false;
459 }
460 
461 static int SignatureCreateMask(Signature *s)
462 {
463  SCEnter();
464 
465  if (SignatureNeedsDCERPCMask(s)) {
467  SCLogDebug("sig requires DCERPC");
468  }
469 
470  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
472  SCLogDebug("sig requires payload");
473  }
474 
475  SigMatch *sm;
476  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
477  switch(sm->type) {
478  case DETECT_FLOWBITS:
479  {
480  /* figure out what flowbit action */
482  if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
483  /* not a mask flag, but still set it here */
485 
486  SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has "
487  "flowbit isset option.");
488  }
489 
490  /* flow is required for any flowbit manipulation */
492  SCLogDebug("sig requires flow to be able to manipulate "
493  "flowbit(s)");
494  break;
495  }
496  case DETECT_FLOWINT:
497  /* flow is required for any flowint manipulation */
499  SCLogDebug("sig requires flow to be able to manipulate "
500  "flowint(s)");
501  break;
502  case DETECT_FLAGS:
503  {
504  DetectFlagsData *fl = (DetectFlagsData *)sm->ctx;
505 
506  if (fl->flags & TH_SYN) {
508  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
509  }
510  if (fl->flags & TH_RST) {
512  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
513  }
514  if (fl->flags & TH_FIN) {
516  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
517  }
518  if (fl->flags & TH_URG) {
520  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
521  }
522  if (fl->flags & TH_ECN) {
524  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
525  }
526  if (fl->flags & TH_CWR) {
528  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
529  }
530  break;
531  }
532  case DETECT_DSIZE:
533  {
534  DetectDsizeData *ds = (DetectDsizeData *)sm->ctx;
535  switch (ds->mode) {
536  case DETECTDSIZE_LT:
537  /* LT will include 0, so no payload.
538  * if GT is used in the same rule the
539  * flag will be set anyway. */
540  break;
541  case DETECTDSIZE_RA:
542  case DETECTDSIZE_GT:
544  SCLogDebug("sig requires payload");
545  break;
546  case DETECTDSIZE_EQ:
547  if (ds->dsize > 0) {
549  SCLogDebug("sig requires payload");
550  } else if (ds->dsize == 0) {
552  SCLogDebug("sig requires no payload");
553  }
554  break;
555  }
556  break;
557  }
560  break;
561  case DETECT_ENGINE_EVENT:
563  break;
564  }
565  }
566 
569  SCLogDebug("sig requires flow");
570  }
571 
572  if (s->flags & SIG_FLAG_APPLAYER) {
574  SCLogDebug("sig requires flow");
575  }
576 
577  SCLogDebug("mask %02X", s->mask);
578  SCReturnInt(0);
579 }
580 
581 static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
582 {
584  DetectMpmSetupAppMpms(de_ctx);
585 
586  return;
587 }
588 
589 /** \brief Pure-PCRE or bytetest rule */
590 static int RuleInspectsPayloadHasNoMpm(const Signature *s)
591 {
592  if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
593  return 1;
594  return 0;
595 }
596 
597 static int RuleGetMpmPatternSize(const Signature *s)
598 {
599  if (s->init_data->mpm_sm == NULL)
600  return -1;
601  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
602  if (mpm_list < 0)
603  return -1;
604  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
605  if (cd == NULL)
606  return -1;
607  return (int)cd->content_len;
608 }
609 
610 static int RuleMpmIsNegated(const Signature *s)
611 {
612  if (s->init_data->mpm_sm == NULL)
613  return 0;
614  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
615  if (mpm_list < 0)
616  return 0;
617  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
618  if (cd == NULL)
619  return 0;
620  return (cd->flags & DETECT_CONTENT_NEGATED);
621 }
622 
623 #ifdef HAVE_LIBJANSSON
624 static json_t *RulesGroupPrintSghStats(const SigGroupHead *sgh,
625  const int add_rules, const int add_mpm_stats)
626 {
627  uint32_t mpm_cnt = 0;
628  uint32_t nonmpm_cnt = 0;
629  uint32_t negmpm_cnt = 0;
630  uint32_t any5_cnt = 0;
631  uint32_t payload_no_mpm_cnt = 0;
632  uint32_t syn_cnt = 0;
633 
634  uint32_t mpms_total = 0;
635  uint32_t mpms_min = 0;
636  uint32_t mpms_max = 0;
637 
638  struct {
639  uint32_t total;
640  uint32_t cnt;
641  uint32_t min;
642  uint32_t max;
643  } mpm_stats[DETECT_SM_LIST_MAX];
644  memset(mpm_stats, 0x00, sizeof(mpm_stats));
645 
646  uint32_t alstats[ALPROTO_MAX] = {0};
647  uint32_t mpm_sizes[DETECT_SM_LIST_MAX][256];
648  memset(mpm_sizes, 0, sizeof(mpm_sizes));
649  uint32_t alproto_mpm_bufs[ALPROTO_MAX][DETECT_SM_LIST_MAX];
650  memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs));
651 
652  json_t *js = json_object();
653  if (unlikely(js == NULL))
654  return NULL;
655 
656  json_object_set_new(js, "id", json_integer(sgh->id));
657 
658  json_t *js_array = json_array();
659 
660  const Signature *s;
661  uint32_t x;
662  for (x = 0; x < sgh->sig_cnt; x++) {
663  s = sgh->match_array[x];
664  if (s == NULL)
665  continue;
666 
667  int any = 0;
668  if (s->proto.flags & DETECT_PROTO_ANY) {
669  any++;
670  }
671  if (s->flags & SIG_FLAG_DST_ANY) {
672  any++;
673  }
674  if (s->flags & SIG_FLAG_SRC_ANY) {
675  any++;
676  }
677  if (s->flags & SIG_FLAG_DP_ANY) {
678  any++;
679  }
680  if (s->flags & SIG_FLAG_SP_ANY) {
681  any++;
682  }
683  if (any == 5) {
684  any5_cnt++;
685  }
686 
687  if (s->init_data->mpm_sm == NULL) {
688  nonmpm_cnt++;
689 
690  if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) {
691  SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id);
692  }
693 
694  DetectPort *sp = s->sp;
695  DetectPort *dp = s->dp;
696 
697  if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) {
698  SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id);
699  }
700  if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) {
701  SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id);
702  }
703 
705  syn_cnt++;
706  }
707 
708  } else {
709  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
710  BUG_ON(mpm_list < 0);
711  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
712  uint32_t size = cd->content_len < 256 ? cd->content_len : 255;
713 
714  mpm_sizes[mpm_list][size]++;
715  if (s->alproto != ALPROTO_UNKNOWN) {
716  alproto_mpm_bufs[s->alproto][mpm_list]++;
717  }
718 
719  if (mpm_list == DETECT_SM_LIST_PMATCH) {
720  if (size == 1) {
721  DetectPort *sp = s->sp;
722  DetectPort *dp = s->dp;
723  if (s->flags & SIG_FLAG_TOSERVER) {
724  if (dp->port == 0 && dp->port2 == 65535) {
725  SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
726  } else {
727  SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id);
728  }
729  }
730  if (s->flags & SIG_FLAG_TOCLIENT) {
731  if (sp->port == 0 && sp->port2 == 65535) {
732  SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
733  } else {
734  SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id);
735  }
736  }
737  }
738  }
739 
740  uint32_t w = PatternStrength(cd->content, cd->content_len);
741  mpms_total += w;
742  if (mpms_min == 0)
743  mpms_min = w;
744  if (w < mpms_min)
745  mpms_min = w;
746  if (w > mpms_max)
747  mpms_max = w;
748 
749  mpm_stats[mpm_list].total += w;
750  mpm_stats[mpm_list].cnt++;
751  if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min)
752  mpm_stats[mpm_list].min = w;
753  if (w > mpm_stats[mpm_list].max)
754  mpm_stats[mpm_list].max = w;
755 
756  mpm_cnt++;
757 
758  if (w < 10) {
759  SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id);
760  }
761  if (w < 10 && any == 5) {
762  SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id);
763  }
764 
765  if (cd->flags & DETECT_CONTENT_NEGATED) {
766  SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id);
767  negmpm_cnt++;
768  }
769  }
770 
771  if (RuleInspectsPayloadHasNoMpm(s)) {
772  SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id);
773  payload_no_mpm_cnt++;
774  }
775 
776  if (s->alproto != ALPROTO_UNKNOWN) {
777  alstats[s->alproto]++;
778  }
779 
780  if (add_rules) {
781  json_t *js_sig = json_object();
782  if (unlikely(js == NULL))
783  continue;
784  json_object_set_new(js_sig, "sig_id", json_integer(s->id));
785  json_array_append_new(js_array, js_sig);
786  }
787  }
788 
789  json_object_set_new(js, "rules", js_array);
790 
791  json_t *stats = json_object();
792  json_object_set_new(stats, "total", json_integer(sgh->sig_cnt));
793 
794  json_t *types = json_object();
795  json_object_set_new(types, "mpm", json_integer(mpm_cnt));
796  json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt));
797  json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt));
798  json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt));
799  json_object_set_new(types, "syn", json_integer(syn_cnt));
800  json_object_set_new(types, "any5", json_integer(any5_cnt));
801  json_object_set_new(stats, "types", types);
802 
803  int i;
804  for (i = 0; i < ALPROTO_MAX; i++) {
805  if (alstats[i] > 0) {
806  json_t *app = json_object();
807  json_object_set_new(app, "total", json_integer(alstats[i]));
808 
809  for (x = 0; x < DETECT_SM_LIST_MAX; x++) {
810  if (alproto_mpm_bufs[i][x] == 0)
811  continue;
812  json_object_set_new(app, DetectListToHumanString(x), json_integer(alproto_mpm_bufs[i][x]));
813  }
814 
815  json_object_set_new(stats, AppProtoToString(i), app);
816  }
817  }
818 
819  if (add_mpm_stats) {
820  json_t *mpm_js = json_object();
821 
822  for (i = 0; i < DETECT_SM_LIST_MAX; i++) {
823  if (mpm_stats[i].cnt > 0) {
824 
825  json_t *mpm_sizes_array = json_array();
826  for (x = 0; x < 256; x++) {
827  if (mpm_sizes[i][x] == 0)
828  continue;
829 
830  json_t *e = json_object();
831  json_object_set_new(e, "size", json_integer(x));
832  json_object_set_new(e, "count", json_integer(mpm_sizes[i][x]));
833  json_array_append_new(mpm_sizes_array, e);
834  }
835 
836  json_t *buf = json_object();
837  json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt));
838  json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt));
839  json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min));
840  json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max));
841 
842  json_object_set_new(buf, "sizes", mpm_sizes_array);
843 
844  json_object_set_new(mpm_js, DetectListToHumanString(i), buf);
845  }
846  }
847 
848  json_object_set_new(stats, "mpm", mpm_js);
849  }
850  json_object_set_new(js, "stats", stats);
851 
852  json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist));
853 
854  return js;
855 }
856 #endif /* HAVE_LIBJANSSON */
857 
858 static void RulesDumpGrouping(const DetectEngineCtx *de_ctx,
859  const int add_rules, const int add_mpm_stats)
860 {
861 #ifdef HAVE_LIBJANSSON
862  json_t *js = json_object();
863  if (unlikely(js == NULL))
864  return;
865 
866  int p;
867  for (p = 0; p < 256; p++) {
868  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
869  const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp";
870 
871  json_t *tcp = json_object();
872 
873  json_t *ts_array = json_array();
874  DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp :
875  de_ctx->flow_gh[1].udp;
876  while (list != NULL) {
877  json_t *port = json_object();
878  json_object_set_new(port, "port", json_integer(list->port));
879  json_object_set_new(port, "port2", json_integer(list->port2));
880 
881  json_t *tcp_ts = RulesGroupPrintSghStats(list->sh,
882  add_rules, add_mpm_stats);
883  json_object_set_new(port, "rulegroup", tcp_ts);
884  json_array_append_new(ts_array, port);
885 
886  list = list->next;
887  }
888  json_object_set_new(tcp, "toserver", ts_array);
889 
890  json_t *tc_array = json_array();
891  list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp :
892  de_ctx->flow_gh[0].udp;
893  while (list != NULL) {
894  json_t *port = json_object();
895  json_object_set_new(port, "port", json_integer(list->port));
896  json_object_set_new(port, "port2", json_integer(list->port2));
897 
898  json_t *tcp_tc = RulesGroupPrintSghStats(list->sh,
899  add_rules, add_mpm_stats);
900  json_object_set_new(port, "rulegroup", tcp_tc);
901  json_array_append_new(tc_array, port);
902 
903  list = list->next;
904  }
905  json_object_set_new(tcp, "toclient", tc_array);
906 
907  json_object_set_new(js, name, tcp);
908  }
909 
910  }
911 
912  const char *filename = "rule_group.json";
913  const char *log_dir = ConfigGetLogDirectory();
914  char log_path[PATH_MAX] = "";
915 
916  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
917 
918  FILE *fp = fopen(log_path, "w");
919  if (fp == NULL) {
920  return;
921  }
922 
923  char *js_s = json_dumps(js,
924  JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH);
925  if (unlikely(js_s == NULL)) {
926  fclose(fp);
927  return;
928  }
929 
930  json_object_clear(js);
931  json_decref(js);
932 
933  fprintf(fp, "%s\n", js_s);
934  free(js_s);
935  fclose(fp);
936 #endif
937  return;
938 }
939 
940 static int RulesGroupByProto(DetectEngineCtx *de_ctx)
941 {
942  Signature *s = de_ctx->sig_list;
943 
944  uint32_t max_idx = 0;
945  SigGroupHead *sgh_ts[256] = {NULL};
946  SigGroupHead *sgh_tc[256] = {NULL};
947 
948  for ( ; s != NULL; s = s->next) {
949  if (s->flags & SIG_FLAG_IPONLY)
950  continue;
951 
952  int p;
953  for (p = 0; p < 256; p++) {
954  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
955  continue;
956  }
957  if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) {
958  continue;
959  }
960 
961  if (s->flags & SIG_FLAG_TOCLIENT) {
962  SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s);
963  max_idx = s->num;
964  }
965  if (s->flags & SIG_FLAG_TOSERVER) {
966  SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s);
967  max_idx = s->num;
968  }
969  }
970  }
971  SCLogDebug("max_idx %u", max_idx);
972 
973  /* lets look at deduplicating this list */
974  SigGroupHeadHashFree(de_ctx);
975  SigGroupHeadHashInit(de_ctx);
976 
977  uint32_t cnt = 0;
978  uint32_t own = 0;
979  uint32_t ref = 0;
980  int p;
981  for (p = 0; p < 256; p++) {
982  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
983  continue;
984  if (sgh_ts[p] == NULL)
985  continue;
986 
987  cnt++;
988 
989  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]);
990  if (lookup_sgh == NULL) {
991  SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]);
992 
993  SigGroupHeadSetSigCnt(sgh_ts[p], max_idx);
994  SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx);
995 
996  SigGroupHeadHashAdd(de_ctx, sgh_ts[p]);
997  SigGroupHeadStore(de_ctx, sgh_ts[p]);
998 
999  de_ctx->gh_unique++;
1000  own++;
1001  } else {
1002  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]);
1003 
1004  SigGroupHeadFree(de_ctx, sgh_ts[p]);
1005  sgh_ts[p] = lookup_sgh;
1006 
1007  de_ctx->gh_reuse++;
1008  ref++;
1009  }
1010  }
1011  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1012  "toserver", cnt, own, ref);
1013 
1014  cnt = 0;
1015  own = 0;
1016  ref = 0;
1017  for (p = 0; p < 256; p++) {
1018  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1019  continue;
1020  if (sgh_tc[p] == NULL)
1021  continue;
1022 
1023  cnt++;
1024 
1025  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]);
1026  if (lookup_sgh == NULL) {
1027  SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]);
1028 
1029  SigGroupHeadSetSigCnt(sgh_tc[p], max_idx);
1030  SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx);
1031 
1032  SigGroupHeadHashAdd(de_ctx, sgh_tc[p]);
1033  SigGroupHeadStore(de_ctx, sgh_tc[p]);
1034 
1035  de_ctx->gh_unique++;
1036  own++;
1037 
1038  } else {
1039  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]);
1040 
1041  SigGroupHeadFree(de_ctx, sgh_tc[p]);
1042  sgh_tc[p] = lookup_sgh;
1043 
1044  de_ctx->gh_reuse++;
1045  ref++;
1046  }
1047  }
1048  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1049  "toclient", cnt, own, ref);
1050 
1051  for (p = 0; p < 256; p++) {
1052  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1053  continue;
1054 
1055  de_ctx->flow_gh[0].sgh[p] = sgh_tc[p];
1056  de_ctx->flow_gh[1].sgh[p] = sgh_ts[p];
1057  }
1058 
1059  return 0;
1060 }
1061 
1062 static int PortIsWhitelisted(const DetectEngineCtx *de_ctx,
1063  const DetectPort *a, int ipproto)
1064 {
1065  DetectPort *w = de_ctx->tcp_whitelist;
1066  if (ipproto == IPPROTO_UDP)
1067  w = de_ctx->udp_whitelist;
1068 
1069  while (w) {
1070  if (a->port >= w->port && a->port2 <= w->port) {
1071  SCLogDebug("port group %u:%u whitelisted -> %d", a->port, a->port2, w->port);
1072  return 1;
1073  }
1074  w = w->next;
1075  }
1076 
1077  return 0;
1078 }
1079 
1080 static int RuleSetWhitelist(Signature *s)
1081 {
1082  DetectPort *p = NULL;
1083  if (s->flags & SIG_FLAG_TOSERVER)
1084  p = s->dp;
1085  else if (s->flags & SIG_FLAG_TOCLIENT)
1086  p = s->sp;
1087  else
1088  return 0;
1089 
1090  /* for sigs that don't use 'any' as port, see if we want to
1091  * whitelist poor sigs */
1092  int wl = 0;
1093  if (!(p->port == 0 && p->port2 == 65535)) {
1094  /* pure pcre, bytetest, etc rules */
1095  if (RuleInspectsPayloadHasNoMpm(s)) {
1096  SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id);
1097  wl = 99;
1098 
1099  } else if (RuleMpmIsNegated(s)) {
1100  SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id);
1101  wl = 77;
1102 
1103  /* one byte pattern in packet/stream payloads */
1104  } else if (s->init_data->mpm_sm != NULL &&
1106  RuleGetMpmPatternSize(s) == 1)
1107  {
1108  SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id);
1109  wl = 55;
1110 
1111  } else if (DetectFlagsSignatureNeedsSynPackets(s) &&
1113  {
1114  SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id);
1115  wl = 33;
1116  }
1117  }
1118 
1119  s->init_data->whitelist = wl;
1120  return wl;
1121 }
1122 
1123 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx);
1125 
1126 static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint32_t direction) {
1127  /* step 1: create a hash of 'DetectPort' objects based on all the
1128  * rules. Each object will have a SGH with the sigs added
1129  * that belong to the SGH. */
1130  DetectPortHashInit(de_ctx);
1131 
1132  uint32_t max_idx = 0;
1133  const Signature *s = de_ctx->sig_list;
1134  DetectPort *list = NULL;
1135  while (s) {
1136  /* IP Only rules are handled separately */
1137  if (s->flags & SIG_FLAG_IPONLY)
1138  goto next;
1139  if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY)))
1140  goto next;
1141  if (direction == SIG_FLAG_TOSERVER) {
1142  if (!(s->flags & SIG_FLAG_TOSERVER))
1143  goto next;
1144  } else if (direction == SIG_FLAG_TOCLIENT) {
1145  if (!(s->flags & SIG_FLAG_TOCLIENT))
1146  goto next;
1147  }
1148 
1149  DetectPort *p = NULL;
1150  if (direction == SIG_FLAG_TOSERVER)
1151  p = s->dp;
1152  else if (direction == SIG_FLAG_TOCLIENT)
1153  p = s->sp;
1154  else
1155  BUG_ON(1);
1156 
1157  /* see if we want to exclude directionless sigs that really care only for
1158  * to_server syn scans/floods */
1159  if ((direction == SIG_FLAG_TOCLIENT) &&
1163  (!(s->dp->port == 0 && s->dp->port2 == 65535)))
1164  {
1165  SCLogWarning(SC_WARN_POOR_RULE, "rule %u: SYN-only to port(s) %u:%u "
1166  "w/o direction specified, disabling for toclient direction",
1167  s->id, s->dp->port, s->dp->port2);
1168  goto next;
1169  }
1170 
1171  int wl = s->init_data->whitelist;
1172  while (p) {
1173  int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? 111 : 0;
1174  pwl = MAX(wl,pwl);
1175 
1176  DetectPort *lookup = DetectPortHashLookup(de_ctx, p);
1177  if (lookup) {
1178  SigGroupHeadAppendSig(de_ctx, &lookup->sh, s);
1179  lookup->sh->init->whitelist = MAX(lookup->sh->init->whitelist, pwl);
1180  } else {
1181  DetectPort *tmp2 = DetectPortCopySingle(de_ctx, p);
1182  BUG_ON(tmp2 == NULL);
1183  SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s);
1184  tmp2->sh->init->whitelist = pwl;
1185  DetectPortHashAdd(de_ctx, tmp2);
1186  }
1187 
1188  p = p->next;
1189  }
1190  max_idx = s->num;
1191  next:
1192  s = s->next;
1193  }
1194 
1195  /* step 2: create a list of DetectPort objects */
1196  HashListTableBucket *htb = NULL;
1197  for (htb = HashListTableGetListHead(de_ctx->dport_hash_table);
1198  htb != NULL;
1199  htb = HashListTableGetListNext(htb))
1200  {
1202  DetectPort *tmp = DetectPortCopySingle(de_ctx, p);
1203  BUG_ON(tmp == NULL);
1204  int r = DetectPortInsert(de_ctx, &list , tmp);
1205  BUG_ON(r == -1);
1206  }
1207  DetectPortHashFree(de_ctx);
1208  de_ctx->dport_hash_table = NULL;
1209 
1210  SCLogDebug("rules analyzed");
1211 
1212  /* step 3: group the list and shrink it if necessary */
1213  DetectPort *newlist = NULL;
1214  uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups :
1215  de_ctx->max_uniq_toserver_groups;
1216  CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx);
1217  list = newlist;
1218 
1219  /* step 4: deduplicate the SGH's */
1220  SigGroupHeadHashFree(de_ctx);
1221  SigGroupHeadHashInit(de_ctx);
1222 
1223  uint32_t cnt = 0;
1224  uint32_t own = 0;
1225  uint32_t ref = 0;
1226  DetectPort *iter;
1227  for (iter = list ; iter != NULL; iter = iter->next) {
1228  BUG_ON (iter->sh == NULL);
1229  cnt++;
1230 
1231  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh);
1232  if (lookup_sgh == NULL) {
1233  SCLogDebug("port group %p sgh %p is the original", iter, iter->sh);
1234 
1235  SigGroupHeadSetSigCnt(iter->sh, max_idx);
1236  SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx);
1237  SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction);
1238  SigGroupHeadHashAdd(de_ctx, iter->sh);
1239  SigGroupHeadStore(de_ctx, iter->sh);
1240  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1241  de_ctx->gh_unique++;
1242  own++;
1243  } else {
1244  SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh);
1245 
1246  SigGroupHeadFree(de_ctx, iter->sh);
1247  iter->sh = lookup_sgh;
1248  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1249 
1250  de_ctx->gh_reuse++;
1251  ref++;
1252  }
1253  }
1254 #if 0
1255  for (iter = list ; iter != NULL; iter = iter->next) {
1256  SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)",
1257  iter->port, iter->port2, iter->sh,
1258  iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own",
1259  iter->sh->init->whitelist ? "true" : "false",
1260  iter->sh->init->whitelist);
1261  }
1262 #endif
1263  SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies",
1264  ipproto == 6 ? "TCP" : "UDP",
1265  direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1266  cnt, own, ref);
1267  return list;
1268 }
1269 
1271 {
1272  /* see if the sig is dp only */
1273  if (SignatureIsPDOnly(de_ctx, s) == 1) {
1274  s->flags |= SIG_FLAG_PDONLY;
1275 
1276  /* see if the sig is ip only */
1277  } else if (SignatureIsIPOnly(de_ctx, s) == 1) {
1278  s->flags |= SIG_FLAG_IPONLY;
1279 
1280  } else if (SignatureIsDEOnly(de_ctx, s) == 1) {
1282  }
1283 }
1284 
1285 /**
1286  * \brief Preprocess signature, classify ip-only, etc, build sig array
1287  *
1288  * \param de_ctx Pointer to the Detection Engine Context
1289  *
1290  * \retval 0 on success
1291  * \retval -1 on failure
1292  */
1294 {
1295  uint32_t cnt_iponly = 0;
1296  uint32_t cnt_payload = 0;
1297  uint32_t cnt_applayer = 0;
1298  uint32_t cnt_deonly = 0;
1299 
1300  if (!(de_ctx->flags & DE_QUIET)) {
1301  SCLogDebug("building signature grouping structure, stage 1: "
1302  "preprocessing rules...");
1303  }
1304 
1305  de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx);
1306  de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *));
1307  de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size);
1308  if (de_ctx->sig_array == NULL)
1309  goto error;
1310  memset(de_ctx->sig_array,0,de_ctx->sig_array_size);
1311 
1312  SCLogDebug("signature lookup array: %" PRIu32 " sigs, %" PRIu32 " bytes",
1313  de_ctx->sig_array_len, de_ctx->sig_array_size);
1314 
1315  /* now for every rule add the source group */
1316  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1317  de_ctx->sig_array[s->num] = s;
1318 
1319  SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", s->id, s->num, s, de_ctx->sig_array[s->num]);
1320 
1321  if (s->flags & SIG_FLAG_PDONLY) {
1322  SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", s->id);
1323  } else if (s->flags & SIG_FLAG_IPONLY) {
1324  SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", s->id);
1325  cnt_iponly++;
1326  } else if (SignatureIsInspectingPayload(de_ctx, s) == 1) {
1327  SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", s->id);
1328  cnt_payload++;
1329  } else if (s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) {
1330  SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", s->id);
1331  cnt_deonly++;
1332  } else if (s->flags & SIG_FLAG_APPLAYER) {
1333  SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", s->id);
1334  cnt_applayer++;
1335  }
1336 
1337 #ifdef DEBUG
1338  if (SCLogDebugEnabled()) {
1339  uint16_t colen = 0;
1340  char copresent = 0;
1341  SigMatch *sm;
1342  DetectContentData *co;
1343  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) {
1344  if (sm->type != DETECT_CONTENT)
1345  continue;
1346 
1347  copresent = 1;
1348  co = (DetectContentData *)sm->ctx;
1349  if (co->content_len > colen)
1350  colen = co->content_len;
1351  }
1352 
1353  if (copresent && colen == 1) {
1354  SCLogDebug("signature %8u content maxlen 1", s->id);
1355  int proto;
1356  for (proto = 0; proto < 256; proto++) {
1357  if (s->proto.proto[(proto/8)] & (1<<(proto%8)))
1358  SCLogDebug("=> proto %" PRId32 "", proto);
1359  }
1360  }
1361  }
1362 #endif /* DEBUG */
1363 
1364  if (RuleMpmIsNegated(s)) {
1365  s->flags |= SIG_FLAG_MPM_NEG;
1366  }
1367 
1368  SignatureCreateMask(s);
1371 
1372  RuleSetWhitelist(s);
1373 
1374  /* if keyword engines are enabled in the config, handle them here */
1375  if (de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
1376  !(s->flags & SIG_FLAG_PREFILTER))
1377  {
1378  int i;
1379  int prefilter_list = DETECT_TBLSIZE;
1380 
1381  /* get the keyword supporting prefilter with the lowest type */
1382  for (i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1383  SigMatch *sm = s->init_data->smlists[i];
1384  while (sm != NULL) {
1385  if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
1386  if (sigmatch_table[sm->type].SupportsPrefilter(s) == TRUE) {
1387  prefilter_list = MIN(prefilter_list, sm->type);
1388  }
1389  }
1390  sm = sm->next;
1391  }
1392  }
1393 
1394  /* apply that keyword as prefilter */
1395  if (prefilter_list != DETECT_TBLSIZE) {
1396  for (i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1397  SigMatch *sm = s->init_data->smlists[i];
1398  while (sm != NULL) {
1399  if (sm->type == prefilter_list) {
1400  s->init_data->prefilter_sm = sm;
1401  s->flags |= SIG_FLAG_PREFILTER;
1402  SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name);
1403  break;
1404  }
1405  sm = sm->next;
1406  }
1407  }
1408  }
1409  }
1410 
1411  /* run buffer type callbacks if any */
1412  int x;
1413  for (x = 0; x < (int)s->init_data->smlists_array_size; x++) {
1414  if (s->init_data->smlists[x])
1415  DetectBufferRunSetupCallback(de_ctx, x, s);
1416  }
1417 
1418  de_ctx->sig_cnt++;
1419  }
1420 
1421  if (!(de_ctx->flags & DE_QUIET)) {
1422  SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only "
1423  "rules, %" PRIu32 " are inspecting packet payload, %"PRIu32
1424  " inspect application layer, %"PRIu32" are decoder event only",
1425  de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer,
1426  cnt_deonly);
1427 
1428  SCLogConfig("building signature grouping structure, stage 1: "
1429  "preprocessing rules... complete");
1430  }
1431  DetectFlowbitsAnalyze(de_ctx);
1432 
1433  return 0;
1434 
1435 error:
1436  return -1;
1437 }
1438 
1439 static int PortGroupWhitelist(const DetectPort *a)
1440 {
1441  return a->sh->init->whitelist;
1442 }
1443 
1445 {
1446  if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) {
1447  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1448  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1449  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1450  return 1;
1451  } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) {
1452  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1453  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1454  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1455  return 0;
1456  } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) {
1457  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1458  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1459  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1460  return 1;
1461  } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) {
1462  if (a->sh->sig_cnt > b->sh->sig_cnt) {
1463  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1464  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1465  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1466  return 1;
1467  }
1468  }
1469 
1470  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1471  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1472  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1473  return 0;
1474 }
1475 
1476 /** \internal
1477  * \brief Create a list of DetectPort objects sorted based on CompareFunc's
1478  * logic.
1479  *
1480  * List can limit the number of groups. In this case an extra "join" group
1481  * is created that contains the sigs belonging to that. It's *appended* to
1482  * the list, meaning that if the list is walked linearly it's found last.
1483  * The joingr is meant to be a catch all.
1484  *
1485  */
1486 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
1487 {
1488  DetectPort *tmplist = NULL, *joingr = NULL;
1489  char insert = 0;
1490  uint32_t groups = 0;
1491  DetectPort *list;
1492 
1493  /* insert the addresses into the tmplist, where it will
1494  * be sorted descending on 'cnt' and on wehther a group
1495  * is whitelisted. */
1496 
1497  DetectPort *oldhead = port_list;
1498  while (oldhead) {
1499  /* take the top of the list */
1500  list = oldhead;
1501  oldhead = oldhead->next;
1502  list->next = NULL;
1503 
1504  groups++;
1505 
1506  SigGroupHeadSetSigCnt(list->sh, max_idx);
1507 
1508  /* insert it */
1509  DetectPort *tmpgr = tmplist, *prevtmpgr = NULL;
1510  if (tmplist == NULL) {
1511  /* empty list, set head */
1512  tmplist = list;
1513  } else {
1514  /* look for the place to insert */
1515  for ( ; tmpgr != NULL && !insert; tmpgr = tmpgr->next) {
1516  if (CompareFunc(list, tmpgr) == 1) {
1517  if (tmpgr == tmplist) {
1518  list->next = tmplist;
1519  tmplist = list;
1520  SCLogDebug("new list top: %u:%u", tmplist->port, tmplist->port2);
1521  } else {
1522  list->next = prevtmpgr->next;
1523  prevtmpgr->next = list;
1524  }
1525  insert = 1;
1526  break;
1527  }
1528  prevtmpgr = tmpgr;
1529  }
1530  if (insert == 0) {
1531  list->next = NULL;
1532  prevtmpgr->next = list;
1533  }
1534  insert = 0;
1535  }
1536  }
1537 
1538  uint32_t left = unique_groups;
1539  if (left == 0)
1540  left = groups;
1541 
1542  /* create another list: take the port groups from above
1543  * and add them to the 2nd list until we have met our
1544  * count. The rest is added to the 'join' group. */
1545  DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL;
1546  DetectPort *gr, *next_gr;
1547  for (gr = tmplist; gr != NULL; ) {
1548  next_gr = gr->next;
1549 
1550  SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2);
1551  DetectPortPrint(gr);
1552 
1553  /* if we've set up all the unique groups, add the rest to the
1554  * catch-all joingr */
1555  if (left == 0) {
1556  if (joingr == NULL) {
1557  DetectPortParse(de_ctx, &joingr, "0:65535");
1558  if (joingr == NULL) {
1559  goto error;
1560  }
1561  SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2);
1562  joingr->next = NULL;
1563  }
1564  SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh);
1565 
1566  /* when a group's sigs are added to the joingr, we can free it */
1567  gr->next = NULL;
1568  DetectPortFree(de_ctx, gr);
1569  gr = NULL;
1570 
1571  /* append */
1572  } else {
1573  gr->next = NULL;
1574 
1575  if (tmplist2 == NULL) {
1576  tmplist2 = gr;
1577  tmplist2_tail = gr;
1578  } else {
1579  tmplist2_tail->next = gr;
1580  tmplist2_tail = gr;
1581  }
1582  }
1583 
1584  if (left > 0)
1585  left--;
1586 
1587  gr = next_gr;
1588  }
1589 
1590  /* if present, append the joingr that covers the rest */
1591  if (joingr != NULL) {
1592  SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2);
1593 
1594  if (tmplist2 == NULL) {
1595  tmplist2 = joingr;
1596  //tmplist2_tail = joingr;
1597  } else {
1598  tmplist2_tail->next = joingr;
1599  //tmplist2_tail = joingr;
1600  }
1601  } else {
1602  SCLogDebug("no joingr");
1603  }
1604 
1605  /* pass back our new list to the caller */
1606  *newhead = tmplist2;
1607  DetectPortPrintList(*newhead);
1608 
1609  return 0;
1610 error:
1611  return -1;
1612 }
1613 
1614 /**
1615  * \internal
1616  * \brief add a decoder event signature to the detection engine ctx
1617  */
1618 static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s)
1619 {
1620  SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id);
1621  SigGroupHeadAppendSig(de_ctx, &de_ctx->decoder_event_sgh, s);
1622 }
1623 
1624 /**
1625  * \brief Fill the global src group head, with the sigs included
1626  *
1627  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1628  * to be processed
1629  *
1630  * \retval 0 On success
1631  * \retval -1 On failure
1632  */
1634 {
1635  Signature *tmp_s = NULL;
1636  uint32_t sigs = 0;
1637 
1638  if (!(de_ctx->flags & DE_QUIET)) {
1639  SCLogDebug("building signature grouping structure, stage 2: "
1640  "building source address lists...");
1641  }
1642 
1643  IPOnlyInit(de_ctx, &de_ctx->io_ctx);
1644 
1645  de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER);
1646  de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT);
1647  de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER);
1648  de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT);
1649 
1650  /* Setup the other IP Protocols (so not TCP/UDP) */
1651  RulesGroupByProto(de_ctx);
1652 
1653  /* now for every rule add the source group to our temp lists */
1654  for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) {
1655  SCLogDebug("tmp_s->id %"PRIu32, tmp_s->id);
1656  if (tmp_s->flags & SIG_FLAG_IPONLY) {
1657  IPOnlyAddSignature(de_ctx, &de_ctx->io_ctx, tmp_s);
1658  }
1659 
1660  if (tmp_s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) {
1661  DetectEngineAddDecoderEventSig(de_ctx, tmp_s);
1662  }
1663 
1664  sigs++;
1665  }
1666 
1667  IPOnlyPrepare(de_ctx);
1668  IPOnlyPrint(de_ctx, &de_ctx->io_ctx);
1669 
1670  return 0;
1671 }
1672 
1673 static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx)
1674 {
1675  if (de_ctx->decoder_event_sgh == NULL)
1676  return;
1677 
1678  uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx);
1679  SigGroupHeadSetSigCnt(de_ctx->decoder_event_sgh, max_idx);
1680  SigGroupHeadBuildMatchArray(de_ctx, de_ctx->decoder_event_sgh, max_idx);
1681 }
1682 
1684 {
1685  /* prepare the decoder event sgh */
1686  DetectEngineBuildDecoderEventSgh(de_ctx);
1687  return 0;
1688 }
1689 
1691 {
1692  BUG_ON(de_ctx == NULL);
1693 
1694  if (!(de_ctx->flags & DE_QUIET)) {
1695  SCLogDebug("cleaning up signature grouping structure...");
1696  }
1697  if (de_ctx->decoder_event_sgh)
1698  SigGroupHeadFree(de_ctx, de_ctx->decoder_event_sgh);
1699  de_ctx->decoder_event_sgh = NULL;
1700 
1701  int f;
1702  for (f = 0; f < FLOW_STATES; f++) {
1703  int p;
1704  for (p = 0; p < 256; p++) {
1705  de_ctx->flow_gh[f].sgh[p] = NULL;
1706  }
1707 
1708  /* free lookup lists */
1709  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].tcp);
1710  de_ctx->flow_gh[f].tcp = NULL;
1711  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].udp);
1712  de_ctx->flow_gh[f].udp = NULL;
1713  }
1714 
1715  uint32_t idx;
1716  for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1717  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1718  if (sgh == NULL)
1719  continue;
1720 
1721  SCLogDebug("sgh %p", sgh);
1722  SigGroupHeadFree(de_ctx, sgh);
1723  }
1724  SCFree(de_ctx->sgh_array);
1725  de_ctx->sgh_array = NULL;
1726  de_ctx->sgh_array_cnt = 0;
1727  de_ctx->sgh_array_size = 0;
1728 
1729  IPOnlyDeinit(de_ctx, &de_ctx->io_ctx);
1730 
1731  if (!(de_ctx->flags & DE_QUIET)) {
1732  SCLogInfo("cleaning up signature grouping structure... complete");
1733  }
1734  return 0;
1735 }
1736 
1737 #if 0
1738 static void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1739 {
1740  if (sgh == NULL) {
1741  printf("\n");
1742  return;
1743  }
1744 
1745  uint32_t sig;
1746  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1747  printf("%" PRIu32 " ", sgh->match_array[sig]->id);
1748  }
1749  printf("\n");
1750 }
1751 
1752 static void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1753 {
1754  if (sgh == NULL || sgh->init == NULL) {
1755  printf("\n");
1756  return;
1757  }
1758 
1759  uint32_t sig;
1760  for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
1761  if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) {
1762  printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id);
1763  }
1764  }
1765  printf("\n");
1766 }
1767 #endif
1768 
1769 /** \brief finalize preparing sgh's */
1771 {
1772  SCEnter();
1773 
1774  //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt);
1775 
1776  uint32_t cnt = 0;
1777  uint32_t idx = 0;
1778  for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1779  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1780  if (sgh == NULL)
1781  continue;
1782 
1783  SCLogDebug("sgh %p", sgh);
1784 
1785  SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
1786  SigGroupHeadSetFileHashFlag(de_ctx, sgh);
1787  SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
1788  SigGroupHeadSetFilestoreCount(de_ctx, sgh);
1789  SCLogDebug("filestore count %u", sgh->filestore_cnt);
1790 
1791  PrefilterSetupRuleGroup(de_ctx, sgh);
1792 
1794 
1796  sgh->init = NULL;
1797 
1798  sgh->id = idx;
1799  cnt++;
1800  }
1801  SCLogPerf("Unique rule groups: %u", cnt);
1802 
1803  MpmStoreReportStats(de_ctx);
1804 
1805  if (de_ctx->decoder_event_sgh != NULL) {
1806  /* no need to set filestore count here as that would make a
1807  * signature not decode event only. */
1808  }
1809 
1810  /* cleanup the hashes now since we won't need them
1811  * after the initialization phase. */
1812  SigGroupHeadHashFree(de_ctx);
1813 
1814  int dump_grouping = 0;
1815  (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping);
1816 
1817  if (dump_grouping) {
1818  int add_rules = 0;
1819  (void)ConfGetBool("detect.profiling.grouping.include-rules", &add_rules);
1820  int add_mpm_stats = 0;
1821  (void)ConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_rules);
1822 
1823  RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats);
1824  }
1825 
1826 #ifdef PROFILING
1828 #endif
1829  SCReturnInt(0);
1830 }
1831 
1832 extern int rule_engine_analysis_set;
1833 /** \internal
1834  * \brief perform final per signature setup tasks
1835  *
1836  * - Create SigMatchData arrays from the init only SigMatch lists
1837  * - Setup per signature inspect engines
1838  * - remove signature init data.
1839  */
1840 static int SigMatchPrepare(DetectEngineCtx *de_ctx)
1841 {
1842  SCEnter();
1843 
1844  Signature *s = de_ctx->sig_list;
1845  for (; s != NULL; s = s->next) {
1846  /* set up inspect engines */
1848 
1849  /* built-ins */
1850  int type;
1851  for (type = 0; type < DETECT_SM_LIST_MAX; type++) {
1852  /* skip PMATCH if it is used in a stream 'app engine' instead */
1854  continue;
1855  SigMatch *sm = s->init_data->smlists[type];
1857  }
1858  if (rule_engine_analysis_set) {
1859 #ifdef HAVE_LIBJANSSON
1860  EngineAnalysisRules2(de_ctx, s);
1861 #endif
1862  }
1863  /* free lists. Ctx' are xferred to sm_arrays so won't get freed */
1864  uint32_t i;
1865  for (i = 0; i < s->init_data->smlists_array_size; i++) {
1866  SigMatch *sm = s->init_data->smlists[i];
1867  while (sm != NULL) {
1868  SigMatch *nsm = sm->next;
1869  SigMatchFree(sm);
1870  sm = nsm;
1871  }
1872  }
1873  SCFree(s->init_data->smlists);
1875  SCFree(s->init_data);
1876  s->init_data = NULL;
1877  }
1878 
1879 
1880  SCReturnInt(0);
1881 }
1882 
1883 /**
1884  * \brief Convert the signature list into the runtime match structure.
1885  *
1886  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1887  * to be processed
1888  *
1889  * \retval 0 On Success.
1890  * \retval -1 On failure.
1891  */
1893 {
1894  Signature *s = de_ctx->sig_list;
1895 
1896  /* Assign the unique order id of signatures after sorting,
1897  * so the IP Only engine process them in order too. Also
1898  * reset the old signums and assign new signums. We would
1899  * have experienced Sig reordering by now, hence the new
1900  * signums. */
1901  de_ctx->signum = 0;
1902  while (s != NULL) {
1903  s->num = de_ctx->signum++;
1904 
1905  s = s->next;
1906  }
1907 
1908  if (DetectSetFastPatternAndItsId(de_ctx) < 0)
1909  return -1;
1910 
1911  SigInitStandardMpmFactoryContexts(de_ctx);
1912 
1913  if (SigAddressPrepareStage1(de_ctx) != 0) {
1914  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1915  exit(EXIT_FAILURE);
1916  }
1917 
1918  if (SigAddressPrepareStage2(de_ctx) != 0) {
1919  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1920  exit(EXIT_FAILURE);
1921  }
1922 
1923  if (SigAddressPrepareStage3(de_ctx) != 0) {
1924  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1925  exit(EXIT_FAILURE);
1926  }
1927  if (SigAddressPrepareStage4(de_ctx) != 0) {
1928  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1929  exit(EXIT_FAILURE);
1930  }
1931 
1932  int r = DetectMpmPrepareBuiltinMpms(de_ctx);
1933  r |= DetectMpmPrepareAppMpms(de_ctx);
1934  if (r != 0) {
1935  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1936  exit(EXIT_FAILURE);
1937  }
1938 
1939  if (SigMatchPrepare(de_ctx) != 0) {
1940  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1941  exit(EXIT_FAILURE);
1942  }
1943 
1944 #ifdef PROFILING
1947  de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled
1948 
1949  intmax_t v = 0;
1950  if (ConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1)
1951  de_ctx->profile_match_logging_threshold = (uint32_t)v;
1952 
1954 #endif
1955  SCFree(de_ctx->app_mpms);
1956  de_ctx->app_mpms = NULL;
1957 
1960  }
1961  return 0;
1962 }
1963 
1965 {
1966  SigAddressCleanupStage1(de_ctx);
1967 
1968  return 0;
1969 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:239
void DetectMpmSetupAppMpms(DetectEngineCtx *de_ctx)
uint16_t filestore_cnt
Definition: detect.h:1314
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1406
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
Predict a strength value for patterns.
DetectProto proto
Definition: detect.h:512
#define DETECTDSIZE_LT
Definition: detect-dsize.h:27
SigIntId sig_cnt
Definition: detect.h:1303
SignatureInitData * init_data
Definition: detect.h:563
int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
Preprocess signature, classify ip-only, etc, build sig array.
#define SIG_FLAG_PDONLY
Definition: detect.h:252
#define SCLogDebug(...)
Definition: util-debug.h:335
#define MAX(x, y)
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Free a SigGroupHead and its members.
Signature ** sig_array
Definition: detect.h:738
void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx, Signature *s)
Add a signature to the lists of Adrresses in CIDR format (sorted) this step is necesary to build the ...
#define SIG_FLAG_PREFILTER
Definition: detect.h:248
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
Figured out the FP and their respective content ids for all the sigs in the engine.
#define DETECTDSIZE_GT
Definition: detect-dsize.h:29
#define SignatureMask
Definition: detect.h:282
uint32_t sig_array_len
Definition: detect.h:740
struct HtpBodyChunk_ * next
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
#define SIGMATCH_DEONLY_COMPAT
Definition: detect.h:1335
void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
SigMatch * prefilter_sm
Definition: detect.h:468
#define BUG_ON(x)
uint32_t flags
Definition: detect.h:496
_Bool(* SupportsPrefilter)(const Signature *s)
Definition: detect.h:1151
uint32_t id
Definition: detect.h:528
#define DETECT_PROTO_ANY
struct SigGroupHead_ * sh
Definition: detect.h:207
Signature ** match_array
Definition: detect.h:1323
#define DETECTDSIZE_RA
Definition: detect-dsize.h:30
#define unlikely(expr)
Definition: util-optimize.h:35
int SigAddressPrepareStage2(DetectEngineCtx *de_ctx)
Fill the global src group head, with the sigs included.
DetectPort * tcp_whitelist
Definition: detect.h:869
void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:59
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
DetectPort * udp_whitelist
Definition: detect.h:870
DetectMpmAppLayerKeyword * app_mpms
Definition: detect.h:898
void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx)
Updates the SigGroupHead->sig_cnt with the total count of all the Signatures present in this SigGroup...
int rule_engine_analysis_set
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
#define TH_RST
Definition: decode-tcp.h:37
Signature * sig_list
Definition: detect.h:729
#define TH_FIN
Definition: decode-tcp.h:35
uint32_t sig_array_size
Definition: detect.h:739
#define MIN(x, y)
int SigAddressPrepareStage3(DetectEngineCtx *de_ctx)
void SCProfilingSghInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
uint8_t proto[256/8]
void SigCleanSignatures(DetectEngineCtx *de_ctx)
int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
finalize preparing sgh&#39;s
Container for matching data for a signature group.
Definition: detect.h:1298
#define DETECT_FLOWBITS_CMD_ISSET
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:237
const char * name
Definition: detect.h:1163
uint16_t AppProto
TCPHdr * tcph
Definition: decode.h:520
Signature container.
Definition: detect.h:495
#define SIG_MASK_REQUIRE_ENGINE_EVENT
Definition: detect.h:279
#define TRUE
uint32_t id
Definition: detect.h:1316
uint32_t signum
Definition: detect.h:742
#define SIG_FLAG_APPLAYER
Definition: detect.h:223
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:821
void IPOnlyPrepare(DetectEngineCtx *de_ctx)
Build the radix trees from the lists of parsed adresses in CIDR format the result should be 4 radix t...
int DetectEngineMultiTenantEnabled(void)
void VarNameStoreActivateStaging(void)
int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t max_idx)
Create an array with all the internal ids of the sigs that this sig group head will check for...
SigIntId num
Definition: detect.h:505
struct SigMatch_ * next
Definition: detect.h:326
main detection engine ctx
Definition: detect.h:723
int SCLogDebugEnabled(void)
Returns whether debug messages are enabled to be logged or not.
Definition: util-debug.c:631
#define SIG_MASK_REQUIRE_FLAGS_INITDEINIT
Definition: detect.h:274
void IPOnlyDeinit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Deinitialize the IP Only detection engine context.
DetectPort * udp
Definition: detect.h:662
#define SIG_FLAG_DST_ANY
Definition: detect.h:217
int SignatureIsFilesizeInspecting(const Signature *s)
Check if a signature contains the filesize keyword.
#define SIG_FLAG_MPM_NEG
Definition: detect.h:231
bool src_contains_negation
Definition: detect.h:455
uint32_t sgh_array_cnt
Definition: detect.h:805
#define DE_QUIET
Definition: detect.h:296
DetectPort * DetectPortHashLookup(DetectEngineCtx *de_ctx, DetectPort *dp)
Used to lookup a DetectPort hash from the detection engine context DetectPort hash table...
#define DetectEngineGetMaxSigId(de_ctx)
Definition: detect-engine.h:81
int DetectBufferTypeGetByName(const char *name)
void SigGroupHeadStore(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
#define PORT_SIGGROUPHEAD_COPY
Definition: detect.h:193
uint16_t max_uniq_toserver_groups
Definition: detect.h:782
int SignatureIsFileSha256Inspecting(const Signature *s)
Check if a signature contains the filesha256 keyword.
#define SIG_FLAG_TOCLIENT
Definition: detect.h:242
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:58
DetectPort * dp
Definition: detect.h:534
void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need size flag in the sgh.
#define SIG_FLAG_SP_ANY
Definition: detect.h:218
DetectPort * tcp
Definition: detect.h:661
uint8_t flags
Definition: detect.h:724
struct SigGroupHead_ * sgh[256]
Definition: detect.h:663
#define SIG_FLAG_IPONLY
Definition: detect.h:224
#define SIG_MASK_REQUIRE_FLOW
Definition: detect.h:273
int SigAddressCleanupStage1(DetectEngineCtx *de_ctx)
void SigGroupHeadHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by SigGroupHeadHashInit() function...
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SIG_FLAG_INIT_DEONLY
Definition: detect.h:261
uint8_t type
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1122
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void SigFree(Signature *)
int SignatureIsFileSha1Inspecting(const Signature *s)
Check if a signature contains the filesha1 keyword.
int SigGroupBuild(DetectEngineCtx *de_ctx)
Convert the signature list into the runtime match structure.
#define SIG_FLAG_INIT_FLOW
Definition: detect.h:263
bool dst_contains_negation
Definition: detect.h:456
#define FILE_SIG_NEED_MAGIC
Definition: detect.h:288
Signature * SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
Find a specific signature by sid and gid.
#define TH_URG
Definition: decode-tcp.h:40
#define SIG_MASK_REQUIRE_FLAGS_UNUSUAL
Definition: detect.h:275
#define SIG_FLAG_TOSERVER
Definition: detect.h:241
#define SCEnter(...)
Definition: util-debug.h:337
int DetectPortHashAdd(DetectEngineCtx *de_ctx, DetectPort *dp)
Adds a DetectPort to the detection engine context DetectPort hash table.
int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
build an array of rule id&#39;s for sigs with no prefilter Also updated de_ctx::non_pf_store_cnt_max to t...
void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid)
uint32_t profile_match_logging_threshold
Definition: detect.h:848
PacketEngineEvents events
Definition: decode.h:565
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:547
void SigParseApplyDsizeToContent(Signature *s)
Apply dsize as depth to content matches in the rule.
Definition: detect-dsize.c:440
void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need magic flag in the sgh.
AppProto alproto
Definition: detect.h:499
struct SigMatch_ ** smlists_tail
Definition: detect.h:491
#define TH_ECN
Definition: decode-tcp.h:42
uint16_t max_uniq_toclient_groups
Definition: detect.h:781
#define DETECTDSIZE_EQ
Definition: detect-dsize.h:28
int SigGroupCleanup(DetectEngineCtx *de_ctx)
void DetectPortHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by DetectPortInit() function...
#define DETECT_FLOW_FLAG_TOCLIENT
Definition: detect-flow.h:28
struct Signature_ * next
Definition: detect.h:566
uint8_t type
Definition: detect.h:323
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
const char * DetectListToString(int list)
Definition: detect-parse.c:128
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:267
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:769
#define FILE_SIG_NEED_MD5
Definition: detect.h:290
SigGroupHead * SigGroupHeadHashLookup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Used to lookup a SigGroupHead hash from the detection engine context SigGroupHead hash table...
int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHead **dst)
Copies the bitarray holding the sids from the source SigGroupHead to the destination SigGroupHead...
void DetectPortCleanupList(const DetectEngineCtx *de_ctx, DetectPort *head)
Free a DetectPort list and each of its members.
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
uint32_t gh_unique
Definition: detect.h:759
struct SigMatch_ ** smlists
Definition: detect.h:489
int DetectPortHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the DetectPort hash...
uint32_t sgh_array_size
Definition: detect.h:806
#define SIG_MASK_REQUIRE_PAYLOAD
Definition: detect.h:272
int SignatureIsFilestoring(const Signature *s)
Check if a signature contains the filestore keyword.
uint32_t smlists_array_size
Definition: detect.h:487
void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
enum DetectEnginePrefilterSetting prefilter_setting
Definition: detect.h:865
SigMatchCtx * ctx
Definition: detect.h:325
#define MASK_TCP_INITDEINIT_FLAGS
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
int SigGroupHeadHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the SigGroupHeads.
int SigGroupHeadAppendSig(const DetectEngineCtx *de_ctx, SigGroupHead **sgh, const Signature *s)
Add a Signature to a SigGroupHead.
#define SCMalloc(a)
Definition: util-mem.h:166
void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
uint8_t flags
Definition: detect.h:200
int SignatureIsFileMd5Inspecting(const Signature *s)
Check if a signature contains the filemd5 keyword.
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define DETECT_CONTENT_NEGATED
#define TH_SYN
Definition: decode-tcp.h:36
#define SCFree(a)
Definition: util-mem.h:228
struct SigGroupHead_ ** sgh_array
Definition: detect.h:804
void IPOnlyPrint(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Print stats of the IP Only engine.
DetectPort * sp
Definition: detect.h:534
#define PKT_IS_TCP(p)
Definition: decode.h:251
uint16_t port
Definition: detect.h:197
void SigGroupHeadSetFileHashFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need hash flag in the sgh.
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for builtin buffers that are in "single or "shared" mode. ...
SigMatchData * SigMatchList2DataArray(SigMatch *head)
convert SigMatch list to SigMatchData array
uint32_t init_flags
Definition: detect.h:459
#define FILE_SIG_NEED_SHA1
Definition: detect.h:291
int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
Test is a initialized signature is IP only.
#define DETECT_FLOWBITS_CMD_SET
int DetectPortInsert(DetectEngineCtx *de_ctx, DetectPort **head, DetectPort *new)
function for inserting a port group object. This also makes sure SigGroupContainer lists are handled ...
void SigMatchFree(SigMatch *sm)
free a SigMatch
Definition: detect-parse.c:247
void DetectPortFree(const DetectEngineCtx *de_ctx, DetectPort *dp)
Free a DetectPort and its members.
void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
#define SCLogPerf(...)
Definition: util-debug.h:261
int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int(*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
struct DetectPort_ * next
Definition: detect.h:210
#define FILE_SIG_NEED_SHA256
Definition: detect.h:292
int SignatureIsFilemagicInspecting(const Signature *s)
Check if a signature contains the filemagic keyword.
void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
void SCProfilingPrefilterInitCounters(DetectEngineCtx *de_ctx)
Register the prefilter profiling counters.
#define PKT_HAS_FLOW
Definition: decode.h:1092
uint32_t gh_reuse
Definition: detect.h:759
int SigGroupHeadHashAdd(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Adds a SigGroupHead to the detection engine context SigGroupHead hash table.
void DetectBufferRunSetupCallback(const DetectEngineCtx *de_ctx, const int id, Signature *s)
#define SIGMATCH_IPONLY_COMPAT
Definition: detect.h:1333
const char * DetectListToHumanString(int list)
Definition: detect-parse.c:111
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1131
#define SIG_FLAG_DP_ANY
Definition: detect.h:219
void SCProfilingRuleInitCounters(DetectEngineCtx *de_ctx)
Register the rule profiling counters.
void IPOnlyInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Setup the IP Only detection engine context.
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:757
#define DETECT_FLOW_FLAG_TOSERVER
Definition: detect-flow.h:27
SignatureMask mask
Definition: detect.h:504
uint8_t file_flags
Definition: detect.h:509
int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s)
#define PKT_NOPAYLOAD_INSPECTION
Definition: decode.h:1086
uint32_t flags
Definition: decode.h:441
uint16_t port2
Definition: detect.h:198
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:561
uint16_t payload_len
Definition: decode.h:541
uint32_t sig_cnt
Definition: detect.h:730
int DetectFlagsSignatureNeedsSynOnlyPackets(const Signature *s)
Definition: detect-flags.c:526
void SigGroupHeadSetProtoAndDirection(SigGroupHead *sgh, uint8_t ipproto, int dir)
void DetectPortPrintList(DetectPort *head)
Helper function used to print the list of ports present in this DetectPort list.
#define SIG_FLAG_SRC_ANY
Definition: detect.h:216
DetectPort * DetectPortCopySingle(DetectEngineCtx *de_ctx, DetectPort *src)
Function that return a copy of DetectPort src sigs.
uint16_t flags
Definition: detect.h:1157
#define SIG_MASK_REQUIRE_DCERPC
Definition: detect.h:277
#define TH_CWR
Definition: decode-tcp.h:44
int DetectPortParse(const DetectEngineCtx *de_ctx, DetectPort **head, const char *str)
Function for parsing port strings.
const char * DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
void SigGroupHeadSetFilestoreCount(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the filestore_cnt in the sgh.
#define SIG_MASK_REQUIRE_NO_PAYLOAD
Definition: detect.h:276
SigMatch * mpm_sm
Definition: detect.h:466
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
HashListTable * dport_hash_table
Definition: detect.h:867
uint8_t * sig_array
Definition: detect.h:1280
#define FLOW_STATES
Definition: detect.h:671
int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
int DetectFlagsSignatureNeedsSynPackets(const Signature *s)
Definition: detect-flags.c:507
Port structure for detection engine.
Definition: detect.h:196
a single match condition for a signature
Definition: detect.h:322
void DetectPortPrint(DetectPort *dp)
Helper function that print the DetectPort info.
void DetectContentPropagateLimits(Signature *s)
apply depth/offset and distance/within to content matches
#define FILE_SIG_NEED_SIZE
Definition: detect.h:293
#define MASK_TCP_UNUSUAL_FLAGS
SigGroupHeadInitData * init
Definition: detect.h:1326