suricata
detect-engine-build.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2018 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "detect.h"
20 #include "detect-engine.h"
21 #include "detect-parse.h"
22 
23 #include "detect-engine-address.h"
24 #include "detect-engine-analyzer.h"
25 #include "detect-engine-iponly.h"
26 #include "detect-engine-mpm.h"
27 #include "detect-engine-siggroup.h"
28 #include "detect-engine-port.h"
30 #include "detect-engine-proto.h"
31 
32 #include "detect-dsize.h"
33 #include "detect-tcp-flags.h"
34 #include "detect-flow.h"
35 #include "detect-flowbits.h"
36 
37 #include "util-profiling.h"
38 
40 {
41  if (de_ctx == NULL)
42  return;
43 
44  for (Signature *s = de_ctx->sig_list; s != NULL;) {
45  Signature *ns = s->next;
46  SigFree(s);
47  s = ns;
48  }
49  de_ctx->sig_list = NULL;
50 
52  de_ctx->sig_list = NULL;
53 }
54 
55 /** \brief Find a specific signature by sid and gid
56  * \param de_ctx detection engine ctx
57  * \param sid the signature id
58  * \param gid the signature group id
59  *
60  * \retval s sig found
61  * \retval NULL sig not found
62  */
63 Signature *SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
64 {
65  if (de_ctx == NULL)
66  return NULL;
67 
68  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
69  if (s->id == sid && s->gid == gid)
70  return s;
71  }
72 
73  return NULL;
74 }
75 
76 /**
77  * \brief Check if a signature contains the filestore keyword.
78  *
79  * \param s signature
80  *
81  * \retval 0 no
82  * \retval 1 yes
83  */
85 {
86  if (s == NULL)
87  return 0;
88 
89  if (s->flags & SIG_FLAG_FILESTORE)
90  return 1;
91 
92  return 0;
93 }
94 
95 /**
96  * \brief Check if a signature contains the filemagic keyword.
97  *
98  * \param s signature
99  *
100  * \retval 0 no
101  * \retval 1 yes
102  */
104 {
105  if (s == NULL)
106  return 0;
107 
109  return 1;
110 
111  return 0;
112 }
113 
114 /**
115  * \brief Check if a signature contains the filemd5 keyword.
116  *
117  * \param s signature
118  *
119  * \retval 0 no
120  * \retval 1 yes
121  */
123 {
124  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5))
125  return 1;
126 
127  return 0;
128 }
129 
130 /**
131  * \brief Check if a signature contains the filesha1 keyword.
132  *
133  * \param s signature
134  *
135  * \retval 0 no
136  * \retval 1 yes
137  */
139 {
140  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1))
141  return 1;
142 
143  return 0;
144 }
145 
146 /**
147  * \brief Check if a signature contains the filesha256 keyword.
148  *
149  * \param s signature
150  *
151  * \retval 0 no
152  * \retval 1 yes
153  */
155 {
156  if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256))
157  return 1;
158 
159  return 0;
160 }
161 
162 /**
163  * \brief Check if a signature contains the filesize keyword.
164  *
165  * \param s signature
166  *
167  * \retval 0 no
168  * \retval 1 yes
169  */
171 {
172  if (s == NULL)
173  return 0;
174 
176  return 1;
177 
178  return 0;
179 }
180 
181 /** \brief Test is a initialized signature is IP only
182  * \param de_ctx detection engine ctx
183  * \param s the signature
184  * \retval 1 sig is ip only
185  * \retval 0 sig is not ip only
186  */
188 {
189  if (s->alproto != ALPROTO_UNKNOWN)
190  return 0;
191 
192  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
193  return 0;
194 
195  /* if flow dir is set we can't process it in ip-only */
196  if (!(((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) ||
199  return 0;
200 
201  /* for now assume that all registered buffer types are incompatible */
202  const int nlists = s->init_data->smlists_array_size;
203  for (int i = 0; i < nlists; i++) {
204  if (s->init_data->smlists[i] == NULL)
205  continue;
206  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
207  continue;
208 
209  SCReturnInt(0);
210  }
211 
212  /* TMATCH list can be ignored, it contains TAGs and
213  * tags are compatible to IP-only. */
214 
215  /* if any of the addresses uses negation, we don't support
216  * it in ip-only */
218  return 0;
220  return 0;
221 
223  if (sm == NULL)
224  goto iponly;
225 
226  for ( ; sm != NULL; sm = sm->next) {
228  return 0;
229  /* we have enabled flowbits to be compatible with ip only sigs, as long
230  * as the sig only has a "set" flowbits */
231  if (sm->type == DETECT_FLOWBITS &&
232  (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
233  return 0;
234  }
235  }
236 
237 iponly:
238  if (!(de_ctx->flags & DE_QUIET)) {
239  SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
240  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
241  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
242  }
243  return 1;
244 }
245 
246 /** \internal
247  * \brief Test is a initialized signature is inspecting protocol detection only
248  * \param de_ctx detection engine ctx
249  * \param s the signature
250  * \retval 1 sig is dp only
251  * \retval 0 sig is not dp only
252  */
253 static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s)
254 {
255  if (s->alproto != ALPROTO_UNKNOWN)
256  return 0;
257 
258  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
259  return 0;
260 
261  /* for now assume that all registered buffer types are incompatible */
262  const int nlists = s->init_data->smlists_array_size;
263  for (int i = 0; i < nlists; i++) {
264  if (s->init_data->smlists[i] == NULL)
265  continue;
266  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
267  continue;
268 
269  SCReturnInt(0);
270  }
271 
272  /* TMATCH list can be ignored, it contains TAGs and
273  * tags are compatible to DP-only. */
274 
275  /* match list matches may be compatible to DP only. We follow the same
276  * logic as IP-only so we can use that flag */
277 
279  if (sm == NULL)
280  return 0;
281 
282  int pd = 0;
283  for ( ; sm != NULL; sm = sm->next) {
284  if (sm->type == DETECT_AL_APP_LAYER_PROTOCOL) {
285  pd = 1;
286  } else {
287  /* flowbits are supported for dp only sigs, as long
288  * as the sig only has a "set" flowbits */
289  if (sm->type == DETECT_FLOWBITS) {
290  if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
291  SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id);
292  return 0;
293  }
294  } else if (sm->type == DETECT_FLOW) {
296  SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id);
297  return 0;
298  }
299  } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) {
300  SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name);
301  return 0;
302  }
303  }
304  }
305 
306  if (pd) {
307  SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id);
308  }
309  return pd;
310 }
311 
312 /**
313  * \internal
314  * \brief Check if the initialized signature is inspecting the packet payload
315  * \param de_ctx detection engine ctx
316  * \param s the signature
317  * \retval 1 sig is inspecting the payload
318  * \retval 0 sig is not inspecting the payload
319  */
320 static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s)
321 {
322 
323  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
324  return 1;
325  }
326  return 0;
327 }
328 
329 /**
330  * \internal
331  * \brief check if a signature is decoder event matching only
332  * \param de_ctx detection engine
333  * \param s the signature to test
334  * \retval 0 not a DEOnly sig
335  * \retval 1 DEOnly sig
336  */
337 static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
338 {
339  if (s->alproto != ALPROTO_UNKNOWN) {
340  SCReturnInt(0);
341  }
342 
343  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
344  {
345  SCReturnInt(0);
346  }
347 
348  /* for now assume that all registered buffer types are incompatible */
349  const int nlists = s->init_data->smlists_array_size;
350  for (int i = 0; i < nlists; i++) {
351  if (s->init_data->smlists[i] == NULL)
352  continue;
353  if (!(DetectBufferTypeGetNameById(de_ctx, i)))
354  continue;
355 
356  SCReturnInt(0);
357  }
358 
359  /* check for conflicting keywords */
361  for ( ;sm != NULL; sm = sm->next) {
363  SCReturnInt(0);
364  }
365 
366  /* need at least one decode event keyword to be considered decode event. */
368  for ( ;sm != NULL; sm = sm->next) {
369  if (sm->type == DETECT_DECODE_EVENT)
370  goto deonly;
371  if (sm->type == DETECT_ENGINE_EVENT)
372  goto deonly;
373  if (sm->type == DETECT_STREAM_EVENT)
374  goto deonly;
375  }
376 
377  SCReturnInt(0);
378 
379 deonly:
380  if (!(de_ctx->flags & DE_QUIET)) {
381  SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
382  s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
383  s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
384  }
385 
386  SCReturnInt(1);
387 }
388 
389 #define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN)
390 #define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR)
391 
392 /* Create mask for this packet + it's flow if it has one
393  */
394 void
396  bool app_decoder_events)
397 {
398  if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
399  SCLogDebug("packet has payload");
400  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
401  } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
402  SCLogDebug("stream data available");
403  (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
404  } else {
405  SCLogDebug("packet has no payload");
406  (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
407  }
408 
409  if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) {
410  SCLogDebug("packet/flow has events set");
412  }
413 
414  if (!(PKT_IS_PSEUDOPKT(p)) && PKT_IS_TCP(p)) {
415  if ((p->tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) {
417  }
418  if ((p->tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) {
420  }
421  }
422 
423  if (p->flags & PKT_HAS_FLOW) {
424  SCLogDebug("packet has flow");
425  (*mask) |= SIG_MASK_REQUIRE_FLOW;
426  }
427 
428  if (alproto == ALPROTO_SMB || alproto == ALPROTO_DCERPC) {
429  SCLogDebug("packet will be inspected for DCERPC");
430  (*mask) |= SIG_MASK_REQUIRE_DCERPC;
431  }
432 }
433 
434 static int g_dce_generic_list_id = -1;
435 static int g_dce_stub_data_buffer_id = -1;
436 
437 static bool SignatureNeedsDCERPCMask(const Signature *s)
438 {
439  if (g_dce_generic_list_id == -1) {
440  g_dce_generic_list_id = DetectBufferTypeGetByName("dce_generic");
441  SCLogDebug("g_dce_generic_list_id %d", g_dce_generic_list_id);
442  }
443  if (g_dce_stub_data_buffer_id == -1) {
444  g_dce_stub_data_buffer_id = DetectBufferTypeGetByName("dce_stub_data");
445  SCLogDebug("g_dce_stub_data_buffer_id %d", g_dce_stub_data_buffer_id);
446  }
447 
448  if (g_dce_generic_list_id >= 0 &&
449  s->init_data->smlists[g_dce_generic_list_id] != NULL)
450  {
451  return true;
452  }
453  if (g_dce_stub_data_buffer_id >= 0 &&
454  s->init_data->smlists[g_dce_stub_data_buffer_id] != NULL)
455  {
456  return true;
457  }
458  return false;
459 }
460 
461 static int SignatureCreateMask(Signature *s)
462 {
463  SCEnter();
464 
465  if (SignatureNeedsDCERPCMask(s)) {
467  SCLogDebug("sig requires DCERPC");
468  }
469 
470  if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
472  SCLogDebug("sig requires payload");
473  }
474 
475  SigMatch *sm;
476  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
477  switch(sm->type) {
478  case DETECT_FLOWBITS:
479  {
480  /* figure out what flowbit action */
482  if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
483  /* not a mask flag, but still set it here */
485 
486  SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has "
487  "flowbit isset option.");
488  }
489 
490  /* flow is required for any flowbit manipulation */
492  SCLogDebug("sig requires flow to be able to manipulate "
493  "flowbit(s)");
494  break;
495  }
496  case DETECT_FLOWINT:
497  /* flow is required for any flowint manipulation */
499  SCLogDebug("sig requires flow to be able to manipulate "
500  "flowint(s)");
501  break;
502  case DETECT_FLAGS:
503  {
504  DetectFlagsData *fl = (DetectFlagsData *)sm->ctx;
505 
506  if (fl->flags & TH_SYN) {
508  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
509  }
510  if (fl->flags & TH_RST) {
512  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
513  }
514  if (fl->flags & TH_FIN) {
516  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
517  }
518  if (fl->flags & TH_URG) {
520  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
521  }
522  if (fl->flags & TH_ECN) {
524  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
525  }
526  if (fl->flags & TH_CWR) {
528  SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
529  }
530  break;
531  }
532  case DETECT_DSIZE:
533  {
534  DetectDsizeData *ds = (DetectDsizeData *)sm->ctx;
535  switch (ds->mode) {
536  case DETECTDSIZE_LT:
537  /* LT will include 0, so no payload.
538  * if GT is used in the same rule the
539  * flag will be set anyway. */
540  break;
541  case DETECTDSIZE_RA:
542  case DETECTDSIZE_GT:
544  SCLogDebug("sig requires payload");
545  break;
546  case DETECTDSIZE_EQ:
547  if (ds->dsize > 0) {
549  SCLogDebug("sig requires payload");
550  } else if (ds->dsize == 0) {
552  SCLogDebug("sig requires no payload");
553  }
554  break;
555  }
556  break;
557  }
560  break;
561  case DETECT_ENGINE_EVENT:
563  break;
564  }
565  }
566 
569  SCLogDebug("sig requires flow");
570  }
571 
572  if (s->flags & SIG_FLAG_APPLAYER) {
574  SCLogDebug("sig requires flow");
575  }
576 
577  SCLogDebug("mask %02X", s->mask);
578  SCReturnInt(0);
579 }
580 
581 static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
582 {
584 }
585 
586 /** \brief Pure-PCRE or bytetest rule */
587 static int RuleInspectsPayloadHasNoMpm(const Signature *s)
588 {
589  if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
590  return 1;
591  return 0;
592 }
593 
594 static int RuleGetMpmPatternSize(const Signature *s)
595 {
596  if (s->init_data->mpm_sm == NULL)
597  return -1;
598  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
599  if (mpm_list < 0)
600  return -1;
601  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
602  if (cd == NULL)
603  return -1;
604  return (int)cd->content_len;
605 }
606 
607 static int RuleMpmIsNegated(const Signature *s)
608 {
609  if (s->init_data->mpm_sm == NULL)
610  return 0;
611  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
612  if (mpm_list < 0)
613  return 0;
614  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
615  if (cd == NULL)
616  return 0;
617  return (cd->flags & DETECT_CONTENT_NEGATED);
618 }
619 
620 static json_t *RulesGroupPrintSghStats(const SigGroupHead *sgh,
621  const int add_rules, const int add_mpm_stats)
622 {
623  uint32_t mpm_cnt = 0;
624  uint32_t nonmpm_cnt = 0;
625  uint32_t negmpm_cnt = 0;
626  uint32_t any5_cnt = 0;
627  uint32_t payload_no_mpm_cnt = 0;
628  uint32_t syn_cnt = 0;
629 
630  uint32_t mpms_total = 0;
631  uint32_t mpms_min = 0;
632  uint32_t mpms_max = 0;
633 
634  struct {
635  uint32_t total;
636  uint32_t cnt;
637  uint32_t min;
638  uint32_t max;
639  } mpm_stats[DETECT_SM_LIST_MAX];
640  memset(mpm_stats, 0x00, sizeof(mpm_stats));
641 
642  uint32_t alstats[ALPROTO_MAX] = {0};
643  uint32_t mpm_sizes[DETECT_SM_LIST_MAX][256];
644  memset(mpm_sizes, 0, sizeof(mpm_sizes));
645  uint32_t alproto_mpm_bufs[ALPROTO_MAX][DETECT_SM_LIST_MAX];
646  memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs));
647 
648  json_t *js = json_object();
649  if (unlikely(js == NULL))
650  return NULL;
651 
652  json_object_set_new(js, "id", json_integer(sgh->id));
653 
654  json_t *js_array = json_array();
655 
656  const Signature *s;
657  uint32_t x;
658  for (x = 0; x < sgh->sig_cnt; x++) {
659  s = sgh->match_array[x];
660  if (s == NULL)
661  continue;
662 
663  int any = 0;
664  if (s->proto.flags & DETECT_PROTO_ANY) {
665  any++;
666  }
667  if (s->flags & SIG_FLAG_DST_ANY) {
668  any++;
669  }
670  if (s->flags & SIG_FLAG_SRC_ANY) {
671  any++;
672  }
673  if (s->flags & SIG_FLAG_DP_ANY) {
674  any++;
675  }
676  if (s->flags & SIG_FLAG_SP_ANY) {
677  any++;
678  }
679  if (any == 5) {
680  any5_cnt++;
681  }
682 
683  if (s->init_data->mpm_sm == NULL) {
684  nonmpm_cnt++;
685 
686  if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) {
687  SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id);
688  }
689 
690  DetectPort *sp = s->sp;
691  DetectPort *dp = s->dp;
692 
693  if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) {
694  SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id);
695  }
696  if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) {
697  SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id);
698  }
699 
701  syn_cnt++;
702  }
703 
704  } else {
705  int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
706  BUG_ON(mpm_list < 0);
707  const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
708  uint32_t size = cd->content_len < 256 ? cd->content_len : 255;
709 
710  mpm_sizes[mpm_list][size]++;
711  if (s->alproto != ALPROTO_UNKNOWN) {
712  alproto_mpm_bufs[s->alproto][mpm_list]++;
713  }
714 
715  if (mpm_list == DETECT_SM_LIST_PMATCH) {
716  if (size == 1) {
717  DetectPort *sp = s->sp;
718  DetectPort *dp = s->dp;
719  if (s->flags & SIG_FLAG_TOSERVER) {
720  if (dp->port == 0 && dp->port2 == 65535) {
721  SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
722  } else {
723  SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id);
724  }
725  }
726  if (s->flags & SIG_FLAG_TOCLIENT) {
727  if (sp->port == 0 && sp->port2 == 65535) {
728  SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
729  } else {
730  SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id);
731  }
732  }
733  }
734  }
735 
736  uint32_t w = PatternStrength(cd->content, cd->content_len);
737  mpms_total += w;
738  if (mpms_min == 0)
739  mpms_min = w;
740  if (w < mpms_min)
741  mpms_min = w;
742  if (w > mpms_max)
743  mpms_max = w;
744 
745  mpm_stats[mpm_list].total += w;
746  mpm_stats[mpm_list].cnt++;
747  if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min)
748  mpm_stats[mpm_list].min = w;
749  if (w > mpm_stats[mpm_list].max)
750  mpm_stats[mpm_list].max = w;
751 
752  mpm_cnt++;
753 
754  if (w < 10) {
755  SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id);
756  }
757  if (w < 10 && any == 5) {
758  SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id);
759  }
760 
761  if (cd->flags & DETECT_CONTENT_NEGATED) {
762  SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id);
763  negmpm_cnt++;
764  }
765  }
766 
767  if (RuleInspectsPayloadHasNoMpm(s)) {
768  SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id);
769  payload_no_mpm_cnt++;
770  }
771 
772  if (s->alproto != ALPROTO_UNKNOWN) {
773  alstats[s->alproto]++;
774  }
775 
776  if (add_rules) {
777  json_t *js_sig = json_object();
778  if (unlikely(js == NULL))
779  continue;
780  json_object_set_new(js_sig, "sig_id", json_integer(s->id));
781  json_array_append_new(js_array, js_sig);
782  }
783  }
784 
785  json_object_set_new(js, "rules", js_array);
786 
787  json_t *stats = json_object();
788  json_object_set_new(stats, "total", json_integer(sgh->sig_cnt));
789 
790  json_t *types = json_object();
791  json_object_set_new(types, "mpm", json_integer(mpm_cnt));
792  json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt));
793  json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt));
794  json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt));
795  json_object_set_new(types, "syn", json_integer(syn_cnt));
796  json_object_set_new(types, "any5", json_integer(any5_cnt));
797  json_object_set_new(stats, "types", types);
798 
799  int i;
800  for (i = 0; i < ALPROTO_MAX; i++) {
801  if (alstats[i] > 0) {
802  json_t *app = json_object();
803  json_object_set_new(app, "total", json_integer(alstats[i]));
804 
805  for (x = 0; x < DETECT_SM_LIST_MAX; x++) {
806  if (alproto_mpm_bufs[i][x] == 0)
807  continue;
808  json_object_set_new(app, DetectListToHumanString(x), json_integer(alproto_mpm_bufs[i][x]));
809  }
810 
811  json_object_set_new(stats, AppProtoToString(i), app);
812  }
813  }
814 
815  if (add_mpm_stats) {
816  json_t *mpm_js = json_object();
817 
818  for (i = 0; i < DETECT_SM_LIST_MAX; i++) {
819  if (mpm_stats[i].cnt > 0) {
820 
821  json_t *mpm_sizes_array = json_array();
822  for (x = 0; x < 256; x++) {
823  if (mpm_sizes[i][x] == 0)
824  continue;
825 
826  json_t *e = json_object();
827  json_object_set_new(e, "size", json_integer(x));
828  json_object_set_new(e, "count", json_integer(mpm_sizes[i][x]));
829  json_array_append_new(mpm_sizes_array, e);
830  }
831 
832  json_t *buf = json_object();
833  json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt));
834  json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt));
835  json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min));
836  json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max));
837 
838  json_object_set_new(buf, "sizes", mpm_sizes_array);
839 
840  json_object_set_new(mpm_js, DetectListToHumanString(i), buf);
841  }
842  }
843 
844  json_object_set_new(stats, "mpm", mpm_js);
845  }
846  json_object_set_new(js, "stats", stats);
847 
848  json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist));
849 
850  return js;
851 }
852 
853 static void RulesDumpGrouping(const DetectEngineCtx *de_ctx,
854  const int add_rules, const int add_mpm_stats)
855 {
856  json_t *js = json_object();
857  if (unlikely(js == NULL))
858  return;
859 
860  int p;
861  for (p = 0; p < 256; p++) {
862  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
863  const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp";
864 
865  json_t *tcp = json_object();
866 
867  json_t *ts_array = json_array();
868  DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp :
869  de_ctx->flow_gh[1].udp;
870  while (list != NULL) {
871  json_t *port = json_object();
872  json_object_set_new(port, "port", json_integer(list->port));
873  json_object_set_new(port, "port2", json_integer(list->port2));
874 
875  json_t *tcp_ts = RulesGroupPrintSghStats(list->sh,
876  add_rules, add_mpm_stats);
877  json_object_set_new(port, "rulegroup", tcp_ts);
878  json_array_append_new(ts_array, port);
879 
880  list = list->next;
881  }
882  json_object_set_new(tcp, "toserver", ts_array);
883 
884  json_t *tc_array = json_array();
885  list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp :
886  de_ctx->flow_gh[0].udp;
887  while (list != NULL) {
888  json_t *port = json_object();
889  json_object_set_new(port, "port", json_integer(list->port));
890  json_object_set_new(port, "port2", json_integer(list->port2));
891 
892  json_t *tcp_tc = RulesGroupPrintSghStats(list->sh,
893  add_rules, add_mpm_stats);
894  json_object_set_new(port, "rulegroup", tcp_tc);
895  json_array_append_new(tc_array, port);
896 
897  list = list->next;
898  }
899  json_object_set_new(tcp, "toclient", tc_array);
900 
901  json_object_set_new(js, name, tcp);
902  }
903 
904  }
905 
906  const char *filename = "rule_group.json";
907  const char *log_dir = ConfigGetLogDirectory();
908  char log_path[PATH_MAX] = "";
909 
910  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
911 
912  FILE *fp = fopen(log_path, "w");
913  if (fp == NULL) {
914  return;
915  }
916 
917  char *js_s = json_dumps(js,
918  JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH);
919  if (unlikely(js_s == NULL)) {
920  fclose(fp);
921  return;
922  }
923 
924  json_object_clear(js);
925  json_decref(js);
926 
927  fprintf(fp, "%s\n", js_s);
928  free(js_s);
929  fclose(fp);
930  return;
931 }
932 
933 static int RulesGroupByProto(DetectEngineCtx *de_ctx)
934 {
935  Signature *s = de_ctx->sig_list;
936 
937  uint32_t max_idx = 0;
938  SigGroupHead *sgh_ts[256] = {NULL};
939  SigGroupHead *sgh_tc[256] = {NULL};
940 
941  for ( ; s != NULL; s = s->next) {
942  if (s->flags & SIG_FLAG_IPONLY)
943  continue;
944 
945  int p;
946  for (p = 0; p < 256; p++) {
947  if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
948  continue;
949  }
950  if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) {
951  continue;
952  }
953 
954  if (s->flags & SIG_FLAG_TOCLIENT) {
955  SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s);
956  max_idx = s->num;
957  }
958  if (s->flags & SIG_FLAG_TOSERVER) {
959  SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s);
960  max_idx = s->num;
961  }
962  }
963  }
964  SCLogDebug("max_idx %u", max_idx);
965 
966  /* lets look at deduplicating this list */
967  SigGroupHeadHashFree(de_ctx);
968  SigGroupHeadHashInit(de_ctx);
969 
970  uint32_t cnt = 0;
971  uint32_t own = 0;
972  uint32_t ref = 0;
973  int p;
974  for (p = 0; p < 256; p++) {
975  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
976  continue;
977  if (sgh_ts[p] == NULL)
978  continue;
979 
980  cnt++;
981 
982  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]);
983  if (lookup_sgh == NULL) {
984  SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]);
985 
986  SigGroupHeadSetSigCnt(sgh_ts[p], max_idx);
987  SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx);
988 
989  SigGroupHeadHashAdd(de_ctx, sgh_ts[p]);
990  SigGroupHeadStore(de_ctx, sgh_ts[p]);
991 
992  de_ctx->gh_unique++;
993  own++;
994  } else {
995  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]);
996 
997  SigGroupHeadFree(de_ctx, sgh_ts[p]);
998  sgh_ts[p] = lookup_sgh;
999 
1000  de_ctx->gh_reuse++;
1001  ref++;
1002  }
1003  }
1004  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1005  "toserver", cnt, own, ref);
1006 
1007  cnt = 0;
1008  own = 0;
1009  ref = 0;
1010  for (p = 0; p < 256; p++) {
1011  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1012  continue;
1013  if (sgh_tc[p] == NULL)
1014  continue;
1015 
1016  cnt++;
1017 
1018  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]);
1019  if (lookup_sgh == NULL) {
1020  SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]);
1021 
1022  SigGroupHeadSetSigCnt(sgh_tc[p], max_idx);
1023  SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx);
1024 
1025  SigGroupHeadHashAdd(de_ctx, sgh_tc[p]);
1026  SigGroupHeadStore(de_ctx, sgh_tc[p]);
1027 
1028  de_ctx->gh_unique++;
1029  own++;
1030 
1031  } else {
1032  SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]);
1033 
1034  SigGroupHeadFree(de_ctx, sgh_tc[p]);
1035  sgh_tc[p] = lookup_sgh;
1036 
1037  de_ctx->gh_reuse++;
1038  ref++;
1039  }
1040  }
1041  SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1042  "toclient", cnt, own, ref);
1043 
1044  for (p = 0; p < 256; p++) {
1045  if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1046  continue;
1047 
1048  de_ctx->flow_gh[0].sgh[p] = sgh_tc[p];
1049  de_ctx->flow_gh[1].sgh[p] = sgh_ts[p];
1050  }
1051 
1052  return 0;
1053 }
1054 
1055 static int PortIsWhitelisted(const DetectEngineCtx *de_ctx,
1056  const DetectPort *a, int ipproto)
1057 {
1058  DetectPort *w = de_ctx->tcp_whitelist;
1059  if (ipproto == IPPROTO_UDP)
1060  w = de_ctx->udp_whitelist;
1061 
1062  while (w) {
1063  if (a->port >= w->port && a->port2 <= w->port) {
1064  SCLogDebug("port group %u:%u whitelisted -> %d", a->port, a->port2, w->port);
1065  return 1;
1066  }
1067  w = w->next;
1068  }
1069 
1070  return 0;
1071 }
1072 
1073 static int RuleSetWhitelist(Signature *s)
1074 {
1075  DetectPort *p = NULL;
1076  if (s->flags & SIG_FLAG_TOSERVER)
1077  p = s->dp;
1078  else if (s->flags & SIG_FLAG_TOCLIENT)
1079  p = s->sp;
1080  else
1081  return 0;
1082 
1083  /* for sigs that don't use 'any' as port, see if we want to
1084  * whitelist poor sigs */
1085  int wl = 0;
1086  if (!(p->port == 0 && p->port2 == 65535)) {
1087  /* pure pcre, bytetest, etc rules */
1088  if (RuleInspectsPayloadHasNoMpm(s)) {
1089  SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id);
1090  wl = 99;
1091 
1092  } else if (RuleMpmIsNegated(s)) {
1093  SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id);
1094  wl = 77;
1095 
1096  /* one byte pattern in packet/stream payloads */
1097  } else if (s->init_data->mpm_sm != NULL &&
1099  RuleGetMpmPatternSize(s) == 1)
1100  {
1101  SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id);
1102  wl = 55;
1103 
1104  } else if (DetectFlagsSignatureNeedsSynPackets(s) &&
1106  {
1107  SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id);
1108  wl = 33;
1109  }
1110  }
1111 
1112  s->init_data->whitelist = wl;
1113  return wl;
1114 }
1115 
1116 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx);
1118 
1119 static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint32_t direction) {
1120  /* step 1: create a hash of 'DetectPort' objects based on all the
1121  * rules. Each object will have a SGH with the sigs added
1122  * that belong to the SGH. */
1123  DetectPortHashInit(de_ctx);
1124 
1125  uint32_t max_idx = 0;
1126  const Signature *s = de_ctx->sig_list;
1127  DetectPort *list = NULL;
1128  while (s) {
1129  /* IP Only rules are handled separately */
1130  if (s->flags & SIG_FLAG_IPONLY)
1131  goto next;
1132  if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY)))
1133  goto next;
1134  if (direction == SIG_FLAG_TOSERVER) {
1135  if (!(s->flags & SIG_FLAG_TOSERVER))
1136  goto next;
1137  } else if (direction == SIG_FLAG_TOCLIENT) {
1138  if (!(s->flags & SIG_FLAG_TOCLIENT))
1139  goto next;
1140  }
1141 
1142  DetectPort *p = NULL;
1143  if (direction == SIG_FLAG_TOSERVER)
1144  p = s->dp;
1145  else if (direction == SIG_FLAG_TOCLIENT)
1146  p = s->sp;
1147  else
1148  BUG_ON(1);
1149 
1150  /* see if we want to exclude directionless sigs that really care only for
1151  * to_server syn scans/floods */
1152  if ((direction == SIG_FLAG_TOCLIENT) &&
1156  (!(s->dp->port == 0 && s->dp->port2 == 65535)))
1157  {
1158  SCLogWarning(SC_WARN_POOR_RULE, "rule %u: SYN-only to port(s) %u:%u "
1159  "w/o direction specified, disabling for toclient direction",
1160  s->id, s->dp->port, s->dp->port2);
1161  goto next;
1162  }
1163 
1164  int wl = s->init_data->whitelist;
1165  while (p) {
1166  int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? 111 : 0;
1167  pwl = MAX(wl,pwl);
1168 
1169  DetectPort *lookup = DetectPortHashLookup(de_ctx, p);
1170  if (lookup) {
1171  SigGroupHeadAppendSig(de_ctx, &lookup->sh, s);
1172  lookup->sh->init->whitelist = MAX(lookup->sh->init->whitelist, pwl);
1173  } else {
1174  DetectPort *tmp2 = DetectPortCopySingle(de_ctx, p);
1175  BUG_ON(tmp2 == NULL);
1176  SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s);
1177  tmp2->sh->init->whitelist = pwl;
1178  DetectPortHashAdd(de_ctx, tmp2);
1179  }
1180 
1181  p = p->next;
1182  }
1183  max_idx = s->num;
1184  next:
1185  s = s->next;
1186  }
1187 
1188  /* step 2: create a list of DetectPort objects */
1189  HashListTableBucket *htb = NULL;
1190  for (htb = HashListTableGetListHead(de_ctx->dport_hash_table);
1191  htb != NULL;
1192  htb = HashListTableGetListNext(htb))
1193  {
1195  DetectPort *tmp = DetectPortCopySingle(de_ctx, p);
1196  BUG_ON(tmp == NULL);
1197  int r = DetectPortInsert(de_ctx, &list , tmp);
1198  BUG_ON(r == -1);
1199  }
1200  DetectPortHashFree(de_ctx);
1201  de_ctx->dport_hash_table = NULL;
1202 
1203  SCLogDebug("rules analyzed");
1204 
1205  /* step 3: group the list and shrink it if necessary */
1206  DetectPort *newlist = NULL;
1207  uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups :
1208  de_ctx->max_uniq_toserver_groups;
1209  CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx);
1210  list = newlist;
1211 
1212  /* step 4: deduplicate the SGH's */
1213  SigGroupHeadHashFree(de_ctx);
1214  SigGroupHeadHashInit(de_ctx);
1215 
1216  uint32_t cnt = 0;
1217  uint32_t own = 0;
1218  uint32_t ref = 0;
1219  DetectPort *iter;
1220  for (iter = list ; iter != NULL; iter = iter->next) {
1221  BUG_ON (iter->sh == NULL);
1222  cnt++;
1223 
1224  SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh);
1225  if (lookup_sgh == NULL) {
1226  SCLogDebug("port group %p sgh %p is the original", iter, iter->sh);
1227 
1228  SigGroupHeadSetSigCnt(iter->sh, max_idx);
1229  SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx);
1230  SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction);
1231  SigGroupHeadHashAdd(de_ctx, iter->sh);
1232  SigGroupHeadStore(de_ctx, iter->sh);
1233  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1234  de_ctx->gh_unique++;
1235  own++;
1236  } else {
1237  SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh);
1238 
1239  SigGroupHeadFree(de_ctx, iter->sh);
1240  iter->sh = lookup_sgh;
1241  iter->flags |= PORT_SIGGROUPHEAD_COPY;
1242 
1243  de_ctx->gh_reuse++;
1244  ref++;
1245  }
1246  }
1247 #if 0
1248  for (iter = list ; iter != NULL; iter = iter->next) {
1249  SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)",
1250  iter->port, iter->port2, iter->sh,
1251  iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own",
1252  iter->sh->init->whitelist ? "true" : "false",
1253  iter->sh->init->whitelist);
1254  }
1255 #endif
1256  SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies",
1257  ipproto == 6 ? "TCP" : "UDP",
1258  direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1259  cnt, own, ref);
1260  return list;
1261 }
1262 
1264 {
1265  /* see if the sig is dp only */
1266  if (SignatureIsPDOnly(de_ctx, s) == 1) {
1267  s->flags |= SIG_FLAG_PDONLY;
1268 
1269  /* see if the sig is ip only */
1270  } else if (SignatureIsIPOnly(de_ctx, s) == 1) {
1271  s->flags |= SIG_FLAG_IPONLY;
1272 
1273  } else if (SignatureIsDEOnly(de_ctx, s) == 1) {
1275  }
1276 }
1277 
1278 /**
1279  * \brief Preprocess signature, classify ip-only, etc, build sig array
1280  *
1281  * \param de_ctx Pointer to the Detection Engine Context
1282  *
1283  * \retval 0 on success
1284  * \retval -1 on failure
1285  */
1287 {
1288  uint32_t cnt_iponly = 0;
1289  uint32_t cnt_payload = 0;
1290  uint32_t cnt_applayer = 0;
1291  uint32_t cnt_deonly = 0;
1292 
1293  if (!(de_ctx->flags & DE_QUIET)) {
1294  SCLogDebug("building signature grouping structure, stage 1: "
1295  "preprocessing rules...");
1296  }
1297 
1298  de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx);
1299  de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *));
1300  de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size);
1301  if (de_ctx->sig_array == NULL)
1302  goto error;
1303  memset(de_ctx->sig_array,0,de_ctx->sig_array_size);
1304 
1305  SCLogDebug("signature lookup array: %" PRIu32 " sigs, %" PRIu32 " bytes",
1306  de_ctx->sig_array_len, de_ctx->sig_array_size);
1307 
1308  /* now for every rule add the source group */
1309  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1310  de_ctx->sig_array[s->num] = s;
1311 
1312  SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", s->id, s->num, s, de_ctx->sig_array[s->num]);
1313 
1314  if (s->flags & SIG_FLAG_PDONLY) {
1315  SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", s->id);
1316  } else if (s->flags & SIG_FLAG_IPONLY) {
1317  SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", s->id);
1318  cnt_iponly++;
1319  } else if (SignatureIsInspectingPayload(de_ctx, s) == 1) {
1320  SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", s->id);
1321  cnt_payload++;
1322  } else if (s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) {
1323  SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", s->id);
1324  cnt_deonly++;
1325  } else if (s->flags & SIG_FLAG_APPLAYER) {
1326  SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", s->id);
1327  cnt_applayer++;
1328  }
1329 
1330 #ifdef DEBUG
1331  if (SCLogDebugEnabled()) {
1332  uint16_t colen = 0;
1333  char copresent = 0;
1334  SigMatch *sm;
1335  DetectContentData *co;
1336  for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) {
1337  if (sm->type != DETECT_CONTENT)
1338  continue;
1339 
1340  copresent = 1;
1341  co = (DetectContentData *)sm->ctx;
1342  if (co->content_len > colen)
1343  colen = co->content_len;
1344  }
1345 
1346  if (copresent && colen == 1) {
1347  SCLogDebug("signature %8u content maxlen 1", s->id);
1348  for (int proto = 0; proto < 256; proto++) {
1349  if (s->proto.proto[(proto/8)] & (1<<(proto%8)))
1350  SCLogDebug("=> proto %" PRId32 "", proto);
1351  }
1352  }
1353  }
1354 #endif /* DEBUG */
1355 
1356  if (RuleMpmIsNegated(s)) {
1357  s->flags |= SIG_FLAG_MPM_NEG;
1358  }
1359 
1360  SignatureCreateMask(s);
1363 
1364  RuleSetWhitelist(s);
1365 
1366  /* if keyword engines are enabled in the config, handle them here */
1367  if (de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
1368  !(s->flags & SIG_FLAG_PREFILTER))
1369  {
1370  int prefilter_list = DETECT_TBLSIZE;
1371 
1372  /* get the keyword supporting prefilter with the lowest type */
1373  for (int i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1374  SigMatch *sm = s->init_data->smlists[i];
1375  while (sm != NULL) {
1376  if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
1377  if (sigmatch_table[sm->type].SupportsPrefilter(s) == TRUE) {
1378  prefilter_list = MIN(prefilter_list, sm->type);
1379  }
1380  }
1381  sm = sm->next;
1382  }
1383  }
1384 
1385  /* apply that keyword as prefilter */
1386  if (prefilter_list != DETECT_TBLSIZE) {
1387  for (int i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1388  SigMatch *sm = s->init_data->smlists[i];
1389  while (sm != NULL) {
1390  if (sm->type == prefilter_list) {
1391  s->init_data->prefilter_sm = sm;
1392  s->flags |= SIG_FLAG_PREFILTER;
1393  SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name);
1394  break;
1395  }
1396  sm = sm->next;
1397  }
1398  }
1399  }
1400  }
1401 
1402  /* run buffer type callbacks if any */
1403  for (int x = 0; x < (int)s->init_data->smlists_array_size; x++) {
1404  if (s->init_data->smlists[x])
1405  DetectBufferRunSetupCallback(de_ctx, x, s);
1406  }
1407 
1408  de_ctx->sig_cnt++;
1409  }
1410 
1411  if (!(de_ctx->flags & DE_QUIET)) {
1412  SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only "
1413  "rules, %" PRIu32 " are inspecting packet payload, %"PRIu32
1414  " inspect application layer, %"PRIu32" are decoder event only",
1415  de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer,
1416  cnt_deonly);
1417 
1418  SCLogConfig("building signature grouping structure, stage 1: "
1419  "preprocessing rules... complete");
1420  }
1421  DetectFlowbitsAnalyze(de_ctx);
1422 
1423  return 0;
1424 
1425 error:
1426  return -1;
1427 }
1428 
1429 static int PortGroupWhitelist(const DetectPort *a)
1430 {
1431  return a->sh->init->whitelist;
1432 }
1433 
1435 {
1436  if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) {
1437  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1438  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1439  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1440  return 1;
1441  } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) {
1442  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1443  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1444  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1445  return 0;
1446  } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) {
1447  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1448  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1449  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1450  return 1;
1451  } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) {
1452  if (a->sh->sig_cnt > b->sh->sig_cnt) {
1453  SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1454  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1455  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1456  return 1;
1457  }
1458  }
1459 
1460  SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1461  a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1462  b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1463  return 0;
1464 }
1465 
1466 /** \internal
1467  * \brief Create a list of DetectPort objects sorted based on CompareFunc's
1468  * logic.
1469  *
1470  * List can limit the number of groups. In this case an extra "join" group
1471  * is created that contains the sigs belonging to that. It's *appended* to
1472  * the list, meaning that if the list is walked linearly it's found last.
1473  * The joingr is meant to be a catch all.
1474  *
1475  */
1476 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
1477 {
1478  DetectPort *tmplist = NULL, *joingr = NULL;
1479  char insert = 0;
1480  uint32_t groups = 0;
1481  DetectPort *list;
1482 
1483  /* insert the addresses into the tmplist, where it will
1484  * be sorted descending on 'cnt' and on wehther a group
1485  * is whitelisted. */
1486 
1487  DetectPort *oldhead = port_list;
1488  while (oldhead) {
1489  /* take the top of the list */
1490  list = oldhead;
1491  oldhead = oldhead->next;
1492  list->next = NULL;
1493 
1494  groups++;
1495 
1496  SigGroupHeadSetSigCnt(list->sh, max_idx);
1497 
1498  /* insert it */
1499  DetectPort *tmpgr = tmplist, *prevtmpgr = NULL;
1500  if (tmplist == NULL) {
1501  /* empty list, set head */
1502  tmplist = list;
1503  } else {
1504  /* look for the place to insert */
1505  for ( ; tmpgr != NULL && !insert; tmpgr = tmpgr->next) {
1506  if (CompareFunc(list, tmpgr) == 1) {
1507  if (tmpgr == tmplist) {
1508  list->next = tmplist;
1509  tmplist = list;
1510  SCLogDebug("new list top: %u:%u", tmplist->port, tmplist->port2);
1511  } else {
1512  list->next = prevtmpgr->next;
1513  prevtmpgr->next = list;
1514  }
1515  insert = 1;
1516  break;
1517  }
1518  prevtmpgr = tmpgr;
1519  }
1520  if (insert == 0) {
1521  list->next = NULL;
1522  prevtmpgr->next = list;
1523  }
1524  insert = 0;
1525  }
1526  }
1527 
1528  uint32_t left = unique_groups;
1529  if (left == 0)
1530  left = groups;
1531 
1532  /* create another list: take the port groups from above
1533  * and add them to the 2nd list until we have met our
1534  * count. The rest is added to the 'join' group. */
1535  DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL;
1536  DetectPort *gr, *next_gr;
1537  for (gr = tmplist; gr != NULL; ) {
1538  next_gr = gr->next;
1539 
1540  SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2);
1541  DetectPortPrint(gr);
1542 
1543  /* if we've set up all the unique groups, add the rest to the
1544  * catch-all joingr */
1545  if (left == 0) {
1546  if (joingr == NULL) {
1547  DetectPortParse(de_ctx, &joingr, "0:65535");
1548  if (joingr == NULL) {
1549  goto error;
1550  }
1551  SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2);
1552  joingr->next = NULL;
1553  }
1554  SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh);
1555 
1556  /* when a group's sigs are added to the joingr, we can free it */
1557  gr->next = NULL;
1558  DetectPortFree(de_ctx, gr);
1559  gr = NULL;
1560 
1561  /* append */
1562  } else {
1563  gr->next = NULL;
1564 
1565  if (tmplist2 == NULL) {
1566  tmplist2 = gr;
1567  tmplist2_tail = gr;
1568  } else {
1569  tmplist2_tail->next = gr;
1570  tmplist2_tail = gr;
1571  }
1572  }
1573 
1574  if (left > 0)
1575  left--;
1576 
1577  gr = next_gr;
1578  }
1579 
1580  /* if present, append the joingr that covers the rest */
1581  if (joingr != NULL) {
1582  SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2);
1583 
1584  if (tmplist2 == NULL) {
1585  tmplist2 = joingr;
1586  //tmplist2_tail = joingr;
1587  } else {
1588  tmplist2_tail->next = joingr;
1589  //tmplist2_tail = joingr;
1590  }
1591  } else {
1592  SCLogDebug("no joingr");
1593  }
1594 
1595  /* pass back our new list to the caller */
1596  *newhead = tmplist2;
1597  DetectPortPrintList(*newhead);
1598 
1599  return 0;
1600 error:
1601  return -1;
1602 }
1603 
1604 /**
1605  * \internal
1606  * \brief add a decoder event signature to the detection engine ctx
1607  */
1608 static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s)
1609 {
1610  SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id);
1611  SigGroupHeadAppendSig(de_ctx, &de_ctx->decoder_event_sgh, s);
1612 }
1613 
1614 /**
1615  * \brief Fill the global src group head, with the sigs included
1616  *
1617  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1618  * to be processed
1619  *
1620  * \retval 0 On success
1621  * \retval -1 On failure
1622  */
1624 {
1625  uint32_t sigs = 0;
1626 
1627  SCLogDebug("building signature grouping structure, stage 2: "
1628  "building source address lists...");
1629 
1630  IPOnlyInit(de_ctx, &de_ctx->io_ctx);
1631 
1632  de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER);
1633  de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT);
1634  de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER);
1635  de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT);
1636 
1637  /* Setup the other IP Protocols (so not TCP/UDP) */
1638  RulesGroupByProto(de_ctx);
1639 
1640  /* now for every rule add the source group to our temp lists */
1641  for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1642  SCLogDebug("s->id %"PRIu32, s->id);
1643  if (s->flags & SIG_FLAG_IPONLY) {
1644  IPOnlyAddSignature(de_ctx, &de_ctx->io_ctx, s);
1645  }
1646 
1648  DetectEngineAddDecoderEventSig(de_ctx, s);
1649  }
1650 
1651  sigs++;
1652  }
1653 
1654  IPOnlyPrepare(de_ctx);
1655  IPOnlyPrint(de_ctx, &de_ctx->io_ctx);
1656 
1657  return 0;
1658 }
1659 
1660 static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx)
1661 {
1662  if (de_ctx->decoder_event_sgh == NULL)
1663  return;
1664 
1665  uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx);
1666  SigGroupHeadSetSigCnt(de_ctx->decoder_event_sgh, max_idx);
1667  SigGroupHeadBuildMatchArray(de_ctx, de_ctx->decoder_event_sgh, max_idx);
1668 }
1669 
1671 {
1672  /* prepare the decoder event sgh */
1673  DetectEngineBuildDecoderEventSgh(de_ctx);
1674  return 0;
1675 }
1676 
1678 {
1679  BUG_ON(de_ctx == NULL);
1680 
1681  if (!(de_ctx->flags & DE_QUIET)) {
1682  SCLogDebug("cleaning up signature grouping structure...");
1683  }
1684  if (de_ctx->decoder_event_sgh)
1685  SigGroupHeadFree(de_ctx, de_ctx->decoder_event_sgh);
1686  de_ctx->decoder_event_sgh = NULL;
1687 
1688  for (int f = 0; f < FLOW_STATES; f++) {
1689  for (int p = 0; p < 256; p++) {
1690  de_ctx->flow_gh[f].sgh[p] = NULL;
1691  }
1692 
1693  /* free lookup lists */
1694  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].tcp);
1695  de_ctx->flow_gh[f].tcp = NULL;
1696  DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].udp);
1697  de_ctx->flow_gh[f].udp = NULL;
1698  }
1699 
1700  for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1701  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1702  if (sgh == NULL)
1703  continue;
1704 
1705  SCLogDebug("sgh %p", sgh);
1706  SigGroupHeadFree(de_ctx, sgh);
1707  }
1708  SCFree(de_ctx->sgh_array);
1709  de_ctx->sgh_array = NULL;
1710  de_ctx->sgh_array_cnt = 0;
1711  de_ctx->sgh_array_size = 0;
1712 
1713  IPOnlyDeinit(de_ctx, &de_ctx->io_ctx);
1714 
1715  if (!(de_ctx->flags & DE_QUIET)) {
1716  SCLogInfo("cleaning up signature grouping structure... complete");
1717  }
1718  return 0;
1719 }
1720 
1721 #if 0
1722 static void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1723 {
1724  if (sgh == NULL) {
1725  printf("\n");
1726  return;
1727  }
1728 
1729  uint32_t sig;
1730  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1731  printf("%" PRIu32 " ", sgh->match_array[sig]->id);
1732  }
1733  printf("\n");
1734 }
1735 
1736 static void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1737 {
1738  if (sgh == NULL || sgh->init == NULL) {
1739  printf("\n");
1740  return;
1741  }
1742 
1743  uint32_t sig;
1744  for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
1745  if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) {
1746  printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id);
1747  }
1748  }
1749  printf("\n");
1750 }
1751 #endif
1752 
1753 /** \brief finalize preparing sgh's */
1755 {
1756  SCEnter();
1757 
1758  //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt);
1759 
1760  uint32_t cnt = 0;
1761  for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1762  SigGroupHead *sgh = de_ctx->sgh_array[idx];
1763  if (sgh == NULL)
1764  continue;
1765 
1766  SCLogDebug("sgh %p", sgh);
1767 
1768  SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
1769  SigGroupHeadSetFileHashFlag(de_ctx, sgh);
1770  SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
1771  SigGroupHeadSetFilestoreCount(de_ctx, sgh);
1772  SCLogDebug("filestore count %u", sgh->filestore_cnt);
1773 
1774  PrefilterSetupRuleGroup(de_ctx, sgh);
1775 
1777 
1779  sgh->init = NULL;
1780 
1781  sgh->id = idx;
1782  cnt++;
1783  }
1784  SCLogPerf("Unique rule groups: %u", cnt);
1785 
1786  MpmStoreReportStats(de_ctx);
1787 
1788  if (de_ctx->decoder_event_sgh != NULL) {
1789  /* no need to set filestore count here as that would make a
1790  * signature not decode event only. */
1791  }
1792 
1793  /* cleanup the hashes now since we won't need them
1794  * after the initialization phase. */
1795  SigGroupHeadHashFree(de_ctx);
1796 
1797  int dump_grouping = 0;
1798  (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping);
1799 
1800  if (dump_grouping) {
1801  int add_rules = 0;
1802  (void)ConfGetBool("detect.profiling.grouping.include-rules", &add_rules);
1803  int add_mpm_stats = 0;
1804  (void)ConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_rules);
1805 
1806  RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats);
1807  }
1808 
1809 #ifdef PROFILING
1811 #endif
1812  SCReturnInt(0);
1813 }
1814 
1815 extern int rule_engine_analysis_set;
1816 /** \internal
1817  * \brief perform final per signature setup tasks
1818  *
1819  * - Create SigMatchData arrays from the init only SigMatch lists
1820  * - Setup per signature inspect engines
1821  * - remove signature init data.
1822  */
1823 static int SigMatchPrepare(DetectEngineCtx *de_ctx)
1824 {
1825  SCEnter();
1826 
1827  Signature *s = de_ctx->sig_list;
1828  for (; s != NULL; s = s->next) {
1829  /* set up inspect engines */
1831 
1832  /* built-ins */
1833  int type;
1834  for (type = 0; type < DETECT_SM_LIST_MAX; type++) {
1835  /* skip PMATCH if it is used in a stream 'app engine' instead */
1837  continue;
1838  SigMatch *sm = s->init_data->smlists[type];
1840  }
1841  /* set up the pkt inspection engines */
1843 
1844  if (rule_engine_analysis_set) {
1845  EngineAnalysisRules2(de_ctx, s);
1846  }
1847  /* free lists. Ctx' are xferred to sm_arrays so won't get freed */
1848  uint32_t i;
1849  for (i = 0; i < s->init_data->smlists_array_size; i++) {
1850  SigMatch *sm = s->init_data->smlists[i];
1851  while (sm != NULL) {
1852  SigMatch *nsm = sm->next;
1853  SigMatchFree(sm);
1854  sm = nsm;
1855  }
1856  }
1857  SCFree(s->init_data->smlists);
1859  SCFree(s->init_data);
1860  s->init_data = NULL;
1861  }
1862 
1863 
1864  SCReturnInt(0);
1865 }
1866 
1867 /**
1868  * \brief Convert the signature list into the runtime match structure.
1869  *
1870  * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1871  * to be processed
1872  *
1873  * \retval 0 On Success.
1874  * \retval -1 On failure.
1875  */
1877 {
1878  Signature *s = de_ctx->sig_list;
1879 
1880  /* Assign the unique order id of signatures after sorting,
1881  * so the IP Only engine process them in order too. Also
1882  * reset the old signums and assign new signums. We would
1883  * have experienced Sig reordering by now, hence the new
1884  * signums. */
1885  de_ctx->signum = 0;
1886  while (s != NULL) {
1887  s->num = de_ctx->signum++;
1888 
1889  s = s->next;
1890  }
1891 
1892  if (DetectSetFastPatternAndItsId(de_ctx) < 0)
1893  return -1;
1894 
1895  SigInitStandardMpmFactoryContexts(de_ctx);
1896 
1897  if (SigAddressPrepareStage1(de_ctx) != 0) {
1898  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1899  exit(EXIT_FAILURE);
1900  }
1901 
1902  if (SigAddressPrepareStage2(de_ctx) != 0) {
1903  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1904  exit(EXIT_FAILURE);
1905  }
1906 
1907  if (SigAddressPrepareStage3(de_ctx) != 0) {
1908  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1909  exit(EXIT_FAILURE);
1910  }
1911  if (SigAddressPrepareStage4(de_ctx) != 0) {
1912  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1913  exit(EXIT_FAILURE);
1914  }
1915 
1916  int r = DetectMpmPrepareBuiltinMpms(de_ctx);
1917  r |= DetectMpmPrepareAppMpms(de_ctx);
1918  r |= DetectMpmPreparePktMpms(de_ctx);
1919  if (r != 0) {
1920  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1921  exit(EXIT_FAILURE);
1922  }
1923 
1924  if (SigMatchPrepare(de_ctx) != 0) {
1925  SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
1926  exit(EXIT_FAILURE);
1927  }
1928 
1929 #ifdef PROFILING
1932  de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled
1933 
1934  intmax_t v = 0;
1935  if (ConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1)
1936  de_ctx->profile_match_logging_threshold = (uint32_t)v;
1937 
1939 #endif
1940 
1943  }
1944  return 0;
1945 }
1946 
1948 {
1949  SigAddressCleanupStage1(de_ctx);
1950 
1951  return 0;
1952 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:234
uint16_t filestore_cnt
Definition: detect.h:1352
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1448
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
Predict a strength value for patterns.
DetectProto proto
Definition: detect.h:539
#define DETECTDSIZE_LT
Definition: detect-dsize.h:27
SigIntId sig_cnt
Definition: detect.h:1341
SignatureInitData * init_data
Definition: detect.h:591
int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
Preprocess signature, classify ip-only, etc, build sig array.
#define SIG_FLAG_PDONLY
Definition: detect.h:247
#define SCLogDebug(...)
Definition: util-debug.h:335
#define MAX(x, y)
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Free a SigGroupHead and its members.
Signature ** sig_array
Definition: detect.h:776
void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx, Signature *s)
Add a signature to the lists of Adrresses in CIDR format (sorted) this step is necesary to build the ...
#define SIG_FLAG_PREFILTER
Definition: detect.h:243
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
Figured out the FP and their respective content ids for all the sigs in the engine.
#define DETECTDSIZE_GT
Definition: detect-dsize.h:29
#define SignatureMask
Definition: detect.h:278
uint32_t sig_array_len
Definition: detect.h:778
struct HtpBodyChunk_ * next
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
#define SIGMATCH_DEONLY_COMPAT
Definition: detect.h:1373
void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
SigMatch * prefilter_sm
Definition: detect.h:495
#define BUG_ON(x)
uint32_t flags
Definition: detect.h:523
_Bool(* SupportsPrefilter)(const Signature *s)
Definition: detect.h:1188
uint32_t id
Definition: detect.h:555
#define DETECT_PROTO_ANY
struct SigGroupHead_ * sh
Definition: detect.h:202
Signature ** match_array
Definition: detect.h:1361
#define DETECTDSIZE_RA
Definition: detect-dsize.h:30
#define unlikely(expr)
Definition: util-optimize.h:35
int SigAddressPrepareStage2(DetectEngineCtx *de_ctx)
Fill the global src group head, with the sigs included.
DetectPort * tcp_whitelist
Definition: detect.h:908
void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:59
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
DetectPort * udp_whitelist
Definition: detect.h:909
void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx)
Updates the SigGroupHead->sig_cnt with the total count of all the Signatures present in this SigGroup...
int rule_engine_analysis_set
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
#define TH_RST
Definition: decode-tcp.h:37
Signature * sig_list
Definition: detect.h:767
#define TH_FIN
Definition: decode-tcp.h:35
uint32_t sig_array_size
Definition: detect.h:777
#define MIN(x, y)
int SigAddressPrepareStage3(DetectEngineCtx *de_ctx)
void SCProfilingSghInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
uint8_t proto[256/8]
void SigCleanSignatures(DetectEngineCtx *de_ctx)
int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
finalize preparing sgh&#39;s
Container for matching data for a signature group.
Definition: detect.h:1336
#define DETECT_FLOWBITS_CMD_ISSET
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:232
const char * name
Definition: detect.h:1200
uint16_t AppProto
TCPHdr * tcph
Definition: decode.h:522
Signature container.
Definition: detect.h:522
int DetectFlagsSignatureNeedsSynPackets(const Signature *s)
int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
#define SIG_MASK_REQUIRE_ENGINE_EVENT
Definition: detect.h:275
#define TRUE
uint32_t id
Definition: detect.h:1354
uint32_t signum
Definition: detect.h:780
#define SIG_FLAG_APPLAYER
Definition: detect.h:218
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:859
void IPOnlyPrepare(DetectEngineCtx *de_ctx)
Build the radix trees from the lists of parsed adresses in CIDR format the result should be 4 radix t...
int DetectEngineMultiTenantEnabled(void)
void VarNameStoreActivateStaging(void)
int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t max_idx)
Create an array with all the internal ids of the sigs that this sig group head will check for...
SigIntId num
Definition: detect.h:532
struct SigMatch_ * next
Definition: detect.h:322
main detection engine ctx
Definition: detect.h:761
int SCLogDebugEnabled(void)
Returns whether debug messages are enabled to be logged or not.
Definition: util-debug.c:624
#define SIG_MASK_REQUIRE_FLAGS_INITDEINIT
Definition: detect.h:270
void IPOnlyDeinit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Deinitialize the IP Only detection engine context.
DetectPort * udp
Definition: detect.h:699
#define SIG_FLAG_DST_ANY
Definition: detect.h:212
int SignatureIsFilesizeInspecting(const Signature *s)
Check if a signature contains the filesize keyword.
#define SIG_FLAG_MPM_NEG
Definition: detect.h:226
bool src_contains_negation
Definition: detect.h:482
uint32_t sgh_array_cnt
Definition: detect.h:843
#define DE_QUIET
Definition: detect.h:292
DetectPort * DetectPortHashLookup(DetectEngineCtx *de_ctx, DetectPort *dp)
Used to lookup a DetectPort hash from the detection engine context DetectPort hash table...
#define DetectEngineGetMaxSigId(de_ctx)
Definition: detect-engine.h:81
int DetectBufferTypeGetByName(const char *name)
void SigGroupHeadStore(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
#define PORT_SIGGROUPHEAD_COPY
Definition: detect.h:188
uint16_t max_uniq_toserver_groups
Definition: detect.h:820
int SignatureIsFileSha256Inspecting(const Signature *s)
Check if a signature contains the filesha256 keyword.
#define SIG_FLAG_TOCLIENT
Definition: detect.h:237
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:58
DetectPort * dp
Definition: detect.h:561
void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need size flag in the sgh.
#define SIG_FLAG_SP_ANY
Definition: detect.h:213
DetectPort * tcp
Definition: detect.h:698
uint8_t flags
Definition: detect.h:762
struct SigGroupHead_ * sgh[256]
Definition: detect.h:700
#define SIG_FLAG_IPONLY
Definition: detect.h:219
#define SIG_MASK_REQUIRE_FLOW
Definition: detect.h:269
int SigAddressCleanupStage1(DetectEngineCtx *de_ctx)
void SigGroupHeadHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by SigGroupHeadHashInit() function...
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
#define SIG_FLAG_INIT_DEONLY
Definition: detect.h:256
uint8_t type
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1123
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void SigFree(Signature *)
int SignatureIsFileSha1Inspecting(const Signature *s)
Check if a signature contains the filesha1 keyword.
int SigGroupBuild(DetectEngineCtx *de_ctx)
Convert the signature list into the runtime match structure.
#define SIG_FLAG_INIT_FLOW
Definition: detect.h:258
bool dst_contains_negation
Definition: detect.h:483
#define FILE_SIG_NEED_MAGIC
Definition: detect.h:284
Signature * SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
Find a specific signature by sid and gid.
#define TH_URG
Definition: decode-tcp.h:40
#define SIG_MASK_REQUIRE_FLAGS_UNUSUAL
Definition: detect.h:271
#define SIG_FLAG_TOSERVER
Definition: detect.h:236
#define SCEnter(...)
Definition: util-debug.h:337
int DetectPortHashAdd(DetectEngineCtx *de_ctx, DetectPort *dp)
Adds a DetectPort to the detection engine context DetectPort hash table.
int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
build an array of rule id&#39;s for sigs with no prefilter Also updated de_ctx::non_pf_store_cnt_max to t...
void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid)
uint32_t profile_match_logging_threshold
Definition: detect.h:887
PacketEngineEvents events
Definition: decode.h:565
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:575
void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
void SigParseApplyDsizeToContent(Signature *s)
Apply dsize as depth to content matches in the rule.
Definition: detect-dsize.c:440
void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need magic flag in the sgh.
AppProto alproto
Definition: detect.h:526
struct SigMatch_ ** smlists_tail
Definition: detect.h:518
#define TH_ECN
Definition: decode-tcp.h:42
uint16_t max_uniq_toclient_groups
Definition: detect.h:819
#define DETECTDSIZE_EQ
Definition: detect-dsize.h:28
int SigGroupCleanup(DetectEngineCtx *de_ctx)
void DetectPortHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by DetectPortInit() function...
#define DETECT_FLOW_FLAG_TOCLIENT
Definition: detect-flow.h:28
struct Signature_ * next
Definition: detect.h:594
uint8_t type
Definition: detect.h:319
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
const char * DetectListToString(int list)
Definition: detect-parse.c:128
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:262
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:807
#define FILE_SIG_NEED_MD5
Definition: detect.h:286
SigGroupHead * SigGroupHeadHashLookup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Used to lookup a SigGroupHead hash from the detection engine context SigGroupHead hash table...
int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHead **dst)
Copies the bitarray holding the sids from the source SigGroupHead to the destination SigGroupHead...
void DetectPortCleanupList(const DetectEngineCtx *de_ctx, DetectPort *head)
Free a DetectPort list and each of its members.
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
uint32_t gh_unique
Definition: detect.h:797
struct SigMatch_ ** smlists
Definition: detect.h:516
int DetectPortHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the DetectPort hash...
uint32_t sgh_array_size
Definition: detect.h:844
#define SIG_MASK_REQUIRE_PAYLOAD
Definition: detect.h:268
int SignatureIsFilestoring(const Signature *s)
Check if a signature contains the filestore keyword.
uint32_t smlists_array_size
Definition: detect.h:514
void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
enum DetectEnginePrefilterSetting prefilter_setting
Definition: detect.h:904
SigMatchCtx * ctx
Definition: detect.h:321
#define MASK_TCP_INITDEINIT_FLAGS
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
int SigGroupHeadHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the SigGroupHeads.
int SigGroupHeadAppendSig(const DetectEngineCtx *de_ctx, SigGroupHead **sgh, const Signature *s)
Add a Signature to a SigGroupHead.
#define SCMalloc(a)
Definition: util-mem.h:222
void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
uint8_t flags
Definition: detect.h:195
int SignatureIsFileMd5Inspecting(const Signature *s)
Check if a signature contains the filemd5 keyword.
#define JSON_ESCAPE_SLASH
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define DETECT_CONTENT_NEGATED
#define TH_SYN
Definition: decode-tcp.h:36
#define SCFree(a)
Definition: util-mem.h:322
struct SigGroupHead_ ** sgh_array
Definition: detect.h:842
void IPOnlyPrint(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Print stats of the IP Only engine.
DetectPort * sp
Definition: detect.h:561
#define PKT_IS_TCP(p)
Definition: decode.h:253
int DetectEnginePktInspectionSetup(Signature *s)
uint16_t port
Definition: detect.h:192
void SigGroupHeadSetFileHashFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the need hash flag in the sgh.
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for builtin buffers that are in "single or "shared" mode. ...
SigMatchData * SigMatchList2DataArray(SigMatch *head)
convert SigMatch list to SigMatchData array
uint32_t init_flags
Definition: detect.h:486
#define FILE_SIG_NEED_SHA1
Definition: detect.h:287
int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
Test is a initialized signature is IP only.
#define DETECT_FLOWBITS_CMD_SET
int DetectPortInsert(DetectEngineCtx *de_ctx, DetectPort **head, DetectPort *new)
function for inserting a port group object. This also makes sure SigGroupContainer lists are handled ...
void SigMatchFree(SigMatch *sm)
free a SigMatch
Definition: detect-parse.c:247
void DetectPortFree(const DetectEngineCtx *de_ctx, DetectPort *dp)
Free a DetectPort and its members.
void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
#define SCLogPerf(...)
Definition: util-debug.h:261
int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int(*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
struct DetectPort_ * next
Definition: detect.h:205
#define FILE_SIG_NEED_SHA256
Definition: detect.h:288
int SignatureIsFilemagicInspecting(const Signature *s)
Check if a signature contains the filemagic keyword.
void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
void SCProfilingPrefilterInitCounters(DetectEngineCtx *de_ctx)
Register the prefilter profiling counters.
#define PKT_HAS_FLOW
Definition: decode.h:1093
uint32_t gh_reuse
Definition: detect.h:797
int SigGroupHeadHashAdd(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Adds a SigGroupHead to the detection engine context SigGroupHead hash table.
void DetectBufferRunSetupCallback(const DetectEngineCtx *de_ctx, const int id, Signature *s)
#define SIGMATCH_IPONLY_COMPAT
Definition: detect.h:1371
const char * DetectListToHumanString(int list)
Definition: detect-parse.c:111
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1132
#define SIG_FLAG_DP_ANY
Definition: detect.h:214
void SCProfilingRuleInitCounters(DetectEngineCtx *de_ctx)
Register the rule profiling counters.
void IPOnlyInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
Setup the IP Only detection engine context.
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:795
#define DETECT_FLOW_FLAG_TOSERVER
Definition: detect-flow.h:27
SignatureMask mask
Definition: detect.h:531
uint8_t file_flags
Definition: detect.h:536
int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s)
#define PKT_NOPAYLOAD_INSPECTION
Definition: decode.h:1087
uint32_t flags
Definition: decode.h:443
uint16_t port2
Definition: detect.h:193
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:619
uint16_t payload_len
Definition: decode.h:541
uint32_t sig_cnt
Definition: detect.h:768
void SigGroupHeadSetProtoAndDirection(SigGroupHead *sgh, uint8_t ipproto, int dir)
void DetectPortPrintList(DetectPort *head)
Helper function used to print the list of ports present in this DetectPort list.
#define SIG_FLAG_SRC_ANY
Definition: detect.h:211
DetectPort * DetectPortCopySingle(DetectEngineCtx *de_ctx, DetectPort *src)
Function that return a copy of DetectPort src sigs.
uint16_t flags
Definition: detect.h:1194
#define SIG_MASK_REQUIRE_DCERPC
Definition: detect.h:273
#define TH_CWR
Definition: decode-tcp.h:44
int DetectPortParse(const DetectEngineCtx *de_ctx, DetectPort **head, const char *str)
Function for parsing port strings.
const char * DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
int DetectFlagsSignatureNeedsSynOnlyPackets(const Signature *s)
void SigGroupHeadSetFilestoreCount(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
Set the filestore_cnt in the sgh.
#define SIG_MASK_REQUIRE_NO_PAYLOAD
Definition: detect.h:272
SigMatch * mpm_sm
Definition: detect.h:493
void SCProfilingKeywordInitCounters(DetectEngineCtx *de_ctx)
Register the keyword profiling counters.
HashListTable * dport_hash_table
Definition: detect.h:906
uint8_t * sig_array
Definition: detect.h:1317
#define FLOW_STATES
Definition: detect.h:758
int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
Port structure for detection engine.
Definition: detect.h:191
a single match condition for a signature
Definition: detect.h:318
void DetectPortPrint(DetectPort *dp)
Helper function that print the DetectPort info.
void DetectContentPropagateLimits(Signature *s)
apply depth/offset and distance/within to content matches
#define FILE_SIG_NEED_SIZE
Definition: detect.h:289
#define MASK_TCP_UNUSUAL_FLAGS
SigGroupHeadInitData * init
Definition: detect.h:1364