suricata
detect-engine-mpm.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
23  *
24  * Multi pattern matcher
25  */
26 
27 #include "suricata.h"
28 #include "suricata-common.h"
29 
30 #include "app-layer-protos.h"
31 
32 #include "decode.h"
33 #include "detect.h"
34 #include "detect-engine.h"
35 #include "detect-engine-siggroup.h"
36 #include "detect-engine-mpm.h"
37 #include "detect-engine-iponly.h"
38 #include "detect-parse.h"
40 #include "util-mpm.h"
41 #include "util-memcmp.h"
42 #include "util-memcpy.h"
43 #include "conf.h"
44 #include "detect-fast-pattern.h"
45 
46 #include "flow.h"
47 #include "flow-var.h"
48 #include "detect-flow.h"
49 
50 #include "detect-content.h"
51 
52 #include "detect-engine-payload.h"
53 #include "detect-engine-dns.h"
54 
55 #include "stream.h"
56 
57 #include "util-enum.h"
58 #include "util-debug.h"
59 #include "util-print.h"
60 #include "util-validate.h"
61 
62 const char *builtin_mpms[] = {
63  "toserver TCP packet",
64  "toclient TCP packet",
65  "toserver TCP stream",
66  "toclient TCP stream",
67  "toserver UDP packet",
68  "toclient UDP packet",
69  "other IP packet",
70 
71  NULL };
72 
73 /* Registery for mpm keywords
74  *
75  * Keywords are registered at engine start up
76  */
77 
78 static DetectMpmAppLayerRegistery *g_app_mpms_list = NULL;
79 static int g_app_mpms_list_cnt = 0;
80 
81 /** \brief register a MPM engine
82  *
83  * \note to be used at start up / registration only. Errors are fatal.
84  */
85 void DetectAppLayerMpmRegister2(const char *name,
86  int direction, int priority,
87  int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
88  SigGroupHead *sgh, MpmCtx *mpm_ctx,
89  const DetectMpmAppLayerRegistery *mpm_reg, int list_id),
91  AppProto alproto, int tx_min_progress)
92 {
93  SCLogDebug("registering %s/%d/%d/%p/%p/%u/%d", name, direction, priority,
94  PrefilterRegister, GetData, alproto, tx_min_progress);
95 
96  if (PrefilterRegister == PrefilterGenericMpmRegister && GetData == NULL) {
97  // must register GetData with PrefilterGenericMpmRegister
98  abort();
99  }
100 
103  int sm_list = DetectBufferTypeGetByName(name);
104  if (sm_list == -1) {
106  "MPM engine registration for %s failed", name);
107  }
108 
109  DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
110  BUG_ON(am == NULL);
111  am->name = name;
112  snprintf(am->pname, sizeof(am->pname), "%s", am->name);
113  am->direction = direction;
114  am->sm_list = sm_list;
115  am->priority = priority;
116 
117  am->v2.PrefilterRegisterWithListId = PrefilterRegister;
118  am->v2.GetData = GetData;
119  am->v2.alproto = alproto;
120  am->v2.tx_min_progress = tx_min_progress;
121 
122  if (g_app_mpms_list == NULL) {
123  g_app_mpms_list = am;
124  } else {
125  DetectMpmAppLayerRegistery *t = g_app_mpms_list;
126  while (t->next != NULL) {
127  t = t->next;
128  }
129 
130  t->next = am;
131  am->id = t->id + 1;
132  }
133  g_app_mpms_list_cnt++;
134 
135  SupportFastPatternForSigMatchList(sm_list, priority);
136 }
137 
138 void DetectAppLayerMpmRegister(const char *name,
139  int direction, int priority,
140  int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
141  SigGroupHead *sgh, MpmCtx *mpm_ctx))
142 {
143  SCLogDebug("registering %s/%d/%d/%p",
144  name, direction, priority, PrefilterRegister);
145 
147  int sm_list = DetectBufferTypeGetByName(name);
148  BUG_ON(sm_list == -1);
149 
150  DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
151  BUG_ON(am == NULL);
152  am->name = name;
153  snprintf(am->pname, sizeof(am->pname), "%s", am->name);
154  am->direction = direction;
155  am->sm_list = sm_list;
156  am->priority = priority;
157  am->PrefilterRegister = PrefilterRegister;
158 
159  if (g_app_mpms_list == NULL) {
160  g_app_mpms_list = am;
161  } else {
162  DetectMpmAppLayerRegistery *t = g_app_mpms_list;
163  while (t->next != NULL) {
164  t = t->next;
165  }
166 
167  t->next = am;
168  am->id = t->id + 1;
169  }
170  g_app_mpms_list_cnt++;
171 
172  SupportFastPatternForSigMatchList(sm_list, priority);
173 }
174 
175 /** \brief copy a mpm engine from parent_id, add in transforms */
177  const int id, const int parent_id,
178  DetectEngineTransforms *transforms)
179 {
180  SCLogDebug("registering %d/%d", id, parent_id);
181 
183  while (t) {
184  if (t->sm_list == parent_id) {
185  DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
186  BUG_ON(am == NULL);
187  am->name = t->name;
188  snprintf(am->pname, sizeof(am->pname), "%s#%d", am->name, id);
189  am->direction = t->direction;
190  am->sm_list = id; // use new id
193  am->v2.GetData = t->v2.GetData;
194  am->v2.alproto = t->v2.alproto;
196  am->priority = t->priority;
197  am->next = t->next;
198  if (transforms) {
199  memcpy(&am->v2.transforms, transforms, sizeof(*transforms));
200  }
201  am->id = de_ctx->app_mpms_list_cnt++;
202 
204  t->next = am;
205  SCLogDebug("copied mpm registration for %s id %u "
206  "with parent %u and GetData %p",
207  t->name, id, parent_id, am->v2.GetData);
208  t = am;
209  }
210  t = t->next;
211  }
212 }
213 
215 {
216  const DetectMpmAppLayerRegistery *list = g_app_mpms_list;
217  while (list != NULL) {
218  DetectMpmAppLayerRegistery *n = SCCalloc(1, sizeof(*n));
219  BUG_ON(n == NULL);
220 
221  *n = *list;
222  n->next = NULL;
223 
224  if (de_ctx->app_mpms_list == NULL) {
225  de_ctx->app_mpms_list = n;
226  } else {
228  while (t->next != NULL) {
229  t = t->next;
230  }
231 
232  t->next = n;
233  }
234 
235  list = list->next;
236  }
237  de_ctx->app_mpms_list_cnt = g_app_mpms_list_cnt;
238  SCLogDebug("mpm: de_ctx app_mpms_list %p %u",
239  de_ctx->app_mpms_list, de_ctx->app_mpms_list_cnt);
240 }
241 
243 {
244  BUG_ON(de_ctx->app_mpms_list_cnt == 0);
245 
246  de_ctx->app_mpms = SCCalloc(de_ctx->app_mpms_list_cnt + 1, sizeof(DetectMpmAppLayerKeyword));
247  BUG_ON(de_ctx->app_mpms == NULL);
248 
250  while (list != NULL) {
251  DetectMpmAppLayerKeyword *am = &de_ctx->app_mpms[list->id];
252  am->reg = list;
253 
254  /* default to whatever the global setting is */
255  int shared = (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);
256 
257  /* see if we use a unique or shared mpm ctx for this type */
258  int confshared = 0;
259  char confstring[256] = "detect.mpm.";
260  strlcat(confstring, am->reg->name, sizeof(confstring));
261  strlcat(confstring, ".shared", sizeof(confstring));
262  if (ConfGetBool(confstring, &confshared) == 1)
263  shared = confshared;
264 
265  if (shared == 0) {
266  if (!(de_ctx->flags & DE_QUIET)) {
267  SCLogPerf("using unique mpm ctx' for %s", am->reg->name);
268  }
270  } else {
271  if (!(de_ctx->flags & DE_QUIET)) {
272  SCLogPerf("using shared mpm ctx' for %s", am->reg->name);
273  }
275  }
276 
277  SCLogDebug("AppLayer MPM %s: %u", am->reg->name, am->sgh_mpm_context);
278 
279  list = list->next;
280  }
281 }
282 
283 /**
284  * \brief initialize mpm contexts for applayer buffers that are in
285  * "single or "shared" mode.
286  */
288 {
289  int r = 0;
290  DetectMpmAppLayerKeyword *am = de_ctx->app_mpms;
291  while (am->reg != NULL) {
292  int dir = (am->reg->direction == SIG_FLAG_TOSERVER) ? 1 : 0;
293 
295  {
296  MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
297  if (mpm_ctx != NULL) {
298  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
299  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
300  }
301  }
302  }
303  am++;
304  }
305  return r;
306 }
307 
308 static int32_t SetupBuiltinMpm(DetectEngineCtx *de_ctx, const char *name)
309 {
310  /* default to whatever the global setting is */
311  int shared = (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);
312 
313  /* see if we use a unique or shared mpm ctx for this type */
314  int confshared = 0;
315  char confstring[256] = "detect.mpm.";
316  strlcat(confstring, name, sizeof(confstring));
317  strlcat(confstring, ".shared", sizeof(confstring));
318  if (ConfGetBool(confstring, &confshared) == 1)
319  shared = confshared;
320 
321  int32_t ctx;
322  if (shared == 0) {
324  SCLogPerf("using unique mpm ctx' for %s", name);
325  } else {
326  ctx = MpmFactoryRegisterMpmCtxProfile(de_ctx, name);
327  SCLogPerf("using shared mpm ctx' for %s", name);
328  }
329  return ctx;
330 }
331 
333 {
334  de_ctx->sgh_mpm_context_proto_tcp_packet = SetupBuiltinMpm(de_ctx, "tcp-packet");
335  de_ctx->sgh_mpm_context_stream = SetupBuiltinMpm(de_ctx, "tcp-stream");
336 
337  de_ctx->sgh_mpm_context_proto_udp_packet = SetupBuiltinMpm(de_ctx, "udp-packet");
338  de_ctx->sgh_mpm_context_proto_other_packet = SetupBuiltinMpm(de_ctx, "other-ip");
339 }
340 
341 /**
342  * \brief initialize mpm contexts for builtin buffers that are in
343  * "single or "shared" mode.
344  */
346 {
347  int r = 0;
348  MpmCtx *mpm_ctx = NULL;
349 
352  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
353  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
354  }
356  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
357  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
358  }
359  }
360 
363  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
364  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
365  }
367  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
368  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
369  }
370  }
371 
374  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
375  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
376  }
377  }
378 
380  mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0);
381  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
382  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
383  }
384  mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1);
385  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
386  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
387  }
388  }
389 
390  return r;
391 }
392 
393 /**
394  * \brief check if a signature has patterns that are to be inspected
395  * against a packets payload (as opposed to the stream payload)
396  *
397  * \param s signature
398  *
399  * \retval 1 true
400  * \retval 0 false
401  */
403 {
404  SCEnter();
405 
406  if (s == NULL) {
407  SCReturnInt(0);
408  }
409 
410  if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) {
411  SCReturnInt(1);
412  }
413 
414  if ((s->init_data != NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) ||
415  (s->init_data == NULL && s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL))
416  {
417  SCLogDebug("no mpm");
418  SCReturnInt(0);
419  }
420 
421  if (!(s->flags & SIG_FLAG_REQUIRE_PACKET)) {
422  SCReturnInt(0);
423  }
424 
425  SCReturnInt(1);
426 }
427 
428 /**
429  * \brief check if a signature has patterns that are to be inspected
430  * against the stream payload (as opposed to the individual packets
431  * payload(s))
432  *
433  * \param s signature
434  *
435  * \retval 1 true
436  * \retval 0 false
437  */
439 {
440  SCEnter();
441 
442  if (s == NULL) {
443  SCReturnInt(0);
444  }
445 
446  if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) {
447  SCReturnInt(0);
448  }
449 
450  if ((s->init_data != NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) ||
451  (s->init_data == NULL && s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL))
452  {
453  SCLogDebug("no mpm");
454  SCReturnInt(0);
455  }
456 
457  if (!(s->flags & SIG_FLAG_REQUIRE_STREAM)) {
458  SCReturnInt(0);
459  }
460 
461  SCReturnInt(1);
462 }
463 
464 
465 /**
466  * \brief Function to return the multi pattern matcher algorithm to be
467  * used by the engine, based on the mpm-algo setting in yaml
468  * Use the default mpm if none is specified in the yaml file.
469  *
470  * \retval mpm algo value
471  */
473 {
474  const char *mpm_algo;
475  uint16_t mpm_algo_val = mpm_default_matcher;
476 
477  /* Get the mpm algo defined in config file by the user */
478  if ((ConfGet("mpm-algo", &mpm_algo)) == 1) {
479  uint16_t u;
480 
481  if (mpm_algo != NULL) {
482 #if __BYTE_ORDER == __BIG_ENDIAN
483  if (strcmp(mpm_algo, "ac-ks") == 0) {
485  "not work on big endian systems at this time.");
486  exit(EXIT_FAILURE);
487  }
488 #endif
489  if (strcmp("auto", mpm_algo) == 0) {
490  goto done;
491  }
492  for (u = 0; u < MPM_TABLE_SIZE; u++) {
493  if (mpm_table[u].name == NULL)
494  continue;
495 
496  if (strcmp(mpm_table[u].name, mpm_algo) == 0) {
497  mpm_algo_val = u;
498  goto done;
499  }
500  }
501  }
502 
503  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid mpm algo supplied "
504  "in the yaml conf file: \"%s\"", mpm_algo);
505  exit(EXIT_FAILURE);
506  }
507 
508  done:
509  return mpm_algo_val;
510 }
511 
512 /** \brief cleans up the mpm instance after a match */
514 {
515  PmqReset(&det_ctx->pmq);
516 }
517 
518 void PatternMatchDestroy(MpmCtx *mpm_ctx, uint16_t mpm_matcher)
519 {
520  SCLogDebug("mpm_ctx %p, mpm_matcher %"PRIu16"", mpm_ctx, mpm_matcher);
521  mpm_table[mpm_matcher].DestroyCtx(mpm_ctx);
522 }
523 
524 void PatternMatchThreadPrint(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
525 {
526  SCLogDebug("mpm_thread_ctx %p, mpm_matcher %"PRIu16" defunct", mpm_thread_ctx, mpm_matcher);
527  //mpm_table[mpm_matcher].PrintThreadCtx(mpm_thread_ctx);
528 }
529 void PatternMatchThreadDestroy(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
530 {
531  SCLogDebug("mpm_thread_ctx %p, mpm_matcher %"PRIu16"", mpm_thread_ctx, mpm_matcher);
532  if (mpm_table[mpm_matcher].DestroyThreadCtx != NULL)
533  mpm_table[mpm_matcher].DestroyThreadCtx(NULL, mpm_thread_ctx);
534 }
535 void PatternMatchThreadPrepare(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
536 {
537  SCLogDebug("mpm_thread_ctx %p, type %"PRIu16, mpm_thread_ctx, mpm_matcher);
538  MpmInitThreadCtx(mpm_thread_ctx, mpm_matcher);
539 }
540 
541 /** \brief Predict a strength value for patterns
542  *
543  * Patterns with high character diversity score higher.
544  * Alpha chars score not so high
545  * Other printable + a few common codes a little higher
546  * Everything else highest.
547  * Longer patterns score better than short patters.
548  *
549  * \param pat pattern
550  * \param patlen length of the patternn
551  *
552  * \retval s pattern score
553  */
554 uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
555 {
556  uint8_t a[256];
557  memset(&a, 0 ,sizeof(a));
558 
559  uint32_t s = 0;
560  uint16_t u = 0;
561  for (u = 0; u < patlen; u++) {
562  if (a[pat[u]] == 0) {
563  if (isalpha(pat[u]))
564  s += 3;
565  else if (isprint(pat[u]) || pat[u] == 0x00 || pat[u] == 0x01 || pat[u] == 0xFF)
566  s += 4;
567  else
568  s += 6;
569 
570  a[pat[u]] = 1;
571  } else {
572  s++;
573  }
574  }
575 
576  return s;
577 }
578 
579 static void PopulateMpmHelperAddPattern(MpmCtx *mpm_ctx,
580  const DetectContentData *cd,
581  const Signature *s, uint8_t flags,
582  int chop)
583 {
584  uint16_t pat_offset = cd->offset;
585  uint16_t pat_depth = cd->depth;
586 
587  /* recompute offset/depth to cope with chop */
588  if (chop && (pat_depth || pat_offset)) {
589  pat_offset += cd->fp_chop_offset;
590  if (pat_depth) {
591  pat_depth -= cd->content_len;
592  pat_depth += cd->fp_chop_offset + cd->fp_chop_len;
593  }
594  }
595 
596  if (cd->flags & DETECT_CONTENT_NOCASE) {
597  if (chop) {
598  MpmAddPatternCI(mpm_ctx,
599  cd->content + cd->fp_chop_offset, cd->fp_chop_len,
600  pat_offset, pat_depth,
601  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
602  } else {
603  MpmAddPatternCI(mpm_ctx,
604  cd->content, cd->content_len,
605  pat_offset, pat_depth,
606  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
607  }
608  } else {
609  if (chop) {
610  MpmAddPatternCS(mpm_ctx,
611  cd->content + cd->fp_chop_offset, cd->fp_chop_len,
612  pat_offset, pat_depth,
613  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
614  } else {
615  MpmAddPatternCS(mpm_ctx,
616  cd->content, cd->content_len,
617  pat_offset, pat_depth,
618  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
619  }
620  }
621 
622  return;
623 }
624 
625 #define SGH_PROTO(sgh, p) ((sgh)->init->protos[(p)] == 1)
626 #define SGH_DIRECTION_TS(sgh) ((sgh)->init->direction & SIG_FLAG_TOSERVER)
627 #define SGH_DIRECTION_TC(sgh) ((sgh)->init->direction & SIG_FLAG_TOCLIENT)
628 
629 static void SetMpm(Signature *s, SigMatch *mpm_sm)
630 {
631  if (s == NULL || mpm_sm == NULL)
632  return;
633 
634  DetectContentData *cd = (DetectContentData *)mpm_sm->ctx;
636  if (DETECT_CONTENT_IS_SINGLE(cd) &&
637  !(cd->flags & DETECT_CONTENT_NEGATED) &&
638  !(cd->flags & DETECT_CONTENT_REPLACE) &&
639  cd->content_len == cd->fp_chop_len)
640  {
642  }
643  } else {
644  if (DETECT_CONTENT_IS_SINGLE(cd) &&
645  !(cd->flags & DETECT_CONTENT_NEGATED) &&
646  !(cd->flags & DETECT_CONTENT_REPLACE))
647  {
649  }
650  }
651  cd->flags |= DETECT_CONTENT_MPM;
652  s->init_data->mpm_sm = mpm_sm;
653  return;
654 }
655 
656 static SigMatch *GetMpmForList(const Signature *s, const int list, SigMatch *mpm_sm,
657  uint16_t max_len, bool skip_negated_content)
658 {
659  for (SigMatch *sm = s->init_data->smlists[list]; sm != NULL; sm = sm->next) {
660  if (sm->type != DETECT_CONTENT)
661  continue;
662 
663  const DetectContentData *cd = (DetectContentData *)sm->ctx;
664  /* skip_negated_content is only set if there's absolutely no
665  * non-negated content present in the sig */
666  if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
667  continue;
668  if (cd->content_len != max_len)
669  continue;
670 
671  if (mpm_sm == NULL) {
672  mpm_sm = sm;
673  } else {
674  DetectContentData *data1 = (DetectContentData *)sm->ctx;
675  DetectContentData *data2 = (DetectContentData *)mpm_sm->ctx;
676  uint32_t ls = PatternStrength(data1->content, data1->content_len);
677  uint32_t ss = PatternStrength(data2->content, data2->content_len);
678  if (ls > ss) {
679  mpm_sm = sm;
680  } else if (ls == ss) {
681  /* if 2 patterns are of equal strength, we pick the longest */
682  if (data1->content_len > data2->content_len)
683  mpm_sm = sm;
684  } else {
685  SCLogDebug("sticking with mpm_sm");
686  }
687  }
688  }
689  return mpm_sm;
690 }
691 
693 {
694  if (s->init_data->mpm_sm != NULL)
695  return;
696 
697  SigMatch *mpm_sm = NULL, *sm = NULL;
698  const int nlists = s->init_data->smlists_array_size;
699  int nn_sm_list[nlists];
700  int n_sm_list[nlists];
701  memset(nn_sm_list, 0, nlists * sizeof(int));
702  memset(n_sm_list, 0, nlists * sizeof(int));
703  int count_nn_sm_list = 0;
704  int count_n_sm_list = 0;
705 
706  /* inspect rule to see if we have the fast_pattern reg to
707  * force using a sig, otherwise keep stats about the patterns */
708  for (int list_id = 0; list_id < nlists; list_id++) {
709  if (s->init_data->smlists[list_id] == NULL)
710  continue;
711 
712  if (!FastPatternSupportEnabledForSigMatchList(de_ctx, list_id))
713  continue;
714 
715  for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) {
716  if (sm->type != DETECT_CONTENT)
717  continue;
718 
719  const DetectContentData *cd = (DetectContentData *)sm->ctx;
720  /* fast_pattern set in rule, so using this pattern */
721  if ((cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
722  SetMpm(s, sm);
723  return;
724  }
725 
726  if (cd->flags & DETECT_CONTENT_NEGATED) {
727  n_sm_list[list_id] = 1;
728  count_n_sm_list++;
729  } else {
730  nn_sm_list[list_id] = 1;
731  count_nn_sm_list++;
732  }
733  }
734  }
735 
736  /* prefer normal not-negated over negated */
737  int *curr_sm_list = NULL;
738  int skip_negated_content = 1;
739  if (count_nn_sm_list > 0) {
740  curr_sm_list = nn_sm_list;
741  } else if (count_n_sm_list > 0) {
742  curr_sm_list = n_sm_list;
743  skip_negated_content = 0;
744  } else {
745  return;
746  }
747 
748  int final_sm_list[nlists];
749  memset(&final_sm_list, 0, (nlists * sizeof(int)));
750 
751  int count_final_sm_list = 0;
752  int priority;
753 
755  while (tmp != NULL) {
756  for (priority = tmp->priority;
757  tmp != NULL && priority == tmp->priority;
758  tmp = tmp->next)
759  {
760  if (tmp->list_id >= nlists)
761  continue;
762  if (curr_sm_list[tmp->list_id] == 0)
763  continue;
764  final_sm_list[count_final_sm_list++] = tmp->list_id;
765  }
766  if (count_final_sm_list != 0)
767  break;
768  }
769 
770  BUG_ON(count_final_sm_list == 0);
771 
772  uint16_t max_len = 0;
773  for (int i = 0; i < count_final_sm_list; i++) {
774  if (final_sm_list[i] >= (int)s->init_data->smlists_array_size)
775  continue;
776 
777  for (sm = s->init_data->smlists[final_sm_list[i]]; sm != NULL; sm = sm->next) {
778  if (sm->type != DETECT_CONTENT)
779  continue;
780 
781  const DetectContentData *cd = (DetectContentData *)sm->ctx;
782  /* skip_negated_content is only set if there's absolutely no
783  * non-negated content present in the sig */
784  if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
785  continue;
786  if (max_len < cd->content_len)
787  max_len = cd->content_len;
788  }
789  }
790 
791  for (int i = 0; i < count_final_sm_list; i++) {
792  if (final_sm_list[i] >= (int)s->init_data->smlists_array_size)
793  continue;
794 
795  mpm_sm = GetMpmForList(s, final_sm_list[i], mpm_sm, max_len, skip_negated_content);
796  }
797 
798  /* assign to signature */
799  SetMpm(s, mpm_sm);
800  return;
801 }
802 
803 /** \internal
804  * \brief The hash function for MpmStore
805  *
806  * \param ht Pointer to the hash table.
807  * \param data Pointer to the MpmStore.
808  * \param datalen Not used in our case.
809  *
810  * \retval hash The generated hash value.
811  */
812 static uint32_t MpmStoreHashFunc(HashListTable *ht, void *data, uint16_t datalen)
813 {
814  const MpmStore *ms = (MpmStore *)data;
815  uint32_t hash = 0;
816  uint32_t b = 0;
817 
818  for (b = 0; b < ms->sid_array_size; b++)
819  hash += ms->sid_array[b];
820 
821  return hash % ht->array_size;
822 }
823 
824 /**
825  * \brief The Compare function for MpmStore
826  *
827  * \param data1 Pointer to the first MpmStore.
828  * \param len1 Not used.
829  * \param data2 Pointer to the second MpmStore.
830  * \param len2 Not used.
831  *
832  * \retval 1 If the 2 MpmStores sent as args match.
833  * \retval 0 If the 2 MpmStores sent as args do not match.
834  */
835 static char MpmStoreCompareFunc(void *data1, uint16_t len1, void *data2,
836  uint16_t len2)
837 {
838  const MpmStore *ms1 = (MpmStore *)data1;
839  const MpmStore *ms2 = (MpmStore *)data2;
840 
841  if (ms1->sid_array_size != ms2->sid_array_size)
842  return 0;
843 
844  if (ms1->buffer != ms2->buffer)
845  return 0;
846 
847  if (ms1->direction != ms2->direction)
848  return 0;
849 
850  if (ms1->sm_list != ms2->sm_list)
851  return 0;
852 
853  if (SCMemcmp(ms1->sid_array, ms2->sid_array,
854  ms1->sid_array_size) != 0)
855  {
856  return 0;
857  }
858 
859  return 1;
860 }
861 
862 static void MpmStoreFreeFunc(void *ptr)
863 {
864  MpmStore *ms = ptr;
865  if (ms != NULL) {
866  if (ms->mpm_ctx != NULL && !(ms->mpm_ctx->flags & MPMCTX_FLAGS_GLOBAL))
867  {
868  SCLogDebug("destroying mpm_ctx %p", ms->mpm_ctx);
870  SCFree(ms->mpm_ctx);
871  }
872  ms->mpm_ctx = NULL;
873 
874  SCFree(ms->sid_array);
875  SCFree(ms);
876  }
877 }
878 
879 /**
880  * \brief Initializes the MpmStore mpm hash table to be used by the detection
881  * engine context.
882  *
883  * \param de_ctx Pointer to the detection engine context.
884  *
885  * \retval 0 On success.
886  * \retval -1 On failure.
887  */
889 {
890  de_ctx->mpm_hash_table = HashListTableInit(4096,
891  MpmStoreHashFunc,
892  MpmStoreCompareFunc,
893  MpmStoreFreeFunc);
894  if (de_ctx->mpm_hash_table == NULL)
895  goto error;
896 
897  return 0;
898 
899 error:
900  return -1;
901 }
902 
903 /**
904  * \brief Adds a MpmStore to the detection engine context MpmStore
905  *
906  * \param de_ctx Pointer to the detection engine context.
907  * \param sgh Pointer to the MpmStore.
908  *
909  * \retval ret 0 on Successfully adding the argument sgh; -1 on failure.
910  */
911 static int MpmStoreAdd(DetectEngineCtx *de_ctx, MpmStore *s)
912 {
913  int ret = HashListTableAdd(de_ctx->mpm_hash_table, (void *)s, 0);
914  return ret;
915 }
916 
917 /**
918  * \brief Used to lookup a MpmStore from the MpmStore
919  *
920  * \param de_ctx Pointer to the detection engine context.
921  * \param sgh Pointer to the MpmStore.
922  *
923  * \retval rsgh On success a pointer to the MpmStore if the MpmStore is
924  * found in the hash table; NULL on failure.
925  */
926 static MpmStore *MpmStoreLookup(DetectEngineCtx *de_ctx, MpmStore *s)
927 {
929  (void *)s, 0);
930  return rs;
931 }
932 
934 {
935  HashListTableBucket *htb = NULL;
936 
937  int app_mpms_cnt = 0;
938  DetectMpmAppLayerKeyword *a = de_ctx->app_mpms;
939  while (a->reg != NULL) {
940  a++;
941  app_mpms_cnt++;
942  }
943  uint32_t stats[MPMB_MAX] = {0};
944  uint32_t appstats[app_mpms_cnt + 1]; // +1 to silence scan-build
945  memset(&appstats, 0x00, sizeof(appstats));
946 
947  for (htb = HashListTableGetListHead(de_ctx->mpm_hash_table);
948  htb != NULL;
949  htb = HashListTableGetListNext(htb))
950  {
951  const MpmStore *ms = (MpmStore *)HashListTableGetListData(htb);
952  if (ms == NULL) {
953  continue;
954  }
955  if (ms->buffer < MPMB_MAX)
956  stats[ms->buffer]++;
957  else if (ms->sm_list != DETECT_SM_LIST_PMATCH) {
958  int i = 0;
959  DetectMpmAppLayerKeyword *am = de_ctx->app_mpms;
960  while (am->reg != NULL) {
961  if (ms->sm_list == am->reg->sm_list &&
962  ms->direction == am->reg->direction)
963  {
964  SCLogDebug("%s %s: %u patterns. Min %u, Max %u. Ctx %p",
965  am->reg->name,
966  am->reg->direction == SIG_FLAG_TOSERVER ? "toserver":"toclient",
967  ms->mpm_ctx->pattern_cnt,
968  ms->mpm_ctx->minlen, ms->mpm_ctx->maxlen,
969  ms->mpm_ctx);
970  appstats[i]++;
971  break;
972  }
973  i++;
974  am++;
975  }
976  }
977  }
978 
979  if (!(de_ctx->flags & DE_QUIET)) {
980  int x;
981  for (x = 0; x < MPMB_MAX; x++) {
982  SCLogPerf("Builtin MPM \"%s\": %u", builtin_mpms[x], stats[x]);
983  }
984  for (x = 0; x < app_mpms_cnt; x++) {
985  if (appstats[x] == 0)
986  continue;
987  const char *name = de_ctx->app_mpms[x].reg->name;
988  const char *direction = de_ctx->app_mpms[x].reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient";
989  SCLogPerf("AppLayer MPM \"%s %s\": %u", direction, name, appstats[x]);
990  }
991  }
992 }
993 
994 /**
995  * \brief Frees the hash table - DetectEngineCtx->mpm_hash_table, allocated by
996  * MpmStoreInit() function.
997  *
998  * \param de_ctx Pointer to the detection engine context.
999  */
1001 {
1002  if (de_ctx->mpm_hash_table == NULL)
1003  return;
1004 
1006  de_ctx->mpm_hash_table = NULL;
1007  return;
1008 }
1009 
1010 static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
1011 {
1012  const Signature *s = NULL;
1013  uint32_t sig;
1014 
1015  int dir = 0;
1016 
1017  if (ms->buffer != MPMB_MAX) {
1019 
1020  switch (ms->buffer) {
1021  /* TS is 1 */
1022  case MPMB_TCP_PKT_TS:
1023  case MPMB_TCP_STREAM_TS:
1024  case MPMB_UDP_TS:
1025  dir = 1;
1026  break;
1027 
1028  /* TC is 0 */
1029  default:
1030  case MPMB_UDP_TC:
1031  case MPMB_TCP_STREAM_TC:
1032  case MPMB_TCP_PKT_TC:
1033  case MPMB_OTHERIP: /**< use 0 for other */
1034  dir = 0;
1035  break;
1036  }
1037  } else {
1039  BUG_ON(ms->direction == 0);
1041 
1042  if (ms->direction == SIG_FLAG_TOSERVER)
1043  dir = 1;
1044  else
1045  dir = 0;
1046  }
1047 
1048  ms->mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, ms->sgh_mpm_context, dir);
1049  if (ms->mpm_ctx == NULL)
1050  return;
1051 
1052  MpmInitCtx(ms->mpm_ctx, de_ctx->mpm_matcher);
1053 
1054  /* add the patterns */
1055  for (sig = 0; sig < (ms->sid_array_size * 8); sig++) {
1056  if (ms->sid_array[sig / 8] & (1 << (sig % 8))) {
1057  s = de_ctx->sig_array[sig];
1058  if (s == NULL)
1059  continue;
1060  if ((s->flags & ms->direction) == 0)
1061  continue;
1062  if (s->init_data->mpm_sm == NULL)
1063  continue;
1064  int list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
1065  if (list < 0)
1066  continue;
1067  if (list != ms->sm_list)
1068  continue;
1069 
1070  SCLogDebug("adding %u", s->id);
1071 
1073 
1074  int skip = 0;
1075  /* negated logic: if mpm match can't be used to be sure about this
1076  * pattern, we have to inspect the rule fully regardless of mpm
1077  * match. So in this case there is no point of adding it at all.
1078  * The non-mpm list entry for the sig will make sure the sig is
1079  * inspected. */
1080  if ((cd->flags & DETECT_CONTENT_NEGATED) &&
1082  {
1083  skip = 1;
1084  SCLogDebug("not adding negated mpm as it's not 'single'");
1085  }
1086 
1087  if (!skip) {
1088  PopulateMpmHelperAddPattern(ms->mpm_ctx,
1089  cd, s, 0, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
1090  }
1091  }
1092  }
1093 
1094  if (ms->mpm_ctx->pattern_cnt == 0) {
1095  MpmFactoryReClaimMpmCtx(de_ctx, ms->mpm_ctx);
1096  ms->mpm_ctx = NULL;
1097  } else {
1099  if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) {
1101  }
1102  }
1103  }
1104 }
1105 
1106 
1107 /** \brief Get MpmStore for a built-in buffer type
1108  *
1109  */
1111  enum MpmBuiltinBuffers buf)
1112 {
1113  const Signature *s = NULL;
1114  uint32_t sig;
1115  uint32_t cnt = 0;
1116  int direction = 0;
1117  uint32_t max_sid = DetectEngineGetMaxSigId(de_ctx) / 8 + 1;
1118  uint8_t sids_array[max_sid];
1119  memset(sids_array, 0x00, max_sid);
1120  int sgh_mpm_context = 0;
1121 
1122  switch (buf) {
1123  case MPMB_TCP_PKT_TS:
1124  case MPMB_TCP_PKT_TC:
1125  sgh_mpm_context = de_ctx->sgh_mpm_context_proto_tcp_packet;
1126  break;
1127  case MPMB_TCP_STREAM_TS:
1128  case MPMB_TCP_STREAM_TC:
1129  sgh_mpm_context = de_ctx->sgh_mpm_context_stream;
1130  break;
1131  case MPMB_UDP_TS:
1132  case MPMB_UDP_TC:
1133  sgh_mpm_context = de_ctx->sgh_mpm_context_proto_udp_packet;
1134  break;
1135  case MPMB_OTHERIP:
1136  sgh_mpm_context = de_ctx->sgh_mpm_context_proto_other_packet;
1137  break;
1138  default:
1139  break;
1140  }
1141 
1142  switch(buf) {
1143  case MPMB_TCP_PKT_TS:
1144  case MPMB_TCP_STREAM_TS:
1145  case MPMB_UDP_TS:
1146  direction = SIG_FLAG_TOSERVER;
1147  break;
1148 
1149  case MPMB_TCP_PKT_TC:
1150  case MPMB_TCP_STREAM_TC:
1151  case MPMB_UDP_TC:
1152  direction = SIG_FLAG_TOCLIENT;
1153  break;
1154 
1155  case MPMB_OTHERIP:
1156  direction = (SIG_FLAG_TOCLIENT|SIG_FLAG_TOSERVER);
1157  break;
1158 
1159  case MPMB_MAX:
1160  BUG_ON(1);
1161  break;
1162  }
1163 
1164  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1165  s = sgh->match_array[sig];
1166  if (s == NULL)
1167  continue;
1168 
1169  if (s->init_data->mpm_sm == NULL)
1170  continue;
1171 
1172  int list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
1173  if (list < 0)
1174  continue;
1175 
1176  if (list != DETECT_SM_LIST_PMATCH)
1177  continue;
1178 
1179  switch (buf) {
1180  case MPMB_TCP_PKT_TS:
1181  case MPMB_TCP_PKT_TC:
1182  if (SignatureHasPacketContent(s) == 1)
1183  {
1184  sids_array[s->num / 8] |= 1 << (s->num % 8);
1185  cnt++;
1186  }
1187  break;
1188  case MPMB_TCP_STREAM_TS:
1189  case MPMB_TCP_STREAM_TC:
1190  if (SignatureHasStreamContent(s) == 1)
1191  {
1192  sids_array[s->num / 8] |= 1 << (s->num % 8);
1193  cnt++;
1194  }
1195  break;
1196  case MPMB_UDP_TS:
1197  case MPMB_UDP_TC:
1198  sids_array[s->num / 8] |= 1 << (s->num % 8);
1199  cnt++;
1200  break;
1201  case MPMB_OTHERIP:
1202  sids_array[s->num / 8] |= 1 << (s->num % 8);
1203  cnt++;
1204  break;
1205  default:
1206  break;
1207  }
1208  }
1209 
1210  if (cnt == 0)
1211  return NULL;
1212 
1213  MpmStore lookup = { sids_array, max_sid, direction, buf, DETECT_SM_LIST_PMATCH, 0, NULL};
1214 
1215  MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
1216  if (result == NULL) {
1217  MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
1218  if (copy == NULL)
1219  return NULL;
1220  uint8_t *sids = SCCalloc(1, max_sid);
1221  if (sids == NULL) {
1222  SCFree(copy);
1223  return NULL;
1224  }
1225 
1226  memcpy(sids, sids_array, max_sid);
1227  copy->sid_array = sids;
1228  copy->sid_array_size = max_sid;
1229  copy->buffer = buf;
1230  copy->direction = direction;
1232  copy->sgh_mpm_context = sgh_mpm_context;
1233 
1234  MpmStoreSetup(de_ctx, copy);
1235  MpmStoreAdd(de_ctx, copy);
1236  return copy;
1237  } else {
1238  return result;
1239  }
1240 }
1241 
1242 static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
1244 {
1245  const Signature *s = NULL;
1246  uint32_t sig;
1247  uint32_t cnt = 0;
1248  uint32_t max_sid = DetectEngineGetMaxSigId(de_ctx) / 8 + 1;
1249  uint8_t sids_array[max_sid];
1250  memset(sids_array, 0x00, max_sid);
1251 
1252  SCLogDebug("handling %s direction %s for list %d", am->reg->name,
1253  am->reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1254  am->reg->sm_list);
1255 
1256  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1257  s = sgh->match_array[sig];
1258  if (s == NULL)
1259  continue;
1260 
1261  if (s->init_data->mpm_sm == NULL)
1262  continue;
1263 
1264  int list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
1265  if (list < 0)
1266  continue;
1267 
1268  if ((s->flags & am->reg->direction) == 0)
1269  continue;
1270 
1271  if (list != am->reg->sm_list)
1272  continue;
1273 
1274  sids_array[s->num / 8] |= 1 << (s->num % 8);
1275  cnt++;
1276  }
1277 
1278  if (cnt == 0)
1279  return NULL;
1280 
1281  MpmStore lookup = { sids_array, max_sid, am->reg->direction,
1282  MPMB_MAX, am->reg->sm_list, 0, NULL};
1283  SCLogDebug("am->direction %d am->sm_list %d",
1284  am->reg->direction, am->reg->sm_list);
1285 
1286  MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
1287  if (result == NULL) {
1288  SCLogDebug("new unique mpm for %s %s: %u patterns",
1289  am->reg->name,
1290  am->reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1291  cnt);
1292 
1293  MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
1294  if (copy == NULL)
1295  return NULL;
1296  uint8_t *sids = SCCalloc(1, max_sid);
1297  if (sids == NULL) {
1298  SCFree(copy);
1299  return NULL;
1300  }
1301 
1302  memcpy(sids, sids_array, max_sid);
1303  copy->sid_array = sids;
1304  copy->sid_array_size = max_sid;
1305  copy->buffer = MPMB_MAX;
1306  copy->direction = am->reg->direction;
1307  copy->sm_list = am->reg->sm_list;
1308  copy->sgh_mpm_context = am->sgh_mpm_context;
1309 
1310  MpmStoreSetup(de_ctx, copy);
1311  MpmStoreAdd(de_ctx, copy);
1312  return copy;
1313  } else {
1314  SCLogDebug("using existing mpm %p", result);
1315  return result;
1316  }
1317  return NULL;
1318 }
1319 
1320 static void SetRawReassemblyFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1321 {
1322  const Signature *s = NULL;
1323  uint32_t sig;
1324 
1325  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1326  s = sgh->match_array[sig];
1327  if (s == NULL)
1328  continue;
1329 
1330  if (SignatureHasStreamContent(s) == 1) {
1332  SCLogDebug("rule group %p has SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
1333  return;
1334  }
1335  }
1336  SCLogDebug("rule group %p does NOT have SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
1337 }
1338 
1339 /** \brief Prepare the pattern matcher ctx in a sig group head.
1340  *
1341  */
1343 {
1344  MpmStore *mpm_store = NULL;
1345  if (SGH_PROTO(sh, IPPROTO_TCP)) {
1346  if (SGH_DIRECTION_TS(sh)) {
1347  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TS);
1348  if (mpm_store != NULL) {
1349  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1350  }
1351 
1352  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_STREAM_TS);
1353  if (mpm_store != NULL) {
1354  PrefilterPktStreamRegister(de_ctx, sh, mpm_store->mpm_ctx);
1355  }
1356 
1357  SetRawReassemblyFlag(de_ctx, sh);
1358  }
1359  if (SGH_DIRECTION_TC(sh)) {
1360  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TC);
1361  if (mpm_store != NULL) {
1362  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1363  }
1364 
1365  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_STREAM_TC);
1366  if (mpm_store != NULL) {
1367  PrefilterPktStreamRegister(de_ctx, sh, mpm_store->mpm_ctx);
1368  }
1369 
1370  SetRawReassemblyFlag(de_ctx, sh);
1371  }
1372  } else if (SGH_PROTO(sh, IPPROTO_UDP)) {
1373  if (SGH_DIRECTION_TS(sh)) {
1374  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_UDP_TS);
1375  if (mpm_store != NULL) {
1376  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1377  }
1378  }
1379  if (SGH_DIRECTION_TC(sh)) {
1380  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_UDP_TC);
1381  if (mpm_store != NULL) {
1382  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1383  }
1384  }
1385  } else {
1386  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_OTHERIP);
1387  if (mpm_store != NULL) {
1388  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1389  }
1390  }
1391 
1392  int i = 0;
1393  DetectMpmAppLayerKeyword *a = de_ctx->app_mpms;
1394  while (a->reg != NULL) {
1395  i++;
1396  a++;
1397  }
1398  if (i == 0)
1399  return 0;
1400 
1401  sh->init->app_mpms = SCCalloc(i, sizeof(MpmCtx *));
1402  BUG_ON(sh->init->app_mpms == NULL);
1403 
1404  a = de_ctx->app_mpms;
1405  while (a->reg != NULL) {
1406  if ((a->reg->direction == SIG_FLAG_TOSERVER && SGH_DIRECTION_TS(sh)) ||
1408  {
1409  mpm_store = MpmStorePrepareBufferAppLayer(de_ctx, sh, a);
1410  if (mpm_store != NULL) {
1411  sh->init->app_mpms[a->reg->id] = mpm_store->mpm_ctx;
1412 
1413  SCLogDebug("a->reg->PrefilterRegister %p mpm_store->mpm_ctx %p",
1414  a->reg->PrefilterRegister, mpm_store->mpm_ctx);
1415  SCLogDebug("a %p a->reg->name %s a->reg->PrefilterRegisterWithListId %p "
1416  "mpm_store->mpm_ctx %p", a, a->reg->name,
1417  a->reg->v2.PrefilterRegisterWithListId, mpm_store->mpm_ctx);
1418 
1419  /* if we have just certain types of negated patterns,
1420  * mpm_ctx can be NULL */
1421  if (a->reg->v2.PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
1423  sh, mpm_store->mpm_ctx,
1424  a->reg, a->reg->sm_list) != 0);
1425  SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
1426  }
1427  else if (a->reg->PrefilterRegister && mpm_store->mpm_ctx) {
1428  BUG_ON(a->reg->PrefilterRegister(de_ctx, sh, mpm_store->mpm_ctx) != 0);
1429  SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
1430  }
1431  }
1432  }
1433  a++;
1434  }
1435 
1436  return 0;
1437 }
1438 
1439 typedef struct DetectFPAndItsId_ {
1441  uint16_t content_len;
1442  uint32_t flags;
1443  int sm_list;
1444 
1445  uint8_t *content;
1447 
1448 /**
1449  * \brief Figured out the FP and their respective content ids for all the
1450  * sigs in the engine.
1451  *
1452  * \param de_ctx Detection engine context.
1453  *
1454  * \retval 0 On success.
1455  * \retval -1 On failure.
1456  */
1458 {
1459  uint32_t struct_total_size = 0;
1460  uint32_t content_total_size = 0;
1461  Signature *s = NULL;
1462 
1463  /* Count the amount of memory needed to store all the structures
1464  * and the content of those structures. This will over estimate the
1465  * true size, since duplicates are removed below, but counted here.
1466  */
1467  for (s = de_ctx->sig_list; s != NULL; s = s->next) {
1468  if (s->flags & SIG_FLAG_PREFILTER)
1469  continue;
1470 
1471  RetrieveFPForSig(de_ctx, s);
1472  if (s->init_data->mpm_sm != NULL) {
1474  struct_total_size += sizeof(DetectFPAndItsId);
1475  content_total_size += cd->content_len;
1476 
1477  s->flags |= SIG_FLAG_PREFILTER;
1478  }
1479  }
1480  /* no rules */
1481  if (struct_total_size + content_total_size == 0)
1482  return 0;
1483 
1484  /* array hash buffer - i've run out of ideas to name it */
1485  uint8_t *ahb = SCMalloc(sizeof(uint8_t) * (struct_total_size + content_total_size));
1486  if (unlikely(ahb == NULL))
1487  return -1;
1488 
1489  uint8_t *content = NULL;
1490  uint16_t content_len = 0;
1491  PatIntId max_id = 0;
1492  DetectFPAndItsId *struct_offset = (DetectFPAndItsId *)ahb;
1493  uint8_t *content_offset = ahb + struct_total_size;
1494 
1495  for (s = de_ctx->sig_list; s != NULL; s = s->next) {
1496  if (s->init_data->mpm_sm != NULL) {
1498  BUG_ON(sm_list == -1);
1499 
1501  DetectFPAndItsId *dup = (DetectFPAndItsId *)ahb;
1503  content = cd->content + cd->fp_chop_offset;
1504  content_len = cd->fp_chop_len;
1505  } else {
1506  content = cd->content;
1507  content_len = cd->content_len;
1508  }
1509  uint32_t flags = cd->flags & DETECT_CONTENT_NOCASE;
1510  /* Check for content already found on the same list */
1511  for (; dup != struct_offset; dup++) {
1512  if (dup->content_len != content_len)
1513  continue;
1514  if (dup->sm_list != sm_list)
1515  continue;
1516  if (dup->flags != flags)
1517  continue;
1518  /* Check for pattern matching a duplicate. Use case insensitive matching
1519  * for case insensitive patterns. */
1520  if (flags & DETECT_CONTENT_NOCASE) {
1521  if (SCMemcmpLowercase(dup->content, content, content_len) != 0)
1522  continue;
1523  } else {
1524  /* Case sensitive matching */
1525  if (SCMemcmp(dup->content, content, content_len) != 0)
1526  continue;
1527  }
1528  /* Found a match with a previous pattern. */
1529  break;
1530  }
1531  if (dup != struct_offset) {
1532  /* Exited for-loop before the end, so found an existing match.
1533  * Use its ID. */
1534  cd->id = dup->id;
1535  continue;
1536  }
1537 
1538  /* Not found, so new content. Give it a new ID and add it
1539  * to the array. Copy the content at the end of the
1540  * content array.
1541  */
1542  struct_offset->id = max_id++;
1543  cd->id = struct_offset->id;
1544  struct_offset->content_len = content_len;
1545  struct_offset->sm_list = sm_list;
1546  struct_offset->content = content_offset;
1547  struct_offset->flags = flags;
1548 
1549  content_offset += content_len;
1550 
1551  if (flags & DETECT_CONTENT_NOCASE) {
1552  /* Need to store case-insensitive patterns as lower case
1553  * because SCMemcmpLowercase() above assumes that all
1554  * patterns are stored lower case so that it doesn't
1555  * need to relower its first argument.
1556  */
1557  memcpy_tolower(struct_offset->content, content, content_len);
1558  } else {
1559  memcpy(struct_offset->content, content, content_len);
1560  }
1561 
1562  struct_offset++;
1563  } /* if (s->mpm_sm != NULL) */
1564  } /* for */
1565 
1566  de_ctx->max_fp_id = max_id;
1567 
1568  SCFree(ahb);
1569 
1570  return 0;
1571 }
void DetectMpmSetupAppMpms(DetectEngineCtx *de_ctx)
void MpmFactoryReClaimMpmCtx(const DetectEngineCtx *de_ctx, MpmCtx *mpm_ctx)
Definition: util-mpm.c:215
#define PatIntId
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
Predict a strength value for patterns.
DetectProto proto
Definition: detect.h:513
SigIntId sig_cnt
Definition: detect.h:1304
SignatureInitData * init_data
Definition: detect.h:564
uint16_t flags
#define SCMemcmp(a, b, c)
Definition: util-memcmp.h:369
#define DETECT_CONTENT_MPM_IS_CONCLUSIVE(c)
#define SCLogDebug(...)
Definition: util-debug.h:335
const char * builtin_mpms[]
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
uint8_t flags
Definition: util-mpm.h:92
int MpmAddPatternCS(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags)
Definition: util-mpm.c:302
Signature ** sig_array
Definition: detect.h:739
uint16_t minlen
Definition: util-mpm.h:99
#define SIG_FLAG_PREFILTER
Definition: detect.h:248
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
Figured out the FP and their respective content ids for all the sigs in the engine.
void PacketPatternCleanup(DetectEngineThreadCtx *det_ctx)
cleans up the mpm instance after a match
int32_t sgh_mpm_context
Definition: detect.h:1219
#define BUG_ON(x)
uint32_t pattern_cnt
Definition: util-mpm.h:97
uint32_t flags
Definition: detect.h:497
void DetectAppLayerMpmRegister(const char *name, int direction, int priority, int(*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx))
register an app layer keyword for mpm
uint32_t id
Definition: detect.h:529
void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s)
Signature ** match_array
Definition: detect.h:1324
#define DETECT_CONTENT_FAST_PATTERN
uint32_t sid_array_size
Definition: detect.h:1214
#define unlikely(expr)
Definition: util-optimize.h:35
void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:59
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
DetectMpmAppLayerKeyword * app_mpms
Definition: detect.h:899
uint16_t PatternMatchDefaultMatcher(void)
Function to return the multi pattern matcher algorithm to be used by the engine, based on the mpm-alg...
InspectionBufferGetDataPtr GetData
Definition: detect.h:586
Signature * sig_list
Definition: detect.h:730
one time registration of keywords at start up
Definition: detect.h:571
int PrefilterGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
int32_t sgh_mpm_context_proto_tcp_packet
Definition: detect.h:809
uint8_t * sid_array
Definition: detect.h:1213
size_t strlcat(char *, const char *src, size_t siz)
Definition: util-strlcatu.c:45
#define SIG_FLAG_REQUIRE_STREAM
Definition: detect.h:229
void MpmInitThreadCtx(MpmThreadCtx *mpm_thread_ctx, uint16_t matcher)
Definition: util-mpm.c:256
uint8_t proto[256/8]
void(* DestroyThreadCtx)(struct MpmCtx_ *, struct MpmThreadCtx_ *)
Definition: util-mpm.h:146
struct DetectFPAndItsId_ DetectFPAndItsId
Container for matching data for a signature group.
Definition: detect.h:1299
void * HashListTableLookup(HashListTable *ht, void *data, uint16_t datalen)
#define SGH_PROTO(sgh, p)
void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
#define SIG_FLAG_REQUIRE_PACKET
Definition: detect.h:228
void PatternMatchThreadDestroy(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
uint16_t AppProto
Signature container.
Definition: detect.h:496
int32_t sgh_mpm_context_proto_other_packet
Definition: detect.h:811
HashListTable * mpm_hash_table
Definition: detect.h:765
int sm_list
Definition: detect.h:1218
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
Definition: conf.c:331
int32_t MpmFactoryRegisterMpmCtxProfile(DetectEngineCtx *de_ctx, const char *name)
Register a new Mpm Context.
Definition: util-mpm.c:55
structure for storing per detect engine mpm keyword settings
Definition: detect.h:599
#define DETECT_CONTENT_IS_SINGLE(c)
#define SGH_DIRECTION_TS(sgh)
SigIntId num
Definition: detect.h:506
struct SigMatch_ * next
Definition: detect.h:326
main detection engine ctx
Definition: detect.h:724
uint32_t max_fp_id
Definition: detect.h:791
#define MPMCTX_FLAGS_GLOBAL
Definition: util-mpm.h:85
MpmBuiltinBuffers
Definition: detect.h:1201
struct DetectMpmAppLayerRegistery_ * next
Definition: detect.h:594
SCFPSupportSMList * sm_fp_support_smlist_list
#define DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED
int HashListTableAdd(HashListTable *ht, void *data, uint16_t datalen)
#define DE_QUIET
Definition: detect.h:296
int(* Prepare)(struct MpmCtx_ *)
Definition: util-mpm.h:161
#define DetectEngineGetMaxSigId(de_ctx)
Definition: detect-engine.h:81
int DetectBufferTypeGetByName(const char *name)
#define SCCalloc(nm, a)
Definition: util-mem.h:197
#define SIG_FLAG_TOCLIENT
Definition: detect.h:242
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:58
int32_t sgh_mpm_context_stream
Definition: detect.h:812
void PatternMatchDestroy(MpmCtx *mpm_ctx, uint16_t mpm_matcher)
uint8_t flags
Definition: detect.h:725
int direction
Definition: detect.h:1216
MpmCtx ** app_mpms
Definition: detect.h:1288
struct SCFPSupportSMList_ * next
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
uint8_t mpm_type
Definition: util-mpm.h:90
uint16_t mpm_matcher
Definition: detect.h:773
#define SIG_FLAG_TOSERVER
Definition: detect.h:241
int PrefilterPktStreamRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx)
DetectMpmAppLayerRegistery * app_mpms_list
Definition: detect.h:890
#define SCEnter(...)
Definition: util-debug.h:337
PrefilterRuleStore pmq
Definition: detect.h:1065
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:548
int SignatureHasPacketContent(const Signature *s)
check if a signature has patterns that are to be inspected against a packets payload (as opposed to t...
struct DetectMpmAppLayerRegistery_::@93 v2
void MpmStoreFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->mpm_hash_table, allocated by MpmStoreInit() function...
HashListTable * HashListTableInit(uint32_t size, uint32_t(*Hash)(struct HashListTable_ *, void *, uint16_t), char(*Compare)(void *, uint16_t, void *, uint16_t), void(*Free)(void *))
Definition: util-hashlist.c:35
void(* DestroyCtx)(struct MpmCtx_ *)
Definition: util-mpm.h:145
uint32_t flags
Definition: detect.h:1300
#define MPM_PATTERN_CTX_OWNS_ID
Definition: util-mpm.h:139
struct Signature_ * next
Definition: detect.h:567
#define SCReturnInt(x)
Definition: util-debug.h:341
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
int32_t sgh_mpm_context_proto_udp_packet
Definition: detect.h:810
#define SIG_GROUP_HEAD_HAVERAWSTREAM
Definition: detect.h:1192
void PatternMatchThreadPrepare(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
struct SigMatch_ ** smlists
Definition: detect.h:490
int(* PrefilterRegister)(struct DetectEngineCtx_ *de_ctx, struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx)
Definition: detect.h:577
void DetectBufferTypeSupportsMpm(const char *name)
uint32_t smlists_array_size
Definition: detect.h:488
#define DETECT_CONTENT_MPM
SigMatchCtx * ctx
Definition: detect.h:325
#define SCMalloc(a)
Definition: util-mem.h:166
#define MPM_CTX_FACTORY_UNIQUE_CONTEXT
Definition: util-mpm.h:113
int mpm_default_matcher
Definition: util-mpm.h:170
uint16_t maxlen
Definition: util-mpm.h:100
void MpmInitCtx(MpmCtx *mpm_ctx, uint16_t matcher)
Definition: util-mpm.c:261
#define DETECT_CONTENT_NEGATED
MpmTableElmt mpm_table[MPM_TABLE_SIZE]
Definition: util-mpm.h:169
const DetectMpmAppLayerRegistery * reg
Definition: detect.h:600
#define SCFree(a)
Definition: util-mem.h:228
#define DETECT_CONTENT_REPLACE
void DetectBufferTypeSupportsTransformations(const char *name)
void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx, const int id, const int parent_id, DetectEngineTransforms *transforms)
copy a mpm engine from parent_id, add in transforms
MpmCtx * mpm_ctx
Definition: detect.h:1221
int(* PrefilterRegisterWithListId)(struct DetectEngineCtx_ *de_ctx, struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx, const struct DetectMpmAppLayerRegistery_ *mpm_reg, int list_id)
Definition: detect.h:583
int FastPatternSupportEnabledForSigMatchList(const DetectEngineCtx *de_ctx, const int list_id)
Checks if a particular list(Signature->sm_lists[]) is in the list of lists that need to be searched f...
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for builtin buffers that are in "single or "shared" mode. ...
DetectEngineTransforms transforms
Definition: detect.h:589
void SupportFastPatternForSigMatchList(int list_id, int priority)
Lets one add a sm list id to be searched for potential fp supported keywords later.
int MpmStoreInit(DetectEngineCtx *de_ctx)
Initializes the MpmStore mpm hash table to be used by the detection engine context.
void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
#define SCLogPerf(...)
Definition: util-debug.h:261
MpmCtx * MpmFactoryGetMpmCtxForProfile(const DetectEngineCtx *de_ctx, int32_t id, int direction)
Definition: util-mpm.c:192
#define FatalError(x,...)
Definition: util-debug.h:539
int PrefilterPktPayloadRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx)
uint32_t array_size
Definition: util-hashlist.h:41
int SignatureHasStreamContent(const Signature *s)
check if a signature has patterns that are to be inspected against the stream payload (as opposed to ...
uint8_t sgh_mpm_context
Definition: detect.h:786
int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags)
Definition: util-mpm.c:311
void HashListTableFree(HashListTable *ht)
Definition: util-hashlist.c:82
#define DETECT_CONTENT_FAST_PATTERN_CHOP
#define DETECT_CONTENT_NOCASE
enum MpmBuiltinBuffers buffer
Definition: detect.h:1217
void PatternMatchThreadPrint(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
uint32_t app_mpms_list_cnt
Definition: detect.h:891
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:561
InspectionBuffer *(* InspectionBufferGetDataPtr)(struct DetectEngineThreadCtx_ *det_ctx, const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv, const int list_id)
Definition: detect.h:379
#define SGH_DIRECTION_TC(sgh)
SigMatch * mpm_sm
Definition: detect.h:467
int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
Prepare the pattern matcher ctx in a sig group head.
MpmStore * MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh, enum MpmBuiltinBuffers buf)
Get MpmStore for a built-in buffer type.
a single match condition for a signature
Definition: detect.h:322
void PmqReset(PrefilterRuleStore *)
Reset a Pmq for reusage. Meant to be called after a single search.
void DetectAppLayerMpmRegister2(const char *name, int direction, int priority, int(*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectMpmAppLayerRegistery *mpm_reg, int list_id), InspectionBufferGetDataPtr GetData, AppProto alproto, int tx_min_progress)
register a MPM engine
SigGroupHeadInitData * init
Definition: detect.h:1327