suricata
detect-engine-mpm.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
23  *
24  * Multi pattern matcher
25  */
26 
27 #include "suricata.h"
28 #include "suricata-common.h"
29 
30 #include "app-layer-protos.h"
31 
32 #include "decode.h"
33 #include "detect.h"
34 #include "detect-engine.h"
35 #include "detect-engine-siggroup.h"
36 #include "detect-engine-mpm.h"
37 #include "detect-engine-iponly.h"
38 #include "detect-parse.h"
40 #include "util-mpm.h"
41 #include "util-memcmp.h"
42 #include "util-memcpy.h"
43 #include "conf.h"
44 #include "detect-fast-pattern.h"
45 
46 #include "flow.h"
47 #include "flow-var.h"
48 #include "detect-flow.h"
49 
50 #include "detect-content.h"
51 
52 #include "detect-engine-payload.h"
53 #include "detect-engine-dns.h"
54 
55 #include "stream.h"
56 
57 #include "util-enum.h"
58 #include "util-debug.h"
59 #include "util-print.h"
60 #include "util-validate.h"
61 
62 const char *builtin_mpms[] = {
63  "toserver TCP packet",
64  "toclient TCP packet",
65  "toserver TCP stream",
66  "toclient TCP stream",
67  "toserver UDP packet",
68  "toclient UDP packet",
69  "other IP packet",
70 
71  NULL };
72 
73 /* Registery for mpm keywords
74  *
75  * Keywords are registered at engine start up
76  */
77 
78 static DetectMpmAppLayerRegistery *g_app_mpms_list = NULL;
79 static int g_app_mpms_list_cnt = 0;
80 
81 /** \brief register a MPM engine
82  *
83  * \note to be used at start up / registration only. Errors are fatal.
84  */
85 void DetectAppLayerMpmRegister2(const char *name,
86  int direction, int priority,
87  int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
88  SigGroupHead *sgh, MpmCtx *mpm_ctx,
89  const DetectMpmAppLayerRegistery *mpm_reg, int list_id),
91  AppProto alproto, int tx_min_progress)
92 {
93  SCLogDebug("registering %s/%d/%d/%p/%p/%u/%d", name, direction, priority,
94  PrefilterRegister, GetData, alproto, tx_min_progress);
95 
96  if (PrefilterRegister == PrefilterGenericMpmRegister && GetData == NULL) {
97  // must register GetData with PrefilterGenericMpmRegister
98  abort();
99  }
100 
103  int sm_list = DetectBufferTypeGetByName(name);
104  if (sm_list == -1) {
106  "MPM engine registration for %s failed", name);
107  }
108 
109  DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
110  BUG_ON(am == NULL);
111  am->name = name;
112  snprintf(am->pname, sizeof(am->pname), "%s", am->name);
113  am->direction = direction;
114  am->sm_list = sm_list;
115  am->priority = priority;
116 
117  am->v2.PrefilterRegisterWithListId = PrefilterRegister;
118  am->v2.GetData = GetData;
119  am->v2.alproto = alproto;
120  am->v2.tx_min_progress = tx_min_progress;
121 
122  if (g_app_mpms_list == NULL) {
123  g_app_mpms_list = am;
124  } else {
125  DetectMpmAppLayerRegistery *t = g_app_mpms_list;
126  while (t->next != NULL) {
127  t = t->next;
128  }
129 
130  t->next = am;
131  am->id = t->id + 1;
132  }
133  g_app_mpms_list_cnt++;
134 
135  SupportFastPatternForSigMatchList(sm_list, priority);
136 }
137 
138 void DetectAppLayerMpmRegister(const char *name,
139  int direction, int priority,
140  int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
141  SigGroupHead *sgh, MpmCtx *mpm_ctx))
142 {
143  SCLogDebug("registering %s/%d/%d/%p",
144  name, direction, priority, PrefilterRegister);
145 
147  int sm_list = DetectBufferTypeGetByName(name);
148  BUG_ON(sm_list == -1);
149 
150  DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
151  BUG_ON(am == NULL);
152  am->name = name;
153  snprintf(am->pname, sizeof(am->pname), "%s", am->name);
154  am->direction = direction;
155  am->sm_list = sm_list;
156  am->priority = priority;
157  am->PrefilterRegister = PrefilterRegister;
158 
159  if (g_app_mpms_list == NULL) {
160  g_app_mpms_list = am;
161  } else {
162  DetectMpmAppLayerRegistery *t = g_app_mpms_list;
163  while (t->next != NULL) {
164  t = t->next;
165  }
166 
167  t->next = am;
168  am->id = t->id + 1;
169  }
170  g_app_mpms_list_cnt++;
171 
172  SupportFastPatternForSigMatchList(sm_list, priority);
173 }
174 
175 /** \brief copy a mpm engine from parent_id, add in transforms */
177  const int id, const int parent_id,
178  DetectEngineTransforms *transforms)
179 {
180  SCLogDebug("registering %d/%d", id, parent_id);
181 
183  while (t) {
184  if (t->sm_list == parent_id) {
185  DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
186  BUG_ON(am == NULL);
187  am->name = t->name;
188  snprintf(am->pname, sizeof(am->pname), "%s#%d", am->name, id);
189  am->direction = t->direction;
190  am->sm_list = id; // use new id
193  am->v2.GetData = t->v2.GetData;
194  am->v2.alproto = t->v2.alproto;
196  am->priority = t->priority;
197  am->next = t->next;
198  if (transforms) {
199  memcpy(&am->v2.transforms, transforms, sizeof(*transforms));
200  }
201  am->id = de_ctx->app_mpms_list_cnt++;
202 
204  t->next = am;
205  SCLogDebug("copied mpm registration for %s id %u "
206  "with parent %u and GetData %p",
207  t->name, id, parent_id, am->v2.GetData);
208  t = am;
209  }
210  t = t->next;
211  }
212 }
213 
215 {
216  const DetectMpmAppLayerRegistery *list = g_app_mpms_list;
217  while (list != NULL) {
218  DetectMpmAppLayerRegistery *n = SCCalloc(1, sizeof(*n));
219  BUG_ON(n == NULL);
220 
221  *n = *list;
222  n->next = NULL;
223 
224  if (de_ctx->app_mpms_list == NULL) {
225  de_ctx->app_mpms_list = n;
226  } else {
228  while (t->next != NULL) {
229  t = t->next;
230  }
231 
232  t->next = n;
233  }
234 
235  list = list->next;
236  }
237  de_ctx->app_mpms_list_cnt = g_app_mpms_list_cnt;
238  SCLogDebug("mpm: de_ctx app_mpms_list %p %u",
239  de_ctx->app_mpms_list, de_ctx->app_mpms_list_cnt);
240 }
241 
243 {
244  BUG_ON(de_ctx->app_mpms_list_cnt == 0);
245 
246  de_ctx->app_mpms = SCCalloc(de_ctx->app_mpms_list_cnt + 1, sizeof(DetectMpmAppLayerKeyword));
247  BUG_ON(de_ctx->app_mpms == NULL);
248 
250  while (list != NULL) {
251  DetectMpmAppLayerKeyword *am = &de_ctx->app_mpms[list->id];
252  am->reg = list;
253 
254  /* default to whatever the global setting is */
255  int shared = (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);
256 
257  /* see if we use a unique or shared mpm ctx for this type */
258  int confshared = 0;
259  char confstring[256] = "detect.mpm.";
260  strlcat(confstring, am->reg->name, sizeof(confstring));
261  strlcat(confstring, ".shared", sizeof(confstring));
262  if (ConfGetBool(confstring, &confshared) == 1)
263  shared = confshared;
264 
265  if (shared == 0) {
266  if (!(de_ctx->flags & DE_QUIET)) {
267  SCLogPerf("using unique mpm ctx' for %s", am->reg->name);
268  }
270  } else {
271  if (!(de_ctx->flags & DE_QUIET)) {
272  SCLogPerf("using shared mpm ctx' for %s", am->reg->name);
273  }
275  }
276 
277  SCLogDebug("AppLayer MPM %s: %u", am->reg->name, am->sgh_mpm_context);
278 
279  list = list->next;
280  }
281 }
282 
283 /**
284  * \brief initialize mpm contexts for applayer buffers that are in
285  * "single or "shared" mode.
286  */
288 {
289  int r = 0;
290  DetectMpmAppLayerKeyword *am = de_ctx->app_mpms;
291  while (am->reg != NULL) {
292  int dir = (am->reg->direction == SIG_FLAG_TOSERVER) ? 1 : 0;
293 
295  {
296  MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
297  if (mpm_ctx != NULL) {
298  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
299  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
300  }
301  }
302  }
303  am++;
304  }
305  return r;
306 }
307 
308 static int32_t SetupBuiltinMpm(DetectEngineCtx *de_ctx, const char *name)
309 {
310  /* default to whatever the global setting is */
311  int shared = (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);
312 
313  /* see if we use a unique or shared mpm ctx for this type */
314  int confshared = 0;
315  char confstring[256] = "detect.mpm.";
316  strlcat(confstring, name, sizeof(confstring));
317  strlcat(confstring, ".shared", sizeof(confstring));
318  if (ConfGetBool(confstring, &confshared) == 1)
319  shared = confshared;
320 
321  int32_t ctx;
322  if (shared == 0) {
324  SCLogPerf("using unique mpm ctx' for %s", name);
325  } else {
326  ctx = MpmFactoryRegisterMpmCtxProfile(de_ctx, name);
327  SCLogPerf("using shared mpm ctx' for %s", name);
328  }
329  return ctx;
330 }
331 
333 {
334  de_ctx->sgh_mpm_context_proto_tcp_packet = SetupBuiltinMpm(de_ctx, "tcp-packet");
335  de_ctx->sgh_mpm_context_stream = SetupBuiltinMpm(de_ctx, "tcp-stream");
336 
337  de_ctx->sgh_mpm_context_proto_udp_packet = SetupBuiltinMpm(de_ctx, "udp-packet");
338  de_ctx->sgh_mpm_context_proto_other_packet = SetupBuiltinMpm(de_ctx, "other-ip");
339 }
340 
341 /**
342  * \brief initialize mpm contexts for builtin buffers that are in
343  * "single or "shared" mode.
344  */
346 {
347  int r = 0;
348  MpmCtx *mpm_ctx = NULL;
349 
352  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
353  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
354  }
356  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
357  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
358  }
359  }
360 
363  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
364  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
365  }
367  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
368  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
369  }
370  }
371 
374  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
375  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
376  }
377  }
378 
380  mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0);
381  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
382  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
383  }
384  mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1);
385  if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
386  r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
387  }
388  }
389 
390  return r;
391 }
392 
393 /**
394  * \brief check if a signature has patterns that are to be inspected
395  * against a packets payload (as opposed to the stream payload)
396  *
397  * \param s signature
398  *
399  * \retval 1 true
400  * \retval 0 false
401  */
403 {
404  SCEnter();
405 
406  if (s == NULL) {
407  SCReturnInt(0);
408  }
409 
410  if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) {
411  SCReturnInt(1);
412  }
413 
414  if ((s->init_data != NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) ||
415  (s->init_data == NULL && s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL))
416  {
417  SCLogDebug("no mpm");
418  SCReturnInt(0);
419  }
420 
421  if (!(s->flags & SIG_FLAG_REQUIRE_PACKET)) {
422  SCReturnInt(0);
423  }
424 
425  SCReturnInt(1);
426 }
427 
428 /**
429  * \brief check if a signature has patterns that are to be inspected
430  * against the stream payload (as opposed to the individual packets
431  * payload(s))
432  *
433  * \param s signature
434  *
435  * \retval 1 true
436  * \retval 0 false
437  */
439 {
440  SCEnter();
441 
442  if (s == NULL) {
443  SCReturnInt(0);
444  }
445 
446  if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) {
447  SCReturnInt(0);
448  }
449 
450  if ((s->init_data != NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) ||
451  (s->init_data == NULL && s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL))
452  {
453  SCLogDebug("no mpm");
454  SCReturnInt(0);
455  }
456 
457  if (!(s->flags & SIG_FLAG_REQUIRE_STREAM)) {
458  SCReturnInt(0);
459  }
460 
461  SCReturnInt(1);
462 }
463 
464 
465 /**
466  * \brief Function to return the multi pattern matcher algorithm to be
467  * used by the engine, based on the mpm-algo setting in yaml
468  * Use the default mpm if none is specified in the yaml file.
469  *
470  * \retval mpm algo value
471  */
473 {
474  const char *mpm_algo;
475  uint16_t mpm_algo_val = mpm_default_matcher;
476 
477  /* Get the mpm algo defined in config file by the user */
478  if ((ConfGet("mpm-algo", &mpm_algo)) == 1) {
479  uint16_t u;
480 
481  if (mpm_algo != NULL) {
482 #if __BYTE_ORDER == __BIG_ENDIAN
483  if (strcmp(mpm_algo, "ac-ks") == 0) {
485  "not work on big endian systems at this time.");
486  exit(EXIT_FAILURE);
487  }
488 #endif
489  if (strcmp("auto", mpm_algo) == 0) {
490  goto done;
491  }
492  for (u = 0; u < MPM_TABLE_SIZE; u++) {
493  if (mpm_table[u].name == NULL)
494  continue;
495 
496  if (strcmp(mpm_table[u].name, mpm_algo) == 0) {
497  mpm_algo_val = u;
498  goto done;
499  }
500  }
501  }
502 
503  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid mpm algo supplied "
504  "in the yaml conf file: \"%s\"", mpm_algo);
505  exit(EXIT_FAILURE);
506  }
507 
508  done:
509 #ifdef __tile__
510  if (mpm_algo_val == MPM_AC)
511  mpm_algo_val = MPM_AC_TILE;
512 #endif
513 
514  return mpm_algo_val;
515 }
516 
517 /** \brief cleans up the mpm instance after a match */
519 {
520  PmqReset(&det_ctx->pmq);
521 }
522 
523 void PatternMatchDestroy(MpmCtx *mpm_ctx, uint16_t mpm_matcher)
524 {
525  SCLogDebug("mpm_ctx %p, mpm_matcher %"PRIu16"", mpm_ctx, mpm_matcher);
526  mpm_table[mpm_matcher].DestroyCtx(mpm_ctx);
527 }
528 
529 void PatternMatchThreadPrint(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
530 {
531  SCLogDebug("mpm_thread_ctx %p, mpm_matcher %"PRIu16" defunct", mpm_thread_ctx, mpm_matcher);
532  //mpm_table[mpm_matcher].PrintThreadCtx(mpm_thread_ctx);
533 }
534 void PatternMatchThreadDestroy(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
535 {
536  SCLogDebug("mpm_thread_ctx %p, mpm_matcher %"PRIu16"", mpm_thread_ctx, mpm_matcher);
537  if (mpm_table[mpm_matcher].DestroyThreadCtx != NULL)
538  mpm_table[mpm_matcher].DestroyThreadCtx(NULL, mpm_thread_ctx);
539 }
540 void PatternMatchThreadPrepare(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
541 {
542  SCLogDebug("mpm_thread_ctx %p, type %"PRIu16, mpm_thread_ctx, mpm_matcher);
543  MpmInitThreadCtx(mpm_thread_ctx, mpm_matcher);
544 }
545 
546 /** \brief Predict a strength value for patterns
547  *
548  * Patterns with high character diversity score higher.
549  * Alpha chars score not so high
550  * Other printable + a few common codes a little higher
551  * Everything else highest.
552  * Longer patterns score better than short patters.
553  *
554  * \param pat pattern
555  * \param patlen length of the patternn
556  *
557  * \retval s pattern score
558  */
559 uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
560 {
561  uint8_t a[256];
562  memset(&a, 0 ,sizeof(a));
563 
564  uint32_t s = 0;
565  uint16_t u = 0;
566  for (u = 0; u < patlen; u++) {
567  if (a[pat[u]] == 0) {
568  if (isalpha(pat[u]))
569  s += 3;
570  else if (isprint(pat[u]) || pat[u] == 0x00 || pat[u] == 0x01 || pat[u] == 0xFF)
571  s += 4;
572  else
573  s += 6;
574 
575  a[pat[u]] = 1;
576  } else {
577  s++;
578  }
579  }
580 
581  return s;
582 }
583 
584 static void PopulateMpmHelperAddPattern(MpmCtx *mpm_ctx,
585  const DetectContentData *cd,
586  const Signature *s, uint8_t flags,
587  int chop)
588 {
589  uint16_t pat_offset = cd->offset;
590  uint16_t pat_depth = cd->depth;
591 
592  /* recompute offset/depth to cope with chop */
593  if (chop && (pat_depth || pat_offset)) {
594  pat_offset += cd->fp_chop_offset;
595  if (pat_depth) {
596  pat_depth -= cd->content_len;
597  pat_depth += cd->fp_chop_offset + cd->fp_chop_len;
598  }
599  }
600 
601  if (cd->flags & DETECT_CONTENT_NOCASE) {
602  if (chop) {
603  MpmAddPatternCI(mpm_ctx,
604  cd->content + cd->fp_chop_offset, cd->fp_chop_len,
605  pat_offset, pat_depth,
606  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
607  } else {
608  MpmAddPatternCI(mpm_ctx,
609  cd->content, cd->content_len,
610  pat_offset, pat_depth,
611  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
612  }
613  } else {
614  if (chop) {
615  MpmAddPatternCS(mpm_ctx,
616  cd->content + cd->fp_chop_offset, cd->fp_chop_len,
617  pat_offset, pat_depth,
618  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
619  } else {
620  MpmAddPatternCS(mpm_ctx,
621  cd->content, cd->content_len,
622  pat_offset, pat_depth,
623  cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
624  }
625  }
626 
627  return;
628 }
629 
630 #define SGH_PROTO(sgh, p) ((sgh)->init->protos[(p)] == 1)
631 #define SGH_DIRECTION_TS(sgh) ((sgh)->init->direction & SIG_FLAG_TOSERVER)
632 #define SGH_DIRECTION_TC(sgh) ((sgh)->init->direction & SIG_FLAG_TOCLIENT)
633 
634 static void SetMpm(Signature *s, SigMatch *mpm_sm)
635 {
636  if (s == NULL || mpm_sm == NULL)
637  return;
638 
639  DetectContentData *cd = (DetectContentData *)mpm_sm->ctx;
641  if (DETECT_CONTENT_IS_SINGLE(cd) &&
642  !(cd->flags & DETECT_CONTENT_NEGATED) &&
643  !(cd->flags & DETECT_CONTENT_REPLACE) &&
644  cd->content_len == cd->fp_chop_len)
645  {
647  }
648  } else {
649  if (DETECT_CONTENT_IS_SINGLE(cd) &&
650  !(cd->flags & DETECT_CONTENT_NEGATED) &&
651  !(cd->flags & DETECT_CONTENT_REPLACE))
652  {
654  }
655  }
656  cd->flags |= DETECT_CONTENT_MPM;
657  s->init_data->mpm_sm = mpm_sm;
658  return;
659 }
660 
661 static SigMatch *GetMpmForList(const Signature *s, const int list, SigMatch *mpm_sm,
662  uint16_t max_len, bool skip_negated_content)
663 {
664  for (SigMatch *sm = s->init_data->smlists[list]; sm != NULL; sm = sm->next) {
665  if (sm->type != DETECT_CONTENT)
666  continue;
667 
668  const DetectContentData *cd = (DetectContentData *)sm->ctx;
669  /* skip_negated_content is only set if there's absolutely no
670  * non-negated content present in the sig */
671  if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
672  continue;
673  if (cd->content_len != max_len)
674  continue;
675 
676  if (mpm_sm == NULL) {
677  mpm_sm = sm;
678  } else {
679  DetectContentData *data1 = (DetectContentData *)sm->ctx;
680  DetectContentData *data2 = (DetectContentData *)mpm_sm->ctx;
681  uint32_t ls = PatternStrength(data1->content, data1->content_len);
682  uint32_t ss = PatternStrength(data2->content, data2->content_len);
683  if (ls > ss) {
684  mpm_sm = sm;
685  } else if (ls == ss) {
686  /* if 2 patterns are of equal strength, we pick the longest */
687  if (data1->content_len > data2->content_len)
688  mpm_sm = sm;
689  } else {
690  SCLogDebug("sticking with mpm_sm");
691  }
692  }
693  }
694  return mpm_sm;
695 }
696 
698 {
699  if (s->init_data->mpm_sm != NULL)
700  return;
701 
702  SigMatch *mpm_sm = NULL, *sm = NULL;
703  const int nlists = s->init_data->smlists_array_size;
704  int nn_sm_list[nlists];
705  int n_sm_list[nlists];
706  memset(nn_sm_list, 0, nlists * sizeof(int));
707  memset(n_sm_list, 0, nlists * sizeof(int));
708  int count_nn_sm_list = 0;
709  int count_n_sm_list = 0;
710 
711  /* inspect rule to see if we have the fast_pattern reg to
712  * force using a sig, otherwise keep stats about the patterns */
713  for (int list_id = 0; list_id < nlists; list_id++) {
714  if (s->init_data->smlists[list_id] == NULL)
715  continue;
716 
717  if (!FastPatternSupportEnabledForSigMatchList(de_ctx, list_id))
718  continue;
719 
720  for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) {
721  if (sm->type != DETECT_CONTENT)
722  continue;
723 
724  const DetectContentData *cd = (DetectContentData *)sm->ctx;
725  /* fast_pattern set in rule, so using this pattern */
726  if ((cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
727  SetMpm(s, sm);
728  return;
729  }
730 
731  if (cd->flags & DETECT_CONTENT_NEGATED) {
732  n_sm_list[list_id] = 1;
733  count_n_sm_list++;
734  } else {
735  nn_sm_list[list_id] = 1;
736  count_nn_sm_list++;
737  }
738  }
739  }
740 
741  /* prefer normal not-negated over negated */
742  int *curr_sm_list = NULL;
743  int skip_negated_content = 1;
744  if (count_nn_sm_list > 0) {
745  curr_sm_list = nn_sm_list;
746  } else if (count_n_sm_list > 0) {
747  curr_sm_list = n_sm_list;
748  skip_negated_content = 0;
749  } else {
750  return;
751  }
752 
753  int final_sm_list[nlists];
754  memset(&final_sm_list, 0, (nlists * sizeof(int)));
755 
756  int count_final_sm_list = 0;
757  int priority;
758 
760  while (tmp != NULL) {
761  for (priority = tmp->priority;
762  tmp != NULL && priority == tmp->priority;
763  tmp = tmp->next)
764  {
765  if (tmp->list_id >= nlists)
766  continue;
767  if (curr_sm_list[tmp->list_id] == 0)
768  continue;
769  final_sm_list[count_final_sm_list++] = tmp->list_id;
770  }
771  if (count_final_sm_list != 0)
772  break;
773  }
774 
775  BUG_ON(count_final_sm_list == 0);
776 
777  uint16_t max_len = 0;
778  for (int i = 0; i < count_final_sm_list; i++) {
779  if (final_sm_list[i] >= (int)s->init_data->smlists_array_size)
780  continue;
781 
782  for (sm = s->init_data->smlists[final_sm_list[i]]; sm != NULL; sm = sm->next) {
783  if (sm->type != DETECT_CONTENT)
784  continue;
785 
786  const DetectContentData *cd = (DetectContentData *)sm->ctx;
787  /* skip_negated_content is only set if there's absolutely no
788  * non-negated content present in the sig */
789  if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
790  continue;
791  if (max_len < cd->content_len)
792  max_len = cd->content_len;
793  }
794  }
795 
796  for (int i = 0; i < count_final_sm_list; i++) {
797  if (final_sm_list[i] >= (int)s->init_data->smlists_array_size)
798  continue;
799 
800  mpm_sm = GetMpmForList(s, final_sm_list[i], mpm_sm, max_len, skip_negated_content);
801  }
802 
803  /* assign to signature */
804  SetMpm(s, mpm_sm);
805  return;
806 }
807 
808 /** \internal
809  * \brief The hash function for MpmStore
810  *
811  * \param ht Pointer to the hash table.
812  * \param data Pointer to the MpmStore.
813  * \param datalen Not used in our case.
814  *
815  * \retval hash The generated hash value.
816  */
817 static uint32_t MpmStoreHashFunc(HashListTable *ht, void *data, uint16_t datalen)
818 {
819  const MpmStore *ms = (MpmStore *)data;
820  uint32_t hash = 0;
821  uint32_t b = 0;
822 
823  for (b = 0; b < ms->sid_array_size; b++)
824  hash += ms->sid_array[b];
825 
826  return hash % ht->array_size;
827 }
828 
829 /**
830  * \brief The Compare function for MpmStore
831  *
832  * \param data1 Pointer to the first MpmStore.
833  * \param len1 Not used.
834  * \param data2 Pointer to the second MpmStore.
835  * \param len2 Not used.
836  *
837  * \retval 1 If the 2 MpmStores sent as args match.
838  * \retval 0 If the 2 MpmStores sent as args do not match.
839  */
840 static char MpmStoreCompareFunc(void *data1, uint16_t len1, void *data2,
841  uint16_t len2)
842 {
843  const MpmStore *ms1 = (MpmStore *)data1;
844  const MpmStore *ms2 = (MpmStore *)data2;
845 
846  if (ms1->sid_array_size != ms2->sid_array_size)
847  return 0;
848 
849  if (ms1->buffer != ms2->buffer)
850  return 0;
851 
852  if (ms1->direction != ms2->direction)
853  return 0;
854 
855  if (ms1->sm_list != ms2->sm_list)
856  return 0;
857 
858  if (SCMemcmp(ms1->sid_array, ms2->sid_array,
859  ms1->sid_array_size) != 0)
860  {
861  return 0;
862  }
863 
864  return 1;
865 }
866 
867 static void MpmStoreFreeFunc(void *ptr)
868 {
869  MpmStore *ms = ptr;
870  if (ms != NULL) {
871  if (ms->mpm_ctx != NULL && !ms->mpm_ctx->global)
872  {
873  SCLogDebug("destroying mpm_ctx %p", ms->mpm_ctx);
875  SCFree(ms->mpm_ctx);
876  }
877  ms->mpm_ctx = NULL;
878 
879  SCFree(ms->sid_array);
880  SCFree(ms);
881  }
882 }
883 
884 /**
885  * \brief Initializes the MpmStore mpm hash table to be used by the detection
886  * engine context.
887  *
888  * \param de_ctx Pointer to the detection engine context.
889  *
890  * \retval 0 On success.
891  * \retval -1 On failure.
892  */
894 {
895  de_ctx->mpm_hash_table = HashListTableInit(4096,
896  MpmStoreHashFunc,
897  MpmStoreCompareFunc,
898  MpmStoreFreeFunc);
899  if (de_ctx->mpm_hash_table == NULL)
900  goto error;
901 
902  return 0;
903 
904 error:
905  return -1;
906 }
907 
908 /**
909  * \brief Adds a MpmStore to the detection engine context MpmStore
910  *
911  * \param de_ctx Pointer to the detection engine context.
912  * \param sgh Pointer to the MpmStore.
913  *
914  * \retval ret 0 on Successfully adding the argument sgh; -1 on failure.
915  */
916 static int MpmStoreAdd(DetectEngineCtx *de_ctx, MpmStore *s)
917 {
918  int ret = HashListTableAdd(de_ctx->mpm_hash_table, (void *)s, 0);
919  return ret;
920 }
921 
922 /**
923  * \brief Used to lookup a MpmStore from the MpmStore
924  *
925  * \param de_ctx Pointer to the detection engine context.
926  * \param sgh Pointer to the MpmStore.
927  *
928  * \retval rsgh On success a pointer to the MpmStore if the MpmStore is
929  * found in the hash table; NULL on failure.
930  */
931 static MpmStore *MpmStoreLookup(DetectEngineCtx *de_ctx, MpmStore *s)
932 {
934  (void *)s, 0);
935  return rs;
936 }
937 
939 {
940  HashListTableBucket *htb = NULL;
941 
942  int app_mpms_cnt = 0;
943  DetectMpmAppLayerKeyword *a = de_ctx->app_mpms;
944  while (a->reg != NULL) {
945  a++;
946  app_mpms_cnt++;
947  }
948  uint32_t stats[MPMB_MAX] = {0};
949  uint32_t appstats[app_mpms_cnt + 1]; // +1 to silence scan-build
950  memset(&appstats, 0x00, sizeof(appstats));
951 
952  for (htb = HashListTableGetListHead(de_ctx->mpm_hash_table);
953  htb != NULL;
954  htb = HashListTableGetListNext(htb))
955  {
956  const MpmStore *ms = (MpmStore *)HashListTableGetListData(htb);
957  if (ms == NULL) {
958  continue;
959  }
960  if (ms->buffer < MPMB_MAX)
961  stats[ms->buffer]++;
962  else if (ms->sm_list != DETECT_SM_LIST_PMATCH) {
963  int i = 0;
964  DetectMpmAppLayerKeyword *am = de_ctx->app_mpms;
965  while (am->reg != NULL) {
966  if (ms->sm_list == am->reg->sm_list &&
967  ms->direction == am->reg->direction)
968  {
969  SCLogDebug("%s %s: %u patterns. Min %u, Max %u. Ctx %p",
970  am->reg->name,
971  am->reg->direction == SIG_FLAG_TOSERVER ? "toserver":"toclient",
972  ms->mpm_ctx->pattern_cnt,
973  ms->mpm_ctx->minlen, ms->mpm_ctx->maxlen,
974  ms->mpm_ctx);
975  appstats[i]++;
976  break;
977  }
978  i++;
979  am++;
980  }
981  }
982  }
983 
984  if (!(de_ctx->flags & DE_QUIET)) {
985  int x;
986  for (x = 0; x < MPMB_MAX; x++) {
987  SCLogPerf("Builtin MPM \"%s\": %u", builtin_mpms[x], stats[x]);
988  }
989  for (x = 0; x < app_mpms_cnt; x++) {
990  if (appstats[x] == 0)
991  continue;
992  const char *name = de_ctx->app_mpms[x].reg->name;
993  const char *direction = de_ctx->app_mpms[x].reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient";
994  SCLogPerf("AppLayer MPM \"%s %s\": %u", direction, name, appstats[x]);
995  }
996  }
997 }
998 
999 /**
1000  * \brief Frees the hash table - DetectEngineCtx->mpm_hash_table, allocated by
1001  * MpmStoreInit() function.
1002  *
1003  * \param de_ctx Pointer to the detection engine context.
1004  */
1006 {
1007  if (de_ctx->mpm_hash_table == NULL)
1008  return;
1009 
1011  de_ctx->mpm_hash_table = NULL;
1012  return;
1013 }
1014 
1015 static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
1016 {
1017  const Signature *s = NULL;
1018  uint32_t sig;
1019 
1020  int dir = 0;
1021 
1022  if (ms->buffer != MPMB_MAX) {
1024 
1025  switch (ms->buffer) {
1026  /* TS is 1 */
1027  case MPMB_TCP_PKT_TS:
1028  case MPMB_TCP_STREAM_TS:
1029  case MPMB_UDP_TS:
1030  dir = 1;
1031  break;
1032 
1033  /* TC is 0 */
1034  default:
1035  case MPMB_UDP_TC:
1036  case MPMB_TCP_STREAM_TC:
1037  case MPMB_TCP_PKT_TC:
1038  case MPMB_OTHERIP: /**< use 0 for other */
1039  dir = 0;
1040  break;
1041  }
1042  } else {
1044  BUG_ON(ms->direction == 0);
1046 
1047  if (ms->direction == SIG_FLAG_TOSERVER)
1048  dir = 1;
1049  else
1050  dir = 0;
1051  }
1052 
1053  ms->mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, ms->sgh_mpm_context, dir);
1054  if (ms->mpm_ctx == NULL)
1055  return;
1056 
1057  MpmInitCtx(ms->mpm_ctx, de_ctx->mpm_matcher);
1058 
1059  /* add the patterns */
1060  for (sig = 0; sig < (ms->sid_array_size * 8); sig++) {
1061  if (ms->sid_array[sig / 8] & (1 << (sig % 8))) {
1062  s = de_ctx->sig_array[sig];
1063  if (s == NULL)
1064  continue;
1065  if ((s->flags & ms->direction) == 0)
1066  continue;
1067  if (s->init_data->mpm_sm == NULL)
1068  continue;
1069  int list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
1070  if (list < 0)
1071  continue;
1072  if (list != ms->sm_list)
1073  continue;
1074 
1075  SCLogDebug("adding %u", s->id);
1076 
1078 
1079  int skip = 0;
1080  /* negated logic: if mpm match can't be used to be sure about this
1081  * pattern, we have to inspect the rule fully regardless of mpm
1082  * match. So in this case there is no point of adding it at all.
1083  * The non-mpm list entry for the sig will make sure the sig is
1084  * inspected. */
1085  if ((cd->flags & DETECT_CONTENT_NEGATED) &&
1087  {
1088  skip = 1;
1089  SCLogDebug("not adding negated mpm as it's not 'single'");
1090  }
1091 
1092  if (!skip) {
1093  PopulateMpmHelperAddPattern(ms->mpm_ctx,
1094  cd, s, 0, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
1095  }
1096  }
1097  }
1098 
1099  if (ms->mpm_ctx->pattern_cnt == 0) {
1100  MpmFactoryReClaimMpmCtx(de_ctx, ms->mpm_ctx);
1101  ms->mpm_ctx = NULL;
1102  } else {
1104  if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) {
1106  }
1107  }
1108  }
1109 }
1110 
1111 
1112 /** \brief Get MpmStore for a built-in buffer type
1113  *
1114  */
1116  enum MpmBuiltinBuffers buf)
1117 {
1118  const Signature *s = NULL;
1119  uint32_t sig;
1120  uint32_t cnt = 0;
1121  int direction = 0;
1122  uint32_t max_sid = DetectEngineGetMaxSigId(de_ctx) / 8 + 1;
1123  uint8_t sids_array[max_sid];
1124  memset(sids_array, 0x00, max_sid);
1125  int sgh_mpm_context = 0;
1126 
1127  switch (buf) {
1128  case MPMB_TCP_PKT_TS:
1129  case MPMB_TCP_PKT_TC:
1130  sgh_mpm_context = de_ctx->sgh_mpm_context_proto_tcp_packet;
1131  break;
1132  case MPMB_TCP_STREAM_TS:
1133  case MPMB_TCP_STREAM_TC:
1134  sgh_mpm_context = de_ctx->sgh_mpm_context_stream;
1135  break;
1136  case MPMB_UDP_TS:
1137  case MPMB_UDP_TC:
1138  sgh_mpm_context = de_ctx->sgh_mpm_context_proto_udp_packet;
1139  break;
1140  case MPMB_OTHERIP:
1141  sgh_mpm_context = de_ctx->sgh_mpm_context_proto_other_packet;
1142  break;
1143  default:
1144  break;
1145  }
1146 
1147  switch(buf) {
1148  case MPMB_TCP_PKT_TS:
1149  case MPMB_TCP_STREAM_TS:
1150  case MPMB_UDP_TS:
1151  direction = SIG_FLAG_TOSERVER;
1152  break;
1153 
1154  case MPMB_TCP_PKT_TC:
1155  case MPMB_TCP_STREAM_TC:
1156  case MPMB_UDP_TC:
1157  direction = SIG_FLAG_TOCLIENT;
1158  break;
1159 
1160  case MPMB_OTHERIP:
1161  direction = (SIG_FLAG_TOCLIENT|SIG_FLAG_TOSERVER);
1162  break;
1163 
1164  case MPMB_MAX:
1165  BUG_ON(1);
1166  break;
1167  }
1168 
1169  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1170  s = sgh->match_array[sig];
1171  if (s == NULL)
1172  continue;
1173 
1174  if (s->init_data->mpm_sm == NULL)
1175  continue;
1176 
1177  int list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
1178  if (list < 0)
1179  continue;
1180 
1181  if (list != DETECT_SM_LIST_PMATCH)
1182  continue;
1183 
1184  switch (buf) {
1185  case MPMB_TCP_PKT_TS:
1186  case MPMB_TCP_PKT_TC:
1187  if (SignatureHasPacketContent(s) == 1)
1188  {
1189  sids_array[s->num / 8] |= 1 << (s->num % 8);
1190  cnt++;
1191  }
1192  break;
1193  case MPMB_TCP_STREAM_TS:
1194  case MPMB_TCP_STREAM_TC:
1195  if (SignatureHasStreamContent(s) == 1)
1196  {
1197  sids_array[s->num / 8] |= 1 << (s->num % 8);
1198  cnt++;
1199  }
1200  break;
1201  case MPMB_UDP_TS:
1202  case MPMB_UDP_TC:
1203  sids_array[s->num / 8] |= 1 << (s->num % 8);
1204  cnt++;
1205  break;
1206  case MPMB_OTHERIP:
1207  sids_array[s->num / 8] |= 1 << (s->num % 8);
1208  cnt++;
1209  break;
1210  default:
1211  break;
1212  }
1213  }
1214 
1215  if (cnt == 0)
1216  return NULL;
1217 
1218  MpmStore lookup = { sids_array, max_sid, direction, buf, DETECT_SM_LIST_PMATCH, 0, NULL};
1219 
1220  MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
1221  if (result == NULL) {
1222  MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
1223  if (copy == NULL)
1224  return NULL;
1225  uint8_t *sids = SCCalloc(1, max_sid);
1226  if (sids == NULL) {
1227  SCFree(copy);
1228  return NULL;
1229  }
1230 
1231  memcpy(sids, sids_array, max_sid);
1232  copy->sid_array = sids;
1233  copy->sid_array_size = max_sid;
1234  copy->buffer = buf;
1235  copy->direction = direction;
1237  copy->sgh_mpm_context = sgh_mpm_context;
1238 
1239  MpmStoreSetup(de_ctx, copy);
1240  MpmStoreAdd(de_ctx, copy);
1241  return copy;
1242  } else {
1243  return result;
1244  }
1245 }
1246 
1247 static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
1249 {
1250  const Signature *s = NULL;
1251  uint32_t sig;
1252  uint32_t cnt = 0;
1253  uint32_t max_sid = DetectEngineGetMaxSigId(de_ctx) / 8 + 1;
1254  uint8_t sids_array[max_sid];
1255  memset(sids_array, 0x00, max_sid);
1256 
1257  SCLogDebug("handling %s direction %s for list %d", am->reg->name,
1258  am->reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1259  am->reg->sm_list);
1260 
1261  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1262  s = sgh->match_array[sig];
1263  if (s == NULL)
1264  continue;
1265 
1266  if (s->init_data->mpm_sm == NULL)
1267  continue;
1268 
1269  int list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
1270  if (list < 0)
1271  continue;
1272 
1273  if ((s->flags & am->reg->direction) == 0)
1274  continue;
1275 
1276  if (list != am->reg->sm_list)
1277  continue;
1278 
1279  sids_array[s->num / 8] |= 1 << (s->num % 8);
1280  cnt++;
1281  }
1282 
1283  if (cnt == 0)
1284  return NULL;
1285 
1286  MpmStore lookup = { sids_array, max_sid, am->reg->direction,
1287  MPMB_MAX, am->reg->sm_list, 0, NULL};
1288  SCLogDebug("am->direction %d am->sm_list %d",
1289  am->reg->direction, am->reg->sm_list);
1290 
1291  MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
1292  if (result == NULL) {
1293  SCLogDebug("new unique mpm for %s %s: %u patterns",
1294  am->reg->name,
1295  am->reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1296  cnt);
1297 
1298  MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
1299  if (copy == NULL)
1300  return NULL;
1301  uint8_t *sids = SCCalloc(1, max_sid);
1302  if (sids == NULL) {
1303  SCFree(copy);
1304  return NULL;
1305  }
1306 
1307  memcpy(sids, sids_array, max_sid);
1308  copy->sid_array = sids;
1309  copy->sid_array_size = max_sid;
1310  copy->buffer = MPMB_MAX;
1311  copy->direction = am->reg->direction;
1312  copy->sm_list = am->reg->sm_list;
1313  copy->sgh_mpm_context = am->sgh_mpm_context;
1314 
1315  MpmStoreSetup(de_ctx, copy);
1316  MpmStoreAdd(de_ctx, copy);
1317  return copy;
1318  } else {
1319  SCLogDebug("using existing mpm %p", result);
1320  return result;
1321  }
1322  return NULL;
1323 }
1324 
1325 static void SetRawReassemblyFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1326 {
1327  const Signature *s = NULL;
1328  uint32_t sig;
1329 
1330  for (sig = 0; sig < sgh->sig_cnt; sig++) {
1331  s = sgh->match_array[sig];
1332  if (s == NULL)
1333  continue;
1334 
1335  if (SignatureHasStreamContent(s) == 1) {
1337  SCLogDebug("rule group %p has SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
1338  return;
1339  }
1340  }
1341  SCLogDebug("rule group %p does NOT have SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
1342 }
1343 
1344 /** \brief Prepare the pattern matcher ctx in a sig group head.
1345  *
1346  */
1348 {
1349  MpmStore *mpm_store = NULL;
1350  if (SGH_PROTO(sh, IPPROTO_TCP)) {
1351  if (SGH_DIRECTION_TS(sh)) {
1352  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TS);
1353  if (mpm_store != NULL) {
1354  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1355  }
1356 
1357  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_STREAM_TS);
1358  if (mpm_store != NULL) {
1359  PrefilterPktStreamRegister(de_ctx, sh, mpm_store->mpm_ctx);
1360  }
1361 
1362  SetRawReassemblyFlag(de_ctx, sh);
1363  }
1364  if (SGH_DIRECTION_TC(sh)) {
1365  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TC);
1366  if (mpm_store != NULL) {
1367  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1368  }
1369 
1370  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_STREAM_TC);
1371  if (mpm_store != NULL) {
1372  PrefilterPktStreamRegister(de_ctx, sh, mpm_store->mpm_ctx);
1373  }
1374 
1375  SetRawReassemblyFlag(de_ctx, sh);
1376  }
1377  } else if (SGH_PROTO(sh, IPPROTO_UDP)) {
1378  if (SGH_DIRECTION_TS(sh)) {
1379  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_UDP_TS);
1380  if (mpm_store != NULL) {
1381  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1382  }
1383  }
1384  if (SGH_DIRECTION_TC(sh)) {
1385  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_UDP_TC);
1386  if (mpm_store != NULL) {
1387  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1388  }
1389  }
1390  } else {
1391  mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_OTHERIP);
1392  if (mpm_store != NULL) {
1393  PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
1394  }
1395  }
1396 
1397  int i = 0;
1398  DetectMpmAppLayerKeyword *a = de_ctx->app_mpms;
1399  while (a->reg != NULL) {
1400  i++;
1401  a++;
1402  }
1403  if (i == 0)
1404  return 0;
1405 
1406  sh->init->app_mpms = SCCalloc(i, sizeof(MpmCtx *));
1407  BUG_ON(sh->init->app_mpms == NULL);
1408 
1409  a = de_ctx->app_mpms;
1410  while (a->reg != NULL) {
1411  if ((a->reg->direction == SIG_FLAG_TOSERVER && SGH_DIRECTION_TS(sh)) ||
1413  {
1414  mpm_store = MpmStorePrepareBufferAppLayer(de_ctx, sh, a);
1415  if (mpm_store != NULL) {
1416  sh->init->app_mpms[a->reg->id] = mpm_store->mpm_ctx;
1417 
1418  SCLogDebug("a->reg->PrefilterRegister %p mpm_store->mpm_ctx %p",
1419  a->reg->PrefilterRegister, mpm_store->mpm_ctx);
1420  SCLogDebug("a %p a->reg->name %s a->reg->PrefilterRegisterWithListId %p "
1421  "mpm_store->mpm_ctx %p", a, a->reg->name,
1422  a->reg->v2.PrefilterRegisterWithListId, mpm_store->mpm_ctx);
1423 
1424  /* if we have just certain types of negated patterns,
1425  * mpm_ctx can be NULL */
1426  if (a->reg->v2.PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
1428  sh, mpm_store->mpm_ctx,
1429  a->reg, a->reg->sm_list) != 0);
1430  SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
1431  }
1432  else if (a->reg->PrefilterRegister && mpm_store->mpm_ctx) {
1433  BUG_ON(a->reg->PrefilterRegister(de_ctx, sh, mpm_store->mpm_ctx) != 0);
1434  SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
1435  }
1436  }
1437  }
1438  a++;
1439  }
1440 
1441  return 0;
1442 }
1443 
1444 typedef struct DetectFPAndItsId_ {
1446  uint16_t content_len;
1447  uint32_t flags;
1448  int sm_list;
1449 
1450  uint8_t *content;
1452 
1453 /**
1454  * \brief Figured out the FP and their respective content ids for all the
1455  * sigs in the engine.
1456  *
1457  * \param de_ctx Detection engine context.
1458  *
1459  * \retval 0 On success.
1460  * \retval -1 On failure.
1461  */
1463 {
1464  uint32_t struct_total_size = 0;
1465  uint32_t content_total_size = 0;
1466  Signature *s = NULL;
1467 
1468  /* Count the amount of memory needed to store all the structures
1469  * and the content of those structures. This will over estimate the
1470  * true size, since duplicates are removed below, but counted here.
1471  */
1472  for (s = de_ctx->sig_list; s != NULL; s = s->next) {
1473  if (s->flags & SIG_FLAG_PREFILTER)
1474  continue;
1475 
1476  RetrieveFPForSig(de_ctx, s);
1477  if (s->init_data->mpm_sm != NULL) {
1479  struct_total_size += sizeof(DetectFPAndItsId);
1480  content_total_size += cd->content_len;
1481 
1482  s->flags |= SIG_FLAG_PREFILTER;
1483  }
1484  }
1485  /* no rules */
1486  if (struct_total_size + content_total_size == 0)
1487  return 0;
1488 
1489  /* array hash buffer - i've run out of ideas to name it */
1490  uint8_t *ahb = SCMalloc(sizeof(uint8_t) * (struct_total_size + content_total_size));
1491  if (unlikely(ahb == NULL))
1492  return -1;
1493 
1494  uint8_t *content = NULL;
1495  uint16_t content_len = 0;
1496  PatIntId max_id = 0;
1497  DetectFPAndItsId *struct_offset = (DetectFPAndItsId *)ahb;
1498  uint8_t *content_offset = ahb + struct_total_size;
1499 
1500  for (s = de_ctx->sig_list; s != NULL; s = s->next) {
1501  if (s->init_data->mpm_sm != NULL) {
1503  BUG_ON(sm_list == -1);
1504 
1506  DetectFPAndItsId *dup = (DetectFPAndItsId *)ahb;
1508  content = cd->content + cd->fp_chop_offset;
1509  content_len = cd->fp_chop_len;
1510  } else {
1511  content = cd->content;
1512  content_len = cd->content_len;
1513  }
1514  uint32_t flags = cd->flags & DETECT_CONTENT_NOCASE;
1515  /* Check for content already found on the same list */
1516  for (; dup != struct_offset; dup++) {
1517  if (dup->content_len != content_len)
1518  continue;
1519  if (dup->sm_list != sm_list)
1520  continue;
1521  if (dup->flags != flags)
1522  continue;
1523  /* Check for pattern matching a duplicate. Use case insensitive matching
1524  * for case insensitive patterns. */
1525  if (flags & DETECT_CONTENT_NOCASE) {
1526  if (SCMemcmpLowercase(dup->content, content, content_len) != 0)
1527  continue;
1528  } else {
1529  /* Case sensitive matching */
1530  if (SCMemcmp(dup->content, content, content_len) != 0)
1531  continue;
1532  }
1533  /* Found a match with a previous pattern. */
1534  break;
1535  }
1536  if (dup != struct_offset) {
1537  /* Exited for-loop before the end, so found an existing match.
1538  * Use its ID. */
1539  cd->id = dup->id;
1540  continue;
1541  }
1542 
1543  /* Not found, so new content. Give it a new ID and add it
1544  * to the array. Copy the content at the end of the
1545  * content array.
1546  */
1547  struct_offset->id = max_id++;
1548  cd->id = struct_offset->id;
1549  struct_offset->content_len = content_len;
1550  struct_offset->sm_list = sm_list;
1551  struct_offset->content = content_offset;
1552  struct_offset->flags = flags;
1553 
1554  content_offset += content_len;
1555 
1556  if (flags & DETECT_CONTENT_NOCASE) {
1557  /* Need to store case-insensitive patterns as lower case
1558  * because SCMemcmpLowercase() above assumes that all
1559  * patterns are stored lower case so that it doesn't
1560  * need to relower its first argument.
1561  */
1562  memcpy_tolower(struct_offset->content, content, content_len);
1563  } else {
1564  memcpy(struct_offset->content, content, content_len);
1565  }
1566 
1567  struct_offset++;
1568  } /* if (s->mpm_sm != NULL) */
1569  } /* for */
1570 
1571  de_ctx->max_fp_id = max_id;
1572 
1573  SCFree(ahb);
1574 
1575  return 0;
1576 }
void DetectMpmSetupAppMpms(DetectEngineCtx *de_ctx)
void MpmFactoryReClaimMpmCtx(const DetectEngineCtx *de_ctx, MpmCtx *mpm_ctx)
Definition: util-mpm.c:215
#define PatIntId
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
Predict a strength value for patterns.
DetectProto proto
Definition: detect.h:509
SigIntId sig_cnt
Definition: detect.h:1300
SignatureInitData * init_data
Definition: detect.h:560
uint16_t flags
#define SCMemcmp(a, b, c)
Definition: util-memcmp.h:490
#define DETECT_CONTENT_MPM_IS_CONCLUSIVE(c)
#define SCLogDebug(...)
Definition: util-debug.h:335
const char * builtin_mpms[]
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
int MpmAddPatternCS(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags)
Definition: util-mpm.c:310
Signature ** sig_array
Definition: detect.h:735
uint16_t minlen
Definition: util-mpm.h:95
#define SIG_FLAG_PREFILTER
Definition: detect.h:250
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
Figured out the FP and their respective content ids for all the sigs in the engine.
void PacketPatternCleanup(DetectEngineThreadCtx *det_ctx)
cleans up the mpm instance after a match
int32_t sgh_mpm_context
Definition: detect.h:1215
#define BUG_ON(x)
uint32_t pattern_cnt
Definition: util-mpm.h:93
uint32_t flags
Definition: detect.h:493
void DetectAppLayerMpmRegister(const char *name, int direction, int priority, int(*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx))
register an app layer keyword for mpm
uint32_t id
Definition: detect.h:525
void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s)
Signature ** match_array
Definition: detect.h:1320
#define DETECT_CONTENT_FAST_PATTERN
uint32_t sid_array_size
Definition: detect.h:1210
#define unlikely(expr)
Definition: util-optimize.h:35
void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:59
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
DetectMpmAppLayerKeyword * app_mpms
Definition: detect.h:895
uint16_t PatternMatchDefaultMatcher(void)
Function to return the multi pattern matcher algorithm to be used by the engine, based on the mpm-alg...
InspectionBufferGetDataPtr GetData
Definition: detect.h:582
Signature * sig_list
Definition: detect.h:726
one time registration of keywords at start up
Definition: detect.h:567
int PrefilterGenericMpmRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
int32_t sgh_mpm_context_proto_tcp_packet
Definition: detect.h:805
uint8_t * sid_array
Definition: detect.h:1209
size_t strlcat(char *, const char *src, size_t siz)
Definition: util-strlcatu.c:45
#define SIG_FLAG_REQUIRE_STREAM
Definition: detect.h:231
void MpmInitThreadCtx(MpmThreadCtx *mpm_thread_ctx, uint16_t matcher)
Definition: util-mpm.c:256
uint8_t proto[256/8]
uint16_t global
Definition: util-mpm.h:90
void(* DestroyThreadCtx)(struct MpmCtx_ *, struct MpmThreadCtx_ *)
Definition: util-mpm.h:142
struct DetectFPAndItsId_ DetectFPAndItsId
Container for matching data for a signature group.
Definition: detect.h:1295
void * HashListTableLookup(HashListTable *ht, void *data, uint16_t datalen)
#define SGH_PROTO(sgh, p)
void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
#define SIG_FLAG_REQUIRE_PACKET
Definition: detect.h:230
void PatternMatchThreadDestroy(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
uint16_t AppProto
Signature container.
Definition: detect.h:492
int32_t sgh_mpm_context_proto_other_packet
Definition: detect.h:807
HashListTable * mpm_hash_table
Definition: detect.h:761
int sm_list
Definition: detect.h:1214
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
Definition: conf.c:331
int32_t MpmFactoryRegisterMpmCtxProfile(DetectEngineCtx *de_ctx, const char *name)
Register a new Mpm Context.
Definition: util-mpm.c:55
structure for storing per detect engine mpm keyword settings
Definition: detect.h:595
#define DETECT_CONTENT_IS_SINGLE(c)
#define SGH_DIRECTION_TS(sgh)
SigIntId num
Definition: detect.h:502
struct SigMatch_ * next
Definition: detect.h:328
main detection engine ctx
Definition: detect.h:720
uint32_t max_fp_id
Definition: detect.h:787
MpmBuiltinBuffers
Definition: detect.h:1197
struct DetectMpmAppLayerRegistery_ * next
Definition: detect.h:590
SCFPSupportSMList * sm_fp_support_smlist_list
#define DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED
int HashListTableAdd(HashListTable *ht, void *data, uint16_t datalen)
#define DE_QUIET
Definition: detect.h:298
int(* Prepare)(struct MpmCtx_ *)
Definition: util-mpm.h:157
#define DetectEngineGetMaxSigId(de_ctx)
Definition: detect-engine.h:81
int DetectBufferTypeGetByName(const char *name)
#define SCCalloc(nm, a)
Definition: util-mem.h:205
#define SIG_FLAG_TOCLIENT
Definition: detect.h:244
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:58
int32_t sgh_mpm_context_stream
Definition: detect.h:808
void PatternMatchDestroy(MpmCtx *mpm_ctx, uint16_t mpm_matcher)
uint8_t flags
Definition: detect.h:721
uint16_t mpm_type
Definition: util-mpm.h:84
int direction
Definition: detect.h:1212
MpmCtx ** app_mpms
Definition: detect.h:1284
struct SCFPSupportSMList_ * next
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
uint16_t mpm_matcher
Definition: detect.h:769
#define SIG_FLAG_TOSERVER
Definition: detect.h:243
int PrefilterPktStreamRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx)
DetectMpmAppLayerRegistery * app_mpms_list
Definition: detect.h:886
#define SCEnter(...)
Definition: util-debug.h:337
PrefilterRuleStore pmq
Definition: detect.h:1061
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:544
int SignatureHasPacketContent(const Signature *s)
check if a signature has patterns that are to be inspected against a packets payload (as opposed to t...
void MpmStoreFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->mpm_hash_table, allocated by MpmStoreInit() function...
HashListTable * HashListTableInit(uint32_t size, uint32_t(*Hash)(struct HashListTable_ *, void *, uint16_t), char(*Compare)(void *, uint16_t, void *, uint16_t), void(*Free)(void *))
Definition: util-hashlist.c:35
void(* DestroyCtx)(struct MpmCtx_ *)
Definition: util-mpm.h:141
uint32_t flags
Definition: detect.h:1296
#define MPM_PATTERN_CTX_OWNS_ID
Definition: util-mpm.h:135
struct Signature_ * next
Definition: detect.h:563
#define SCReturnInt(x)
Definition: util-debug.h:341
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for applayer buffers that are in "single or "shared" mode.
int32_t sgh_mpm_context_proto_udp_packet
Definition: detect.h:806
#define SIG_GROUP_HEAD_HAVERAWSTREAM
Definition: detect.h:1188
void PatternMatchThreadPrepare(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
struct SigMatch_ ** smlists
Definition: detect.h:486
int(* PrefilterRegister)(struct DetectEngineCtx_ *de_ctx, struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx)
Definition: detect.h:573
void DetectBufferTypeSupportsMpm(const char *name)
uint32_t smlists_array_size
Definition: detect.h:484
#define DETECT_CONTENT_MPM
SigMatchCtx * ctx
Definition: detect.h:327
#define SCMalloc(a)
Definition: util-mem.h:174
#define MPM_CTX_FACTORY_UNIQUE_CONTEXT
Definition: util-mpm.h:109
int mpm_default_matcher
Definition: util-mpm.h:166
uint16_t maxlen
Definition: util-mpm.h:96
void MpmInitCtx(MpmCtx *mpm_ctx, uint16_t matcher)
Definition: util-mpm.c:261
#define DETECT_CONTENT_NEGATED
MpmTableElmt mpm_table[MPM_TABLE_SIZE]
Definition: util-mpm.h:165
const DetectMpmAppLayerRegistery * reg
Definition: detect.h:596
#define SCFree(a)
Definition: util-mem.h:236
#define DETECT_CONTENT_REPLACE
void DetectBufferTypeSupportsTransformations(const char *name)
void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx, const int id, const int parent_id, DetectEngineTransforms *transforms)
copy a mpm engine from parent_id, add in transforms
MpmCtx * mpm_ctx
Definition: detect.h:1217
int(* PrefilterRegisterWithListId)(struct DetectEngineCtx_ *de_ctx, struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx, const struct DetectMpmAppLayerRegistery_ *mpm_reg, int list_id)
Definition: detect.h:579
int FastPatternSupportEnabledForSigMatchList(const DetectEngineCtx *de_ctx, const int list_id)
Checks if a particular list(Signature->sm_lists[]) is in the list of lists that need to be searched f...
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
initialize mpm contexts for builtin buffers that are in "single or "shared" mode. ...
DetectEngineTransforms transforms
Definition: detect.h:585
void SupportFastPatternForSigMatchList(int list_id, int priority)
Lets one add a sm list id to be searched for potential fp supported keywords later.
int MpmStoreInit(DetectEngineCtx *de_ctx)
Initializes the MpmStore mpm hash table to be used by the detection engine context.
void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
#define SCLogPerf(...)
Definition: util-debug.h:261
MpmCtx * MpmFactoryGetMpmCtxForProfile(const DetectEngineCtx *de_ctx, int32_t id, int direction)
Definition: util-mpm.c:192
#define FatalError(x,...)
Definition: util-debug.h:539
struct DetectMpmAppLayerRegistery_::@101 v2
int PrefilterPktPayloadRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx)
uint32_t array_size
Definition: util-hashlist.h:41
int SignatureHasStreamContent(const Signature *s)
check if a signature has patterns that are to be inspected against the stream payload (as opposed to ...
uint8_t sgh_mpm_context
Definition: detect.h:782
int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags)
Definition: util-mpm.c:319
void HashListTableFree(HashListTable *ht)
Definition: util-hashlist.c:82
#define DETECT_CONTENT_FAST_PATTERN_CHOP
#define DETECT_CONTENT_NOCASE
enum MpmBuiltinBuffers buffer
Definition: detect.h:1213
void PatternMatchThreadPrint(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
uint32_t app_mpms_list_cnt
Definition: detect.h:887
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:555
InspectionBuffer *(* InspectionBufferGetDataPtr)(struct DetectEngineThreadCtx_ *det_ctx, const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv, const int list_id)
Definition: detect.h:380
#define SGH_DIRECTION_TC(sgh)
SigMatch * mpm_sm
Definition: detect.h:463
int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
Prepare the pattern matcher ctx in a sig group head.
MpmStore * MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh, enum MpmBuiltinBuffers buf)
Get MpmStore for a built-in buffer type.
a single match condition for a signature
Definition: detect.h:324
void PmqReset(PrefilterRuleStore *)
Reset a Pmq for reusage. Meant to be called after a single search.
void DetectAppLayerMpmRegister2(const char *name, int direction, int priority, int(*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectMpmAppLayerRegistery *mpm_reg, int list_id), InspectionBufferGetDataPtr GetData, AppProto alproto, int tx_min_progress)
register a MPM engine
SigGroupHeadInitData * init
Definition: detect.h:1323