suricata
detect-engine.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2010 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "debug.h"
27 #include "detect.h"
28 #include "flow.h"
29 #include "flow-private.h"
30 #include "flow-util.h"
31 #include "flow-worker.h"
32 #include "conf.h"
33 #include "conf-yaml-loader.h"
34 
35 #include "app-layer-parser.h"
36 #include "app-layer-htp.h"
37 
38 #include "detect-parse.h"
39 #include "detect-engine-sigorder.h"
40 
41 #include "detect-engine-siggroup.h"
42 #include "detect-engine-address.h"
43 #include "detect-engine-port.h"
45 #include "detect-engine-mpm.h"
46 #include "detect-engine-iponly.h"
47 #include "detect-engine-tag.h"
48 
49 #include "detect-engine-file.h"
50 
51 #include "detect-engine.h"
52 #include "detect-engine-state.h"
53 #include "detect-engine-payload.h"
54 #include "detect-byte-extract.h"
55 #include "detect-content.h"
56 #include "detect-uricontent.h"
57 #include "detect-tcphdr.h"
60 
61 #include "detect-engine-loader.h"
62 
64 #include "util-reference-config.h"
65 #include "util-threshold-config.h"
66 #include "util-error.h"
67 #include "util-hash.h"
68 #include "util-byte.h"
69 #include "util-debug.h"
70 #include "util-unittest.h"
71 #include "util-action.h"
72 #include "util-magic.h"
73 #include "util-signal.h"
74 #include "util-spm.h"
75 #include "util-device.h"
76 #include "util-var-name.h"
77 #include "util-profiling.h"
78 
79 #include "tm-threads.h"
80 #include "runmodes.h"
81 
82 #include "reputation.h"
83 
84 #define DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT 3000
85 
86 static DetectEngineThreadCtx *DetectEngineThreadCtxInitForReload(
87  ThreadVars *tv, DetectEngineCtx *new_de_ctx, int mt);
88 
89 static int DetectEngineCtxLoadConf(DetectEngineCtx *);
90 
91 static DetectEngineMasterCtx g_master_de_ctx = { SCMUTEX_INITIALIZER,
92  0, 99, NULL, NULL, TENANT_SELECTOR_UNKNOWN, NULL, NULL, 0};
93 
94 static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len);
95 static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len);
96 static void TenantIdFree(void *d);
97 static uint32_t DetectEngineTentantGetIdFromLivedev(const void *ctx, const Packet *p);
98 static uint32_t DetectEngineTentantGetIdFromVlanId(const void *ctx, const Packet *p);
99 static uint32_t DetectEngineTentantGetIdFromPcap(const void *ctx, const Packet *p);
100 
101 static DetectEngineAppInspectionEngine *g_app_inspect_engines = NULL;
102 static DetectEnginePktInspectionEngine *g_pkt_inspect_engines = NULL;
103 
105 #ifdef UNITTESTS
106  { "TEST", DET_CTX_EVENT_TEST },
107 #endif
108  { "NO_MEMORY", FILE_DECODER_EVENT_NO_MEM },
109  { "INVALID_SWF_LENGTH", FILE_DECODER_EVENT_INVALID_SWF_LENGTH },
110  { "INVALID_SWF_VERSION", FILE_DECODER_EVENT_INVALID_SWF_VERSION },
111  { "Z_DATA_ERROR", FILE_DECODER_EVENT_Z_DATA_ERROR },
112  { "Z_STREAM_ERROR", FILE_DECODER_EVENT_Z_STREAM_ERROR },
113  { "Z_BUF_ERROR", FILE_DECODER_EVENT_Z_BUF_ERROR },
114  { "Z_UNKNOWN_ERROR", FILE_DECODER_EVENT_Z_UNKNOWN_ERROR },
115  { "LZMA_DECODER_ERROR", FILE_DECODER_EVENT_LZMA_DECODER_ERROR },
116  { "LZMA_MEMLIMIT_ERROR", FILE_DECODER_EVENT_LZMA_MEMLIMIT_ERROR },
117  { "LZMA_OPTIONS_ERROR", FILE_DECODER_EVENT_LZMA_OPTIONS_ERROR },
118  { "LZMA_FORMAT_ERROR", FILE_DECODER_EVENT_LZMA_FORMAT_ERROR },
119  { "LZMA_DATA_ERROR", FILE_DECODER_EVENT_LZMA_DATA_ERROR },
120  { "LZMA_BUF_ERROR", FILE_DECODER_EVENT_LZMA_BUF_ERROR },
121  { "LZMA_UNKNOWN_ERROR", FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR },
122  { NULL, -1 },
123 };
124 
125 /** \brief register inspect engine at start up time
126  *
127  * \note errors are fatal */
128 void DetectPktInspectEngineRegister(const char *name,
131 {
133  const int sm_list = DetectBufferTypeGetByName(name);
134  if (sm_list == -1) {
136  "failed to register inspect engine %s", name);
137  }
138 
139  if ((sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
140  (Callback == NULL))
141  {
142  SCLogError(SC_ERR_INVALID_ARGUMENTS, "Invalid arguments");
143  BUG_ON(1);
144  }
145 
146  DetectEnginePktInspectionEngine *new_engine = SCCalloc(1, sizeof(*new_engine));
147  if (unlikely(new_engine == NULL)) {
149  "failed to register inspect engine %s: %s", name, strerror(errno));
150  }
151  new_engine->sm_list = sm_list;
152  new_engine->v1.Callback = Callback;
153  new_engine->v1.GetData = GetPktData;
154 
155  if (g_pkt_inspect_engines == NULL) {
156  g_pkt_inspect_engines = new_engine;
157  } else {
158  DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
159  while (t->next != NULL) {
160  t = t->next;
161  }
162 
163  t->next = new_engine;
164  }
165 }
166 
167 /** \brief register inspect engine at start up time
168  *
169  * \note errors are fatal */
171  AppProto alproto, uint32_t dir,
172  int progress, InspectEngineFuncPtr Callback)
173 {
175  const int sm_list = DetectBufferTypeGetByName(name);
176  if (sm_list == -1) {
178  "failed to register inspect engine %s", name);
179  }
180 
181  if ((alproto >= ALPROTO_FAILED) ||
182  (!(dir == SIG_FLAG_TOSERVER || dir == SIG_FLAG_TOCLIENT)) ||
183  (sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
184  (progress < 0 || progress >= SHRT_MAX) ||
185  (Callback == NULL))
186  {
187  SCLogError(SC_ERR_INVALID_ARGUMENTS, "Invalid arguments");
188  BUG_ON(1);
189  }
190 
191  int direction;
192  if (dir == SIG_FLAG_TOSERVER) {
193  direction = 0;
194  } else {
195  direction = 1;
196  }
197 
199  if (unlikely(new_engine == NULL)) {
200  exit(EXIT_FAILURE);
201  }
202  memset(new_engine, 0, sizeof(*new_engine));
203  new_engine->alproto = alproto;
204  new_engine->dir = direction;
205  new_engine->sm_list = sm_list;
206  new_engine->progress = progress;
207  new_engine->Callback = Callback;
208 
209  if (g_app_inspect_engines == NULL) {
210  g_app_inspect_engines = new_engine;
211  } else {
212  DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
213  while (t->next != NULL) {
214  t = t->next;
215  }
216 
217  t->next = new_engine;
218  }
219 }
220 
221 /** \brief register inspect engine at start up time
222  *
223  * \note errors are fatal */
225  AppProto alproto, uint32_t dir, int progress,
226  InspectEngineFuncPtr2 Callback2,
228 {
230  const int sm_list = DetectBufferTypeGetByName(name);
231  if (sm_list == -1) {
233  "failed to register inspect engine %s", name);
234  }
235 
236  if ((alproto >= ALPROTO_FAILED) ||
237  (!(dir == SIG_FLAG_TOSERVER || dir == SIG_FLAG_TOCLIENT)) ||
238  (sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
239  (progress < 0 || progress >= SHRT_MAX) ||
240  (Callback2 == NULL))
241  {
242  SCLogError(SC_ERR_INVALID_ARGUMENTS, "Invalid arguments");
243  BUG_ON(1);
244  } else if (Callback2 == DetectEngineInspectBufferGeneric && GetData == NULL) {
245  SCLogError(SC_ERR_INVALID_ARGUMENTS, "Invalid arguments: must register "
246  "GetData with DetectEngineInspectBufferGeneric");
247  BUG_ON(1);
248  }
249 
250  int direction;
251  if (dir == SIG_FLAG_TOSERVER) {
252  direction = 0;
253  } else {
254  direction = 1;
255  }
256 
258  if (unlikely(new_engine == NULL)) {
259  exit(EXIT_FAILURE);
260  }
261  memset(new_engine, 0, sizeof(*new_engine));
262  new_engine->alproto = alproto;
263  new_engine->dir = direction;
264  new_engine->sm_list = sm_list;
265  new_engine->progress = progress;
266  new_engine->v2.Callback = Callback2;
267  new_engine->v2.GetData = GetData;
268 
269  if (g_app_inspect_engines == NULL) {
270  g_app_inspect_engines = new_engine;
271  } else {
272  DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
273  while (t->next != NULL) {
274  t = t->next;
275  }
276 
277  t->next = new_engine;
278  }
279 }
280 
281 /* copy an inspect engine with transforms to a new list id. */
282 static void DetectAppLayerInspectEngineCopy(
283  DetectEngineCtx *de_ctx,
284  int sm_list, int new_list,
285  const DetectEngineTransforms *transforms)
286 {
287  const DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
288  while (t) {
289  if (t->sm_list == sm_list) {
291  if (unlikely(new_engine == NULL)) {
292  exit(EXIT_FAILURE);
293  }
294  new_engine->alproto = t->alproto;
295  new_engine->dir = t->dir;
296  new_engine->sm_list = new_list; /* use new list id */
297  new_engine->progress = t->progress;
298  new_engine->Callback = t->Callback;
299  new_engine->v2 = t->v2;
300  new_engine->v2.transforms = transforms; /* assign transforms */
301 
302  if (de_ctx->app_inspect_engines == NULL) {
303  de_ctx->app_inspect_engines = new_engine;
304  } else {
306  while (list->next != NULL) {
307  list = list->next;
308  }
309 
310  list->next = new_engine;
311  }
312  }
313  t = t->next;
314  }
315 }
316 
317 /* copy inspect engines from global registrations to de_ctx list */
318 static void DetectAppLayerInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
319 {
320  const DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
321  while (t) {
323  if (unlikely(new_engine == NULL)) {
324  exit(EXIT_FAILURE);
325  }
326  new_engine->alproto = t->alproto;
327  new_engine->dir = t->dir;
328  new_engine->sm_list = t->sm_list;
329  new_engine->progress = t->progress;
330  new_engine->Callback = t->Callback;
331  new_engine->v2 = t->v2;
332 
333  if (de_ctx->app_inspect_engines == NULL) {
334  de_ctx->app_inspect_engines = new_engine;
335  } else {
337  while (list->next != NULL) {
338  list = list->next;
339  }
340 
341  list->next = new_engine;
342  }
343 
344  t = t->next;
345  }
346 }
347 
348 /* copy an inspect engine with transforms to a new list id. */
349 static void DetectPktInspectEngineCopy(
350  DetectEngineCtx *de_ctx,
351  int sm_list, int new_list,
352  const DetectEngineTransforms *transforms)
353 {
354  const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
355  while (t) {
356  if (t->sm_list == sm_list) {
358  if (unlikely(new_engine == NULL)) {
359  exit(EXIT_FAILURE);
360  }
361  new_engine->sm_list = new_list; /* use new list id */
362  new_engine->v1 = t->v1;
363  new_engine->v1.transforms = transforms; /* assign transforms */
364 
365  if (de_ctx->pkt_inspect_engines == NULL) {
366  de_ctx->pkt_inspect_engines = new_engine;
367  } else {
369  while (list->next != NULL) {
370  list = list->next;
371  }
372 
373  list->next = new_engine;
374  }
375  }
376  t = t->next;
377  }
378 }
379 
380 /* copy inspect engines from global registrations to de_ctx list */
381 static void DetectPktInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
382 {
383  const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
384  while (t) {
385  SCLogDebug("engine %p", t);
387  if (unlikely(new_engine == NULL)) {
388  exit(EXIT_FAILURE);
389  }
390  new_engine->sm_list = t->sm_list;
391  new_engine->v1 = t->v1;
392 
393  if (de_ctx->pkt_inspect_engines == NULL) {
394  de_ctx->pkt_inspect_engines = new_engine;
395  } else {
397  while (list->next != NULL) {
398  list = list->next;
399  }
400 
401  list->next = new_engine;
402  }
403 
404  t = t->next;
405  }
406 }
407 
408 /** \internal
409  * \brief append the stream inspection
410  *
411  * If stream inspection is MPM, then prepend it.
412  */
413 static void AppendStreamInspectEngine(Signature *s, SigMatchData *stream, int direction, uint32_t id)
414 {
415  bool prepend = false;
416 
418  if (unlikely(new_engine == NULL)) {
419  exit(EXIT_FAILURE);
420  }
422  SCLogDebug("stream is mpm");
423  prepend = true;
424  new_engine->mpm = true;
425  }
426  new_engine->alproto = ALPROTO_UNKNOWN; /* all */
427  new_engine->dir = direction;
428  new_engine->stream = true;
429  new_engine->sm_list = DETECT_SM_LIST_PMATCH;
430  new_engine->smd = stream;
431  new_engine->Callback = DetectEngineInspectStream;
432  new_engine->progress = 0;
433 
434  /* append */
435  if (s->app_inspect == NULL) {
436  s->app_inspect = new_engine;
437  new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
438  } else if (prepend) {
439  new_engine->next = s->app_inspect;
440  s->app_inspect = new_engine;
441  new_engine->id = id;
442 
443  } else {
445  while (a->next != NULL) {
446  a = a->next;
447  }
448 
449  a->next = new_engine;
450  new_engine->id = id;
451  }
452  SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
453 }
454 
455 /**
456  * \note for the file inspect engine, the id DE_STATE_ID_FILE_INSPECT
457  * is assigned.
458  */
460 {
461  const int nlists = s->init_data->smlists_array_size;
462  SigMatchData *ptrs[nlists];
463  memset(&ptrs, 0, (nlists * sizeof(SigMatchData *)));
464 
465  const int mpm_list = s->init_data->mpm_sm ?
467  -1;
468 
469  const int files_id = DetectBufferTypeGetByName("files");
470 
471  /* convert lists to SigMatchData arrays */
472  int i = 0;
473  for (i = DETECT_SM_LIST_DYNAMIC_START; i < nlists; i++) {
474  if (s->init_data->smlists[i] == NULL)
475  continue;
476 
477  ptrs[i] = SigMatchList2DataArray(s->init_data->smlists[i]);
478  SCLogDebug("ptrs[%d] is set", i);
479  }
480 
481  /* set up pkt inspect engines */
483  while (e != NULL) {
484  SCLogDebug("e %p sm_list %u nlists %u ptrs[] %p", e, e->sm_list, nlists, e->sm_list < nlists ? ptrs[e->sm_list] : NULL);
485  if (e->sm_list < nlists && ptrs[e->sm_list] != NULL) {
486  bool prepend = false;
487 
489  if (unlikely(new_engine == NULL)) {
490  exit(EXIT_FAILURE);
491  }
492  if (mpm_list == e->sm_list) {
493  SCLogDebug("%s is mpm", DetectBufferTypeGetNameById(de_ctx, e->sm_list));
494  prepend = true;
495  new_engine->mpm = true;
496  }
497 
498  new_engine->sm_list = e->sm_list;
499  new_engine->smd = ptrs[new_engine->sm_list];
500  new_engine->v1 = e->v1;
501  SCLogDebug("sm_list %d new_engine->v1 %p/%p/%p",
502  new_engine->sm_list, new_engine->v1.Callback,
503  new_engine->v1.GetData, new_engine->v1.transforms);
504 
505  if (s->pkt_inspect == NULL) {
506  s->pkt_inspect = new_engine;
507  } else if (prepend) {
508  new_engine->next = s->pkt_inspect;
509  s->pkt_inspect = new_engine;
510  } else {
512  while (a->next != NULL) {
513  a = a->next;
514  }
515  new_engine->next = a->next;
516  a->next = new_engine;
517  }
518  }
519  e = e->next;
520  }
521 
522  bool head_is_mpm = false;
523  uint32_t last_id = DE_STATE_FLAG_BASE;
525  while (t != NULL) {
526  bool prepend = false;
527 
528  if (t->sm_list >= nlists)
529  goto next;
530 
531  if (ptrs[t->sm_list] == NULL)
532  goto next;
533 
534  SCLogDebug("ptrs[%d] is set", t->sm_list);
535 
536  if (t->alproto == ALPROTO_UNKNOWN) {
537  /* special case, inspect engine applies to all protocols */
538  } else if (s->alproto != ALPROTO_UNKNOWN && s->alproto != t->alproto)
539  goto next;
540 
541  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
542  if (t->dir == 1)
543  goto next;
544  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
545  if (t->dir == 0)
546  goto next;
547  }
549  if (unlikely(new_engine == NULL)) {
550  exit(EXIT_FAILURE);
551  }
552  if (mpm_list == t->sm_list) {
553  SCLogDebug("%s is mpm", DetectBufferTypeGetNameById(de_ctx, t->sm_list));
554  prepend = true;
555  head_is_mpm = true;
556  new_engine->mpm = true;
557  }
558 
559  new_engine->alproto = t->alproto;
560  new_engine->dir = t->dir;
561  new_engine->sm_list = t->sm_list;
562  new_engine->smd = ptrs[new_engine->sm_list];
563  new_engine->Callback = t->Callback;
564  new_engine->progress = t->progress;
565  new_engine->v2 = t->v2;
566  SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p",
567  new_engine->sm_list, new_engine->v2.Callback,
568  new_engine->v2.GetData, new_engine->v2.transforms);
569 
570  if (s->app_inspect == NULL) {
571  s->app_inspect = new_engine;
572  if (new_engine->sm_list == files_id) {
573  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
574  new_engine->id = DE_STATE_ID_FILE_INSPECT;
575  } else {
576  new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
577  }
578 
579  /* prepend engine if forced or if our engine has a lower progress. */
580  } else if (prepend || (!head_is_mpm && s->app_inspect->progress > new_engine->progress)) {
581  new_engine->next = s->app_inspect;
582  s->app_inspect = new_engine;
583  if (new_engine->sm_list == files_id) {
584  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
585  new_engine->id = DE_STATE_ID_FILE_INSPECT;
586  } else {
587  new_engine->id = ++last_id;
588  }
589 
590  } else {
592  while (a->next != NULL) {
593  if (a->next && a->next->progress > new_engine->progress) {
594  break;
595  }
596 
597  a = a->next;
598  }
599 
600  new_engine->next = a->next;
601  a->next = new_engine;
602  if (new_engine->sm_list == files_id) {
603  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
604  new_engine->id = DE_STATE_ID_FILE_INSPECT;
605  } else {
606  new_engine->id = ++last_id;
607  }
608  }
609 
610  SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
611 
613 next:
614  t = t->next;
615  }
616 
619  {
620  /* if engine is added multiple times, we pass it the same list */
622  BUG_ON(stream == NULL);
623  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
624  AppendStreamInspectEngine(s, stream, 0, last_id + 1);
625  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
626  AppendStreamInspectEngine(s, stream, 1, last_id + 1);
627  } else {
628  AppendStreamInspectEngine(s, stream, 0, last_id + 1);
629  AppendStreamInspectEngine(s, stream, 1, last_id + 1);
630  }
631 
633  SCLogDebug("set SIG_FLAG_FLUSH on %u", s->id);
634  s->flags |= SIG_FLAG_FLUSH;
635  }
636  }
637 
638 #ifdef DEBUG
640  while (iter) {
641  SCLogDebug("%u: engine %s id %u progress %d %s", s->id,
642  DetectBufferTypeGetNameById(de_ctx, iter->sm_list), iter->id,
643  iter->progress,
644  iter->sm_list == mpm_list ? "MPM":"");
645  iter = iter->next;
646  }
647 #endif
648  return 0;
649 }
650 
651 /** \brief free app inspect engines for a signature
652  *
653  * For lists that are registered multiple times, like http_header and
654  * http_cookie, making the engines owner of the lists is complicated.
655  * Multiple engines in a sig may be pointing to the same list. To
656  * address this the 'free' code needs to be extra careful about not
657  * double freeing, so it takes an approach to first fill an array
658  * of the to-free pointers before freeing them.
659  */
661 {
662  int nlists = 0;
663 
665  while (ie) {
666  nlists = MAX(ie->sm_list + 1, nlists);
667  ie = ie->next;
668  }
670  while (e) {
671  nlists = MAX(e->sm_list + 1, nlists);
672  e = e->next;
673  }
674  if (nlists == 0) {
675  BUG_ON(s->pkt_inspect);
676  return;
677  }
678 
679  SigMatchData *ptrs[nlists];
680  memset(&ptrs, 0, (nlists * sizeof(SigMatchData *)));
681 
682  /* free engines and put smd in the array */
683  ie = s->app_inspect;
684  while (ie) {
686  BUG_ON(ptrs[ie->sm_list] != NULL && ptrs[ie->sm_list] != ie->smd);
687  ptrs[ie->sm_list] = ie->smd;
688  SCFree(ie);
689  ie = next;
690  }
691  e = s->pkt_inspect;
692  while (e) {
694  ptrs[e->sm_list] = e->smd;
695  SCFree(e);
696  e = next;
697  }
698 
699  /* free the smds */
700  for (int i = 0; i < nlists; i++)
701  {
702  if (ptrs[i] == NULL)
703  continue;
704 
705  SigMatchData *smd = ptrs[i];
706  while(1) {
707  if (sigmatch_table[smd->type].Free != NULL) {
708  sigmatch_table[smd->type].Free(smd->ctx);
709  }
710  if (smd->is_last)
711  break;
712  smd++;
713  }
714  SCFree(ptrs[i]);
715  }
716 }
717 
718 /* code for registering buffers */
719 
720 #include "util-hash-lookup3.h"
721 
722 static HashListTable *g_buffer_type_hash = NULL;
723 static int g_buffer_type_id = DETECT_SM_LIST_DYNAMIC_START;
724 static int g_buffer_type_reg_closed = 0;
725 
726 static DetectEngineTransforms no_transforms = { .transforms = { 0 }, .cnt = 0, };
727 
729 {
730  return g_buffer_type_id;
731 }
732 
733 static uint32_t DetectBufferTypeHashFunc(HashListTable *ht, void *data, uint16_t datalen)
734 {
735  const DetectBufferType *map = (DetectBufferType *)data;
736  uint32_t hash = 0;
737 
738  hash = hashlittle_safe(map->string, strlen(map->string), 0);
739  hash += hashlittle_safe((uint8_t *)&map->transforms, sizeof(map->transforms), 0);
740  hash %= ht->array_size;
741 
742  return hash;
743 }
744 
745 static char DetectBufferTypeCompareFunc(void *data1, uint16_t len1, void *data2,
746  uint16_t len2)
747 {
748  DetectBufferType *map1 = (DetectBufferType *)data1;
749  DetectBufferType *map2 = (DetectBufferType *)data2;
750 
751  int r = (strcmp(map1->string, map2->string) == 0);
752  r &= (memcmp((uint8_t *)&map1->transforms, (uint8_t *)&map2->transforms, sizeof(map2->transforms)) == 0);
753  return r;
754 }
755 
756 static void DetectBufferTypeFreeFunc(void *data)
757 {
758  DetectBufferType *map = (DetectBufferType *)data;
759  if (map != NULL) {
760  SCFree(map);
761  }
762 }
763 
764 static int DetectBufferTypeInit(void)
765 {
766  BUG_ON(g_buffer_type_hash);
767  g_buffer_type_hash = HashListTableInit(256,
768  DetectBufferTypeHashFunc,
769  DetectBufferTypeCompareFunc,
770  DetectBufferTypeFreeFunc);
771  if (g_buffer_type_hash == NULL)
772  return -1;
773 
774  return 0;
775 }
776 #if 0
777 static void DetectBufferTypeFree(void)
778 {
779  if (g_buffer_type_hash == NULL)
780  return;
781 
782  HashListTableFree(g_buffer_type_hash);
783  g_buffer_type_hash = NULL;
784  return;
785 }
786 #endif
787 static int DetectBufferTypeAdd(const char *string)
788 {
789  DetectBufferType *map = SCCalloc(1, sizeof(*map));
790  if (map == NULL)
791  return -1;
792 
793  map->string = string;
794  map->id = g_buffer_type_id++;
795 
796  BUG_ON(HashListTableAdd(g_buffer_type_hash, (void *)map, 0) != 0);
797  SCLogDebug("buffer %s registered with id %d", map->string, map->id);
798  return map->id;
799 }
800 
801 static DetectBufferType *DetectBufferTypeLookupByName(const char *string)
802 {
803  DetectBufferType map = { (char *)string, NULL, 0, 0, 0, 0, false, NULL, NULL, no_transforms };
804 
805  DetectBufferType *res = HashListTableLookup(g_buffer_type_hash, &map, 0);
806  return res;
807 }
808 
809 int DetectBufferTypeRegister(const char *name)
810 {
811  BUG_ON(g_buffer_type_reg_closed);
812  if (g_buffer_type_hash == NULL)
813  DetectBufferTypeInit();
814 
815  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
816  if (!exists) {
817  return DetectBufferTypeAdd(name);
818  } else {
819  return exists->id;
820  }
821 }
822 
823 void DetectBufferTypeSupportsPacket(const char *name)
824 {
825  BUG_ON(g_buffer_type_reg_closed);
827  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
828  BUG_ON(!exists);
829  exists->packet = TRUE;
830  SCLogDebug("%p %s -- %d supports packet inspection", exists, name, exists->id);
831 }
832 
833 void DetectBufferTypeSupportsMpm(const char *name)
834 {
835  BUG_ON(g_buffer_type_reg_closed);
837  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
838  BUG_ON(!exists);
839  exists->mpm = TRUE;
840  SCLogDebug("%p %s -- %d supports mpm", exists, name, exists->id);
841 }
842 
844 {
845  BUG_ON(g_buffer_type_reg_closed);
847  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
848  BUG_ON(!exists);
849  exists->supports_transforms = true;
850  SCLogDebug("%p %s -- %d supports transformations", exists, name, exists->id);
851 }
852 
853 int DetectBufferTypeGetByName(const char *name)
854 {
855  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
856  if (!exists) {
857  return -1;
858  }
859  return exists->id;
860 }
861 
862 const char *DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
863 {
864  BUG_ON(id < 0 || (uint32_t)id >= de_ctx->buffer_type_map_elements);
865  BUG_ON(de_ctx->buffer_type_map == NULL);
866 
867  if (de_ctx->buffer_type_map[id] == NULL)
868  return NULL;
869 
870  return de_ctx->buffer_type_map[id]->string;
871 }
872 
873 static const DetectBufferType *DetectBufferTypeGetById(const DetectEngineCtx *de_ctx, const int id)
874 {
875  BUG_ON(id < 0 || (uint32_t)id >= de_ctx->buffer_type_map_elements);
876  BUG_ON(de_ctx->buffer_type_map == NULL);
877 
878  return de_ctx->buffer_type_map[id];
879 }
880 
881 void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc)
882 {
883  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
884  if (!exists) {
885  return;
886  }
887  exists->description = desc;
888 }
889 
890 const char *DetectBufferTypeGetDescriptionById(const DetectEngineCtx *de_ctx, const int id)
891 {
892  const DetectBufferType *exists = DetectBufferTypeGetById(de_ctx, id);
893  if (!exists) {
894  return NULL;
895  }
896  return exists->description;
897 }
898 
899 const char *DetectBufferTypeGetDescriptionByName(const char *name)
900 {
901  const DetectBufferType *exists = DetectBufferTypeLookupByName(name);
902  if (!exists) {
903  return NULL;
904  }
905  return exists->description;
906 }
907 
909 {
910  const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
911  if (map == NULL)
912  return FALSE;
913  SCLogDebug("map %p id %d packet? %d", map, id, map->packet);
914  return map->packet;
915 }
916 
917 bool DetectBufferTypeSupportsMpmGetById(const DetectEngineCtx *de_ctx, const int id)
918 {
919  const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
920  if (map == NULL)
921  return FALSE;
922  SCLogDebug("map %p id %d mpm? %d", map, id, map->mpm);
923  return map->mpm;
924 }
925 
927  void (*SetupCallback)(const DetectEngineCtx *, Signature *))
928 {
929  BUG_ON(g_buffer_type_reg_closed);
931  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
932  BUG_ON(!exists);
933  exists->SetupCallback = SetupCallback;
934 }
935 
937  const int id, Signature *s)
938 {
939  const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
940  if (map && map->SetupCallback) {
941  map->SetupCallback(de_ctx, s);
942  }
943 }
944 
946  _Bool (*ValidateCallback)(const Signature *, const char **sigerror))
947 {
948  BUG_ON(g_buffer_type_reg_closed);
950  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
951  BUG_ON(!exists);
952  exists->ValidateCallback = ValidateCallback;
953 }
954 
956  const int id, const Signature *s, const char **sigerror)
957 {
958  const DetectBufferType *map = DetectBufferTypeGetById(de_ctx, id);
959  if (map && map->ValidateCallback) {
960  return map->ValidateCallback(s, sigerror);
961  }
962  return TRUE;
963 }
964 
965 int DetectBufferSetActiveList(Signature *s, const int list)
966 {
967  BUG_ON(s->init_data == NULL);
968 
969  if (s->init_data->list && s->init_data->transform_cnt) {
970  return -1;
971  }
972  s->init_data->list = list;
973  s->init_data->list_set = true;
974 
975  return 0;
976 }
977 
979 {
980  BUG_ON(s->init_data == NULL);
981 
982  if (s->init_data->list && s->init_data->transform_cnt) {
983  SCLogDebug("buffer %d has transform(s) registered: %d",
984  s->init_data->list, s->init_data->transforms[0]);
985  int new_list = DetectBufferTypeGetByIdTransforms(de_ctx, s->init_data->list,
987  if (new_list == -1) {
988  return -1;
989  }
990  SCLogDebug("new_list %d", new_list);
991  s->init_data->list = new_list;
992  s->init_data->list_set = false;
993  // reset transforms now that we've set up the list
994  s->init_data->transform_cnt = 0;
995  }
996 
997  return 0;
998 }
999 
1001 {
1002  /* single buffers */
1003  for (uint32_t i = 0; i < det_ctx->inspect.to_clear_idx; i++)
1004  {
1005  const uint32_t idx = det_ctx->inspect.to_clear_queue[i];
1006  InspectionBuffer *buffer = &det_ctx->inspect.buffers[idx];
1007  buffer->inspect = NULL;
1008  }
1009  det_ctx->inspect.to_clear_idx = 0;
1010 
1011  /* multi buffers */
1012  for (uint32_t i = 0; i < det_ctx->multi_inspect.to_clear_idx; i++)
1013  {
1014  const uint32_t idx = det_ctx->multi_inspect.to_clear_queue[i];
1015  InspectionBufferMultipleForList *mbuffer = &det_ctx->multi_inspect.buffers[idx];
1016  for (uint32_t x = 0; x <= mbuffer->max; x++) {
1017  InspectionBuffer *buffer = &mbuffer->inspection_buffers[x];
1018  buffer->inspect = NULL;
1019  }
1020  mbuffer->init = 0;
1021  mbuffer->max = 0;
1022  }
1023  det_ctx->multi_inspect.to_clear_idx = 0;
1024 }
1025 
1027 {
1028  InspectionBuffer *buffer = &det_ctx->inspect.buffers[list_id];
1029  if (buffer->inspect == NULL) {
1030  det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id;
1031  }
1032  return buffer;
1033 }
1034 
1035 /** \brief for a InspectionBufferMultipleForList get a InspectionBuffer
1036  * \param fb the multiple buffer array
1037  * \param local_id the index to get a buffer
1038  * \param buffer the inspect buffer or NULL in case of error */
1040 {
1041  if (local_id >= fb->size) {
1042  uint32_t old_size = fb->size;
1043  uint32_t new_size = local_id + 1;
1044  uint32_t grow_by = new_size - old_size;
1045  SCLogDebug("size is %u, need %u, so growing by %u", old_size, new_size, grow_by);
1046 
1047  SCLogDebug("fb->inspection_buffers %p", fb->inspection_buffers);
1048  void *ptr = SCRealloc(fb->inspection_buffers, (local_id + 1) * sizeof(InspectionBuffer));
1049  if (ptr == NULL)
1050  return NULL;
1051 
1052  InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
1053  SCLogDebug("ptr %p to_zero %p", ptr, to_zero);
1054  memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
1055  fb->inspection_buffers = ptr;
1056  fb->size = new_size;
1057  }
1058 
1059  fb->max = MAX(fb->max, local_id);
1060  InspectionBuffer *buffer = &fb->inspection_buffers[local_id];
1061  SCLogDebug("using file_data buffer %p", buffer);
1062  return buffer;
1063 }
1064 
1066 {
1067  InspectionBufferMultipleForList *buffer = &det_ctx->multi_inspect.buffers[list_id];
1068  if (!buffer->init) {
1069  det_ctx->multi_inspect.to_clear_queue[det_ctx->multi_inspect.to_clear_idx++] = list_id;
1070  buffer->init = 1;
1071  }
1072  return buffer;
1073 }
1074 
1075 void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
1076 {
1077  memset(buffer, 0, sizeof(*buffer));
1078  buffer->buf = SCCalloc(initial_size, sizeof(uint8_t));
1079  if (buffer->buf != NULL) {
1080  buffer->size = initial_size;
1081  }
1082 }
1083 
1084 /** \brief setup the buffer with our initial data */
1085 void InspectionBufferSetup(InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
1086 {
1087  buffer->inspect = buffer->orig = data;
1088  buffer->inspect_len = buffer->orig_len = data_len;
1089  buffer->len = 0;
1090 }
1091 
1093 {
1094  if (buffer->buf != NULL) {
1095  SCFree(buffer->buf);
1096  }
1097  memset(buffer, 0, sizeof(*buffer));
1098 }
1099 
1100 /**
1101  * \brief make sure that the buffer has at least 'min_size' bytes
1102  * Expand the buffer if necessary
1103  */
1104 void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
1105 {
1106  if (likely(buffer->size >= min_size))
1107  return;
1108 
1109  uint32_t new_size = (buffer->size == 0) ? 4096 : buffer->size;
1110  while (new_size < min_size) {
1111  new_size *= 2;
1112  }
1113 
1114  void *ptr = SCRealloc(buffer->buf, new_size);
1115  if (ptr != NULL) {
1116  buffer->buf = ptr;
1117  buffer->size = new_size;
1118  }
1119 }
1120 
1121 void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len)
1122 {
1123  InspectionBufferCheckAndExpand(buffer, buf_len);
1124 
1125  if (buffer->size) {
1126  uint32_t copy_size = MIN(buf_len, buffer->size);
1127  memcpy(buffer->buf, buf, copy_size);
1128  buffer->inspect = buffer->buf;
1129  buffer->inspect_len = copy_size;
1130  }
1131 }
1132 
1134  const DetectEngineTransforms *transforms)
1135 {
1136  if (transforms) {
1137  for (int i = 0; i < DETECT_TRANSFORMS_MAX; i++) {
1138  const int id = transforms->transforms[i];
1139  if (id == 0)
1140  break;
1141  BUG_ON(sigmatch_table[id].Transform == NULL);
1142  sigmatch_table[id].Transform(buffer);
1143  SCLogDebug("applied transform %s", sigmatch_table[id].name);
1144  }
1145  }
1146 }
1147 
1148 static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx)
1149 {
1150  const int size = g_buffer_type_id;
1151  BUG_ON(!(size > 0));
1152 
1153  de_ctx->buffer_type_map = SCCalloc(size, sizeof(DetectBufferType *));
1154  BUG_ON(!de_ctx->buffer_type_map);
1155  de_ctx->buffer_type_map_elements = size;
1156  SCLogDebug("de_ctx->buffer_type_map %p with %u members", de_ctx->buffer_type_map, size);
1157 
1158  SCLogDebug("DETECT_SM_LIST_DYNAMIC_START %u", DETECT_SM_LIST_DYNAMIC_START);
1159  HashListTableBucket *b = HashListTableGetListHead(g_buffer_type_hash);
1160  while (b) {
1162  de_ctx->buffer_type_map[map->id] = map;
1163  SCLogDebug("name %s id %d mpm %s packet %s -- %s. "
1164  "Callbacks: Setup %p Validate %p", map->string, map->id,
1165  map->mpm ? "true" : "false", map->packet ? "true" : "false",
1166  map->description, map->SetupCallback, map->ValidateCallback);
1167  b = HashListTableGetListNext(b);
1168  }
1169 
1170  de_ctx->buffer_type_hash = HashListTableInit(256,
1171  DetectBufferTypeHashFunc,
1172  DetectBufferTypeCompareFunc,
1173  DetectBufferTypeFreeFunc);
1174  if (de_ctx->buffer_type_hash == NULL) {
1175  BUG_ON(1);
1176  }
1177  de_ctx->buffer_type_id = g_buffer_type_id;
1178 
1179  PrefilterInit(de_ctx);
1181  DetectAppLayerInspectEngineCopyListToDetectCtx(de_ctx);
1183  DetectPktInspectEngineCopyListToDetectCtx(de_ctx);
1184 }
1185 
1186 static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
1187 {
1188  if (de_ctx) {
1189  if (de_ctx->buffer_type_map)
1190  SCFree(de_ctx->buffer_type_map);
1191  if (de_ctx->buffer_type_hash)
1193 
1195  while (ilist) {
1197  SCFree(ilist);
1198  ilist = next;
1199  }
1200  DetectBufferMpmRegistery *mlist = de_ctx->app_mpms_list;
1201  while (mlist) {
1203  SCFree(mlist);
1204  mlist = next;
1205  }
1207  while (plist) {
1209  SCFree(plist);
1210  plist = next;
1211  }
1212  DetectBufferMpmRegistery *pmlist = de_ctx->pkt_mpms_list;
1213  while (pmlist) {
1214  DetectBufferMpmRegistery *next = pmlist->next;
1215  SCFree(pmlist);
1216  pmlist = next;
1217  }
1218  PrefilterDeinit(de_ctx);
1219  }
1220 }
1221 
1223 {
1224  BUG_ON(g_buffer_type_hash == NULL);
1225 
1226  g_buffer_type_reg_closed = 1;
1227 }
1228 
1230  int *transforms, int transform_cnt)
1231 {
1232  const DetectBufferType *base_map = DetectBufferTypeGetById(de_ctx, id);
1233  if (!base_map) {
1234  return -1;
1235  }
1236  if (!base_map->supports_transforms) {
1237  SCLogError(SC_ERR_INVALID_SIGNATURE, "buffer '%s' does not support transformations",
1238  base_map->string);
1239  return -1;
1240  }
1241 
1242  SCLogDebug("base_map %s", base_map->string);
1243 
1245  memset(&t, 0, sizeof(t));
1246  for (int i = 0; i < transform_cnt; i++) {
1247  t.transforms[i] = transforms[i];
1248  }
1249  t.cnt = transform_cnt;
1250 
1251  DetectBufferType lookup_map = { (char *)base_map->string, NULL, 0, 0, 0, 0, false, NULL, NULL, t };
1252  DetectBufferType *res = HashListTableLookup(de_ctx->buffer_type_hash, &lookup_map, 0);
1253 
1254  SCLogDebug("res %p", res);
1255  if (res != NULL) {
1256  return res->id;
1257  }
1258 
1259  DetectBufferType *map = SCCalloc(1, sizeof(*map));
1260  if (map == NULL)
1261  return -1;
1262 
1263  map->string = base_map->string;
1264  map->id = de_ctx->buffer_type_id++;
1265  map->parent_id = base_map->id;
1266  map->transforms = t;
1267  map->mpm = base_map->mpm;
1268  map->packet = base_map->packet;
1269  map->SetupCallback = base_map->SetupCallback;
1270  map->ValidateCallback = base_map->ValidateCallback;
1271  if (map->packet) {
1273  map->id, map->parent_id, &map->transforms);
1274  } else {
1276  map->id, map->parent_id, &map->transforms);
1277  }
1278 
1279  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash, (void *)map, 0) != 0);
1280  SCLogDebug("buffer %s registered with id %d, parent %d", map->string, map->id, map->parent_id);
1281 
1282  if (map->id >= 0 && (uint32_t)map->id >= de_ctx->buffer_type_map_elements) {
1283  void *ptr = SCRealloc(de_ctx->buffer_type_map, (map->id + 1) * sizeof(DetectBufferType *));
1284  BUG_ON(ptr == NULL);
1285  SCLogDebug("de_ctx->buffer_type_map resized to %u (was %u)", (map->id + 1), de_ctx->buffer_type_map_elements);
1286  de_ctx->buffer_type_map = ptr;
1287  de_ctx->buffer_type_map[map->id] = map;
1288  de_ctx->buffer_type_map_elements = map->id + 1;
1289 
1290  if (map->packet) {
1291  DetectPktInspectEngineCopy(de_ctx, map->parent_id, map->id,
1292  &map->transforms);
1293  } else {
1294  DetectAppLayerInspectEngineCopy(de_ctx, map->parent_id, map->id,
1295  &map->transforms);
1296  }
1297  }
1298  return map->id;
1299 }
1300 
1301 /* returns false if no match, true if match */
1302 static int DetectEngineInspectRulePacketMatches(
1303  DetectEngineThreadCtx *det_ctx,
1304  const DetectEnginePktInspectionEngine *engine,
1305  const Signature *s,
1306  Packet *p, uint8_t *_alert_flags)
1307 {
1308  SCEnter();
1309 
1310  /* run the packet match functions */
1312  const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_MATCH];
1313 
1314  SCLogDebug("running match functions, sm %p", smd);
1315  while (1) {
1317  if (sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx) <= 0) {
1318  KEYWORD_PROFILING_END(det_ctx, smd->type, 0);
1319  SCLogDebug("no match");
1320  return false;
1321  }
1322  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
1323  if (smd->is_last) {
1324  SCLogDebug("match and is_last");
1325  break;
1326  }
1327  smd++;
1328  }
1329  return true;
1330 }
1331 
1332 static int DetectEngineInspectRulePayloadMatches(
1333  DetectEngineThreadCtx *det_ctx,
1334  const DetectEnginePktInspectionEngine *engine,
1335  const Signature *s, Packet *p, uint8_t *alert_flags)
1336 {
1337  SCEnter();
1338 
1339  DetectEngineCtx *de_ctx = det_ctx->de_ctx;
1340 
1342  /* if we have stream msgs, inspect against those first,
1343  * but not for a "dsize" signature */
1344  if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
1345  int pmatch = 0;
1346  if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
1347  pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, p->flow, p);
1348  if (pmatch) {
1350  /* Tell the engine that this reassembled stream can drop the
1351  * rest of the pkts with no further inspection */
1352  if (s->action & ACTION_DROP)
1353  *alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
1354 
1355  *alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
1356  }
1357  }
1358  /* no match? then inspect packet payload */
1359  if (pmatch == 0) {
1360  SCLogDebug("no match in stream, fall back to packet payload");
1361 
1362  /* skip if we don't have to inspect the packet and segment was
1363  * added to stream */
1364  if (!(s->flags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
1365  return false;
1366  }
1367  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
1368  return false;
1369  }
1370  }
1371  } else {
1372  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
1373  return false;
1374  }
1375  }
1376  return true;
1377 }
1378 
1380  DetectEngineThreadCtx *det_ctx, const Signature *s,
1381  Flow *f, Packet *p,
1382  uint8_t *alert_flags)
1383 {
1384  SCEnter();
1385 
1386  for (DetectEnginePktInspectionEngine *e = s->pkt_inspect; e != NULL; e = e->next) {
1387  if (e->v1.Callback(det_ctx, e, s, p, alert_flags) == false) {
1388  SCLogDebug("sid %u: e %p Callback returned false", s->id, e);
1389  return false;
1390  }
1391  SCLogDebug("sid %u: e %p Callback returned true", s->id, e);
1392  }
1393 
1394  SCLogDebug("sid %u: returning true", s->id);
1395  return true;
1396 }
1397 
1398 /**
1399  * \param data pointer to SigMatchData. Allowed to be NULL.
1400  */
1401 static int DetectEnginePktInspectionAppend(Signature *s,
1403  SigMatchData *data)
1404 {
1405  DetectEnginePktInspectionEngine *e = SCCalloc(1, sizeof(*e));
1406  if (e == NULL)
1407  return -1;
1408 
1409  e->v1.Callback = Callback;
1410  e->smd = data;
1411 
1412  if (s->pkt_inspect == NULL) {
1413  s->pkt_inspect = e;
1414  } else {
1416  while (a->next != NULL) {
1417  a = a->next;
1418  }
1419  a->next = e;
1420  }
1421  return 0;
1422 }
1423 
1425 {
1426  /* only handle PMATCH here if we're not an app inspect rule */
1428  if (DetectEnginePktInspectionAppend(s, DetectEngineInspectRulePayloadMatches,
1429  NULL) < 0)
1430  return -1;
1431  SCLogDebug("sid %u: DetectEngineInspectRulePayloadMatches appended", s->id);
1432  }
1433 
1434  if (s->sm_arrays[DETECT_SM_LIST_MATCH]) {
1435  if (DetectEnginePktInspectionAppend(s, DetectEngineInspectRulePacketMatches,
1436  NULL) < 0)
1437  return -1;
1438  SCLogDebug("sid %u: DetectEngineInspectRulePacketMatches appended", s->id);
1439  }
1440 
1441  return 0;
1442 }
1443 
1444 /* code to control the main thread to do a reload */
1445 
1447  IDLE, /**< ready to start a reload */
1448  RELOAD, /**< command main thread to do the reload */
1449 };
1450 
1451 
1452 typedef struct DetectEngineSyncer_ {
1456 
1457 static DetectEngineSyncer detect_sync = { SCMUTEX_INITIALIZER, IDLE };
1458 
1459 /* tell main to start reloading */
1461 {
1462  int r = 0;
1463  SCMutexLock(&detect_sync.m);
1464  if (detect_sync.state == IDLE) {
1465  detect_sync.state = RELOAD;
1466  } else {
1467  r = -1;
1468  }
1469  SCMutexUnlock(&detect_sync.m);
1470  return r;
1471 }
1472 
1473 /* main thread checks this to see if it should start */
1475 {
1476  int r = 0;
1477  SCMutexLock(&detect_sync.m);
1478  if (detect_sync.state == RELOAD) {
1479  r = 1;
1480  }
1481  SCMutexUnlock(&detect_sync.m);
1482  return r;
1483 }
1484 
1485 /* main thread sets done when it's done */
1487 {
1488  SCMutexLock(&detect_sync.m);
1489  detect_sync.state = IDLE;
1490  SCMutexUnlock(&detect_sync.m);
1491 }
1492 
1493 /* caller loops this until it returns 1 */
1495 {
1496  int r = 0;
1497  SCMutexLock(&detect_sync.m);
1498  if (detect_sync.state == IDLE) {
1499  r = 1;
1500  }
1501  SCMutexUnlock(&detect_sync.m);
1502  return r;
1503 }
1504 
1505 /** \brief Do the content inspection & validation for a signature
1506  *
1507  * \param de_ctx Detection engine context
1508  * \param det_ctx Detection engine thread context
1509  * \param s Signature to inspect
1510  * \param sm SigMatch to inspect
1511  * \param f Flow
1512  * \param flags app layer flags
1513  * \param state App layer state
1514  *
1515  * \retval 0 no match
1516  * \retval 1 match
1517  */
1519  const DetectEngineCtx *de_ctx,
1520  DetectEngineThreadCtx *det_ctx,
1521  const Signature *s, const SigMatchData *smd,
1522  Flow *f, const uint8_t flags,
1523  void *alstate, void *txv, uint64_t tx_id)
1524 {
1525  SCLogDebug("running match functions, sm %p", smd);
1526  if (smd != NULL) {
1527  while (1) {
1528  int match = 0;
1530  match = sigmatch_table[smd->type].
1531  AppLayerTxMatch(det_ctx, f, flags, alstate, txv, s, smd->ctx);
1532  KEYWORD_PROFILING_END(det_ctx, smd->type, (match == 1));
1533  if (match == 0)
1535  if (match == 2) {
1537  }
1538 
1539  if (smd->is_last)
1540  break;
1541  smd++;
1542  }
1543  }
1544 
1546 }
1547 
1548 
1549 /**
1550  * \brief Do the content inspection & validation for a signature
1551  *
1552  * \param de_ctx Detection engine context
1553  * \param det_ctx Detection engine thread context
1554  * \param s Signature to inspect
1555  * \param f Flow
1556  * \param flags app layer flags
1557  * \param state App layer state
1558  *
1559  * \retval 0 no match.
1560  * \retval 1 match.
1561  * \retval 2 Sig can't match.
1562  */
1564  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1565  const DetectEngineAppInspectionEngine *engine,
1566  const Signature *s,
1567  Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
1568 {
1569  const int list_id = engine->sm_list;
1570  SCLogDebug("running inspect on %d", list_id);
1571 
1572  const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
1573 
1574  SCLogDebug("list %d mpm? %s transforms %p",
1575  engine->sm_list, engine->mpm ? "true" : "false", engine->v2.transforms);
1576 
1577  /* if prefilter didn't already run, we need to consider transformations */
1578  const DetectEngineTransforms *transforms = NULL;
1579  if (!engine->mpm) {
1580  transforms = engine->v2.transforms;
1581  }
1582 
1583  const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
1584  f, flags, txv, list_id);
1585  if (unlikely(buffer == NULL)) {
1588  }
1589 
1590  const uint32_t data_len = buffer->inspect_len;
1591  const uint8_t *data = buffer->inspect;
1592  const uint64_t offset = buffer->inspect_offset;
1593 
1594  uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0;
1595  ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0);
1596  ci_flags |= buffer->flags;
1597 
1598  det_ctx->discontinue_matching = 0;
1599  det_ctx->buffer_offset = 0;
1600  det_ctx->inspection_recursion_counter = 0;
1601 
1602  /* Inspect all the uricontents fetched on each
1603  * transaction at the app layer */
1604  int r = DetectEngineContentInspection(de_ctx, det_ctx,
1605  s, engine->smd,
1606  NULL, f,
1607  (uint8_t *)data, data_len, offset, ci_flags,
1609  if (r == 1) {
1611  } else {
1614  }
1615 }
1616 
1617 /**
1618  * \brief Do the content inspection & validation for a signature
1619  *
1620  * \param de_ctx Detection engine context
1621  * \param det_ctx Detection engine thread context
1622  * \param s Signature to inspect
1623  * \param p Packet
1624  *
1625  * \retval 0 no match.
1626  * \retval 1 match.
1627  */
1629  DetectEngineThreadCtx *det_ctx,
1630  const DetectEnginePktInspectionEngine *engine,
1631  const Signature *s, Packet *p, uint8_t *_alert_flags)
1632 {
1633  const int list_id = engine->sm_list;
1634  SCLogDebug("running inspect on %d", list_id);
1635 
1636  SCLogDebug("list %d transforms %p",
1637  engine->sm_list, engine->v1.transforms);
1638 
1639  /* if prefilter didn't already run, we need to consider transformations */
1640  const DetectEngineTransforms *transforms = NULL;
1641  if (!engine->mpm) {
1642  transforms = engine->v1.transforms;
1643  }
1644 
1645  const InspectionBuffer *buffer = engine->v1.GetData(det_ctx, transforms, p,
1646  list_id);
1647  if (unlikely(buffer == NULL)) {
1649  }
1650 
1651  const uint32_t data_len = buffer->inspect_len;
1652  const uint8_t *data = buffer->inspect;
1653  const uint64_t offset = 0;
1654 
1655  uint8_t ci_flags = DETECT_CI_FLAGS_START|DETECT_CI_FLAGS_END;
1656  ci_flags |= buffer->flags;
1657 
1658  det_ctx->discontinue_matching = 0;
1659  det_ctx->buffer_offset = 0;
1660  det_ctx->inspection_recursion_counter = 0;
1661 
1662  /* Inspect all the uricontents fetched on each
1663  * transaction at the app layer */
1664  int r = DetectEngineContentInspection(det_ctx->de_ctx, det_ctx,
1665  s, engine->smd,
1666  p, p->flow,
1667  (uint8_t *)data, data_len, offset, ci_flags,
1669  if (r == 1) {
1671  } else {
1673  }
1674 }
1675 
1676 
1677 /* nudge capture loops to wake up */
1678 static void BreakCapture(void)
1679 {
1681  ThreadVars *tv = tv_root[TVT_PPT];
1682  while (tv) {
1683  /* find the correct slot */
1684  TmSlot *slots = tv->tm_slots;
1685  while (slots != NULL) {
1686  if (suricata_ctl_flags != 0) {
1688  return;
1689  }
1690 
1691  TmModule *tm = TmModuleGetById(slots->tm_id);
1692  if (!(tm->flags & TM_FLAG_RECEIVE_TM)) {
1693  slots = slots->slot_next;
1694  continue;
1695  }
1696 
1697  /* signal capture method that we need a packet. */
1699  /* if the method supports it, BreakLoop. Otherwise we rely on
1700  * the capture method's recv timeout */
1701  if (tm->PktAcqLoop && tm->PktAcqBreakLoop) {
1702  tm->PktAcqBreakLoop(tv, SC_ATOMIC_GET(slots->slot_data));
1703  }
1704 
1705  break;
1706  }
1707  tv = tv->next;
1708  }
1710 }
1711 
1712 /** \internal
1713  * \brief inject a pseudo packet into each detect thread that doesn't use the
1714  * new det_ctx yet
1715  */
1716 static void InjectPackets(ThreadVars **detect_tvs,
1717  DetectEngineThreadCtx **new_det_ctx,
1718  int no_of_detect_tvs)
1719 {
1720  int i;
1721  /* inject a fake packet if the detect thread isn't using the new ctx yet,
1722  * this speeds up the process */
1723  for (i = 0; i < no_of_detect_tvs; i++) {
1724  if (SC_ATOMIC_GET(new_det_ctx[i]->so_far_used_by_detect) != 1) {
1725  if (detect_tvs[i]->inq != NULL) {
1726  Packet *p = PacketGetFromAlloc();
1727  if (p != NULL) {
1729  PacketQueue *q = &trans_q[detect_tvs[i]->inq->id];
1730  SCMutexLock(&q->mutex_q);
1731  PacketEnqueue(q, p);
1732  SCCondSignal(&q->cond_q);
1733  SCMutexUnlock(&q->mutex_q);
1734  }
1735  }
1736  }
1737  }
1738 }
1739 
1740 /** \internal
1741  * \brief Update detect threads with new detect engine
1742  *
1743  * Atomically update each detect thread with a new thread context
1744  * that is associated to the new detection engine(s).
1745  *
1746  * If called in unix socket mode, it's possible that we don't have
1747  * detect threads yet.
1748  *
1749  * \retval -1 error
1750  * \retval 0 no detection threads
1751  * \retval 1 successful reload
1752  */
1753 static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx)
1754 {
1755  SCEnter();
1756  int i = 0;
1757  int no_of_detect_tvs = 0;
1758  ThreadVars *tv = NULL;
1759 
1760  /* count detect threads in use */
1762  tv = tv_root[TVT_PPT];
1763  while (tv) {
1764  /* obtain the slots for this TV */
1765  TmSlot *slots = tv->tm_slots;
1766  while (slots != NULL) {
1767  TmModule *tm = TmModuleGetById(slots->tm_id);
1768 
1769  if (suricata_ctl_flags != 0) {
1770  SCLogInfo("rule reload interupted by engine shutdown");
1772  return -1;
1773  }
1774 
1775  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
1776  slots = slots->slot_next;
1777  continue;
1778  }
1779  no_of_detect_tvs++;
1780  break;
1781  }
1782 
1783  tv = tv->next;
1784  }
1786 
1787  /* can be zero in unix socket mode */
1788  if (no_of_detect_tvs == 0) {
1789  return 0;
1790  }
1791 
1792  /* prepare swap structures */
1793  DetectEngineThreadCtx *old_det_ctx[no_of_detect_tvs];
1794  DetectEngineThreadCtx *new_det_ctx[no_of_detect_tvs];
1795  ThreadVars *detect_tvs[no_of_detect_tvs];
1796  memset(old_det_ctx, 0x00, (no_of_detect_tvs * sizeof(DetectEngineThreadCtx *)));
1797  memset(new_det_ctx, 0x00, (no_of_detect_tvs * sizeof(DetectEngineThreadCtx *)));
1798  memset(detect_tvs, 0x00, (no_of_detect_tvs * sizeof(ThreadVars *)));
1799 
1800  /* start the process of swapping detect threads ctxs */
1801 
1802  /* get reference to tv's and setup new_det_ctx array */
1804  tv = tv_root[TVT_PPT];
1805  while (tv) {
1806  /* obtain the slots for this TV */
1807  TmSlot *slots = tv->tm_slots;
1808  while (slots != NULL) {
1809  TmModule *tm = TmModuleGetById(slots->tm_id);
1810 
1811  if (suricata_ctl_flags != 0) {
1813  goto error;
1814  }
1815 
1816  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
1817  slots = slots->slot_next;
1818  continue;
1819  }
1820 
1821  old_det_ctx[i] = FlowWorkerGetDetectCtxPtr(SC_ATOMIC_GET(slots->slot_data));
1822  detect_tvs[i] = tv;
1823 
1824  new_det_ctx[i] = DetectEngineThreadCtxInitForReload(tv, new_de_ctx, 1);
1825  if (new_det_ctx[i] == NULL) {
1826  SCLogError(SC_ERR_LIVE_RULE_SWAP, "Detect engine thread init "
1827  "failure in live rule swap. Let's get out of here");
1829  goto error;
1830  }
1831  SCLogDebug("live rule swap created new det_ctx - %p and de_ctx "
1832  "- %p\n", new_det_ctx[i], new_de_ctx);
1833  i++;
1834  break;
1835  }
1836 
1837  tv = tv->next;
1838  }
1839  BUG_ON(i != no_of_detect_tvs);
1840 
1841  /* atomicly replace the det_ctx data */
1842  i = 0;
1843  tv = tv_root[TVT_PPT];
1844  while (tv) {
1845  /* find the correct slot */
1846  TmSlot *slots = tv->tm_slots;
1847  while (slots != NULL) {
1848  if (suricata_ctl_flags != 0) {
1850  return -1;
1851  }
1852 
1853  TmModule *tm = TmModuleGetById(slots->tm_id);
1854  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
1855  slots = slots->slot_next;
1856  continue;
1857  }
1858  SCLogDebug("swapping new det_ctx - %p with older one - %p",
1859  new_det_ctx[i], SC_ATOMIC_GET(slots->slot_data));
1860  FlowWorkerReplaceDetectCtx(SC_ATOMIC_GET(slots->slot_data), new_det_ctx[i++]);
1861  break;
1862  }
1863  tv = tv->next;
1864  }
1866 
1867  /* threads now all have new data, however they may not have started using
1868  * it and may still use the old data */
1869 
1870  SCLogDebug("Live rule swap has swapped %d old det_ctx's with new ones, "
1871  "along with the new de_ctx", no_of_detect_tvs);
1872 
1873  InjectPackets(detect_tvs, new_det_ctx, no_of_detect_tvs);
1874 
1875  for (i = 0; i < no_of_detect_tvs; i++) {
1876  int break_out = 0;
1877  usleep(1000);
1878  while (SC_ATOMIC_GET(new_det_ctx[i]->so_far_used_by_detect) != 1) {
1879  if (suricata_ctl_flags != 0) {
1880  break_out = 1;
1881  break;
1882  }
1883 
1884  BreakCapture();
1885  usleep(1000);
1886  }
1887  if (break_out)
1888  break;
1889  SCLogDebug("new_det_ctx - %p used by detect engine", new_det_ctx[i]);
1890  }
1891 
1892  /* this is to make sure that if someone initiated shutdown during a live
1893  * rule swap, the live rule swap won't clean up the old det_ctx and
1894  * de_ctx, till all detect threads have stopped working and sitting
1895  * silently after setting RUNNING_DONE flag and while waiting for
1896  * THV_DEINIT flag */
1897  if (i != no_of_detect_tvs) { // not all threads we swapped
1898  tv = tv_root[TVT_PPT];
1899  while (tv) {
1900  /* obtain the slots for this TV */
1901  TmSlot *slots = tv->tm_slots;
1902  while (slots != NULL) {
1903  TmModule *tm = TmModuleGetById(slots->tm_id);
1904  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
1905  slots = slots->slot_next;
1906  continue;
1907  }
1908 
1909  while (!TmThreadsCheckFlag(tv, THV_RUNNING_DONE)) {
1910  usleep(100);
1911  }
1912 
1913  slots = slots->slot_next;
1914  }
1915 
1916  tv = tv->next;
1917  }
1918  }
1919 
1920  /* free all the ctxs */
1921  for (i = 0; i < no_of_detect_tvs; i++) {
1922  SCLogDebug("Freeing old_det_ctx - %p used by detect",
1923  old_det_ctx[i]);
1924  DetectEngineThreadCtxDeinit(NULL, old_det_ctx[i]);
1925  }
1926 
1928 
1929  return 1;
1930 
1931  error:
1932  for (i = 0; i < no_of_detect_tvs; i++) {
1933  if (new_det_ctx[i] != NULL)
1934  DetectEngineThreadCtxDeinit(NULL, new_det_ctx[i]);
1935  }
1936  return -1;
1937 }
1938 
1939 static DetectEngineCtx *DetectEngineCtxInitReal(enum DetectEngineType type, const char *prefix)
1940 {
1941  DetectEngineCtx *de_ctx = SCMalloc(sizeof(DetectEngineCtx));
1942  if (unlikely(de_ctx == NULL))
1943  goto error;
1944 
1945  memset(de_ctx,0,sizeof(DetectEngineCtx));
1946  memset(&de_ctx->sig_stat, 0, sizeof(SigFileLoaderStat));
1947  TAILQ_INIT(&de_ctx->sig_stat.failed_sigs);
1948  de_ctx->sigerror = NULL;
1949  de_ctx->type = type;
1950 
1951  if (type == DETECT_ENGINE_TYPE_DD_STUB || type == DETECT_ENGINE_TYPE_MT_STUB) {
1952  de_ctx->version = DetectEngineGetVersion();
1953  SCLogDebug("stub %u with version %u", type, de_ctx->version);
1954  return de_ctx;
1955  }
1956 
1957  if (prefix != NULL) {
1958  strlcpy(de_ctx->config_prefix, prefix, sizeof(de_ctx->config_prefix));
1959  }
1960 
1961  if (ConfGetBool("engine.init-failure-fatal", (int *)&(de_ctx->failure_fatal)) != 1) {
1962  SCLogDebug("ConfGetBool could not load the value.");
1963  }
1964 
1967  SCLogConfig("pattern matchers: MPM: %s, SPM: %s",
1968  mpm_table[de_ctx->mpm_matcher].name,
1969  spm_table[de_ctx->spm_matcher].name);
1970 
1972  if (de_ctx->spm_global_thread_ctx == NULL) {
1973  SCLogDebug("Unable to alloc SpmGlobalThreadCtx.");
1974  goto error;
1975  }
1976 
1977  if (DetectEngineCtxLoadConf(de_ctx) == -1) {
1978  goto error;
1979  }
1980 
1981  SigGroupHeadHashInit(de_ctx);
1982  MpmStoreInit(de_ctx);
1983  ThresholdHashInit(de_ctx);
1984  DetectParseDupSigHashInit(de_ctx);
1985  DetectAddressMapInit(de_ctx);
1986  DetectMetadataHashInit(de_ctx);
1987  DetectBufferTypeSetupDetectEngine(de_ctx);
1988 
1989  /* init iprep... ignore errors for now */
1990  (void)SRepInit(de_ctx);
1991 
1993  SCRConfLoadReferenceConfigFile(de_ctx, NULL);
1994 
1995  if (ActionInitConfig() < 0) {
1996  goto error;
1997  }
1998 
1999  de_ctx->version = DetectEngineGetVersion();
2001  SCLogDebug("dectx with version %u", de_ctx->version);
2002  return de_ctx;
2003 error:
2004  if (de_ctx != NULL) {
2005  DetectEngineCtxFree(de_ctx);
2006  }
2007  return NULL;
2008 
2009 }
2010 
2012 {
2013  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_MT_STUB, NULL);
2014 }
2015 
2017 {
2018  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_DD_STUB, NULL);
2019 }
2020 
2022 {
2023  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_NORMAL, NULL);
2024 }
2025 
2027 {
2028  if (prefix == NULL || strlen(prefix) == 0)
2029  return DetectEngineCtxInit();
2030  else
2031  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_NORMAL, prefix);
2032 }
2033 
2034 static void DetectEngineCtxFreeThreadKeywordData(DetectEngineCtx *de_ctx)
2035 {
2037  while (item) {
2039  SCFree(item);
2040  item = next;
2041  }
2042  de_ctx->keyword_list = NULL;
2043 }
2044 
2045 static void DetectEngineCtxFreeFailedSigs(DetectEngineCtx *de_ctx)
2046 {
2047  SigString *item = NULL;
2048  SigString *sitem;
2049 
2050  TAILQ_FOREACH_SAFE(item, &de_ctx->sig_stat.failed_sigs, next, sitem) {
2051  SCFree(item->filename);
2052  SCFree(item->sig_str);
2053  if (item->sig_error) {
2054  SCFree(item->sig_error);
2055  }
2056  TAILQ_REMOVE(&de_ctx->sig_stat.failed_sigs, item, next);
2057  SCFree(item);
2058  }
2059 }
2060 
2061 /**
2062  * \brief Free a DetectEngineCtx::
2063  *
2064  * \param de_ctx DetectEngineCtx:: to be freed
2065  */
2067 {
2068 
2069  if (de_ctx == NULL)
2070  return;
2071 
2072 #ifdef PROFILING
2073  if (de_ctx->profile_ctx != NULL) {
2075  de_ctx->profile_ctx = NULL;
2076  }
2077  if (de_ctx->profile_keyword_ctx != NULL) {
2078  SCProfilingKeywordDestroyCtx(de_ctx);//->profile_keyword_ctx);
2079 // de_ctx->profile_keyword_ctx = NULL;
2080  }
2081  if (de_ctx->profile_sgh_ctx != NULL) {
2082  SCProfilingSghDestroyCtx(de_ctx);
2083  }
2085 #endif
2086 
2087  /* Normally the hashes are freed elsewhere, but
2088  * to be sure look at them again here.
2089  */
2090  SigGroupHeadHashFree(de_ctx);
2091  MpmStoreFree(de_ctx);
2092  DetectParseDupSigHashFree(de_ctx);
2094  ThresholdContextDestroy(de_ctx);
2095  SigCleanSignatures(de_ctx);
2096  if (de_ctx->sig_array)
2097  SCFree(de_ctx->sig_array);
2098 
2099  SCClassConfDeInitContext(de_ctx);
2100  SCRConfDeInitContext(de_ctx);
2101 
2102  SigGroupCleanup(de_ctx);
2103 
2105 
2107 
2108  DetectEngineCtxFreeThreadKeywordData(de_ctx);
2109  SRepDestroy(de_ctx);
2110  DetectEngineCtxFreeFailedSigs(de_ctx);
2111 
2112  DetectAddressMapFree(de_ctx);
2113  DetectMetadataHashFree(de_ctx);
2114 
2115  /* if we have a config prefix, remove the config from the tree */
2116  if (strlen(de_ctx->config_prefix) > 0) {
2117  /* remove config */
2118  ConfNode *node = ConfGetNode(de_ctx->config_prefix);
2119  if (node != NULL) {
2120  ConfNodeRemove(node); /* frees node */
2121  }
2122 #if 0
2123  ConfDump();
2124 #endif
2125  }
2126 
2127  DetectPortCleanupList(de_ctx, de_ctx->tcp_whitelist);
2128  DetectPortCleanupList(de_ctx, de_ctx->udp_whitelist);
2129 
2130  DetectBufferTypeFreeDetectEngine(de_ctx);
2131  /* freed our var name hash */
2132  VarNameStoreFree(de_ctx->version);
2133 
2134  SCFree(de_ctx);
2135  //DetectAddressGroupPrintMemory();
2136  //DetectSigGroupPrintMemory();
2137  //DetectPortPrintMemory();
2138 }
2139 
2140 /** \brief Function that load DetectEngineCtx config for grouping sigs
2141  * used by the engine
2142  * \retval 0 if no config provided, 1 if config was provided
2143  * and loaded successfuly
2144  */
2145 static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
2146 {
2147  uint8_t profile = ENGINE_PROFILE_MEDIUM;
2148  const char *max_uniq_toclient_groups_str = NULL;
2149  const char *max_uniq_toserver_groups_str = NULL;
2150  const char *sgh_mpm_context = NULL;
2151  const char *de_ctx_profile = NULL;
2152 
2153  (void)ConfGet("detect.profile", &de_ctx_profile);
2154  (void)ConfGet("detect.sgh-mpm-context", &sgh_mpm_context);
2155 
2156  ConfNode *de_ctx_custom = ConfGetNode("detect-engine");
2157  ConfNode *opt = NULL;
2158 
2159  if (de_ctx_custom != NULL) {
2160  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2161  if (de_ctx_profile == NULL) {
2162  if (opt->val && strcmp(opt->val, "profile") == 0) {
2163  de_ctx_profile = opt->head.tqh_first->val;
2164  }
2165  }
2166 
2167  if (sgh_mpm_context == NULL) {
2168  if (opt->val && strcmp(opt->val, "sgh-mpm-context") == 0) {
2169  sgh_mpm_context = opt->head.tqh_first->val;
2170  }
2171  }
2172  }
2173  }
2174 
2175  if (de_ctx_profile != NULL) {
2176  if (strcmp(de_ctx_profile, "low") == 0 ||
2177  strcmp(de_ctx_profile, "lowest") == 0) { // legacy
2178  profile = ENGINE_PROFILE_LOW;
2179  } else if (strcmp(de_ctx_profile, "medium") == 0) {
2180  profile = ENGINE_PROFILE_MEDIUM;
2181  } else if (strcmp(de_ctx_profile, "high") == 0 ||
2182  strcmp(de_ctx_profile, "highest") == 0) { // legacy
2183  profile = ENGINE_PROFILE_HIGH;
2184  } else if (strcmp(de_ctx_profile, "custom") == 0) {
2185  profile = ENGINE_PROFILE_CUSTOM;
2186  } else {
2188  "invalid value for detect.profile: '%s'. "
2189  "Valid options: low, medium, high and custom.",
2190  de_ctx_profile);
2191  return -1;
2192  }
2193 
2194  SCLogDebug("Profile for detection engine groups is \"%s\"", de_ctx_profile);
2195  } else {
2196  SCLogDebug("Profile for detection engine groups not provided "
2197  "at suricata.yaml. Using default (\"medium\").");
2198  }
2199 
2200  /* detect-engine.sgh-mpm-context option parsing */
2201  if (sgh_mpm_context == NULL || strcmp(sgh_mpm_context, "auto") == 0) {
2202  /* for now, since we still haven't implemented any intelligence into
2203  * understanding the patterns and distributing mpm_ctx across sgh */
2204  if (de_ctx->mpm_matcher == MPM_AC || de_ctx->mpm_matcher == MPM_AC_KS ||
2205 #ifdef BUILD_HYPERSCAN
2206  de_ctx->mpm_matcher == MPM_HS ||
2207 #endif
2208  de_ctx->mpm_matcher == MPM_AC_BS) {
2210  } else {
2212  }
2213  } else {
2214  if (strcmp(sgh_mpm_context, "single") == 0) {
2216  } else if (strcmp(sgh_mpm_context, "full") == 0) {
2218  } else {
2219  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "You have supplied an "
2220  "invalid conf value for detect-engine.sgh-mpm-context-"
2221  "%s", sgh_mpm_context);
2222  exit(EXIT_FAILURE);
2223  }
2224  }
2225 
2226  if (run_mode == RUNMODE_UNITTEST) {
2228  }
2229 
2230  /* parse profile custom-values */
2231  opt = NULL;
2232  switch (profile) {
2233  case ENGINE_PROFILE_LOW:
2234  de_ctx->max_uniq_toclient_groups = 15;
2235  de_ctx->max_uniq_toserver_groups = 25;
2236  break;
2237 
2238  case ENGINE_PROFILE_HIGH:
2239  de_ctx->max_uniq_toclient_groups = 75;
2240  de_ctx->max_uniq_toserver_groups = 75;
2241  break;
2242 
2243  case ENGINE_PROFILE_CUSTOM:
2244  (void)ConfGet("detect.custom-values.toclient-groups",
2245  &max_uniq_toclient_groups_str);
2246  (void)ConfGet("detect.custom-values.toserver-groups",
2247  &max_uniq_toserver_groups_str);
2248 
2249  if (de_ctx_custom != NULL) {
2250  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2251  if (opt->val && strcmp(opt->val, "custom-values") == 0) {
2252  if (max_uniq_toclient_groups_str == NULL) {
2253  max_uniq_toclient_groups_str = (char *)ConfNodeLookupChildValue
2254  (opt->head.tqh_first, "toclient-sp-groups");
2255  }
2256  if (max_uniq_toclient_groups_str == NULL) {
2257  max_uniq_toclient_groups_str = (char *)ConfNodeLookupChildValue
2258  (opt->head.tqh_first, "toclient-groups");
2259  }
2260  if (max_uniq_toserver_groups_str == NULL) {
2261  max_uniq_toserver_groups_str = (char *)ConfNodeLookupChildValue
2262  (opt->head.tqh_first, "toserver-dp-groups");
2263  }
2264  if (max_uniq_toserver_groups_str == NULL) {
2265  max_uniq_toserver_groups_str = (char *)ConfNodeLookupChildValue
2266  (opt->head.tqh_first, "toserver-groups");
2267  }
2268  }
2269  }
2270  }
2271  if (max_uniq_toclient_groups_str != NULL) {
2273  strlen(max_uniq_toclient_groups_str),
2274  (const char *)max_uniq_toclient_groups_str) <= 0)
2275  {
2276  de_ctx->max_uniq_toclient_groups = 20;
2277 
2278  SCLogWarning(SC_ERR_SIZE_PARSE, "parsing '%s' for "
2279  "toclient-groups failed, using %u",
2280  max_uniq_toclient_groups_str,
2281  de_ctx->max_uniq_toclient_groups);
2282  }
2283  } else {
2284  de_ctx->max_uniq_toclient_groups = 20;
2285  }
2286  SCLogConfig("toclient-groups %u", de_ctx->max_uniq_toclient_groups);
2287 
2288  if (max_uniq_toserver_groups_str != NULL) {
2290  strlen(max_uniq_toserver_groups_str),
2291  (const char *)max_uniq_toserver_groups_str) <= 0)
2292  {
2293  de_ctx->max_uniq_toserver_groups = 40;
2294 
2295  SCLogWarning(SC_ERR_SIZE_PARSE, "parsing '%s' for "
2296  "toserver-groups failed, using %u",
2297  max_uniq_toserver_groups_str,
2298  de_ctx->max_uniq_toserver_groups);
2299  }
2300  } else {
2301  de_ctx->max_uniq_toserver_groups = 40;
2302  }
2303  SCLogConfig("toserver-groups %u", de_ctx->max_uniq_toserver_groups);
2304  break;
2305 
2306  /* Default (or no config provided) is profile medium */
2307  case ENGINE_PROFILE_MEDIUM:
2309  default:
2310  de_ctx->max_uniq_toclient_groups = 20;
2311  de_ctx->max_uniq_toserver_groups = 40;
2312  break;
2313  }
2314 
2315  intmax_t value = 0;
2316  if (ConfGetInt("detect.inspection-recursion-limit", &value) == 1)
2317  {
2318  if (value >= 0 && value <= INT_MAX) {
2319  de_ctx->inspection_recursion_limit = (int)value;
2320  }
2321 
2322  /* fall back to old config parsing */
2323  } else {
2324  ConfNode *insp_recursion_limit_node = NULL;
2325  char *insp_recursion_limit = NULL;
2326 
2327  if (de_ctx_custom != NULL) {
2328  opt = NULL;
2329  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2330  if (opt->val && strcmp(opt->val, "inspection-recursion-limit") != 0)
2331  continue;
2332 
2333  insp_recursion_limit_node = ConfNodeLookupChild(opt, opt->val);
2334  if (insp_recursion_limit_node == NULL) {
2335  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Error retrieving conf "
2336  "entry for detect-engine:inspection-recursion-limit");
2337  break;
2338  }
2339  insp_recursion_limit = insp_recursion_limit_node->val;
2340  SCLogDebug("Found detect-engine.inspection-recursion-limit - %s:%s",
2341  insp_recursion_limit_node->name, insp_recursion_limit_node->val);
2342  break;
2343  }
2344 
2345  if (insp_recursion_limit != NULL) {
2346  de_ctx->inspection_recursion_limit = atoi(insp_recursion_limit);
2347  } else {
2348  de_ctx->inspection_recursion_limit =
2350  }
2351  }
2352  }
2353 
2354  if (de_ctx->inspection_recursion_limit == 0)
2355  de_ctx->inspection_recursion_limit = -1;
2356 
2357  SCLogDebug("de_ctx->inspection_recursion_limit: %d",
2358  de_ctx->inspection_recursion_limit);
2359 
2360  /* parse port grouping whitelisting settings */
2361 
2362  const char *ports = NULL;
2363  (void)ConfGet("detect.grouping.tcp-whitelist", &ports);
2364  if (ports) {
2365  SCLogConfig("grouping: tcp-whitelist %s", ports);
2366  } else {
2367  ports = "53, 80, 139, 443, 445, 1433, 3306, 3389, 6666, 6667, 8080";
2368  SCLogConfig("grouping: tcp-whitelist (default) %s", ports);
2369 
2370  }
2371  if (DetectPortParse(de_ctx, &de_ctx->tcp_whitelist, ports) != 0) {
2372  SCLogWarning(SC_ERR_INVALID_YAML_CONF_ENTRY, "'%s' is not a valid value "
2373  "for detect.grouping.tcp-whitelist", ports);
2374  }
2375  DetectPort *x = de_ctx->tcp_whitelist;
2376  for ( ; x != NULL; x = x->next) {
2377  if (x->port != x->port2) {
2378  SCLogWarning(SC_ERR_INVALID_YAML_CONF_ENTRY, "'%s' is not a valid value "
2379  "for detect.grouping.tcp-whitelist: only single ports allowed", ports);
2380  DetectPortCleanupList(de_ctx, de_ctx->tcp_whitelist);
2381  de_ctx->tcp_whitelist = NULL;
2382  break;
2383  }
2384  }
2385 
2386  ports = NULL;
2387  (void)ConfGet("detect.grouping.udp-whitelist", &ports);
2388  if (ports) {
2389  SCLogConfig("grouping: udp-whitelist %s", ports);
2390  } else {
2391  ports = "53, 135, 5060";
2392  SCLogConfig("grouping: udp-whitelist (default) %s", ports);
2393 
2394  }
2395  if (DetectPortParse(de_ctx, &de_ctx->udp_whitelist, ports) != 0) {
2396  SCLogWarning(SC_ERR_INVALID_YAML_CONF_ENTRY, "'%s' is not a valid value "
2397  "forr detect.grouping.udp-whitelist", ports);
2398  }
2399  for (x = de_ctx->udp_whitelist; x != NULL; x = x->next) {
2400  if (x->port != x->port2) {
2401  SCLogWarning(SC_ERR_INVALID_YAML_CONF_ENTRY, "'%s' is not a valid value "
2402  "for detect.grouping.udp-whitelist: only single ports allowed", ports);
2403  DetectPortCleanupList(de_ctx, de_ctx->udp_whitelist);
2404  de_ctx->udp_whitelist = NULL;
2405  break;
2406  }
2407  }
2408 
2410  const char *pf_setting = NULL;
2411  if (ConfGet("detect.prefilter.default", &pf_setting) == 1 && pf_setting) {
2412  if (strcasecmp(pf_setting, "mpm") == 0) {
2414  } else if (strcasecmp(pf_setting, "auto") == 0) {
2416  }
2417  }
2418  switch (de_ctx->prefilter_setting) {
2419  case DETECT_PREFILTER_MPM:
2420  SCLogConfig("prefilter engines: MPM");
2421  break;
2422  case DETECT_PREFILTER_AUTO:
2423  SCLogConfig("prefilter engines: MPM and keywords");
2424  break;
2425  }
2426 
2427  return 0;
2428 }
2429 
2430 /*
2431  * getting & (re)setting the internal sig i
2432  */
2433 
2434 //inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *de_ctx)
2435 //{
2436 // return de_ctx->signum;
2437 //}
2438 
2440 {
2441  de_ctx->signum = 0;
2442 }
2443 
2444 static int DetectEngineThreadCtxInitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
2445 {
2446  const DetectEngineMasterCtx *master = &g_master_de_ctx;
2447 
2448  if (master->keyword_id > 0) {
2449  // coverity[suspicious_sizeof : FALSE]
2450  det_ctx->global_keyword_ctxs_array = (void **)SCCalloc(master->keyword_id, sizeof(void *));
2451  if (det_ctx->global_keyword_ctxs_array == NULL) {
2452  SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx");
2453  return TM_ECODE_FAILED;
2454  }
2455  det_ctx->global_keyword_ctxs_size = master->keyword_id;
2456 
2457  const DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
2458  while (item) {
2459  det_ctx->global_keyword_ctxs_array[item->id] = item->InitFunc(item->data);
2460  if (det_ctx->global_keyword_ctxs_array[item->id] == NULL) {
2461  SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx "
2462  "for keyword \"%s\" failed", item->name);
2463  return TM_ECODE_FAILED;
2464  }
2465  item = item->next;
2466  }
2467  }
2468  return TM_ECODE_OK;
2469 }
2470 
2471 static void DetectEngineThreadCtxDeinitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
2472 {
2473  if (det_ctx->global_keyword_ctxs_array == NULL ||
2474  det_ctx->global_keyword_ctxs_size == 0) {
2475  return;
2476  }
2477 
2478  const DetectEngineMasterCtx *master = &g_master_de_ctx;
2479  if (master->keyword_id > 0) {
2480  const DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
2481  while (item) {
2482  if (det_ctx->global_keyword_ctxs_array[item->id] != NULL)
2483  item->FreeFunc(det_ctx->global_keyword_ctxs_array[item->id]);
2484 
2485  item = item->next;
2486  }
2487  det_ctx->global_keyword_ctxs_size = 0;
2489  det_ctx->global_keyword_ctxs_array = NULL;
2490  }
2491 }
2492 
2493 static int DetectEngineThreadCtxInitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
2494 {
2495  if (de_ctx->keyword_id > 0) {
2496  // coverity[suspicious_sizeof : FALSE]
2497  det_ctx->keyword_ctxs_array = SCMalloc(de_ctx->keyword_id * sizeof(void *));
2498  if (det_ctx->keyword_ctxs_array == NULL) {
2499  SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx");
2500  return TM_ECODE_FAILED;
2501  }
2502 
2503  memset(det_ctx->keyword_ctxs_array, 0x00, de_ctx->keyword_id * sizeof(void *));
2504 
2505  det_ctx->keyword_ctxs_size = de_ctx->keyword_id;
2506 
2508  while (item) {
2509  det_ctx->keyword_ctxs_array[item->id] = item->InitFunc(item->data);
2510  if (det_ctx->keyword_ctxs_array[item->id] == NULL) {
2511  SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx "
2512  "for keyword \"%s\" failed", item->name);
2513  return TM_ECODE_FAILED;
2514  }
2515  item = item->next;
2516  }
2517  }
2518  return TM_ECODE_OK;
2519 }
2520 
2521 static void DetectEngineThreadCtxDeinitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
2522 {
2523  if (de_ctx->keyword_id > 0) {
2525  while (item) {
2526  if (det_ctx->keyword_ctxs_array[item->id] != NULL)
2527  item->FreeFunc(det_ctx->keyword_ctxs_array[item->id]);
2528 
2529  item = item->next;
2530  }
2531  det_ctx->keyword_ctxs_size = 0;
2532  SCFree(det_ctx->keyword_ctxs_array);
2533  det_ctx->keyword_ctxs_array = NULL;
2534  }
2535 }
2536 
2537 /** NOTE: master MUST be locked before calling this */
2538 static TmEcode DetectEngineThreadCtxInitForMT(ThreadVars *tv, DetectEngineThreadCtx *det_ctx)
2539 {
2540  DetectEngineMasterCtx *master = &g_master_de_ctx;
2541  DetectEngineTenantMapping *map_array = NULL;
2542  uint32_t map_array_size = 0;
2543  uint32_t map_cnt = 0;
2544  int max_tenant_id = 0;
2545  DetectEngineCtx *list = master->list;
2546  HashTable *mt_det_ctxs_hash = NULL;
2547 
2548  if (master->tenant_selector == TENANT_SELECTOR_UNKNOWN) {
2549  SCLogError(SC_ERR_MT_NO_SELECTOR, "no tenant selector set: "
2550  "set using multi-detect.selector");
2551  return TM_ECODE_FAILED;
2552  }
2553 
2554  uint32_t tcnt = 0;
2555  while (list) {
2556  if (list->tenant_id > max_tenant_id)
2557  max_tenant_id = list->tenant_id;
2558 
2559  list = list->next;
2560  tcnt++;
2561  }
2562 
2563  mt_det_ctxs_hash = HashTableInit(tcnt * 2, TenantIdHash, TenantIdCompare, TenantIdFree);
2564  if (mt_det_ctxs_hash == NULL) {
2565  goto error;
2566  }
2567 
2568  if (tcnt == 0) {
2569  SCLogInfo("no tenants left, or none registered yet");
2570  } else {
2571  max_tenant_id++;
2572 
2574  while (map) {
2575  map_cnt++;
2576  map = map->next;
2577  }
2578 
2579  if (map_cnt > 0) {
2580  map_array_size = map_cnt + 1;
2581 
2582  map_array = SCCalloc(map_array_size, sizeof(*map_array));
2583  if (map_array == NULL)
2584  goto error;
2585 
2586  /* fill the array */
2587  map_cnt = 0;
2588  map = master->tenant_mapping_list;
2589  while (map) {
2590  if (map_cnt >= map_array_size) {
2591  goto error;
2592  }
2593  map_array[map_cnt].traffic_id = map->traffic_id;
2594  map_array[map_cnt].tenant_id = map->tenant_id;
2595  map_cnt++;
2596  map = map->next;
2597  }
2598 
2599  }
2600 
2601  /* set up hash for tenant lookup */
2602  list = master->list;
2603  while (list) {
2604  SCLogDebug("tenant-id %u", list->tenant_id);
2605  if (list->tenant_id != 0) {
2606  DetectEngineThreadCtx *mt_det_ctx = DetectEngineThreadCtxInitForReload(tv, list, 0);
2607  if (mt_det_ctx == NULL)
2608  goto error;
2609  if (HashTableAdd(mt_det_ctxs_hash, mt_det_ctx, 0) != 0) {
2610  goto error;
2611  }
2612  }
2613  list = list->next;
2614  }
2615  }
2616 
2617  det_ctx->mt_det_ctxs_hash = mt_det_ctxs_hash;
2618  mt_det_ctxs_hash = NULL;
2619 
2620  det_ctx->mt_det_ctxs_cnt = max_tenant_id;
2621 
2622  det_ctx->tenant_array = map_array;
2623  det_ctx->tenant_array_size = map_array_size;
2624 
2625  switch (master->tenant_selector) {
2627  SCLogDebug("TENANT_SELECTOR_UNKNOWN");
2628  break;
2629  case TENANT_SELECTOR_VLAN:
2630  det_ctx->TenantGetId = DetectEngineTentantGetIdFromVlanId;
2631  SCLogDebug("TENANT_SELECTOR_VLAN");
2632  break;
2634  det_ctx->TenantGetId = DetectEngineTentantGetIdFromLivedev;
2635  SCLogDebug("TENANT_SELECTOR_LIVEDEV");
2636  break;
2638  det_ctx->TenantGetId = DetectEngineTentantGetIdFromPcap;
2639  SCLogDebug("TENANT_SELECTOR_DIRECT");
2640  break;
2641  }
2642 
2643  return TM_ECODE_OK;
2644 error:
2645  if (map_array != NULL)
2646  SCFree(map_array);
2647  if (mt_det_ctxs_hash != NULL)
2648  HashTableFree(mt_det_ctxs_hash);
2649 
2650  return TM_ECODE_FAILED;
2651 }
2652 
2653 /** \internal
2654  * \brief Helper for DetectThread setup functions
2655  */
2656 static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
2657 {
2658  PatternMatchThreadPrepare(&det_ctx->mtc, de_ctx->mpm_matcher);
2659  PatternMatchThreadPrepare(&det_ctx->mtcs, de_ctx->mpm_matcher);
2660  PatternMatchThreadPrepare(&det_ctx->mtcu, de_ctx->mpm_matcher);
2661 
2662  PmqSetup(&det_ctx->pmq);
2663 
2665  if (det_ctx->spm_thread_ctx == NULL) {
2666  return TM_ECODE_FAILED;
2667  }
2668 
2669  /* sized to the max of our sgh settings. A max setting of 0 implies that all
2670  * sgh's have: sgh->non_pf_store_cnt == 0 */
2671  if (de_ctx->non_pf_store_cnt_max > 0) {
2672  det_ctx->non_pf_id_array = SCCalloc(de_ctx->non_pf_store_cnt_max, sizeof(SigIntId));
2673  BUG_ON(det_ctx->non_pf_id_array == NULL);
2674  }
2675 
2676  /* IP-ONLY */
2677  DetectEngineIPOnlyThreadInit(de_ctx,&det_ctx->io_ctx);
2678 
2679  /* DeState */
2680  if (de_ctx->sig_array_len > 0) {
2681  det_ctx->match_array_len = de_ctx->sig_array_len;
2682  det_ctx->match_array = SCMalloc(det_ctx->match_array_len * sizeof(Signature *));
2683  if (det_ctx->match_array == NULL) {
2684  return TM_ECODE_FAILED;
2685  }
2686  memset(det_ctx->match_array, 0,
2687  det_ctx->match_array_len * sizeof(Signature *));
2688 
2689  RuleMatchCandidateTxArrayInit(det_ctx, de_ctx->sig_array_len);
2690  }
2691 
2692  /* byte_extract storage */
2693  det_ctx->bj_values = SCMalloc(sizeof(*det_ctx->bj_values) *
2694  (de_ctx->byte_extract_max_local_id + 1));
2695  if (det_ctx->bj_values == NULL) {
2696  return TM_ECODE_FAILED;
2697  }
2698 
2699  /* Allocate space for base64 decoded data. */
2700  if (de_ctx->base64_decode_max_len) {
2701  det_ctx->base64_decoded = SCMalloc(de_ctx->base64_decode_max_len);
2702  if (det_ctx->base64_decoded == NULL) {
2703  return TM_ECODE_FAILED;
2704  }
2705  det_ctx->base64_decoded_len_max = de_ctx->base64_decode_max_len;
2706  det_ctx->base64_decoded_len = 0;
2707  }
2708 
2709  det_ctx->inspect.buffers_size = de_ctx->buffer_type_id;
2710  det_ctx->inspect.buffers = SCCalloc(det_ctx->inspect.buffers_size, sizeof(InspectionBuffer));
2711  if (det_ctx->inspect.buffers == NULL) {
2712  return TM_ECODE_FAILED;
2713  }
2714  det_ctx->inspect.to_clear_queue = SCCalloc(det_ctx->inspect.buffers_size, sizeof(uint32_t));
2715  if (det_ctx->inspect.to_clear_queue == NULL) {
2716  return TM_ECODE_FAILED;
2717  }
2718  det_ctx->inspect.to_clear_idx = 0;
2719 
2720  det_ctx->multi_inspect.buffers_size = de_ctx->buffer_type_id;
2722  if (det_ctx->multi_inspect.buffers == NULL) {
2723  return TM_ECODE_FAILED;
2724  }
2725  det_ctx->multi_inspect.to_clear_queue = SCCalloc(det_ctx->multi_inspect.buffers_size, sizeof(uint32_t));
2726  if (det_ctx->multi_inspect.to_clear_queue == NULL) {
2727  return TM_ECODE_FAILED;
2728  }
2729  det_ctx->multi_inspect.to_clear_idx = 0;
2730 
2731 
2732  DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
2733  DetectEngineThreadCtxInitGlobalKeywords(det_ctx);
2734 #ifdef PROFILING
2735  SCProfilingRuleThreadSetup(de_ctx->profile_ctx, det_ctx);
2738  SCProfilingSghThreadSetup(de_ctx->profile_sgh_ctx, det_ctx);
2739 #endif
2740  SC_ATOMIC_INIT(det_ctx->so_far_used_by_detect);
2741 
2742  return TM_ECODE_OK;
2743 }
2744 
2745 /** \brief initialize thread specific detection engine context
2746  *
2747  * \note there is a special case when using delayed detect. In this case the
2748  * function is called twice per thread. The first time the rules are not
2749  * yet loaded. de_ctx->delayed_detect_initialized will be 0. The 2nd
2750  * time they will be loaded. de_ctx->delayed_detect_initialized will be 1.
2751  * This is needed to do the per thread counter registration before the
2752  * packet runtime starts. In delayed detect mode, the first call will
2753  * return a NULL ptr through the data ptr.
2754  *
2755  * \param tv ThreadVars for this thread
2756  * \param initdata pointer to de_ctx
2757  * \param data[out] pointer to store our thread detection ctx
2758  *
2759  * \retval TM_ECODE_OK if all went well
2760  * \retval TM_ECODE_FAILED on serious erro
2761  */
2762 TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
2763 {
2765  if (unlikely(det_ctx == NULL))
2766  return TM_ECODE_FAILED;
2767  memset(det_ctx, 0, sizeof(DetectEngineThreadCtx));
2768 
2769  det_ctx->tv = tv;
2770  det_ctx->de_ctx = DetectEngineGetCurrent();
2771  if (det_ctx->de_ctx == NULL) {
2772 #ifdef UNITTESTS
2773  if (RunmodeIsUnittests()) {
2774  det_ctx->de_ctx = (DetectEngineCtx *)initdata;
2775  } else {
2776  DetectEngineThreadCtxDeinit(tv, det_ctx);
2777  return TM_ECODE_FAILED;
2778  }
2779 #else
2780  DetectEngineThreadCtxDeinit(tv, det_ctx);
2781  return TM_ECODE_FAILED;
2782 #endif
2783  }
2784 
2785  if (det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
2786  det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_TENANT)
2787  {
2788  if (ThreadCtxDoInit(det_ctx->de_ctx, det_ctx) != TM_ECODE_OK) {
2789  DetectEngineThreadCtxDeinit(tv, det_ctx);
2790  return TM_ECODE_FAILED;
2791  }
2792  }
2793 
2794  /** alert counter setup */
2795  det_ctx->counter_alerts = StatsRegisterCounter("detect.alert", tv);
2796 #ifdef PROFILING
2797  det_ctx->counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
2798  det_ctx->counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
2799  det_ctx->counter_fnonmpm_list = StatsRegisterAvgCounter("detect.fnonmpm_list", tv);
2800  det_ctx->counter_match_list = StatsRegisterAvgCounter("detect.match_list", tv);
2801 #endif
2802 
2804  if (DetectEngineThreadCtxInitForMT(tv, det_ctx) != TM_ECODE_OK) {
2805  DetectEngineThreadCtxDeinit(tv, det_ctx);
2806  return TM_ECODE_FAILED;
2807  }
2808  }
2809 
2810  /* pass thread data back to caller */
2811  *data = (void *)det_ctx;
2812 
2813  return TM_ECODE_OK;
2814 }
2815 
2816 /**
2817  * \internal
2818  * \brief initialize a det_ctx for reload cases
2819  * \param new_de_ctx the new detection engine
2820  * \param mt flag to indicate if MT should be set up for this det_ctx
2821  * this should only be done for the 'root' det_ctx
2822  *
2823  * \retval det_ctx detection engine thread ctx or NULL in case of error
2824  */
2825 static DetectEngineThreadCtx *DetectEngineThreadCtxInitForReload(
2826  ThreadVars *tv, DetectEngineCtx *new_de_ctx, int mt)
2827 {
2829  if (unlikely(det_ctx == NULL))
2830  return NULL;
2831  memset(det_ctx, 0, sizeof(DetectEngineThreadCtx));
2832 
2833  det_ctx->tenant_id = new_de_ctx->tenant_id;
2834  det_ctx->tv = tv;
2835  det_ctx->de_ctx = DetectEngineReference(new_de_ctx);
2836  if (det_ctx->de_ctx == NULL) {
2837  SCFree(det_ctx);
2838  return NULL;
2839  }
2840 
2841  /* most of the init happens here */
2842  if (det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
2843  det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_TENANT)
2844  {
2845  if (ThreadCtxDoInit(det_ctx->de_ctx, det_ctx) != TM_ECODE_OK) {
2846  DetectEngineDeReference(&det_ctx->de_ctx);
2847  SCFree(det_ctx);
2848  return NULL;
2849  }
2850  }
2851 
2852  /** alert counter setup */
2853  det_ctx->counter_alerts = StatsRegisterCounter("detect.alert", tv);
2854 #ifdef PROFILING
2855  uint16_t counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
2856  uint16_t counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
2857  uint16_t counter_fnonmpm_list = StatsRegisterAvgCounter("detect.fnonmpm_list", tv);
2858  uint16_t counter_match_list = StatsRegisterAvgCounter("detect.match_list", tv);
2859  det_ctx->counter_mpm_list = counter_mpm_list;
2860  det_ctx->counter_nonmpm_list = counter_nonmpm_list;
2861  det_ctx->counter_fnonmpm_list = counter_fnonmpm_list;
2862  det_ctx->counter_match_list = counter_match_list;
2863 #endif
2864 
2865  if (mt && DetectEngineMultiTenantEnabled()) {
2866  if (DetectEngineThreadCtxInitForMT(tv, det_ctx) != TM_ECODE_OK) {
2867  DetectEngineDeReference(&det_ctx->de_ctx);
2868  SCFree(det_ctx);
2869  return NULL;
2870  }
2871  }
2872 
2873  return det_ctx;
2874 }
2875 
2876 static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
2877 {
2878 #if DEBUG
2879  SCLogDebug("PACKET PKT_STREAM_ADD: %"PRIu64, det_ctx->pkt_stream_add_cnt);
2880 
2881  SCLogDebug("PAYLOAD MPM %"PRIu64"/%"PRIu64, det_ctx->payload_mpm_cnt, det_ctx->payload_mpm_size);
2882  SCLogDebug("STREAM MPM %"PRIu64"/%"PRIu64, det_ctx->stream_mpm_cnt, det_ctx->stream_mpm_size);
2883 
2884  SCLogDebug("PAYLOAD SIG %"PRIu64"/%"PRIu64, det_ctx->payload_persig_cnt, det_ctx->payload_persig_size);
2885  SCLogDebug("STREAM SIG %"PRIu64"/%"PRIu64, det_ctx->stream_persig_cnt, det_ctx->stream_persig_size);
2886 #endif
2887 
2888  if (det_ctx->tenant_array != NULL) {
2889  SCFree(det_ctx->tenant_array);
2890  det_ctx->tenant_array = NULL;
2891  }
2892 
2893 #ifdef PROFILING
2897  SCProfilingSghThreadCleanup(det_ctx);
2898 #endif
2899 
2901 
2902  /** \todo get rid of this static */
2903  if (det_ctx->de_ctx != NULL) {
2904  PatternMatchThreadDestroy(&det_ctx->mtc, det_ctx->de_ctx->mpm_matcher);
2905  PatternMatchThreadDestroy(&det_ctx->mtcs, det_ctx->de_ctx->mpm_matcher);
2906  PatternMatchThreadDestroy(&det_ctx->mtcu, det_ctx->de_ctx->mpm_matcher);
2907  }
2908 
2909  PmqFree(&det_ctx->pmq);
2910 
2911  if (det_ctx->spm_thread_ctx != NULL) {
2913  }
2914 
2915  if (det_ctx->non_pf_id_array != NULL)
2916  SCFree(det_ctx->non_pf_id_array);
2917 
2918  if (det_ctx->match_array != NULL)
2919  SCFree(det_ctx->match_array);
2920 
2922 
2923  if (det_ctx->bj_values != NULL)
2924  SCFree(det_ctx->bj_values);
2925 
2926  /* Decoded base64 data. */
2927  if (det_ctx->base64_decoded != NULL) {
2928  SCFree(det_ctx->base64_decoded);
2929  }
2930 
2931  if (det_ctx->inspect.buffers) {
2932  for (uint32_t i = 0; i < det_ctx->inspect.buffers_size; i++) {
2933  InspectionBufferFree(&det_ctx->inspect.buffers[i]);
2934  }
2935  SCFree(det_ctx->inspect.buffers);
2936  }
2937  if (det_ctx->inspect.to_clear_queue) {
2938  SCFree(det_ctx->inspect.to_clear_queue);
2939  }
2940  if (det_ctx->multi_inspect.buffers) {
2941  for (uint32_t i = 0; i < det_ctx->multi_inspect.buffers_size; i++) {
2943  for (uint32_t x = 0; x < fb->size; x++) {
2945  }
2947  }
2948  SCFree(det_ctx->multi_inspect.buffers);
2949  }
2950  if (det_ctx->multi_inspect.to_clear_queue) {
2952  }
2953 
2954  DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx);
2955  if (det_ctx->de_ctx != NULL) {
2956  DetectEngineThreadCtxDeinitKeywords(det_ctx->de_ctx, det_ctx);
2957 #ifdef UNITTESTS
2958  if (!RunmodeIsUnittests() || det_ctx->de_ctx->ref_cnt > 0)
2959  DetectEngineDeReference(&det_ctx->de_ctx);
2960 #else
2961  DetectEngineDeReference(&det_ctx->de_ctx);
2962 #endif
2963  }
2964 
2966 
2967  SCFree(det_ctx);
2968 }
2969 
2971 {
2972  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
2973 
2974  if (det_ctx == NULL) {
2975  SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "argument \"data\" NULL");
2976  return TM_ECODE_OK;
2977  }
2978 
2979  if (det_ctx->mt_det_ctxs_hash != NULL) {
2980  HashTableFree(det_ctx->mt_det_ctxs_hash);
2981  det_ctx->mt_det_ctxs_hash = NULL;
2982  }
2983  DetectEngineThreadCtxFree(det_ctx);
2984 
2985  return TM_ECODE_OK;
2986 }
2987 
2989 {
2990  /* XXX */
2991  PatternMatchThreadPrint(&det_ctx->mtc, det_ctx->de_ctx->mpm_matcher);
2992  PatternMatchThreadPrint(&det_ctx->mtcu, det_ctx->de_ctx->mpm_matcher);
2993 }
2994 
2995 /** \brief Register Thread keyword context Funcs
2996  *
2997  * \param de_ctx detection engine to register in
2998  * \param name keyword name for error printing
2999  * \param InitFunc function ptr
3000  * \param data keyword init data to pass to Func. Can be NULL.
3001  * \param FreeFunc function ptr
3002  * \param mode 0 normal (ctx per keyword instance) 1 shared (one ctx per det_ct)
3003  *
3004  * \retval id for retrieval of ctx at runtime
3005  * \retval -1 on error
3006  *
3007  * \note make sure "data" remains valid and it free'd elsewhere. It's
3008  * recommended to store it in the keywords global ctx so that
3009  * it's freed when the de_ctx is freed.
3010  */
3011 int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int mode)
3012 {
3013  BUG_ON(de_ctx == NULL || InitFunc == NULL || FreeFunc == NULL);
3014 
3015  if (mode) {
3017  while (item != NULL) {
3018  if (strcmp(name, item->name) == 0) {
3019  return item->id;
3020  }
3021 
3022  item = item->next;
3023  }
3024  }
3025 
3027  if (unlikely(item == NULL))
3028  return -1;
3029  memset(item, 0x00, sizeof(DetectEngineThreadKeywordCtxItem));
3030 
3031  item->InitFunc = InitFunc;
3032  item->FreeFunc = FreeFunc;
3033  item->data = data;
3034  item->name = name;
3035 
3036  item->next = de_ctx->keyword_list;
3037  de_ctx->keyword_list = item;
3038  item->id = de_ctx->keyword_id++;
3039 
3040  return item->id;
3041 }
3042 
3043 /** \brief Retrieve thread local keyword ctx by id
3044  *
3045  * \param det_ctx detection engine thread ctx to retrieve the ctx from
3046  * \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
3047  * keyword init.
3048  *
3049  * \retval ctx or NULL on error
3050  */
3052 {
3053  if (id < 0 || id > det_ctx->keyword_ctxs_size || det_ctx->keyword_ctxs_array == NULL)
3054  return NULL;
3055 
3056  return det_ctx->keyword_ctxs_array[id];
3057 }
3058 
3059 
3060 /** \brief Register Thread keyword context Funcs (Global)
3061  *
3062  * IDs stay static over reloads and between tenants
3063  *
3064  * \param name keyword name for error printing
3065  * \param InitFunc function ptr
3066  * \param FreeFunc function ptr
3067  *
3068  * \retval id for retrieval of ctx at runtime
3069  * \retval -1 on error
3070  */
3072  void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *))
3073 {
3074  int id;
3075  BUG_ON(InitFunc == NULL || FreeFunc == NULL);
3076 
3077  DetectEngineMasterCtx *master = &g_master_de_ctx;
3078 
3079  /* if already registered, return existing id */
3081  while (item != NULL) {
3082  if (strcmp(name, item->name) == 0) {
3083  id = item->id;
3084  return id;
3085  }
3086 
3087  item = item->next;
3088  }
3089 
3090  item = SCCalloc(1, sizeof(*item));
3091  if (unlikely(item == NULL)) {
3092  return -1;
3093  }
3094  item->InitFunc = InitFunc;
3095  item->FreeFunc = FreeFunc;
3096  item->name = name;
3097  item->data = data;
3098 
3099  item->next = master->keyword_list;
3100  master->keyword_list = item;
3101  item->id = master->keyword_id++;
3102 
3103  id = item->id;
3104  return id;
3105 }
3106 
3107 /** \brief Retrieve thread local keyword ctx by id
3108  *
3109  * \param det_ctx detection engine thread ctx to retrieve the ctx from
3110  * \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
3111  * keyword init.
3112  *
3113  * \retval ctx or NULL on error
3114  */
3116 {
3117  if (id < 0 || id > det_ctx->global_keyword_ctxs_size ||
3118  det_ctx->global_keyword_ctxs_array == NULL) {
3119  return NULL;
3120  }
3121 
3122  return det_ctx->global_keyword_ctxs_array[id];
3123 }
3124 
3125 /** \brief Check if detection is enabled
3126  * \retval bool true or false */
3128 {
3129  DetectEngineMasterCtx *master = &g_master_de_ctx;
3130  SCMutexLock(&master->lock);
3131 
3132  if (master->list == NULL) {
3133  SCMutexUnlock(&master->lock);
3134  return 0;
3135  }
3136 
3137  SCMutexUnlock(&master->lock);
3138  return 1;
3139 }
3140 
3142 {
3143  uint32_t version;
3144  DetectEngineMasterCtx *master = &g_master_de_ctx;
3145  SCMutexLock(&master->lock);
3146  version = master->version;
3147  SCMutexUnlock(&master->lock);
3148  return version;
3149 }
3150 
3152 {
3153  DetectEngineMasterCtx *master = &g_master_de_ctx;
3154  SCMutexLock(&master->lock);
3155  master->version++;
3156  SCLogDebug("master version now %u", master->version);
3157  SCMutexUnlock(&master->lock);
3158 }
3159 
3161 {
3162  DetectEngineMasterCtx *master = &g_master_de_ctx;
3163  SCMutexLock(&master->lock);
3164 
3165  DetectEngineCtx *de_ctx = master->list;
3166  while (de_ctx) {
3167  if (de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
3168  de_ctx->type == DETECT_ENGINE_TYPE_DD_STUB ||
3169  de_ctx->type == DETECT_ENGINE_TYPE_MT_STUB)
3170  {
3171  de_ctx->ref_cnt++;
3172  SCLogDebug("de_ctx %p ref_cnt %u", de_ctx, de_ctx->ref_cnt);
3173  SCMutexUnlock(&master->lock);
3174  return de_ctx;
3175  }
3176  de_ctx = de_ctx->next;
3177  }
3178 
3179  SCMutexUnlock(&master->lock);
3180  return NULL;
3181 }
3182 
3184 {
3185  if (de_ctx == NULL)
3186  return NULL;
3187  de_ctx->ref_cnt++;
3188  return de_ctx;
3189 }
3190 
3191 /** TODO locking? Not needed if this is a one time setting at startup */
3193 {
3194  DetectEngineMasterCtx *master = &g_master_de_ctx;
3195  return (master->multi_tenant_enabled);
3196 }
3197 
3198 /** \internal
3199  * \brief load a tenant from a yaml file
3200  *
3201  * \param tenant_id the tenant id by which the config is known
3202  * \param filename full path of a yaml file
3203  * \param loader_id id of loader thread or -1
3204  *
3205  * \retval 0 ok
3206  * \retval -1 failed
3207  */
3208 static int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename, int loader_id)
3209 {
3210  DetectEngineCtx *de_ctx = NULL;
3211  char prefix[64];
3212 
3213  snprintf(prefix, sizeof(prefix), "multi-detect.%d", tenant_id);
3214 
3215 #ifdef OS_WIN32
3216  struct _stat st;
3217  if(_stat(filename, &st) != 0) {
3218 #else
3219  struct stat st;
3220  if(stat(filename, &st) != 0) {
3221 #endif /* OS_WIN32 */
3222  SCLogError(SC_ERR_FOPEN, "failed to stat file %s", filename);
3223  goto error;
3224  }
3225 
3226  de_ctx = DetectEngineGetByTenantId(tenant_id);
3227  if (de_ctx != NULL) {
3228  SCLogError(SC_ERR_MT_DUPLICATE_TENANT, "tenant %u already registered",
3229  tenant_id);
3230  DetectEngineDeReference(&de_ctx);
3231  goto error;
3232  }
3233 
3234  ConfNode *node = ConfGetNode(prefix);
3235  if (node == NULL) {
3236  SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to properly setup yaml %s", filename);
3237  goto error;
3238  }
3239 
3240  de_ctx = DetectEngineCtxInitWithPrefix(prefix);
3241  if (de_ctx == NULL) {
3242  SCLogError(SC_ERR_INITIALIZATION, "initializing detection engine "
3243  "context failed.");
3244  goto error;
3245  }
3246  SCLogDebug("de_ctx %p with prefix %s", de_ctx, de_ctx->config_prefix);
3247 
3248  de_ctx->type = DETECT_ENGINE_TYPE_TENANT;
3249  de_ctx->tenant_id = tenant_id;
3250  de_ctx->loader_id = loader_id;
3251 
3252  if (SigLoadSignatures(de_ctx, NULL, 0) < 0) {
3253  SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
3254  goto error;
3255  }
3256 
3257  DetectEngineAddToMaster(de_ctx);
3258 
3259  return 0;
3260 
3261 error:
3262  if (de_ctx != NULL) {
3263  DetectEngineCtxFree(de_ctx);
3264  }
3265  return -1;
3266 }
3267 
3268 static int DetectEngineMultiTenantReloadTenant(uint32_t tenant_id, const char *filename, int reload_cnt)
3269 {
3270  DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
3271  if (old_de_ctx == NULL) {
3272  SCLogError(SC_ERR_INITIALIZATION, "tenant detect engine not found");
3273  return -1;
3274  }
3275 
3276  char prefix[64];
3277  snprintf(prefix, sizeof(prefix), "multi-detect.%d.reload.%d", tenant_id, reload_cnt);
3278  reload_cnt++;
3279  SCLogDebug("prefix %s", prefix);
3280 
3281  if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) {
3282  SCLogError(SC_ERR_INITIALIZATION,"failed to load yaml");
3283  goto error;
3284  }
3285 
3286  ConfNode *node = ConfGetNode(prefix);
3287  if (node == NULL) {
3288  SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to properly setup yaml %s", filename);
3289  goto error;
3290  }
3291 
3292  DetectEngineCtx *new_de_ctx = DetectEngineCtxInitWithPrefix(prefix);
3293  if (new_de_ctx == NULL) {
3294  SCLogError(SC_ERR_INITIALIZATION, "initializing detection engine "
3295  "context failed.");
3296  goto error;
3297  }
3298  SCLogDebug("de_ctx %p with prefix %s", new_de_ctx, new_de_ctx->config_prefix);
3299 
3300  new_de_ctx->type = DETECT_ENGINE_TYPE_TENANT;
3301  new_de_ctx->tenant_id = tenant_id;
3302  new_de_ctx->loader_id = old_de_ctx->loader_id;
3303 
3304  if (SigLoadSignatures(new_de_ctx, NULL, 0) < 0) {
3305  SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
3306  goto error;
3307  }
3308 
3309  DetectEngineAddToMaster(new_de_ctx);
3310 
3311  /* move to free list */
3312  DetectEngineMoveToFreeList(old_de_ctx);
3313  DetectEngineDeReference(&old_de_ctx);
3314  return 0;
3315 
3316 error:
3317  DetectEngineDeReference(&old_de_ctx);
3318  return -1;
3319 }
3320 
3321 
3322 typedef struct TenantLoaderCtx_ {
3323  uint32_t tenant_id;
3324  int reload_cnt; /**< used by reload */
3325  const char *yaml;
3326 } TenantLoaderCtx;
3327 
3328 static int DetectLoaderFuncLoadTenant(void *vctx, int loader_id)
3329 {
3330  TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
3331 
3332  SCLogDebug("loader %d", loader_id);
3333  if (DetectEngineMultiTenantLoadTenant(ctx->tenant_id, ctx->yaml, loader_id) != 0) {
3334  return -1;
3335  }
3336  return 0;
3337 }
3338 
3339 static int DetectLoaderSetupLoadTenant(uint32_t tenant_id, const char *yaml)
3340 {
3341  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
3342  if (t == NULL)
3343  return -ENOMEM;
3344 
3345  t->tenant_id = tenant_id;
3346  t->yaml = yaml;
3347 
3348  return DetectLoaderQueueTask(-1, DetectLoaderFuncLoadTenant, t);
3349 }
3350 
3351 static int DetectLoaderFuncReloadTenant(void *vctx, int loader_id)
3352 {
3353  TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
3354 
3355  SCLogDebug("loader_id %d", loader_id);
3356 
3357  if (DetectEngineMultiTenantReloadTenant(ctx->tenant_id, ctx->yaml, ctx->reload_cnt) != 0) {
3358  return -1;
3359  }
3360  return 0;
3361 }
3362 
3363 static int DetectLoaderSetupReloadTenant(uint32_t tenant_id, const char *yaml, int reload_cnt)
3364 {
3365  DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
3366  if (old_de_ctx == NULL)
3367  return -ENOENT;
3368  int loader_id = old_de_ctx->loader_id;
3369  DetectEngineDeReference(&old_de_ctx);
3370 
3371  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
3372  if (t == NULL)
3373  return -ENOMEM;
3374 
3375  t->tenant_id = tenant_id;
3376  t->yaml = yaml;
3377  t->reload_cnt = reload_cnt;
3378 
3379  SCLogDebug("loader_id %d", loader_id);
3380 
3381  return DetectLoaderQueueTask(loader_id, DetectLoaderFuncReloadTenant, t);
3382 }
3383 
3384 /** \brief Load a tenant and wait for loading to complete
3385  */
3386 int DetectEngineLoadTenantBlocking(uint32_t tenant_id, const char *yaml)
3387 {
3388  int r = DetectLoaderSetupLoadTenant(tenant_id, yaml);
3389  if (r < 0)
3390  return r;
3391 
3392  if (DetectLoadersSync() != 0)
3393  return -1;
3394 
3395  return 0;
3396 }
3397 
3398 /** \brief Reload a tenant and wait for loading to complete
3399  */
3400 int DetectEngineReloadTenantBlocking(uint32_t tenant_id, const char *yaml, int reload_cnt)
3401 {
3402  int r = DetectLoaderSetupReloadTenant(tenant_id, yaml, reload_cnt);
3403  if (r < 0)
3404  return r;
3405 
3406  if (DetectLoadersSync() != 0)
3407  return -1;
3408 
3409  return 0;
3410 }
3411 
3412 static int DetectEngineMultiTenantSetupLoadLivedevMappings(const ConfNode *mappings_root_node,
3413  bool failure_fatal)
3414 {
3415  ConfNode *mapping_node = NULL;
3416 
3417  int mapping_cnt = 0;
3418  if (mappings_root_node != NULL) {
3419  TAILQ_FOREACH(mapping_node, &mappings_root_node->head, next) {
3420  ConfNode *tenant_id_node = ConfNodeLookupChild(mapping_node, "tenant-id");
3421  if (tenant_id_node == NULL)
3422  goto bad_mapping;
3423  ConfNode *device_node = ConfNodeLookupChild(mapping_node, "device");
3424  if (device_node == NULL)
3425  goto bad_mapping;
3426 
3427  uint32_t tenant_id = 0;
3428  if (ByteExtractStringUint32(&tenant_id, 10, strlen(tenant_id_node->val),
3429  tenant_id_node->val) == -1)
3430  {
3431  SCLogError(SC_ERR_INVALID_ARGUMENT, "tenant-id "
3432  "of %s is invalid", tenant_id_node->val);
3433  goto bad_mapping;
3434  }
3435 
3436  const char *dev = device_node->val;
3437  LiveDevice *ld = LiveGetDevice(dev);
3438  if (ld == NULL) {
3439  SCLogWarning(SC_ERR_MT_NO_MAPPING, "device %s not found", dev);
3440  goto bad_mapping;
3441  }
3442 
3443  if (ld->tenant_id_set) {
3444  SCLogWarning(SC_ERR_MT_NO_MAPPING, "device %s already mapped to tenant-id %u",
3445  dev, ld->tenant_id);
3446  goto bad_mapping;
3447  }
3448 
3449  ld->tenant_id = tenant_id;
3450  ld->tenant_id_set = true;
3451 
3452  if (DetectEngineTentantRegisterLivedev(tenant_id, ld->id) != 0) {
3453  goto error;
3454  }
3455 
3456  SCLogConfig("device %s connected to tenant-id %u", dev, tenant_id);
3457  mapping_cnt++;
3458  continue;
3459 
3460  bad_mapping:
3461  if (failure_fatal)
3462  goto error;
3463  }
3464  }
3465  SCLogConfig("%d device - tenant-id mappings defined", mapping_cnt);
3466  return mapping_cnt;
3467 
3468 error:
3469  return 0;
3470 }
3471 
3472 static int DetectEngineMultiTenantSetupLoadVlanMappings(const ConfNode *mappings_root_node,
3473  bool failure_fatal)
3474 {
3475  ConfNode *mapping_node = NULL;
3476 
3477  int mapping_cnt = 0;
3478  if (mappings_root_node != NULL) {
3479  TAILQ_FOREACH(mapping_node, &mappings_root_node->head, next) {
3480  ConfNode *tenant_id_node = ConfNodeLookupChild(mapping_node, "tenant-id");
3481  if (tenant_id_node == NULL)
3482  goto bad_mapping;
3483  ConfNode *vlan_id_node = ConfNodeLookupChild(mapping_node, "vlan-id");
3484  if (vlan_id_node == NULL)
3485  goto bad_mapping;
3486 
3487  uint32_t tenant_id = 0;
3488  if (ByteExtractStringUint32(&tenant_id, 10, strlen(tenant_id_node->val),
3489  tenant_id_node->val) == -1)
3490  {
3491  SCLogError(SC_ERR_INVALID_ARGUMENT, "tenant-id "
3492  "of %s is invalid", tenant_id_node->val);
3493  goto bad_mapping;
3494  }
3495 
3496  uint16_t vlan_id = 0;
3497  if (ByteExtractStringUint16(&vlan_id, 10, strlen(vlan_id_node->val),
3498  vlan_id_node->val) == -1)
3499  {
3501  "of %s is invalid", vlan_id_node->val);
3502  goto bad_mapping;
3503  }
3504  if (vlan_id == 0 || vlan_id >= 4095) {
3506  "of %s is invalid. Valid range 1-4094.", vlan_id_node->val);
3507  goto bad_mapping;
3508  }
3509 
3510  if (DetectEngineTentantRegisterVlanId(tenant_id, (uint32_t)vlan_id) != 0) {
3511  goto error;
3512  }
3513  SCLogConfig("vlan %u connected to tenant-id %u", vlan_id, tenant_id);
3514  mapping_cnt++;
3515  continue;
3516 
3517  bad_mapping:
3518  if (failure_fatal)
3519  goto error;
3520  }
3521  }
3522  return mapping_cnt;
3523 
3524 error:
3525  return 0;
3526 }
3527 
3528 /**
3529  * \brief setup multi-detect / multi-tenancy
3530  *
3531  * See if MT is enabled. If so, setup the selector, tenants and mappings.
3532  * Tenants and mappings are optional, and can also dynamically be added
3533  * and removed from the unix socket.
3534  */
3536 {
3538  DetectEngineMasterCtx *master = &g_master_de_ctx;
3539 
3540  int unix_socket = ConfUnixSocketIsEnable();
3541 
3542  int failure_fatal = 0;
3543  (void)ConfGetBool("engine.init-failure-fatal", &failure_fatal);
3544 
3545  int enabled = 0;
3546  (void)ConfGetBool("multi-detect.enabled", &enabled);
3547  if (enabled == 1) {
3552 
3553  SCMutexLock(&master->lock);
3554  master->multi_tenant_enabled = 1;
3555 
3556  const char *handler = NULL;
3557  if (ConfGet("multi-detect.selector", &handler) == 1) {
3558  SCLogConfig("multi-tenant selector type %s", handler);
3559 
3560  if (strcmp(handler, "vlan") == 0) {
3561  tenant_selector = master->tenant_selector = TENANT_SELECTOR_VLAN;
3562 
3563  int vlanbool = 0;
3564  if ((ConfGetBool("vlan.use-for-tracking", &vlanbool)) == 1 && vlanbool == 0) {
3565  SCLogError(SC_ERR_INVALID_VALUE, "vlan tracking is disabled, "
3566  "can't use multi-detect selector 'vlan'");
3567  SCMutexUnlock(&master->lock);
3568  goto error;
3569  }
3570 
3571  } else if (strcmp(handler, "direct") == 0) {
3572  tenant_selector = master->tenant_selector = TENANT_SELECTOR_DIRECT;
3573  } else if (strcmp(handler, "device") == 0) {
3574  tenant_selector = master->tenant_selector = TENANT_SELECTOR_LIVEDEV;
3575  if (EngineModeIsIPS()) {
3577  "multi-tenant 'device' mode not supported for IPS");
3578  SCMutexUnlock(&master->lock);
3579  goto error;
3580  }
3581 
3582  } else {
3583  SCLogError(SC_ERR_INVALID_VALUE, "unknown value %s "
3584  "multi-detect.selector", handler);
3585  SCMutexUnlock(&master->lock);
3586  goto error;
3587  }
3588  }
3589  SCMutexUnlock(&master->lock);
3590  SCLogConfig("multi-detect is enabled (multi tenancy). Selector: %s", handler);
3591 
3592  /* traffic -- tenant mappings */
3593  ConfNode *mappings_root_node = ConfGetNode("multi-detect.mappings");
3594 
3595  if (tenant_selector == TENANT_SELECTOR_VLAN) {
3596  int mapping_cnt = DetectEngineMultiTenantSetupLoadVlanMappings(mappings_root_node,
3597  failure_fatal);
3598  if (mapping_cnt == 0) {
3599  /* no mappings are valid when we're in unix socket mode,
3600  * they can be added on the fly. Otherwise warn/error
3601  * depending on failure_fatal */
3602 
3603  if (unix_socket) {
3604  SCLogNotice("no tenant traffic mappings defined, "
3605  "tenants won't be used until mappings are added");
3606  } else {
3607  if (failure_fatal) {
3608  SCLogError(SC_ERR_MT_NO_MAPPING, "no multi-detect mappings defined");
3609  goto error;
3610  } else {
3611  SCLogWarning(SC_ERR_MT_NO_MAPPING, "no multi-detect mappings defined");
3612  }
3613  }
3614  }
3615  } else if (tenant_selector == TENANT_SELECTOR_LIVEDEV) {
3616  int mapping_cnt = DetectEngineMultiTenantSetupLoadLivedevMappings(mappings_root_node,
3617  failure_fatal);
3618  if (mapping_cnt == 0) {
3619  if (failure_fatal) {
3620  SCLogError(SC_ERR_MT_NO_MAPPING, "no multi-detect mappings defined");
3621  goto error;
3622  } else {
3623  SCLogWarning(SC_ERR_MT_NO_MAPPING, "no multi-detect mappings defined");
3624  }
3625  }
3626  }
3627 
3628  /* tenants */
3629  ConfNode *tenants_root_node = ConfGetNode("multi-detect.tenants");
3630  ConfNode *tenant_node = NULL;
3631 
3632  if (tenants_root_node != NULL) {
3633  TAILQ_FOREACH(tenant_node, &tenants_root_node->head, next) {
3634  ConfNode *id_node = ConfNodeLookupChild(tenant_node, "id");
3635  if (id_node == NULL) {
3636  goto bad_tenant;
3637  }
3638  ConfNode *yaml_node = ConfNodeLookupChild(tenant_node, "yaml");
3639  if (yaml_node == NULL) {
3640  goto bad_tenant;
3641  }
3642 
3643  uint32_t tenant_id = 0;
3644  if (ByteExtractStringUint32(&tenant_id, 10, strlen(id_node->val),
3645  id_node->val) == -1)
3646  {
3647  SCLogError(SC_ERR_INVALID_ARGUMENT, "tenant_id "
3648  "of %s is invalid", id_node->val);
3649  goto bad_tenant;
3650  }
3651  SCLogDebug("tenant id: %u, %s", tenant_id, yaml_node->val);
3652 
3653  /* setup the yaml in this loop so that it's not done by the loader
3654  * threads. ConfYamlLoadFileWithPrefix is not thread safe. */
3655  char prefix[64];
3656  snprintf(prefix, sizeof(prefix), "multi-detect.%d", tenant_id);
3657  if (ConfYamlLoadFileWithPrefix(yaml_node->val, prefix) != 0) {
3658  SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to load yaml %s", yaml_node->val);
3659  goto bad_tenant;
3660  }
3661 
3662  int r = DetectLoaderSetupLoadTenant(tenant_id, yaml_node->val);
3663  if (r < 0) {
3664  /* error logged already */
3665  goto bad_tenant;
3666  }
3667  continue;
3668 
3669  bad_tenant:
3670  if (failure_fatal)
3671  goto error;
3672  }
3673  }
3674 
3675  /* wait for our loaders to complete their tasks */
3676  if (DetectLoadersSync() != 0) {
3677  goto error;
3678  }
3679 
3681 
3682  } else {
3683  SCLogDebug("multi-detect not enabled (multi tenancy)");
3684  }
3685  return 0;
3686 error:
3687  return -1;
3688 }
3689 
3690 static uint32_t DetectEngineTentantGetIdFromVlanId(const void *ctx, const Packet *p)
3691 {
3692  const DetectEngineThreadCtx *det_ctx = ctx;
3693  uint32_t x = 0;
3694  uint32_t vlan_id = 0;
3695 
3696  if (p->vlan_idx == 0)
3697  return 0;
3698 
3699  vlan_id = p->vlan_id[0];
3700 
3701  if (det_ctx == NULL || det_ctx->tenant_array == NULL || det_ctx->tenant_array_size == 0)
3702  return 0;
3703 
3704  /* not very efficient, but for now we're targeting only limited amounts.
3705  * Can use hash/tree approach later. */
3706  for (x = 0; x < det_ctx->tenant_array_size; x++) {
3707  if (det_ctx->tenant_array[x].traffic_id == vlan_id)
3708  return det_ctx->tenant_array[x].tenant_id;
3709  }
3710 
3711  return 0;
3712 }
3713 
3714 static uint32_t DetectEngineTentantGetIdFromLivedev(const void *ctx, const Packet *p)
3715 {
3716  const DetectEngineThreadCtx *det_ctx = ctx;
3717  const LiveDevice *ld = p->livedev;
3718 
3719  if (ld == NULL || det_ctx == NULL)
3720  return 0;
3721 
3722  SCLogDebug("using tenant-id %u for packet on device %s", ld->tenant_id, ld->dev);
3723  return ld->tenant_id;
3724 }
3725 
3726 static int DetectEngineTentantRegisterSelector(enum DetectEngineTenantSelectors selector,
3727  uint32_t tenant_id, uint32_t traffic_id)
3728 {
3729  DetectEngineMasterCtx *master = &g_master_de_ctx;
3730  SCMutexLock(&master->lock);
3731 
3732  if (!(master->tenant_selector == TENANT_SELECTOR_UNKNOWN || master->tenant_selector == selector)) {
3733  SCLogInfo("conflicting selector already set");
3734  SCMutexUnlock(&master->lock);
3735  return -1;
3736  }
3737 
3739  while (m) {
3740  if (m->traffic_id == traffic_id) {
3741  SCLogInfo("traffic id already registered");
3742  SCMutexUnlock(&master->lock);
3743  return -1;
3744  }
3745  m = m->next;
3746  }
3747 
3748  DetectEngineTenantMapping *map = SCCalloc(1, sizeof(*map));
3749  if (map == NULL) {
3750  SCLogInfo("memory fail");
3751  SCMutexUnlock(&master->lock);
3752  return -1;
3753  }
3754  map->traffic_id = traffic_id;
3755  map->tenant_id = tenant_id;
3756 
3757  map->next = master->tenant_mapping_list;
3758  master->tenant_mapping_list = map;
3759 
3760  master->tenant_selector = selector;
3761 
3762  SCLogDebug("tenant handler %u %u %u registered", selector, tenant_id, traffic_id);
3763  SCMutexUnlock(&master->lock);
3764  return 0;
3765 }
3766 
3767 static int DetectEngineTentantUnregisterSelector(enum DetectEngineTenantSelectors selector,
3768  uint32_t tenant_id, uint32_t traffic_id)
3769 {
3770  DetectEngineMasterCtx *master = &g_master_de_ctx;
3771  SCMutexLock(&master->lock);
3772 
3773  if (master->tenant_mapping_list == NULL) {
3774  SCMutexUnlock(&master->lock);
3775  return -1;
3776  }
3777 
3778  DetectEngineTenantMapping *prev = NULL;
3780  while (map) {
3781  if (map->traffic_id == traffic_id &&
3782  map->tenant_id == tenant_id)
3783  {
3784  if (prev != NULL)
3785  prev->next = map->next;
3786  else
3787  master->tenant_mapping_list = map->next;
3788 
3789  map->next = NULL;
3790  SCFree(map);
3791  SCLogInfo("tenant handler %u %u %u unregistered", selector, tenant_id, traffic_id);
3792  SCMutexUnlock(&master->lock);
3793  return 0;
3794  }
3795  prev = map;
3796  map = map->next;
3797  }
3798 
3799  SCMutexUnlock(&master->lock);
3800  return -1;
3801 }
3802 
3803 int DetectEngineTentantRegisterLivedev(uint32_t tenant_id, int device_id)
3804 {
3805  return DetectEngineTentantRegisterSelector(TENANT_SELECTOR_LIVEDEV, tenant_id, (uint32_t)device_id);
3806 }
3807 
3808 int DetectEngineTentantRegisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
3809 {
3810  return DetectEngineTentantRegisterSelector(TENANT_SELECTOR_VLAN, tenant_id, (uint32_t)vlan_id);
3811 }
3812 
3813 int DetectEngineTentantUnregisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
3814 {
3815  return DetectEngineTentantUnregisterSelector(TENANT_SELECTOR_VLAN, tenant_id, (uint32_t)vlan_id);
3816 }
3817 
3819 {
3820  SCLogInfo("registering %u %d 0", TENANT_SELECTOR_DIRECT, tenant_id);
3821  return DetectEngineTentantRegisterSelector(TENANT_SELECTOR_DIRECT, tenant_id, 0);
3822 }
3823 
3825 {
3826  SCLogInfo("unregistering %u %d 0", TENANT_SELECTOR_DIRECT, tenant_id);
3827  return DetectEngineTentantUnregisterSelector(TENANT_SELECTOR_DIRECT, tenant_id, 0);
3828 }
3829 
3830 static uint32_t DetectEngineTentantGetIdFromPcap(const void *ctx, const Packet *p)
3831 {
3832  return p->pcap_v.tenant_id;
3833 }
3834 
3836 {
3837  DetectEngineMasterCtx *master = &g_master_de_ctx;
3838  SCMutexLock(&master->lock);
3839 
3840  if (master->list == NULL) {
3841  SCMutexUnlock(&master->lock);
3842  return NULL;
3843  }
3844 
3845  DetectEngineCtx *de_ctx = master->list;
3846  while (de_ctx) {
3847  if (de_ctx->type == DETECT_ENGINE_TYPE_TENANT &&
3848  de_ctx->tenant_id == tenant_id)
3849  {
3850  de_ctx->ref_cnt++;
3851  break;
3852  }
3853 
3854  de_ctx = de_ctx->next;
3855  }
3856 
3857  SCMutexUnlock(&master->lock);
3858  return de_ctx;
3859 }
3860 
3862 {
3863  BUG_ON((*de_ctx)->ref_cnt == 0);
3864  (*de_ctx)->ref_cnt--;
3865  *de_ctx = NULL;
3866 }
3867 
3868 static int DetectEngineAddToList(DetectEngineCtx *instance)
3869 {
3870  DetectEngineMasterCtx *master = &g_master_de_ctx;
3871 
3872  if (instance == NULL)
3873  return -1;
3874 
3875  if (master->list == NULL) {
3876  master->list = instance;
3877  } else {
3878  instance->next = master->list;
3879  master->list = instance;
3880  }
3881 
3882  return 0;
3883 }
3884 
3886 {
3887  int r;
3888 
3889  if (de_ctx == NULL)
3890  return -1;
3891 
3892  SCLogDebug("adding de_ctx %p to master", de_ctx);
3893 
3894  DetectEngineMasterCtx *master = &g_master_de_ctx;
3895  SCMutexLock(&master->lock);
3896  r = DetectEngineAddToList(de_ctx);
3897  SCMutexUnlock(&master->lock);
3898  return r;
3899 }
3900 
3902 {
3903  DetectEngineMasterCtx *master = &g_master_de_ctx;
3904 
3905  SCMutexLock(&master->lock);
3906  DetectEngineCtx *instance = master->list;
3907  if (instance == NULL) {
3908  SCMutexUnlock(&master->lock);
3909  return -1;
3910  }
3911 
3912  /* remove from active list */
3913  if (instance == de_ctx) {
3914  master->list = instance->next;
3915  } else {
3916  DetectEngineCtx *prev = instance;
3917  instance = instance->next; /* already checked first element */
3918 
3919  while (instance) {
3920  DetectEngineCtx *next = instance->next;
3921 
3922  if (instance == de_ctx) {
3923  prev->next = instance->next;
3924  break;
3925  }
3926 
3927  prev = instance;
3928  instance = next;
3929  }
3930  if (instance == NULL) {
3931  SCMutexUnlock(&master->lock);
3932  return -1;
3933  }
3934  }
3935 
3936  /* instance is now detached from list */
3937  instance->next = NULL;
3938 
3939  /* add to free list */
3940  if (master->free_list == NULL) {
3941  master->free_list = instance;
3942  } else {
3943  instance->next = master->free_list;
3944  master->free_list = instance;
3945  }
3946  SCLogDebug("detect engine %p moved to free list (%u refs)", de_ctx, de_ctx->ref_cnt);
3947 
3948  SCMutexUnlock(&master->lock);
3949  return 0;
3950 }
3951 
3953 {
3954  DetectEngineMasterCtx *master = &g_master_de_ctx;
3955  SCMutexLock(&master->lock);
3956 
3957  DetectEngineCtx *prev = NULL;
3958  DetectEngineCtx *instance = master->free_list;
3959  while (instance) {
3960  DetectEngineCtx *next = instance->next;
3961 
3962  SCLogDebug("detect engine %p has %u ref(s)", instance, instance->ref_cnt);
3963 
3964  if (instance->ref_cnt == 0) {
3965  if (prev == NULL) {
3966  master->free_list = next;
3967  } else {
3968  prev->next = next;
3969  }
3970 
3971  SCLogDebug("freeing detect engine %p", instance);
3972  DetectEngineCtxFree(instance);
3973  instance = NULL;
3974  }
3975 
3976  prev = instance;
3977  instance = next;
3978  }
3979  SCMutexUnlock(&master->lock);
3980 }
3981 
3982 static int reloads = 0;
3983 
3984 /** \brief Reload the detection engine
3985  *
3986  * \param filename YAML file to load for the detect config
3987  *
3988  * \retval -1 error
3989  * \retval 0 ok
3990  */
3992 {
3993  DetectEngineCtx *new_de_ctx = NULL;
3994  DetectEngineCtx *old_de_ctx = NULL;
3995 
3996  char prefix[128];
3997  memset(prefix, 0, sizeof(prefix));
3998 
3999  SCLogNotice("rule reload starting");
4000 
4001  if (suri->conf_filename != NULL) {
4002  snprintf(prefix, sizeof(prefix), "detect-engine-reloads.%d", reloads++);
4003  if (ConfYamlLoadFileWithPrefix(suri->conf_filename, prefix) != 0) {
4004  SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to load yaml %s",
4005  suri->conf_filename);
4006  return -1;
4007  }
4008 
4009  ConfNode *node = ConfGetNode(prefix);
4010  if (node == NULL) {
4011  SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to properly setup yaml %s",
4012  suri->conf_filename);
4013  return -1;
4014  }
4015 #if 0
4016  ConfDump();
4017 #endif
4018  }
4019 
4020  /* get a reference to the current de_ctx */
4021  old_de_ctx = DetectEngineGetCurrent();
4022  if (old_de_ctx == NULL)
4023  return -1;
4024  SCLogDebug("get ref to old_de_ctx %p", old_de_ctx);
4025 
4026  /* only reload a regular 'normal' and 'delayed detect stub' detect engines */
4027  if (!(old_de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
4028  old_de_ctx->type == DETECT_ENGINE_TYPE_DD_STUB))
4029  {
4030  DetectEngineDeReference(&old_de_ctx);
4031  SCLogNotice("rule reload complete");
4032  return -1;
4033  }
4034 
4035  /* get new detection engine */
4036  new_de_ctx = DetectEngineCtxInitWithPrefix(prefix);
4037  if (new_de_ctx == NULL) {
4038  SCLogError(SC_ERR_INITIALIZATION, "initializing detection engine "
4039  "context failed.");
4040  DetectEngineDeReference(&old_de_ctx);
4041  return -1;
4042  }
4043  if (SigLoadSignatures(new_de_ctx,
4044  suri->sig_file, suri->sig_file_exclusive) != 0) {
4045  DetectEngineCtxFree(new_de_ctx);
4046  DetectEngineDeReference(&old_de_ctx);
4047  return -1;
4048  }
4049  SCLogDebug("set up new_de_ctx %p", new_de_ctx);
4050 
4051  /* add to master */
4052  DetectEngineAddToMaster(new_de_ctx);
4053 
4054  /* move to old free list */
4055  DetectEngineMoveToFreeList(old_de_ctx);
4056  DetectEngineDeReference(&old_de_ctx);
4057 
4058  SCLogDebug("going to reload the threads to use new_de_ctx %p", new_de_ctx);
4059  /* update the threads */
4060  DetectEngineReloadThreads(new_de_ctx);
4061  SCLogDebug("threads now run new_de_ctx %p", new_de_ctx);
4062 
4063  /* walk free list, freeing the old_de_ctx */
4065 
4067 
4068  SCLogDebug("old_de_ctx should have been freed");
4069 
4070  SCLogNotice("rule reload complete");
4071  return 0;
4072 }
4073 
4074 static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len)
4075 {
4076  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
4077  return det_ctx->tenant_id % h->array_size;
4078 }
4079 
4080 static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len)
4081 {
4084  return (det1->tenant_id == det2->tenant_id);
4085 }
4086 
4087 static void TenantIdFree(void *d)
4088 {
4089  DetectEngineThreadCtxFree(d);
4090 }
4091 
4093 {
4094  DetectEngineMasterCtx *master = &g_master_de_ctx;
4095  SCMutexLock(&master->lock);
4096 
4097  if (master->tenant_selector == TENANT_SELECTOR_UNKNOWN) {
4098  SCLogInfo("error, no tenant selector");
4099  SCMutexUnlock(&master->lock);
4100  return -1;
4101  }
4102 
4103  DetectEngineCtx *stub_de_ctx = NULL;
4104  DetectEngineCtx *list = master->list;
4105  for ( ; list != NULL; list = list->next) {
4106  SCLogDebug("list %p tenant %u", list, list->tenant_id);
4107 
4108  if (list->type == DETECT_ENGINE_TYPE_NORMAL ||
4109  list->type == DETECT_ENGINE_TYPE_MT_STUB ||
4111  {
4112  stub_de_ctx = list;
4113  break;
4114  }
4115  }
4116  if (stub_de_ctx == NULL) {
4117  stub_de_ctx = DetectEngineCtxInitStubForMT();
4118  if (stub_de_ctx == NULL) {
4119  SCMutexUnlock(&master->lock);
4120  return -1;
4121  }
4122 
4123  if (master->list == NULL) {
4124  master->list = stub_de_ctx;
4125  } else {
4126  stub_de_ctx->next = master->list;
4127  master->list = stub_de_ctx;
4128  }
4129  }
4130 
4131  /* update the threads */
4132  SCLogDebug("MT reload starting");
4133  DetectEngineReloadThreads(stub_de_ctx);
4134  SCLogDebug("MT reload done");
4135 
4136  SCMutexUnlock(&master->lock);
4137 
4138  /* walk free list, freeing the old_de_ctx */
4140 
4141  SCLogDebug("old_de_ctx should have been freed");
4142  return 0;
4143 }
4144 
4145 static int g_parse_metadata = 0;
4146 
4148 {
4149  g_parse_metadata = 1;
4150 }
4151 
4153 {
4154  g_parse_metadata = 0;
4155 }
4156 
4158 {
4159  return g_parse_metadata;
4160 }
4161 
4163 {
4164  switch (type) {
4165  case DETECT_SM_LIST_MATCH:
4166  return "packet";
4167  case DETECT_SM_LIST_PMATCH:
4168  return "packet/stream payload";
4169 
4170  case DETECT_SM_LIST_TMATCH:
4171  return "tag";
4172 
4174  return "base64_data";
4175 
4177  return "post-match";
4178 
4180  return "suppress";
4182  return "threshold";
4183 
4184  case DETECT_SM_LIST_MAX:
4185  return "max (internal)";
4186  }
4187  return "error";
4188 }
4189 
4190 /* events api */
4192 {
4194  det_ctx->events++;
4195 }
4196 
4198 {
4199  return det_ctx->decoder_events;
4200 }
4201 
4202 int DetectEngineGetEventInfo(const char *event_name, int *event_id,
4204 {
4205  *event_id = SCMapEnumNameToValue(event_name, det_ctx_event_table);
4206  if (*event_id == -1) {
4207  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
4208  "det_ctx's enum map table.", event_name);
4209  /* this should be treated as fatal */
4210  return -1;
4211  }
4212  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
4213 
4214  return 0;
4215 }
4216 
4217 /*************************************Unittest*********************************/
4218 
4219 #ifdef UNITTESTS
4220 
4221 static int DetectEngineInitYamlConf(const char *conf)
4222 {
4224  ConfInit();
4225  return ConfYamlLoadString(conf, strlen(conf));
4226 }
4227 
4228 static void DetectEngineDeInitYamlConf(void)
4229 {
4230  ConfDeInit();
4232 
4233  return;
4234 }
4235 
4236 static int DetectEngineTest01(void)
4237 {
4238  const char *conf =
4239  "%YAML 1.1\n"
4240  "---\n"
4241  "detect-engine:\n"
4242  " - profile: medium\n"
4243  " - custom-values:\n"
4244  " toclient_src_groups: 2\n"
4245  " toclient_dst_groups: 2\n"
4246  " toclient_sp_groups: 2\n"
4247  " toclient_dp_groups: 3\n"
4248  " toserver_src_groups: 2\n"
4249  " toserver_dst_groups: 4\n"
4250  " toserver_sp_groups: 2\n"
4251  " toserver_dp_groups: 25\n"
4252  " - inspection-recursion-limit: 0\n";
4253 
4254  DetectEngineCtx *de_ctx = NULL;
4255  int result = 0;
4256 
4257  if (DetectEngineInitYamlConf(conf) == -1)
4258  return 0;
4259  de_ctx = DetectEngineCtxInit();
4260  if (de_ctx == NULL)
4261  goto end;
4262 
4263  result = (de_ctx->inspection_recursion_limit == -1);
4264 
4265  end:
4266  if (de_ctx != NULL)
4267  DetectEngineCtxFree(de_ctx);
4268 
4269  DetectEngineDeInitYamlConf();
4270 
4271  return result;
4272 }
4273 
4274 static int DetectEngineTest02(void)
4275 {
4276  const char *conf =
4277  "%YAML 1.1\n"
4278  "---\n"
4279  "detect-engine:\n"
4280  " - profile: medium\n"
4281  " - custom-values:\n"
4282  " toclient_src_groups: 2\n"
4283  " toclient_dst_groups: 2\n"
4284  " toclient_sp_groups: 2\n"
4285  " toclient_dp_groups: 3\n"
4286  " toserver_src_groups: 2\n"
4287  " toserver_dst_groups: 4\n"
4288  " toserver_sp_groups: 2\n"
4289  " toserver_dp_groups: 25\n"
4290  " - inspection-recursion-limit:\n";
4291 
4292  DetectEngineCtx *de_ctx = NULL;
4293  int result = 0;
4294 
4295  if (DetectEngineInitYamlConf(conf) == -1)
4296  return 0;
4297  de_ctx = DetectEngineCtxInit();
4298  if (de_ctx == NULL)
4299  goto end;
4300 
4301  result = (de_ctx->inspection_recursion_limit == -1);
4302 
4303  end:
4304  if (de_ctx != NULL)
4305  DetectEngineCtxFree(de_ctx);
4306 
4307  DetectEngineDeInitYamlConf();
4308 
4309  return result;
4310 }
4311 
4312 static int DetectEngineTest03(void)
4313 {
4314  const char *conf =
4315  "%YAML 1.1\n"
4316  "---\n"
4317  "detect-engine:\n"
4318  " - profile: medium\n"
4319  " - custom-values:\n"
4320  " toclient_src_groups: 2\n"
4321  " toclient_dst_groups: 2\n"
4322  " toclient_sp_groups: 2\n"
4323  " toclient_dp_groups: 3\n"
4324  " toserver_src_groups: 2\n"
4325  " toserver_dst_groups: 4\n"
4326  " toserver_sp_groups: 2\n"
4327  " toserver_dp_groups: 25\n";
4328 
4329  DetectEngineCtx *de_ctx = NULL;
4330  int result = 0;
4331 
4332  if (DetectEngineInitYamlConf(conf) == -1)
4333  return 0;
4334  de_ctx = DetectEngineCtxInit();
4335  if (de_ctx == NULL)
4336  goto end;
4337 
4338  result = (de_ctx->inspection_recursion_limit ==
4340 
4341  end:
4342  if (de_ctx != NULL)
4343  DetectEngineCtxFree(de_ctx);
4344 
4345  DetectEngineDeInitYamlConf();
4346 
4347  return result;
4348 }
4349 
4350 static int DetectEngineTest04(void)
4351 {
4352  const char *conf =
4353  "%YAML 1.1\n"
4354  "---\n"
4355  "detect-engine:\n"
4356  " - profile: medium\n"
4357  " - custom-values:\n"
4358  " toclient_src_groups: 2\n"
4359  " toclient_dst_groups: 2\n"
4360  " toclient_sp_groups: 2\n"
4361  " toclient_dp_groups: 3\n"
4362  " toserver_src_groups: 2\n"
4363  " toserver_dst_groups: 4\n"
4364  " toserver_sp_groups: 2\n"
4365  " toserver_dp_groups: 25\n"
4366  " - inspection-recursion-limit: 10\n";
4367 
4368  DetectEngineCtx *de_ctx = NULL;
4369  int result = 0;
4370 
4371  if (DetectEngineInitYamlConf(conf) == -1)
4372  return 0;
4373  de_ctx = DetectEngineCtxInit();
4374  if (de_ctx == NULL)
4375  goto end;
4376 
4377  result = (de_ctx->inspection_recursion_limit == 10);
4378 
4379  end:
4380  if (de_ctx != NULL)
4381  DetectEngineCtxFree(de_ctx);
4382 
4383  DetectEngineDeInitYamlConf();
4384 
4385  return result;
4386 }
4387 
4388 static int DetectEngineTest08(void)
4389 {
4390  const char *conf =
4391  "%YAML 1.1\n"
4392  "---\n"
4393  "detect-engine:\n"
4394  " - profile: custom\n"
4395  " - custom-values:\n"
4396  " toclient-groups: 23\n"
4397  " toserver-groups: 27\n";
4398 
4399  DetectEngineCtx *de_ctx = NULL;
4400  int result = 0;
4401 
4402  if (DetectEngineInitYamlConf(conf) == -1)
4403  return 0;
4404  de_ctx = DetectEngineCtxInit();
4405  if (de_ctx == NULL)
4406  goto end;
4407 
4408  if (de_ctx->max_uniq_toclient_groups == 23 &&
4409  de_ctx->max_uniq_toserver_groups == 27)
4410  result = 1;
4411 
4412  end:
4413  if (de_ctx != NULL)
4414  DetectEngineCtxFree(de_ctx);
4415 
4416  DetectEngineDeInitYamlConf();
4417 
4418  return result;
4419 }
4420 
4421 /** \test bug 892 bad values */
4422 static int DetectEngineTest09(void)
4423 {
4424  const char *conf =
4425  "%YAML 1.1\n"
4426  "---\n"
4427  "detect-engine:\n"
4428  " - profile: custom\n"
4429  " - custom-values:\n"
4430  " toclient-groups: BA\n"
4431  " toserver-groups: BA\n"
4432  " - inspection-recursion-limit: 10\n";
4433 
4434  DetectEngineCtx *de_ctx = NULL;
4435  int result = 0;
4436 
4437  if (DetectEngineInitYamlConf(conf) == -1)
4438  return 0;
4439  de_ctx = DetectEngineCtxInit();
4440  if (de_ctx == NULL)
4441  goto end;
4442 
4443  if (de_ctx->max_uniq_toclient_groups == 20 &&
4444  de_ctx->max_uniq_toserver_groups == 40)
4445  result = 1;
4446 
4447  end:
4448  if (de_ctx != NULL)
4449  DetectEngineCtxFree(de_ctx);
4450 
4451  DetectEngineDeInitYamlConf();
4452 
4453  return result;
4454 }
4455 
4456 #endif
4457 
4459 {
4460 #ifdef UNITTESTS
4461  UtRegisterTest("DetectEngineTest01", DetectEngineTest01);
4462  UtRegisterTest("DetectEngineTest02", DetectEngineTest02);
4463  UtRegisterTest("DetectEngineTest03", DetectEngineTest03);
4464  UtRegisterTest("DetectEngineTest04", DetectEngineTest04);
4465  UtRegisterTest("DetectEngineTest08", DetectEngineTest08);
4466  UtRegisterTest("DetectEngineTest09", DetectEngineTest09);
4467 #endif
4468  return;
4469 }
void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size)
Definition: detect.c:965
TmModule * TmModuleGetById(int id)
Returns a TM Module by its id.
Definition: tm-modules.c:88
void ** keyword_ctxs_array
Definition: detect.h:1129
struct DetectEngineTenantMapping_ * next
Definition: detect.h:1413
const char * name
Definition: util-spm.h:62
#define PACKET_ALERT_FLAG_STREAM_MATCH
Definition: decode.h:284
int SRepInit(DetectEngineCtx *de_ctx)
init reputation
Definition: reputation.c:579
#define SCMutex
int DetectEngineMustParseMetadata(void)
MpmThreadCtx mtcs
Definition: detect.h:1101
enum AppLayerEventType_ AppLayerEventType
const char * description
Definition: detect.h:426
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1448
int ConfYamlLoadFileWithPrefix(const char *filename, const char *prefix)
Load configuration from a YAML file, insert in tree at &#39;prefix&#39;.
SignatureInitData * init_data
Definition: detect.h:591
uint16_t flags
DetectEngineCtx * DetectEngineCtxInitWithPrefix(const char *prefix)
SpmGlobalThreadCtx * spm_global_thread_ctx
Definition: detect.h:815
void ThresholdHashInit(DetectEngineCtx *de_ctx)
Init threshold context hash tables.
InspectionBufferGetDataPtr GetData
Definition: detect.h:413
#define SCLogDebug(...)
Definition: util-debug.h:335
void SCProfilingPrefilterDestroyCtx(DetectEngineCtx *de_ctx)
#define MAX(x, y)
void DetectEnginePruneFreeList(void)
HashListTableBucket * HashListTableGetListHead(HashListTable *ht)
#define DE_STATE_ID_FILE_INSPECT
Signature ** sig_array
Definition: detect.h:776
void SpmDestroyGlobalThreadCtx(SpmGlobalThreadCtx *global_thread_ctx)
Definition: util-spm.c:137
struct Flow_ * flow
Definition: decode.h:445
int HashTableAdd(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:113
DetectEngineCtx * DetectEngineCtxInitStubForDD(void)
struct DetectEngineThreadCtx_::@105 multi_inspect
#define KEYWORD_PROFILING_END(ctx, type, m)
int DetectLoadersSync(void)
wait for loader tasks to complete
void SCProfilingSghThreadSetup(SCProfileSghDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
DetectEngineCtx * DetectEngineGetByTenantId(int tenant_id)
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
uint32_t sig_array_len
Definition: detect.h:778
struct HtpBodyChunk_ * next
SCCondT cond_q
Definition: decode.h:628
InspectionBuffer *(* InspectionBufferGetPktDataPtr)(struct DetectEngineThreadCtx_ *det_ctx, const DetectEngineTransforms *transforms, Packet *p, const int list_id)
Definition: detect.h:449
void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx)
void SCRConfDeInitContext(DetectEngineCtx *de_ctx)
Releases de_ctx resources related to Reference Config API.
InspectionBuffer * inspection_buffers
Definition: detect.h:363
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
struct DetectEngineAppInspectionEngine_::@93 v2
uint16_t spm_matcher
Definition: detect.h:811
void *(* InitFunc)(void *)
Definition: detect.h:732
Signature loader statistics.
Definition: detect.h:723
struct DetectEngineThreadCtx_::@104 inspect
#define SIG_FLAG_INIT_NEED_FLUSH
Definition: detect.h:263
#define KEYWORD_PROFILING_SET_LIST(ctx, list)
int32_t byte_extract_max_local_id
Definition: detect.h:852
uint32_t tenant_array_size
Definition: detect.h:1022
int DetectEngineTentantRegisterPcapFile(uint32_t tenant_id)
#define BUG_ON(x)
const char * string
Definition: detect.h:425
uint8_t flags
Definition: tm-modules.h:70
int PmqSetup(PrefilterRuleStore *)
Setup a pmq.
uint16_t discontinue_matching
Definition: detect.h:1067
uint32_t flags
Definition: detect.h:523
uint32_t event_type
uint8_t proto
Definition: flow.h:344
#define THV_RUNNING_DONE
Definition: threadvars.h:45
int(* InspectionBufferPktInspectFunc)(struct DetectEngineThreadCtx_ *, const struct DetectEnginePktInspectionEngine *engine, const struct Signature_ *s, Packet *p, uint8_t *alert_flags)
Definition: detect.h:442
uint32_t id
Definition: detect.h:555
#define FALSE
void DetectEngineDeReference(DetectEngineCtx **de_ctx)
void SRepDestroy(DetectEngineCtx *de_ctx)
Definition: reputation.c:662
void ConfDeInit(void)
De-initializes the configuration system.
Definition: conf.c:722
int DetectEngineInspectStreamPayload(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p)
Do the content inspection & validation for a signature on the raw stream.
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
uint64_t * bj_values
Definition: detect.h:1112
#define DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH
Definition: detect.h:280
uint16_t counter_fnonmpm_list
Definition: detect.h:1044
void SCProfilingKeywordDestroyCtx(DetectEngineCtx *de_ctx)
#define unlikely(expr)
Definition: util-optimize.h:35
DetectPort * tcp_whitelist
Definition: detect.h:908
uint8_t is_last
Definition: detect.h:329
ThreadVars * tv
Definition: detect.h:1012
#define HashListTableGetListData(hb)
Definition: util-hashlist.h:59
DetectEngineCtx * DetectEngineGetCurrent(void)
int transforms[DETECT_TRANSFORMS_MAX]
Definition: detect.h:370
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void FlowWorkerReplaceDetectCtx(void *flow_worker, void *detect_ctx)
Definition: flow-worker.c:318
void * DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id)
Retrieve thread local keyword ctx by id.
void SCProfilingRuleThreadCleanup(DetectEngineThreadCtx *det_ctx)
DetectPort * udp_whitelist
Definition: detect.h:909
void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx)
Definition: detect.c:978
int DetectEngineTentantUnregisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
int ConfYamlLoadString(const char *string, size_t len)
Load configuration from a YAML string.
uint16_t PatternMatchDefaultMatcher(void)
Function to return the multi pattern matcher algorithm to be used by the engine, based on the mpm-alg...
bool DetectBufferRunValidateCallback(const DetectEngineCtx *de_ctx, const int id, const Signature *s, const char **sigerror)
#define THV_CAPTURE_INJECT_PKT
Definition: threadvars.h:53
char config_prefix[64]
Definition: detect.h:891
int buffer_type_id
Definition: detect.h:923
uint32_t match_array_len
Definition: detect.h:1086
void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
Free a DetectEngineCtx::
uint16_t id
Definition: tm-queues.h:29
void * FlowWorkerGetDetectCtxPtr(void *flow_worker)
Definition: flow-worker.c:325
uint32_t size
Definition: detect.h:350
DetectEngineThreadKeywordCtxItem * keyword_list
Definition: detect.h:871
#define TAILQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:356
#define MIN(x, y)
#define SIG_FLAG_REQUIRE_STREAM
Definition: detect.h:224
uint64_t offset
void DetectMetadataHashFree(DetectEngineCtx *de_ctx)
uint32_t non_pf_store_cnt_max
Definition: detect.h:784
SigIntId * non_pf_id_array
Definition: detect.h:1014
const char * yaml
void ConfNodeRemove(ConfNode *node)
Remove (and SCFree) the provided configuration node.
Definition: conf.c:668
Data needed for Match()
Definition: detect.h:327
void SigCleanSignatures(DetectEngineCtx *de_ctx)
HashTable * HashTableInit(uint32_t size, uint32_t(*Hash)(struct HashTable_ *, void *, uint16_t), char(*Compare)(void *, uint16_t, void *, uint16_t), void(*Free)(void *))
Definition: util-hash.c:34
int ConfUnixSocketIsEnable(void)
Definition: util-conf.c:144
void DetectBufferTypeRegisterSetupCallback(const char *name, void(*SetupCallback)(const DetectEngineCtx *, Signature *))
DetectEngineTransforms transforms
Definition: detect.h:434
int DetectEngineReloadTenantBlocking(uint32_t tenant_id, const char *yaml, int reload_cnt)
Reload a tenant and wait for loading to complete.
uint32_t to_clear_idx
Definition: detect.h:1053
void * HashListTableLookup(HashListTable *ht, void *data, uint16_t datalen)
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
int ByteExtractStringUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:244
volatile uint8_t suricata_ctl_flags
Definition: suricata.c:201
void DetectEngineReloadSetIdle(void)
MpmThreadCtx mtc
Definition: detect.h:1099
uint32_t orig_len
Definition: detect.h:352
int DetectAddressMapInit(DetectEngineCtx *de_ctx)
void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
DetectBufferMpmRegistery * pkt_mpms_list
Definition: detect.h:931
#define SIG_FLAG_REQUIRE_PACKET
Definition: detect.h:223
uint32_t buffer_offset
Definition: detect.h:1032
int SCMapEnumNameToValue(const char *enum_name, SCEnumCharMap *table)
Maps a string name to an enum value from the supplied table. Please specify the last element of any m...
Definition: util-enum.c:41
void PatternMatchThreadDestroy(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
InspectionBuffer * InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id)
InspectionBuffer * buffers
Definition: detect.h:1051
uint16_t AppProto
char * val
Definition: conf.h:34
int DetectEngineAddToMaster(DetectEngineCtx *de_ctx)
int DetectEngineInspectGenericList(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, Flow *f, const uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
Do the content inspection & validation for a signature.
void DetectLoaderThreadSpawn(void)
spawn the detect loader manager thread
#define PACKET_ALERT_FLAG_DROP_FLOW
Definition: decode.h:280
void DetectBufferTypeCloseRegistration(void)
int EngineModeIsIPS(void)
Definition: suricata.c:247
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
Signature container.
Definition: detect.h:522
#define DETECT_TRANSFORMS_MAX
Definition: detect.h:58
void DetectEngineSetEvent(DetectEngineThreadCtx *det_ctx, uint8_t e)
void MpmFactoryDeRegisterAllMpmCtxProfiles(DetectEngineCtx *de_ctx)
Definition: util-mpm.c:229
void DetectBufferTypeSupportsPacket(const char *name)
bool DetectEnginePktInspectionRun(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p, uint8_t *alert_flags)
uint32_t hashlittle_safe(const void *key, size_t length, uint32_t initval)
uint32_t array_size
Definition: util-hash.h:37
uint8_t action
Definition: detect.h:535
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-modules.h:54
#define TRUE
uint32_t * to_clear_queue
Definition: detect.h:1054
#define SCMutexLock(mut)
const char * sigerror
Definition: detect.h:868
uint32_t signum
Definition: detect.h:780
struct DetectEngineTenantMapping_ * tenant_array
Definition: detect.h:1021
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
Definition: conf.c:331
int(* InspectEngineFuncPtr)(ThreadVars *tv, struct DetectEngineCtx_ *de_ctx, struct DetectEngineThreadCtx_ *det_ctx, const struct Signature_ *sig, const SigMatchData *smd, Flow *f, uint8_t flags, void *alstate, void *tx, uint64_t tx_id)
Definition: detect.h:381
int DetectEngineMultiTenantEnabled(void)
void VarNameStoreActivateStaging(void)
uint16_t vlan_id[2]
Definition: decode.h:435
void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
void(* Transform)(InspectionBuffer *)
Definition: detect.h:1183
uint32_t DetectEngineGetVersion(void)
enum DetectEngineTenantSelectors tenant_selector
Definition: detect.h:1434
void DetectPktMpmRegisterByParentId(DetectEngineCtx *de_ctx, const int id, const int parent_id, DetectEngineTransforms *transforms)
copy a mpm engine from parent_id, add in transforms
main detection engine ctx
Definition: detect.h:761
struct DetectEnginePktInspectionEngine * next
Definition: detect.h:464
char * dev
Definition: util-device.h:41
int ByteExtractStringUint16(uint16_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:264
int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s)
#define TM_FLAG_RECEIVE_TM
Definition: tm-modules.h:31
#define DETECT_CI_FLAGS_END
bool(* ValidateCallback)(const struct Signature_ *, const char **sigerror)
Definition: detect.h:433
void PmqFree(PrefilterRuleStore *)
Cleanup and free a Pmq.
int(* InspectEngineFuncPtr2)(struct DetectEngineCtx_ *de_ctx, struct DetectEngineThreadCtx_ *det_ctx, const struct DetectEngineAppInspectionEngine_ *engine, const struct Signature_ *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
Definition: detect.h:389
SpmTableElmt spm_table[SPM_TABLE_SIZE]
Definition: util-spm.h:74
TmEcode(* PktAcqBreakLoop)(ThreadVars *, void *)
Definition: tm-modules.h:57
const uint8_t * orig
Definition: detect.h:353
uint16_t StatsRegisterCounter(const char *name, struct ThreadVars_ *tv)
Registers a normal, unqualified counter.
Definition: counters.c:943
int DetectLoaderQueueTask(int loader_id, LoaderFunc Func, void *func_ctx)
HashListTable * buffer_type_hash
Definition: detect.h:922
int HashListTableAdd(HashListTable *ht, void *data, uint16_t datalen)
int DetectEngineInspectPacketPayload(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p)
Do the content inspection & validation for a signature.
int DetectEngineInspectStream(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
inspect engine for stateful rules
#define KEYWORD_PROFILING_START
void TmThreadsSetFlag(ThreadVars *tv, uint16_t flag)
Set a thread flag.
Definition: tm-threads.c:98
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
void DetectEngineAppInspectionEngineSignatureFree(Signature *s)
free app inspect engines for a signature
DetectEngineCtx * free_list
Definition: detect.h:1432
void ConfCreateContextBackup(void)
Creates a backup of the conf_hash hash_table used by the conf API.
Definition: conf.c:699
struct DetectEnginePktInspectionEngine::@94 v1
int DetectBufferTypeGetByName(const char *name)
#define SCCalloc(nm, a)
Definition: util-mem.h:253
int VarNameStoreSetupStaging(uint32_t de_ctx_version)
setup staging store. Include current store if there is one.
void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
make sure that the buffer has at least &#39;min_size&#39; bytes Expand the buffer if necessary ...
struct TmSlot_ * tm_slots
Definition: threadvars.h:84
Data structure to store app layer decoder events.
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define SCMutexUnlock(mut)
uint16_t max_uniq_toserver_groups
Definition: detect.h:820
int transforms[DETECT_TRANSFORMS_MAX]
Definition: detect.h:501
#define SIG_FLAG_TOCLIENT
Definition: detect.h:237
#define HashListTableGetListNext(hb)
Definition: util-hashlist.h:58
TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
initialize thread specific detection engine context
void DetectEngineIPOnlyThreadInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyThreadCtx *io_tctx)
Setup the IP Only thread detection engine context.
uint8_t flags
Definition: detect.h:346
struct SCProfileSghDetectCtx_ * profile_sgh_ctx
Definition: detect.h:886
SCMutex tv_root_lock
Definition: tm-threads.c:82
DetectEngineTenantSelectors
Definition: detect.h:1399
#define DETECT_ENGINE_INSPECT_SIG_MATCH
#define SCMUTEX_INITIALIZER
struct TenantLoaderCtx_ TenantLoaderCtx
struct DetectEngineSyncer_ DetectEngineSyncer
void ConfDump(void)
Dump configuration to stdout.
Definition: conf.c:780
uint64_t inspect_offset
Definition: detect.h:344
void VarNameStoreFree(uint32_t de_ctx_version)
#define TAILQ_INIT(head)
Definition: queue.h:370
void SigGroupHeadHashFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->sgh_hash_table, allocated by SigGroupHeadHashInit() function...
Data structures and function prototypes for keeping state for the detection engine.
void(* Free)(void *)
Definition: detect.h:1191
SpmThreadCtx * SpmMakeThreadCtx(const SpmGlobalThreadCtx *global_thread_ctx)
Definition: util-spm.c:146
uint8_t type
int DetectEngineMoveToFreeList(DetectEngineCtx *de_ctx)
struct DetectEngineAppInspectionEngine_ * next
Definition: detect.h:421
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1123
void SCProfilingRuleThreadSetup(SCProfileDetectCtx *ctx, DetectEngineThreadCtx *det_ctx)
#define SIG_FLAG_FLUSH
Definition: detect.h:228
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
uint16_t counter_alerts
Definition: detect.h:1040
void SCProfilingPrefilterThreadCleanup(DetectEngineThreadCtx *det_ctx)
uint32_t(* TenantGetId)(const void *, const Packet *p)
Definition: detect.h:1024
struct ThreadVars_ * next
Definition: threadvars.h:111
void DetectEngineThreadCtxInfo(ThreadVars *t, DetectEngineThreadCtx *det_ctx)
char * filename
Definition: detect.h:715
void ConfInit(void)
Initialize the configuration system.
Definition: conf.c:113
uint16_t mpm_matcher
Definition: detect.h:810
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
int DetectEngineLoadTenantBlocking(uint32_t tenant_id, const char *yaml)
Load a tenant and wait for loading to complete.
one time registration of keywords at start up
Definition: detect.h:605
SCEnumCharMap det_ctx_event_table[]
DetectEngineCtx * DetectEngineCtxInit(void)
void SCSigSignatureOrderingModuleCleanup(DetectEngineCtx *de_ctx)
De-registers all the signature ordering functions registered.
struct DetectEngineCtx_ * next
Definition: detect.h:898
#define TAILQ_REMOVE(head, elm, field)
Definition: queue.h:412
#define SIG_FLAG_TOSERVER
Definition: detect.h:236
void DetectPktInspectEngineRegister(const char *name, InspectionBufferGetPktDataPtr GetPktData, InspectionBufferPktInspectFunc Callback)
register inspect engine at start up time
#define SCEnter(...)
Definition: util-debug.h:337
void DetectAppLayerInspectEngineRegister2(const char *name, AppProto alproto, uint32_t dir, int progress, InspectEngineFuncPtr2 Callback2, InspectionBufferGetDataPtr GetData)
register inspect engine at start up time
PrefilterRuleStore pmq
Definition: detect.h:1102
const char * conf_filename
Definition: suricata.h:170
int DetectBufferTypeMaxId(void)
SigFileLoaderStat sig_stat
Definition: detect.h:941
void SCClassConfLoadClassficationConfigFile(DetectEngineCtx *de_ctx, FILE *fd)
Loads the Classtype info from the classification.config file.
uint16_t counter_match_list
Definition: detect.h:1045
int DetectEngineEnabled(void)
Check if detection is enabled.
void SCProfilingRuleDestroyCtx(SCProfileDetectCtx *ctx)
int DetectEngineReloadIsIdle(void)
void DetectParseDupSigHashFree(DetectEngineCtx *de_ctx)
Frees the hash table that is used to cull duplicate sigs.
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:575
int tm_id
Definition: tm-threads.h:81
struct TmSlot_ * slot_next
Definition: tm-threads.h:87
void ConfRestoreContextBackup(void)
Restores the backup of the hash_table present in backup_conf_hash back to conf_hash.
Definition: conf.c:711
uint32_t buffer_type_map_elements
Definition: detect.h:918
struct DetectEngineThreadKeywordCtxItem_ * next
Definition: detect.h:735
HashTable * mt_det_ctxs_hash
Definition: detect.h:1019
void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *det_ctx)
void MpmStoreFree(DetectEngineCtx *de_ctx)
Frees the hash table - DetectEngineCtx->mpm_hash_table, allocated by MpmStoreInit() function...
DetectEngineAppInspectionEngine * app_inspect_engines
Definition: detect.h:927
DetectEngineSyncState
AppProto alproto
Definition: detect.h:526
HashListTable * HashListTableInit(uint32_t size, uint32_t(*Hash)(struct HashListTable_ *, void *, uint16_t), char(*Compare)(void *, uint16_t, void *, uint16_t), void(*Free)(void *))
Definition: util-hashlist.c:35
DetectEngineTenantMapping * tenant_mapping_list
Definition: detect.h:1438
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1094
int ActionInitConfig()
Load the action order from config. If none is provided, it will be default to ACTION_PASS, ACTION_DROP, ACTION_REJECT, ACTION_ALERT (pass has the highest prio)
Definition: util-action.c:98
void SpmDestroyThreadCtx(SpmThreadCtx *thread_ctx)
Definition: util-spm.c:156
struct SCProfilePrefilterDetectCtx_ * profile_prefilter_ctx
Definition: detect.h:884
uint16_t max_uniq_toclient_groups
Definition: detect.h:819
int(* Match)(DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1170
int SigGroupCleanup(DetectEngineCtx *de_ctx)
void DetectEngineIPOnlyThreadDeinit(DetectEngineIPOnlyThreadCtx *io_tctx)
Deinitialize the IP Only thread detection engine context.
uint32_t ref_cnt
Definition: detect.h:896
const char * DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
int RunmodeIsUnittests(void)
Definition: suricata.c:267
#define DE_STATE_FLAG_BASE
int DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine, const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
Do the content inspection & validation for a signature.
LiveDevice * LiveGetDevice(const char *name)
Get a pointer to the device at idx.
Definition: util-device.c:236
InspectEngineFuncPtr Callback
Definition: detect.h:410
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:262
void PrefilterDeinit(DetectEngineCtx *de_ctx)
void DetectPortCleanupList(const DetectEngineCtx *de_ctx, DetectPort *head)
Free a DetectPort list and each of its members.
AppLayerDecoderEvents * DetectEngineGetEvents(DetectEngineThreadCtx *det_ctx)
int DetectEngineTentantUnregisterPcapFile(uint32_t tenant_id)
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
#define SCRealloc(x, a)
Definition: util-mem.h:238
int DetectEngineGetEventInfo(const char *event_name, int *event_id, AppLayerEventType *event_type)
void PatternMatchThreadPrepare(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
struct SigMatch_ ** smlists
Definition: detect.h:516
int DetectBufferTypeRegister(const char *name)
uint8_t vlan_idx
Definition: decode.h:436
int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void(*FreeFunc)(void *), int mode)
Register Thread keyword context Funcs.
void DetectBufferTypeSupportsMpm(const char *name)
uint32_t version
Definition: detect.h:855
void AppLayerDecoderEventsFreeEvents(AppLayerDecoderEvents **events)
DetectEngineCtx * DetectEngineCtxInitStubForMT(void)
const DetectEngineTransforms * transforms
Definition: detect.h:462
uint8_t * buf
Definition: detect.h:349
uint32_t smlists_array_size
Definition: detect.h:514
int DetectEngineTentantRegisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
enum DetectEnginePrefilterSetting prefilter_setting
Definition: detect.h:904
Definition: conf.h:32
void DetectEngineBumpVersion(void)
void(* SetupCallback)(const struct DetectEngineCtx_ *, struct Signature_ *)
Definition: detect.h:432
uint32_t len
Definition: detect.h:348
int SigGroupHeadHashInit(DetectEngineCtx *de_ctx)
Initializes the hash table in the detection engine context to hold the SigGroupHeads.
TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data)
#define DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT
Definition: detect-engine.c:84
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
void HashTableFree(HashTable *ht)
Definition: util-hash.c:79
#define SCMalloc(a)
Definition: util-mem.h:222
char * sig_error
Definition: detect.h:717
DetectEngineCtx * de_ctx
Definition: detect.h:1127
DetectEngineType
Definition: detect.h:746
uint16_t counter_nonmpm_list
Definition: detect.h:1043
DetectBufferType ** buffer_type_map
Definition: detect.h:917
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
uint32_t tenant_id
Definition: source-pcap.h:37
uint8_t type
Definition: detect.h:328
#define DETECT_ENGINE_INSPECT_SIG_NO_MATCH
MpmTableElmt mpm_table[MPM_TABLE_SIZE]
Definition: util-mpm.h:169
uint8_t version
Definition: decode-gre.h:405
struct SCProfileKeywordDetectCtx_ * profile_keyword_ctx
Definition: detect.h:883
void DetectAddressMapFree(DetectEngineCtx *de_ctx)
#define SCFree(a)
Definition: util-mem.h:322
void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len)
PoolThreadReserved res
int DetectEngineMultiTenantSetup(void)
setup multi-detect / multi-tenancy
void DetectMpmInitializePktMpms(DetectEngineCtx *de_ctx)
void DetectBufferTypeSupportsTransformations(const char *name)
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:269
DetectEnginePktInspectionEngine * pkt_inspect
Definition: detect.h:571
void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx, const int id, const int parent_id, DetectEngineTransforms *transforms)
copy a mpm engine from parent_id, add in transforms
#define SCCondSignal
enum DetectEngineType type
Definition: detect.h:893
int DetectEnginePktInspectionSetup(Signature *s)
uint16_t port
Definition: detect.h:192
uint16_t tx_id
void * DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id)
Retrieve thread local keyword ctx by id.
void SRepReloadComplete(void)
Increment effective reputation version after a rule/reputatio reload is complete. ...
Definition: reputation.c:172
int ConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:437
void TmModuleDetectLoaderRegister(void)
SigMatchData * SigMatchList2DataArray(SigMatch *head)
convert SigMatch list to SigMatchData array
uint32_t init_flags
Definition: detect.h:486
uint16_t StatsRegisterAvgCounter(const char *name, struct ThreadVars_ *tv)
Registers a counter, whose value holds the average of all the values assigned to it.
Definition: counters.c:963
void InspectionBufferSetup(InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
setup the buffer with our initial data
SpmGlobalThreadCtx * SpmInitGlobalThreadCtx(uint16_t matcher)
Definition: util-spm.c:131
MpmThreadCtx mtcu
Definition: detect.h:1100
DetectEngineThreadKeywordCtxItem * keyword_list
Definition: detect.h:1443
bool DetectBufferTypeSupportsMpmGetById(const DetectEngineCtx *de_ctx, const int id)
void InspectionBufferApplyTransforms(InspectionBuffer *buffer, const DetectEngineTransforms *transforms)
int MpmStoreInit(DetectEngineCtx *de_ctx)
Initializes the MpmStore mpm hash table to be used by the detection engine context.
DetectEngineAppInspectionEngine * app_inspect
Definition: detect.h:570
char * name
Definition: conf.h:33
int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id, int *transforms, int transform_cnt)
PacketQueue trans_q[256]
Definition: suricata.h:132
void InspectionBufferClean(DetectEngineThreadCtx *det_ctx)
int failure_fatal
Definition: detect.h:763
#define FatalError(x,...)
Definition: util-debug.h:539
struct DetectPort_ * next
Definition: detect.h:205
uint32_t tenant_id
Definition: util-device.h:54