suricata
detect-engine.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "detect.h"
27 #include "flow.h"
28 #include "flow-private.h"
29 #include "flow-util.h"
30 #include "flow-worker.h"
31 #include "conf.h"
32 #include "conf-yaml-loader.h"
33 #include "datasets.h"
34 
35 #include "app-layer-parser.h"
36 #include "app-layer-htp.h"
37 
38 #include "detect-parse.h"
39 #include "detect-engine-sigorder.h"
40 
41 #include "detect-engine-build.h"
42 #include "detect-engine-siggroup.h"
43 #include "detect-engine-address.h"
44 #include "detect-engine-port.h"
46 #include "detect-engine-mpm.h"
47 #include "detect-engine-iponly.h"
48 #include "detect-engine-tag.h"
49 #include "detect-engine-frame.h"
50 
51 #include "detect-engine-file.h"
52 
53 #include "detect-engine.h"
54 #include "detect-engine-state.h"
55 #include "detect-engine-payload.h"
56 #include "detect-fast-pattern.h"
57 #include "detect-byte-extract.h"
58 #include "detect-content.h"
59 #include "detect-uricontent.h"
60 #include "detect-tcphdr.h"
63 
64 #include "detect-engine-loader.h"
65 
67 #include "util-reference-config.h"
68 #include "util-threshold-config.h"
69 #include "util-error.h"
70 #include "util-hash.h"
71 #include "util-byte.h"
72 #include "util-debug.h"
73 #include "util-unittest.h"
74 #include "util-action.h"
75 #include "util-magic.h"
76 #include "util-signal.h"
77 #include "util-spm.h"
78 #include "util-device.h"
79 #include "util-var-name.h"
80 #include "util-path.h"
81 #include "util-profiling.h"
82 #include "util-validate.h"
83 #include "util-hash-string.h"
84 #include "util-enum.h"
85 #include "util-conf.h"
86 
87 #include "tm-threads.h"
88 #include "runmodes.h"
89 
90 #include "reputation.h"
91 
92 #define DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT 3000
93 
94 static int DetectEngineCtxLoadConf(DetectEngineCtx *);
95 
96 static DetectEngineMasterCtx g_master_de_ctx = { SCMUTEX_INITIALIZER,
97  0, 99, NULL, NULL, TENANT_SELECTOR_UNKNOWN, NULL, NULL, 0};
98 
99 static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len);
100 static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len);
101 static void TenantIdFree(void *d);
102 static uint32_t DetectEngineTenantGetIdFromLivedev(const void *ctx, const Packet *p);
103 static uint32_t DetectEngineTenantGetIdFromVlanId(const void *ctx, const Packet *p);
104 static uint32_t DetectEngineTenantGetIdFromPcap(const void *ctx, const Packet *p);
105 
106 static DetectEngineAppInspectionEngine *g_app_inspect_engines = NULL;
107 static DetectEnginePktInspectionEngine *g_pkt_inspect_engines = NULL;
108 static DetectEngineFrameInspectionEngine *g_frame_inspect_engines = NULL;
109 
110 // clang-format off
112  /* SIG_TYPE_NOT_SET */ { SIG_PROP_FLOW_ACTION_PACKET, },
113  /* SIG_TYPE_IPONLY */ { SIG_PROP_FLOW_ACTION_FLOW, },
114  /* SIG_TYPE_LIKE_IPONLY */ { SIG_PROP_FLOW_ACTION_FLOW, },
115  /* SIG_TYPE_PDONLY */ { SIG_PROP_FLOW_ACTION_FLOW, },
116  /* SIG_TYPE_DEONLY */ { SIG_PROP_FLOW_ACTION_PACKET, },
117  /* SIG_TYPE_PKT */ { SIG_PROP_FLOW_ACTION_PACKET, },
118  /* SIG_TYPE_PKT_STREAM */ { SIG_PROP_FLOW_ACTION_FLOW_IF_STATEFUL, },
119  /* SIG_TYPE_STREAM */ { SIG_PROP_FLOW_ACTION_FLOW_IF_STATEFUL, },
120  /* SIG_TYPE_APPLAYER */ { SIG_PROP_FLOW_ACTION_FLOW, },
121  /* SIG_TYPE_APP_TX */ { SIG_PROP_FLOW_ACTION_FLOW, },
122 };
123 // clang-format on
124 
125 /** \brief register inspect engine at start up time
126  *
127  * \note errors are fatal */
128 void DetectPktInspectEngineRegister(const char *name,
131 {
133  const int sm_list = DetectBufferTypeGetByName(name);
134  if (sm_list == -1) {
135  FatalError("failed to register inspect engine %s", name);
136  }
137 
138  if ((sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
139  (Callback == NULL))
140  {
141  SCLogError("Invalid arguments");
142  BUG_ON(1);
143  }
144 
145  DetectEnginePktInspectionEngine *new_engine = SCCalloc(1, sizeof(*new_engine));
146  if (unlikely(new_engine == NULL)) {
147  FatalError("failed to register inspect engine %s: %s", name, strerror(errno));
148  }
149  new_engine->sm_list = (uint16_t)sm_list;
150  new_engine->sm_list_base = (uint16_t)sm_list;
151  new_engine->v1.Callback = Callback;
152  new_engine->v1.GetData = GetPktData;
153 
154  if (g_pkt_inspect_engines == NULL) {
155  g_pkt_inspect_engines = new_engine;
156  } else {
157  DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
158  while (t->next != NULL) {
159  t = t->next;
160  }
161 
162  t->next = new_engine;
163  }
164 }
165 
166 /** \brief register inspect engine at start up time
167  *
168  * \note errors are fatal */
169 static void AppLayerInspectEngineRegisterInternal(const char *name, AppProto alproto, uint32_t dir,
170  int progress, InspectEngineFuncPtr Callback, InspectionBufferGetDataPtr GetData,
171  InspectionMultiBufferGetDataPtr GetMultiData)
172 {
173  BUG_ON(progress >= 48);
174 
176  const int sm_list = DetectBufferTypeGetByName(name);
177  if (sm_list == -1) {
178  FatalError("failed to register inspect engine %s", name);
179  }
180  SCLogDebug("name %s id %d", name, sm_list);
181 
182  if ((alproto >= ALPROTO_FAILED) || (!(dir == SIG_FLAG_TOSERVER || dir == SIG_FLAG_TOCLIENT)) ||
183  (sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
184  (progress < 0 || progress >= SHRT_MAX) || (Callback == NULL)) {
185  SCLogError("Invalid arguments");
186  BUG_ON(1);
187  } else if (Callback == DetectEngineInspectBufferGeneric && GetData == NULL) {
188  SCLogError("Invalid arguments: must register "
189  "GetData with DetectEngineInspectBufferGeneric");
190  BUG_ON(1);
191  } else if (Callback == DetectEngineInspectMultiBufferGeneric && GetMultiData == NULL) {
192  SCLogError("Invalid arguments: must register "
193  "GetData with DetectEngineInspectMultiBufferGeneric");
194  BUG_ON(1);
195  }
196 
197  uint8_t direction;
198  if (dir == SIG_FLAG_TOSERVER) {
199  direction = 0;
200  } else {
201  direction = 1;
202  }
203  // every DNS or HTTP2 can be accessed from DOH2
204  if (alproto == ALPROTO_HTTP2 || alproto == ALPROTO_DNS) {
205  AppLayerInspectEngineRegisterInternal(
206  name, ALPROTO_DOH2, dir, progress, Callback, GetData, GetMultiData);
207  }
208 
209  DetectEngineAppInspectionEngine *new_engine =
211  if (unlikely(new_engine == NULL)) {
212  exit(EXIT_FAILURE);
213  }
214  new_engine->alproto = alproto;
215  new_engine->dir = direction;
216  new_engine->sm_list = (uint16_t)sm_list;
217  new_engine->sm_list_base = (uint16_t)sm_list;
218  new_engine->progress = (int16_t)progress;
219  new_engine->v2.Callback = Callback;
220  if (Callback == DetectEngineInspectBufferGeneric) {
221  new_engine->v2.GetData = GetData;
222  } else if (Callback == DetectEngineInspectMultiBufferGeneric) {
223  new_engine->v2.GetMultiData = GetMultiData;
224  }
225 
226  if (g_app_inspect_engines == NULL) {
227  g_app_inspect_engines = new_engine;
228  } else {
229  DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
230  while (t->next != NULL) {
231  t = t->next;
232  }
233 
234  t->next = new_engine;
235  }
236 }
237 
238 void DetectAppLayerInspectEngineRegister(const char *name, AppProto alproto, uint32_t dir,
239  int progress, InspectEngineFuncPtr Callback, InspectionBufferGetDataPtr GetData)
240 {
241  AppLayerInspectEngineRegisterInternal(name, alproto, dir, progress, Callback, GetData, NULL);
242 }
243 
244 /* copy an inspect engine with transforms to a new list id. */
245 static void DetectAppLayerInspectEngineCopy(
247  int sm_list, int new_list,
248  const DetectEngineTransforms *transforms)
249 {
250  const DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
251  while (t) {
252  if (t->sm_list == sm_list) {
254  if (unlikely(new_engine == NULL)) {
255  exit(EXIT_FAILURE);
256  }
257  new_engine->alproto = t->alproto;
258  new_engine->dir = t->dir;
259  DEBUG_VALIDATE_BUG_ON(new_list < 0 || new_list > UINT16_MAX);
260  new_engine->sm_list = (uint16_t)new_list; /* use new list id */
261  DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > UINT16_MAX);
262  new_engine->sm_list_base = (uint16_t)sm_list;
263  new_engine->progress = t->progress;
264  new_engine->v2 = t->v2;
265  new_engine->v2.transforms = transforms; /* assign transforms */
266 
267  if (de_ctx->app_inspect_engines == NULL) {
268  de_ctx->app_inspect_engines = new_engine;
269  } else {
271  while (list->next != NULL) {
272  list = list->next;
273  }
274 
275  list->next = new_engine;
276  }
277  }
278  t = t->next;
279  }
280 }
281 
282 /* copy inspect engines from global registrations to de_ctx list */
283 static void DetectAppLayerInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
284 {
285  const DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
287  while (t) {
289  if (unlikely(new_engine == NULL)) {
290  exit(EXIT_FAILURE);
291  }
292  new_engine->alproto = t->alproto;
293  new_engine->dir = t->dir;
294  new_engine->sm_list = t->sm_list;
295  new_engine->sm_list_base = t->sm_list;
296  new_engine->progress = t->progress;
297  new_engine->v2 = t->v2;
298 
299  if (list == NULL) {
300  de_ctx->app_inspect_engines = new_engine;
301  } else {
302  list->next = new_engine;
303  }
304  list = new_engine;
305 
306  t = t->next;
307  }
308 }
309 
310 /* copy an inspect engine with transforms to a new list id. */
311 static void DetectPktInspectEngineCopy(
313  int sm_list, int new_list,
314  const DetectEngineTransforms *transforms)
315 {
316  const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
317  while (t) {
318  if (t->sm_list == sm_list) {
320  if (unlikely(new_engine == NULL)) {
321  exit(EXIT_FAILURE);
322  }
323  DEBUG_VALIDATE_BUG_ON(new_list < 0 || new_list > UINT16_MAX);
324  new_engine->sm_list = (uint16_t)new_list; /* use new list id */
325  DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > UINT16_MAX);
326  new_engine->sm_list_base = (uint16_t)sm_list;
327  new_engine->v1 = t->v1;
328  new_engine->v1.transforms = transforms; /* assign transforms */
329 
330  if (de_ctx->pkt_inspect_engines == NULL) {
331  de_ctx->pkt_inspect_engines = new_engine;
332  } else {
334  while (list->next != NULL) {
335  list = list->next;
336  }
337 
338  list->next = new_engine;
339  }
340  }
341  t = t->next;
342  }
343 }
344 
345 /* copy inspect engines from global registrations to de_ctx list */
346 static void DetectPktInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
347 {
348  const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
349  while (t) {
350  SCLogDebug("engine %p", t);
352  if (unlikely(new_engine == NULL)) {
353  exit(EXIT_FAILURE);
354  }
355  new_engine->sm_list = t->sm_list;
356  new_engine->sm_list_base = t->sm_list;
357  new_engine->v1 = t->v1;
358 
359  if (de_ctx->pkt_inspect_engines == NULL) {
360  de_ctx->pkt_inspect_engines = new_engine;
361  } else {
363  while (list->next != NULL) {
364  list = list->next;
365  }
366 
367  list->next = new_engine;
368  }
369 
370  t = t->next;
371  }
372 }
373 
374 /** \brief register inspect engine at start up time
375  *
376  * \note errors are fatal */
378  InspectionBufferFrameInspectFunc Callback, AppProto alproto, uint8_t type)
379 {
380  const int sm_list = DetectEngineBufferTypeRegister(de_ctx, name);
381  if (sm_list < 0) {
382  FatalError("failed to register inspect engine %s", name);
383  }
384 
385  if ((sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) || (Callback == NULL)) {
386  SCLogError("Invalid arguments");
387  BUG_ON(1);
388  }
389 
390  uint8_t direction;
391  if (dir == SIG_FLAG_TOSERVER) {
392  direction = 0;
393  } else {
394  direction = 1;
395  }
396 
397  DetectEngineFrameInspectionEngine *new_engine = SCCalloc(1, sizeof(*new_engine));
398  if (unlikely(new_engine == NULL)) {
399  FatalError("failed to register inspect engine %s: %s", name, strerror(errno));
400  }
401  new_engine->sm_list = (uint16_t)sm_list;
402  new_engine->sm_list_base = (uint16_t)sm_list;
403  new_engine->dir = direction;
404  new_engine->v1.Callback = Callback;
405  new_engine->alproto = alproto;
406  new_engine->type = type;
407 
408  if (de_ctx->frame_inspect_engines == NULL) {
409  de_ctx->frame_inspect_engines = new_engine;
410  } else {
412  while (list->next != NULL) {
413  list = list->next;
414  }
415 
416  list->next = new_engine;
417  }
418 }
419 
420 /* copy an inspect engine with transforms to a new list id. */
421 static void DetectFrameInspectEngineCopy(DetectEngineCtx *de_ctx, int sm_list, int new_list,
422  const DetectEngineTransforms *transforms)
423 {
424  /* take the list from the detect engine as the buffers can be registered
425  * dynamically. */
427  while (t) {
428  if (t->sm_list == sm_list) {
431  if (unlikely(new_engine == NULL)) {
432  exit(EXIT_FAILURE);
433  }
434  DEBUG_VALIDATE_BUG_ON(new_list < 0 || new_list > UINT16_MAX);
435  new_engine->sm_list = (uint16_t)new_list; /* use new list id */
436  DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > UINT16_MAX);
437  new_engine->sm_list_base = (uint16_t)sm_list;
438  new_engine->dir = t->dir;
439  new_engine->alproto = t->alproto;
440  new_engine->type = t->type;
441  new_engine->v1 = t->v1;
442  new_engine->v1.transforms = transforms; /* assign transforms */
443 
444  /* append to the list */
446  while (list->next != NULL) {
447  list = list->next;
448  }
449 
450  list->next = new_engine;
451  }
452  t = t->next;
453  }
454 }
455 
456 /* copy inspect engines from global registrations to de_ctx list */
457 static void DetectFrameInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
458 {
459  const DetectEngineFrameInspectionEngine *t = g_frame_inspect_engines;
460  while (t) {
461  SCLogDebug("engine %p", t);
464  if (unlikely(new_engine == NULL)) {
465  exit(EXIT_FAILURE);
466  }
467  new_engine->sm_list = t->sm_list;
468  new_engine->sm_list_base = t->sm_list;
469  new_engine->dir = t->dir;
470  new_engine->alproto = t->alproto;
471  new_engine->type = t->type;
472  new_engine->v1 = t->v1;
473 
474  if (de_ctx->frame_inspect_engines == NULL) {
475  de_ctx->frame_inspect_engines = new_engine;
476  } else {
478  while (list->next != NULL) {
479  list = list->next;
480  }
481 
482  list->next = new_engine;
483  }
484 
485  t = t->next;
486  }
487 }
488 
489 /** \internal
490  * \brief append the stream inspection
491  *
492  * If stream inspection is MPM, then prepend it.
493  */
494 static void AppendStreamInspectEngine(
495  Signature *s, SigMatchData *stream, uint8_t direction, uint8_t id)
496 {
497  bool prepend = false;
498 
500  if (unlikely(new_engine == NULL)) {
501  exit(EXIT_FAILURE);
502  }
504  SCLogDebug("stream is mpm");
505  prepend = true;
506  new_engine->mpm = true;
507  }
508  new_engine->alproto = ALPROTO_UNKNOWN; /* all */
509  new_engine->dir = direction;
510  new_engine->stream = true;
511  new_engine->sm_list = DETECT_SM_LIST_PMATCH;
512  new_engine->sm_list_base = DETECT_SM_LIST_PMATCH;
513  new_engine->smd = stream;
514  new_engine->v2.Callback = DetectEngineInspectStream;
515  new_engine->progress = 0;
516 
517  /* append */
518  if (s->app_inspect == NULL) {
519  s->app_inspect = new_engine;
520  new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
521  } else if (prepend) {
522  new_engine->next = s->app_inspect;
523  s->app_inspect = new_engine;
524  new_engine->id = id;
525 
526  } else {
528  while (a->next != NULL) {
529  a = a->next;
530  }
531 
532  a->next = new_engine;
533  new_engine->id = id;
534  }
535  SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
536 }
537 
538 static void AppendFrameInspectEngine(DetectEngineCtx *de_ctx,
540  const int mpm_list)
541 {
542  bool prepend = false;
543 
544  if (u->alproto == ALPROTO_UNKNOWN) {
545  /* special case, inspect engine applies to all protocols */
546  } else if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, u->alproto))
547  return;
548 
549  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
550  if (u->dir == 1)
551  return;
552  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
553  if (u->dir == 0)
554  return;
555  }
556 
559  if (unlikely(new_engine == NULL)) {
560  exit(EXIT_FAILURE);
561  }
562  if (mpm_list == u->sm_list) {
564  prepend = true;
565  new_engine->mpm = true;
566  }
567 
568  new_engine->type = u->type;
569  new_engine->sm_list = u->sm_list;
570  new_engine->sm_list_base = u->sm_list_base;
571  new_engine->smd = smd;
572  new_engine->v1 = u->v1;
573  SCLogDebug("sm_list %d new_engine->v1 %p/%p", new_engine->sm_list, new_engine->v1.Callback,
574  new_engine->v1.transforms);
575 
576  if (s->frame_inspect == NULL) {
577  s->frame_inspect = new_engine;
578  } else if (prepend) {
579  new_engine->next = s->frame_inspect;
580  s->frame_inspect = new_engine;
581  } else {
583  while (a->next != NULL) {
584  a = a->next;
585  }
586  new_engine->next = a->next;
587  a->next = new_engine;
588  }
589 }
590 
591 static void AppendPacketInspectEngine(DetectEngineCtx *de_ctx,
593  const int mpm_list)
594 {
595  bool prepend = false;
596 
597  DetectEnginePktInspectionEngine *new_engine =
599  if (unlikely(new_engine == NULL)) {
600  exit(EXIT_FAILURE);
601  }
602  if (mpm_list == e->sm_list) {
604  prepend = true;
605  new_engine->mpm = true;
606  }
607 
608  new_engine->sm_list = e->sm_list;
609  new_engine->sm_list_base = e->sm_list_base;
610  new_engine->smd = smd;
611  new_engine->v1 = e->v1;
612  SCLogDebug("sm_list %d new_engine->v1 %p/%p/%p", new_engine->sm_list, new_engine->v1.Callback,
613  new_engine->v1.GetData, new_engine->v1.transforms);
614 
615  if (s->pkt_inspect == NULL) {
616  s->pkt_inspect = new_engine;
617  } else if (prepend) {
618  new_engine->next = s->pkt_inspect;
619  s->pkt_inspect = new_engine;
620  } else {
622  while (a->next != NULL) {
623  a = a->next;
624  }
625  new_engine->next = a->next;
626  a->next = new_engine;
627  }
628 }
629 
630 static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
632  const int mpm_list, const int files_id, uint8_t *last_id, bool *head_is_mpm)
633 {
634  if (t->alproto == ALPROTO_UNKNOWN) {
635  /* special case, inspect engine applies to all protocols */
636  } else if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, t->alproto))
637  return;
638 
639  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
640  if (t->dir == 1)
641  return;
642  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
643  if (t->dir == 0)
644  return;
645  }
646  SCLogDebug("app engine: t %p t->id %u => alproto:%s files:%s", t, t->id,
647  AppProtoToString(t->alproto), BOOL2STR(t->sm_list == files_id));
648 
649  DetectEngineAppInspectionEngine *new_engine =
651  if (unlikely(new_engine == NULL)) {
652  exit(EXIT_FAILURE);
653  }
654  bool prepend = false;
655  if (mpm_list == t->sm_list) {
657  prepend = true;
658  *head_is_mpm = true;
659  new_engine->mpm = true;
660  }
661 
662  new_engine->alproto = t->alproto;
663  new_engine->dir = t->dir;
664  new_engine->sm_list = t->sm_list;
665  new_engine->sm_list_base = t->sm_list_base;
666  new_engine->smd = smd;
667  new_engine->progress = t->progress;
668  new_engine->v2 = t->v2;
669  SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
670  new_engine->v2.GetData, new_engine->v2.transforms);
671 
672  if (s->app_inspect == NULL) {
673  s->app_inspect = new_engine;
674  if (new_engine->sm_list == files_id) {
675  new_engine->id = DE_STATE_ID_FILE_INSPECT;
676  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
677  } else {
678  new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
679  SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
681  }
682 
683  /* prepend engine if forced or if our engine has a lower progress. */
684  } else if (prepend || (!(*head_is_mpm) && s->app_inspect->progress > new_engine->progress)) {
685  new_engine->next = s->app_inspect;
686  s->app_inspect = new_engine;
687  if (new_engine->sm_list == files_id) {
688  new_engine->id = DE_STATE_ID_FILE_INSPECT;
689  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
690  } else {
691  new_engine->id = ++(*last_id);
692  SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
694  }
695 
696  } else {
698  while (a->next != NULL) {
699  if (a->next && a->next->progress > new_engine->progress) {
700  break;
701  }
702  a = a->next;
703  }
704 
705  new_engine->next = a->next;
706  a->next = new_engine;
707  if (new_engine->sm_list == files_id) {
708  new_engine->id = DE_STATE_ID_FILE_INSPECT;
709  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
710  } else {
711  new_engine->id = ++(*last_id);
712  SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
714  }
715  }
716 
717  SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
718 
720 }
721 
722 /**
723  * \note for the file inspect engine, the id DE_STATE_ID_FILE_INSPECT
724  * is assigned.
725  */
727 {
728  const int mpm_list = s->init_data->mpm_sm ? s->init_data->mpm_sm_list : -1;
729  const int files_id = DetectBufferTypeGetByName("files");
730  bool head_is_mpm = false;
731  uint8_t last_id = DE_STATE_FLAG_BASE;
732 
733  for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
735  SCLogDebug("smd %p, id %u", smd, s->init_data->buffers[x].id);
736 
737  const DetectBufferType *b =
739  if (b == NULL)
740  FatalError("unknown buffer");
741 
742  if (b->frame) {
744  u != NULL; u = u->next) {
745  if (u->sm_list == s->init_data->buffers[x].id) {
746  AppendFrameInspectEngine(de_ctx, u, s, smd, mpm_list);
747  }
748  }
749  } else if (b->packet) {
750  /* set up pkt inspect engines */
751  for (const DetectEnginePktInspectionEngine *e = de_ctx->pkt_inspect_engines; e != NULL;
752  e = e->next) {
753  SCLogDebug("e %p sm_list %u", e, e->sm_list);
754  if (e->sm_list == s->init_data->buffers[x].id) {
755  AppendPacketInspectEngine(de_ctx, e, s, smd, mpm_list);
756  }
757  }
758  } else {
759  SCLogDebug("app %s id %u parent %u rule %u xforms %u", b->name, b->id, b->parent_id,
760  s->init_data->buffers[x].id, b->transforms.cnt);
761  for (const DetectEngineAppInspectionEngine *t = de_ctx->app_inspect_engines; t != NULL;
762  t = t->next) {
763  if (t->sm_list == s->init_data->buffers[x].id) {
764  AppendAppInspectEngine(
765  de_ctx, t, s, smd, mpm_list, files_id, &last_id, &head_is_mpm);
766  }
767  }
768  }
769  }
770 
773  {
774  /* if engine is added multiple times, we pass it the same list */
776  BUG_ON(stream == NULL);
777  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
778  AppendStreamInspectEngine(s, stream, 0, last_id + 1);
779  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
780  AppendStreamInspectEngine(s, stream, 1, last_id + 1);
781  } else {
782  AppendStreamInspectEngine(s, stream, 0, last_id + 1);
783  AppendStreamInspectEngine(s, stream, 1, last_id + 1);
784  }
785 
787  SCLogDebug("set SIG_FLAG_FLUSH on %u", s->id);
788  s->flags |= SIG_FLAG_FLUSH;
789  }
790  }
791 
792 #ifdef DEBUG
794  while (iter) {
795  SCLogDebug("%u: engine %s id %u progress %d %s", s->id,
797  iter->sm_list == mpm_list ? "MPM" : "");
798  iter = iter->next;
799  }
800 #endif
801  return 0;
802 }
803 
804 /** \brief free app inspect engines for a signature
805  *
806  * For lists that are registered multiple times, like http_header and
807  * http_cookie, making the engines owner of the lists is complicated.
808  * Multiple engines in a sig may be pointing to the same list. To
809  * address this the 'free' code needs to be extra careful about not
810  * double freeing, so it takes an approach to first fill an array
811  * of the to-free pointers before freeing them.
812  */
814 {
815  int engines = 0;
816 
818  while (ie) {
819  ie = ie->next;
820  engines++;
821  }
823  while (e) {
824  e = e->next;
825  engines++;
826  }
828  while (u) {
829  u = u->next;
830  engines++;
831  }
832  if (engines == 0) {
833  BUG_ON(s->pkt_inspect);
834  BUG_ON(s->frame_inspect);
835  return;
836  }
837 
838  SigMatchData *bufs[engines];
839  memset(&bufs, 0, (engines * sizeof(SigMatchData *)));
840  int arrays = 0;
841 
842  /* free engines and put smd in the array */
843  ie = s->app_inspect;
844  while (ie) {
846 
847  bool skip = false;
848  for (int i = 0; i < arrays; i++) {
849  if (bufs[i] == ie->smd) {
850  skip = true;
851  break;
852  }
853  }
854  if (!skip) {
855  bufs[arrays++] = ie->smd;
856  }
857  SCFree(ie);
858  ie = next;
859  }
860  e = s->pkt_inspect;
861  while (e) {
863 
864  bool skip = false;
865  for (int i = 0; i < arrays; i++) {
866  if (bufs[i] == e->smd) {
867  skip = true;
868  break;
869  }
870  }
871  if (!skip) {
872  bufs[arrays++] = e->smd;
873  }
874  SCFree(e);
875  e = next;
876  }
877  u = s->frame_inspect;
878  while (u) {
880 
881  bool skip = false;
882  for (int i = 0; i < arrays; i++) {
883  if (bufs[i] == u->smd) {
884  skip = true;
885  break;
886  }
887  }
888  if (!skip) {
889  bufs[arrays++] = u->smd;
890  }
891  SCFree(u);
892  u = next;
893  }
894 
895  for (int i = 0; i < engines; i++) {
896  if (bufs[i] == NULL)
897  continue;
898  SigMatchData *smd = bufs[i];
899  while (1) {
900  if (sigmatch_table[smd->type].Free != NULL) {
901  sigmatch_table[smd->type].Free(de_ctx, smd->ctx);
902  }
903  if (smd->is_last)
904  break;
905  smd++;
906  }
907  SCFree(bufs[i]);
908  }
909 }
910 
911 /* code for registering buffers */
912 
913 #include "util-hash-lookup3.h"
914 
915 static HashListTable *g_buffer_type_hash = NULL;
916 static int g_buffer_type_id = DETECT_SM_LIST_DYNAMIC_START;
917 static int g_buffer_type_reg_closed = 0;
918 
920 {
921  return g_buffer_type_id;
922 }
923 
924 static uint32_t DetectBufferTypeHashNameFunc(HashListTable *ht, void *data, uint16_t datalen)
925 {
926  const DetectBufferType *map = (DetectBufferType *)data;
927  uint32_t hash = hashlittle_safe(map->name, strlen(map->name), 0);
928  hash += hashlittle_safe((uint8_t *)&map->transforms, sizeof(map->transforms), 0);
929  hash %= ht->array_size;
930  return hash;
931 }
932 
933 static uint32_t DetectBufferTypeHashIdFunc(HashListTable *ht, void *data, uint16_t datalen)
934 {
935  const DetectBufferType *map = (DetectBufferType *)data;
936  uint32_t hash = map->id;
937  hash %= ht->array_size;
938  return hash;
939 }
940 
941 static char DetectBufferTypeCompareNameFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
942 {
943  DetectBufferType *map1 = (DetectBufferType *)data1;
944  DetectBufferType *map2 = (DetectBufferType *)data2;
945 
946  char r = (strcmp(map1->name, map2->name) == 0);
947  r &= (memcmp((uint8_t *)&map1->transforms, (uint8_t *)&map2->transforms, sizeof(map2->transforms)) == 0);
948  return r;
949 }
950 
951 static char DetectBufferTypeCompareIdFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
952 {
953  DetectBufferType *map1 = (DetectBufferType *)data1;
954  DetectBufferType *map2 = (DetectBufferType *)data2;
955  return map1->id == map2->id;
956 }
957 
958 static void DetectBufferTypeFreeFunc(void *data)
959 {
960  DetectBufferType *map = (DetectBufferType *)data;
961 
962  if (map == NULL) {
963  return;
964  }
965 
966  /* Release transformation option memory, if any */
967  for (int i = 0; i < map->transforms.cnt; i++) {
968  if (map->transforms.transforms[i].options == NULL)
969  continue;
970  if (sigmatch_table[map->transforms.transforms[i].transform].Free == NULL) {
971  SCLogError("%s allocates transform option memory but has no free routine",
973  continue;
974  }
976  }
977 
978  SCFree(map);
979 }
980 
981 static int DetectBufferTypeInit(void)
982 {
983  BUG_ON(g_buffer_type_hash);
984  g_buffer_type_hash = HashListTableInit(256, DetectBufferTypeHashNameFunc,
985  DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFunc);
986  if (g_buffer_type_hash == NULL)
987  return -1;
988 
989  return 0;
990 }
991 #if 0
992 static void DetectBufferTypeFree(void)
993 {
994  if (g_buffer_type_hash == NULL)
995  return;
996 
997  HashListTableFree(g_buffer_type_hash);
998  g_buffer_type_hash = NULL;
999 }
1000 #endif
1001 static int DetectBufferTypeAdd(const char *string)
1002 {
1003  BUG_ON(string == NULL || strlen(string) >= 32);
1004 
1005  DetectBufferType *map = SCCalloc(1, sizeof(*map));
1006  if (map == NULL)
1007  return -1;
1008 
1009  strlcpy(map->name, string, sizeof(map->name));
1010  map->id = g_buffer_type_id++;
1011 
1012  BUG_ON(HashListTableAdd(g_buffer_type_hash, (void *)map, 0) != 0);
1013  SCLogDebug("buffer %s registered with id %d", map->name, map->id);
1014  return map->id;
1015 }
1016 
1017 static DetectBufferType *DetectBufferTypeLookupByName(const char *string)
1018 {
1019  DetectBufferType map;
1020  memset(&map, 0, sizeof(map));
1021  strlcpy(map.name, string, sizeof(map.name));
1022 
1023  DetectBufferType *res = HashListTableLookup(g_buffer_type_hash, &map, 0);
1024  return res;
1025 }
1026 
1027 int DetectBufferTypeRegister(const char *name)
1028 {
1029  BUG_ON(g_buffer_type_reg_closed);
1030  if (g_buffer_type_hash == NULL)
1031  DetectBufferTypeInit();
1032 
1033  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1034  if (!exists) {
1035  return DetectBufferTypeAdd(name);
1036  } else {
1037  return exists->id;
1038  }
1039 }
1040 
1042 {
1043  BUG_ON(g_buffer_type_reg_closed);
1045  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1046  BUG_ON(!exists);
1047  exists->multi_instance = true;
1048  SCLogDebug("%p %s -- %d supports multi instance", exists, name, exists->id);
1049 }
1050 
1051 void DetectBufferTypeSupportsFrames(const char *name)
1052 {
1053  BUG_ON(g_buffer_type_reg_closed);
1055  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1056  BUG_ON(!exists);
1057  exists->frame = true;
1058  SCLogDebug("%p %s -- %d supports frame inspection", exists, name, exists->id);
1059 }
1060 
1061 void DetectBufferTypeSupportsPacket(const char *name)
1062 {
1063  BUG_ON(g_buffer_type_reg_closed);
1065  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1066  BUG_ON(!exists);
1067  exists->packet = true;
1068  SCLogDebug("%p %s -- %d supports packet inspection", exists, name, exists->id);
1069 }
1070 
1071 void DetectBufferTypeSupportsMpm(const char *name)
1072 {
1073  BUG_ON(g_buffer_type_reg_closed);
1075  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1076  BUG_ON(!exists);
1077  exists->mpm = true;
1078  SCLogDebug("%p %s -- %d supports mpm", exists, name, exists->id);
1079 }
1080 
1082 {
1083  BUG_ON(g_buffer_type_reg_closed);
1085  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1086  BUG_ON(!exists);
1087  exists->supports_transforms = true;
1088  SCLogDebug("%p %s -- %d supports transformations", exists, name, exists->id);
1089 }
1090 
1091 int DetectBufferTypeGetByName(const char *name)
1092 {
1093  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1094  if (!exists) {
1095  return -1;
1096  }
1097  return exists->id;
1098 }
1099 
1100 static DetectBufferType *DetectEngineBufferTypeLookupByName(
1101  const DetectEngineCtx *de_ctx, const char *string)
1102 {
1103  DetectBufferType map;
1104  memset(&map, 0, sizeof(map));
1105  strlcpy(map.name, string, sizeof(map.name));
1106 
1108  return res;
1109 }
1110 
1112 {
1113  DetectBufferType lookup;
1114  memset(&lookup, 0, sizeof(lookup));
1115  lookup.id = id;
1116  const DetectBufferType *res =
1117  HashListTableLookup(de_ctx->buffer_type_hash_id, (void *)&lookup, 0);
1118  return res;
1119 }
1120 
1122 {
1124  return res ? res->name : NULL;
1125 }
1126 
1127 static int DetectEngineBufferTypeAdd(DetectEngineCtx *de_ctx, const char *string)
1128 {
1129  BUG_ON(string == NULL || strlen(string) >= 32);
1130 
1131  DetectBufferType *map = SCCalloc(1, sizeof(*map));
1132  if (map == NULL)
1133  return -1;
1134 
1135  strlcpy(map->name, string, sizeof(map->name));
1136  map->id = de_ctx->buffer_type_id++;
1137 
1138  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_name, (void *)map, 0) != 0);
1139  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_id, (void *)map, 0) != 0);
1140  SCLogDebug("buffer %s registered with id %d", map->name, map->id);
1141  return map->id;
1142 }
1143 
1145  const int direction, const AppProto alproto, const uint8_t frame_type)
1146 {
1147  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1148  if (exists) {
1149  return exists->id;
1150  }
1151 
1152  const int buffer_id = DetectEngineBufferTypeAdd(de_ctx, name);
1153  if (buffer_id < 0) {
1154  return -1;
1155  }
1156 
1157  /* TODO hack we need the map to get the name. Should we return the map at reg? */
1158  const DetectBufferType *map = DetectEngineBufferTypeGetById(de_ctx, buffer_id);
1159  BUG_ON(!map);
1160 
1161  /* register MPM/inspect engines */
1162  if (direction & SIG_FLAG_TOSERVER) {
1164  PrefilterGenericMpmFrameRegister, alproto, frame_type);
1166  DetectEngineInspectFrameBufferGeneric, alproto, frame_type);
1167  }
1168  if (direction & SIG_FLAG_TOCLIENT) {
1170  PrefilterGenericMpmFrameRegister, alproto, frame_type);
1172  DetectEngineInspectFrameBufferGeneric, alproto, frame_type);
1173  }
1174 
1175  return buffer_id;
1176 }
1177 
1179 {
1180  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1181  if (!exists) {
1182  return DetectEngineBufferTypeAdd(de_ctx, name);
1183  } else {
1184  return exists->id;
1185  }
1186 }
1187 
1188 void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc)
1189 {
1190  BUG_ON(desc == NULL || strlen(desc) >= 128);
1191 
1192  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1193  if (!exists) {
1194  return;
1195  }
1196  strlcpy(exists->description, desc, sizeof(exists->description));
1197 }
1198 
1200 {
1202  if (!exists) {
1203  return NULL;
1204  }
1205  return exists->description;
1206 }
1207 
1208 const char *DetectBufferTypeGetDescriptionByName(const char *name)
1209 {
1210  const DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1211  if (!exists) {
1212  return NULL;
1213  }
1214  return exists->description;
1215 }
1216 
1218 {
1219  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1220  BUG_ON(!exists);
1221  exists->frame = true;
1222  SCLogDebug("%p %s -- %d supports frame inspection", exists, name, exists->id);
1223 }
1224 
1226 {
1227  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1228  BUG_ON(!exists);
1229  exists->packet = true;
1230  SCLogDebug("%p %s -- %d supports packet inspection", exists, name, exists->id);
1231 }
1232 
1234 {
1235  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1236  BUG_ON(!exists);
1237  exists->mpm = true;
1238  SCLogDebug("%p %s -- %d supports mpm", exists, name, exists->id);
1239 }
1240 
1242 {
1243  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1244  BUG_ON(!exists);
1245  exists->supports_transforms = true;
1246  SCLogDebug("%p %s -- %d supports transformations", exists, name, exists->id);
1247 }
1248 
1250 {
1252  if (map == NULL)
1253  return false;
1254  SCLogDebug("map %p id %d multi_instance? %s", map, id, BOOL2STR(map->multi_instance));
1255  return map->multi_instance;
1256 }
1257 
1259 {
1261  if (map == NULL)
1262  return false;
1263  SCLogDebug("map %p id %d packet? %d", map, id, map->packet);
1264  return map->packet;
1265 }
1266 
1268 {
1270  if (map == NULL)
1271  return false;
1272  SCLogDebug("map %p id %d mpm? %d", map, id, map->mpm);
1273  return map->mpm;
1274 }
1275 
1277 {
1279  if (map == NULL)
1280  return false;
1281  SCLogDebug("map %p id %d frame? %d", map, id, map->frame);
1282  return map->frame;
1283 }
1284 
1286  void (*SetupCallback)(const DetectEngineCtx *, Signature *))
1287 {
1288  BUG_ON(g_buffer_type_reg_closed);
1290  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1291  BUG_ON(!exists);
1292  exists->SetupCallback = SetupCallback;
1293 }
1294 
1296 {
1298  if (map && map->SetupCallback) {
1299  map->SetupCallback(de_ctx, s);
1300  }
1301 }
1302 
1304  bool (*ValidateCallback)(const Signature *, const char **sigerror))
1305 {
1306  BUG_ON(g_buffer_type_reg_closed);
1308  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1309  BUG_ON(!exists);
1310  exists->ValidateCallback = ValidateCallback;
1311 }
1312 
1314  const DetectEngineCtx *de_ctx, const int id, const Signature *s, const char **sigerror)
1315 {
1317  if (map && map->ValidateCallback) {
1318  return map->ValidateCallback(s, sigerror);
1319  }
1320  return true;
1321 }
1322 
1323 SigMatch *DetectBufferGetFirstSigMatch(const Signature *s, const uint32_t buf_id)
1324 {
1325  for (uint32_t i = 0; i < s->init_data->buffer_index; i++) {
1326  if (buf_id == s->init_data->buffers[i].id) {
1327  return s->init_data->buffers[i].head;
1328  }
1329  }
1330  return NULL;
1331 }
1332 
1333 SigMatch *DetectBufferGetLastSigMatch(const Signature *s, const uint32_t buf_id)
1334 {
1335  SigMatch *last = NULL;
1336  for (uint32_t i = 0; i < s->init_data->buffer_index; i++) {
1337  if (buf_id == s->init_data->buffers[i].id) {
1338  last = s->init_data->buffers[i].tail;
1339  }
1340  }
1341  return last;
1342 }
1343 
1344 bool DetectBufferIsPresent(const Signature *s, const uint32_t buf_id)
1345 {
1346  for (uint32_t i = 0; i < s->init_data->buffer_index; i++) {
1347  if (buf_id == s->init_data->buffers[i].id) {
1348  return true;
1349  }
1350  }
1351  return false;
1352 }
1353 
1355 {
1356  BUG_ON(s->init_data == NULL);
1357 
1359  SCLogError("Rule buffer cannot be reset after base64_data.");
1360  return -1;
1361  }
1362 
1363  if (s->init_data->list && s->init_data->transforms.cnt) {
1364  SCLogError("no matches following transform(s)");
1365  return -1;
1366  }
1367  s->init_data->list = list;
1368  s->init_data->list_set = true;
1369 
1370  // check if last has matches -> if no, error
1371  if (s->init_data->curbuf && s->init_data->curbuf->head == NULL) {
1372  SCLogError("previous sticky buffer has no matches");
1373  return -1;
1374  }
1375 
1376  for (uint32_t x = 0; x < s->init_data->buffers_size; x++) {
1378  for (SigMatch *sm = b->head; sm != NULL; sm = sm->next) {
1379  SCLogDebug(
1380  "buf:%p: id:%u: '%s' pos %u", b, b->id, sigmatch_table[sm->type].name, sm->idx);
1381  }
1382  if ((uint32_t)list == b->id) {
1383  SCLogDebug("found buffer %p for list %d", b, list);
1384  if (s->init_data->buffers[x].sm_init) {
1385  s->init_data->buffers[x].sm_init = false;
1386  SCLogDebug("sm_init was true for %p list %d", b, list);
1387  s->init_data->curbuf = b;
1388  return 0;
1389 
1391  // fall through
1392  } else {
1393  SCLogWarning("duplicate instance for %s in '%s'",
1395  s->init_data->curbuf = b;
1396  return 0;
1397  }
1398  }
1399  }
1400 
1401  if (list < DETECT_SM_LIST_MAX)
1402  return 0;
1403 
1405  SCLogError("failed to expand rule buffer array");
1406  return -1;
1407  }
1408 
1409  /* initialize new buffer */
1411  s->init_data->curbuf->id = list;
1412  s->init_data->curbuf->head = NULL;
1413  s->init_data->curbuf->tail = NULL;
1416  SCLogDebug("new: idx %u list %d set up curbuf %p", s->init_data->buffer_index - 1, list,
1417  s->init_data->curbuf);
1418 
1419  return 0;
1420 }
1421 
1423 {
1424  BUG_ON(s->init_data == NULL);
1425 
1426  if (s->init_data->list && s->init_data->transforms.cnt) {
1427  if (s->init_data->list == DETECT_SM_LIST_NOTSET ||
1429  SCLogError("previous transforms not consumed "
1430  "(list: %u, transform_cnt %u)",
1432  SCReturnInt(-1);
1433  }
1434 
1435  SCLogDebug("buffer %d has transform(s) registered: %d",
1439  if (new_list == -1) {
1440  SCReturnInt(-1);
1441  }
1442  int base_list = s->init_data->list;
1443  SCLogDebug("new_list %d", new_list);
1444  s->init_data->list = new_list;
1445  s->init_data->list_set = false;
1446  // reset transforms now that we've set up the list
1447  s->init_data->transforms.cnt = 0;
1448 
1449  if (s->init_data->curbuf && s->init_data->curbuf->head != NULL) {
1451  SCLogError("failed to expand rule buffer array");
1452  return -1;
1453  }
1457  }
1458  if (s->init_data->curbuf == NULL) {
1459  SCLogError("failed to setup buffer");
1461  SCReturnInt(-1);
1462  }
1463  s->init_data->curbuf->id = new_list;
1464  SCLogDebug("new list after applying transforms: %u", new_list);
1465  }
1466 
1467  SCReturnInt(0);
1468 }
1469 
1471 {
1472  /* single buffers */
1473  for (uint32_t i = 0; i < det_ctx->inspect.to_clear_idx; i++)
1474  {
1475  const uint32_t idx = det_ctx->inspect.to_clear_queue[i];
1476  InspectionBuffer *buffer = &det_ctx->inspect.buffers[idx];
1477  buffer->inspect = NULL;
1478  buffer->initialized = false;
1479  }
1480  det_ctx->inspect.to_clear_idx = 0;
1481 
1482  /* multi buffers */
1483  for (uint32_t i = 0; i < det_ctx->multi_inspect.to_clear_idx; i++)
1484  {
1485  const uint32_t idx = det_ctx->multi_inspect.to_clear_queue[i];
1486  InspectionBufferMultipleForList *mbuffer = &det_ctx->multi_inspect.buffers[idx];
1487  for (uint32_t x = 0; x <= mbuffer->max; x++) {
1488  InspectionBuffer *buffer = &mbuffer->inspection_buffers[x];
1489  buffer->inspect = NULL;
1490  buffer->initialized = false;
1491  }
1492  mbuffer->init = 0;
1493  mbuffer->max = 0;
1494  }
1495  det_ctx->multi_inspect.to_clear_idx = 0;
1496 }
1497 
1499 {
1500  return &det_ctx->inspect.buffers[list_id];
1501 }
1502 
1503 static InspectionBufferMultipleForList *InspectionBufferGetMulti(
1504  DetectEngineThreadCtx *det_ctx, const int list_id)
1505 {
1506  InspectionBufferMultipleForList *buffer = &det_ctx->multi_inspect.buffers[list_id];
1507  if (!buffer->init) {
1508  det_ctx->multi_inspect.to_clear_queue[det_ctx->multi_inspect.to_clear_idx++] = list_id;
1509  buffer->init = 1;
1510  }
1511  return buffer;
1512 }
1513 
1514 /** \brief for a InspectionBufferMultipleForList get a InspectionBuffer
1515  * \param fb the multiple buffer array
1516  * \param local_id the index to get a buffer
1517  * \param buffer the inspect buffer or NULL in case of error */
1519  DetectEngineThreadCtx *det_ctx, const int list_id, const uint32_t local_id)
1520 {
1521  if (unlikely(local_id >= 1024)) {
1523  return NULL;
1524  }
1525 
1526  InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
1527 
1528  if (local_id >= fb->size) {
1529  uint32_t old_size = fb->size;
1530  uint32_t new_size = local_id + 1;
1531  uint32_t grow_by = new_size - old_size;
1532  SCLogDebug("size is %u, need %u, so growing by %u", old_size, new_size, grow_by);
1533 
1534  SCLogDebug("fb->inspection_buffers %p", fb->inspection_buffers);
1535  void *ptr = SCRealloc(fb->inspection_buffers, (local_id + 1) * sizeof(InspectionBuffer));
1536  if (ptr == NULL)
1537  return NULL;
1538 
1539  InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
1540  SCLogDebug("ptr %p to_zero %p", ptr, to_zero);
1541  memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
1542  fb->inspection_buffers = ptr;
1543  fb->size = new_size;
1544  }
1545 
1546  fb->max = MAX(fb->max, local_id);
1547  InspectionBuffer *buffer = &fb->inspection_buffers[local_id];
1548  SCLogDebug("using buffer %p", buffer);
1549 #ifdef DEBUG_VALIDATION
1550  buffer->multi = true;
1551 #endif
1552  return buffer;
1553 }
1554 
1555 void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
1556 {
1557  memset(buffer, 0, sizeof(*buffer));
1558  buffer->buf = SCCalloc(initial_size, sizeof(uint8_t));
1559  if (buffer->buf != NULL) {
1560  buffer->size = initial_size;
1561  }
1562 }
1563 
1564 /** \brief setup the buffer empty */
1566 {
1567 #ifdef DEBUG_VALIDATION
1569  DEBUG_VALIDATE_BUG_ON(!buffer->multi);
1570 #endif
1571  buffer->inspect = NULL;
1572  buffer->inspect_len = 0;
1573  buffer->len = 0;
1574  buffer->initialized = true;
1575 }
1576 
1577 /** \brief setup the buffer with our initial data */
1579  const uint8_t *data, const uint32_t data_len)
1580 {
1581 #ifdef DEBUG_VALIDATION
1582  DEBUG_VALIDATE_BUG_ON(!buffer->multi);
1583 #endif
1584  buffer->inspect = buffer->orig = data;
1585  buffer->inspect_len = buffer->orig_len = data_len;
1586  buffer->len = 0;
1587  buffer->initialized = true;
1588 
1589  InspectionBufferApplyTransforms(buffer, transforms);
1590 }
1591 
1592 /** \brief setup the buffer with our initial data */
1593 void InspectionBufferSetup(DetectEngineThreadCtx *det_ctx, const int list_id,
1594  InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
1595 {
1596 #ifdef DEBUG_VALIDATION
1597  DEBUG_VALIDATE_BUG_ON(buffer->multi);
1598  DEBUG_VALIDATE_BUG_ON(buffer != InspectionBufferGet(det_ctx, list_id));
1599 #endif
1600  if (buffer->inspect == NULL) {
1601 #ifdef UNITTESTS
1602  if (det_ctx && list_id != -1)
1603 #endif
1604  det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id;
1605  }
1606  buffer->inspect = buffer->orig = data;
1607  buffer->inspect_len = buffer->orig_len = data_len;
1608  buffer->len = 0;
1609  buffer->initialized = true;
1610 }
1611 
1613 {
1614  if (buffer->buf != NULL) {
1615  SCFree(buffer->buf);
1616  }
1617  memset(buffer, 0, sizeof(*buffer));
1618 }
1619 
1620 /**
1621  * \brief make sure that the buffer has at least 'min_size' bytes
1622  * Expand the buffer if necessary
1623  */
1624 void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
1625 {
1626  if (likely(buffer->size >= min_size))
1627  return;
1628 
1629  uint32_t new_size = (buffer->size == 0) ? 4096 : buffer->size;
1630  while (new_size < min_size) {
1631  new_size *= 2;
1632  }
1633 
1634  void *ptr = SCRealloc(buffer->buf, new_size);
1635  if (ptr != NULL) {
1636  buffer->buf = ptr;
1637  buffer->size = new_size;
1638  }
1639 }
1640 
1641 void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len)
1642 {
1643  InspectionBufferCheckAndExpand(buffer, buf_len);
1644 
1645  if (buffer->size) {
1646  uint32_t copy_size = MIN(buf_len, buffer->size);
1647  memcpy(buffer->buf, buf, copy_size);
1648  buffer->inspect = buffer->buf;
1649  buffer->inspect_len = copy_size;
1650  buffer->initialized = true;
1651  }
1652 }
1653 
1654 /** \brief Check content byte array compatibility with transforms
1655  *
1656  * The "content" array is presented to the transforms so that each
1657  * transform may validate that it's compatible with the transform.
1658  *
1659  * When a transform indicates the byte array is incompatible, none of the
1660  * subsequent transforms, if any, are invoked. This means the first validation
1661  * failure terminates the loop.
1662  *
1663  * \param de_ctx Detection engine context.
1664  * \param sm_list The SM list id.
1665  * \param content The byte array being validated
1666  * \param namestr returns the name of the transform that is incompatible with
1667  * content.
1668  *
1669  * \retval true (false) If any of the transforms indicate the byte array is
1670  * (is not) compatible.
1671  **/
1673  const uint8_t *content, uint16_t content_len, const char **namestr)
1674 {
1675  const DetectBufferType *dbt = DetectEngineBufferTypeGetById(de_ctx, sm_list);
1676  BUG_ON(dbt == NULL);
1677 
1678  for (int i = 0; i < dbt->transforms.cnt; i++) {
1679  const TransformData *t = &dbt->transforms.transforms[i];
1681  continue;
1682 
1683  if (sigmatch_table[t->transform].TransformValidate(content, content_len, t->options)) {
1684  continue;
1685  }
1686 
1687  if (namestr) {
1688  *namestr = sigmatch_table[t->transform].name;
1689  }
1690 
1691  return false;
1692  }
1693 
1694  return true;
1695 }
1696 
1698  const DetectEngineTransforms *transforms)
1699 {
1700  if (transforms) {
1701  for (int i = 0; i < DETECT_TRANSFORMS_MAX; i++) {
1702  const int id = transforms->transforms[i].transform;
1703  if (id == 0)
1704  break;
1705  BUG_ON(sigmatch_table[id].Transform == NULL);
1706  sigmatch_table[id].Transform(buffer, transforms->transforms[i].options);
1707  SCLogDebug("applied transform %s", sigmatch_table[id].name);
1708  }
1709  }
1710 }
1711 
1712 static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx)
1713 {
1714  const int size = g_buffer_type_id;
1715  BUG_ON(!(size > 0));
1716 
1717  de_ctx->buffer_type_hash_name = HashListTableInit(256, DetectBufferTypeHashNameFunc,
1718  DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFunc);
1721  HashListTableInit(256, DetectBufferTypeHashIdFunc, DetectBufferTypeCompareIdFunc,
1722  NULL); // entries owned by buffer_type_hash_name
1723  BUG_ON(de_ctx->buffer_type_hash_id == NULL);
1724  de_ctx->buffer_type_id = g_buffer_type_id;
1725 
1726  SCLogDebug("DETECT_SM_LIST_DYNAMIC_START %u", DETECT_SM_LIST_DYNAMIC_START);
1727  HashListTableBucket *b = HashListTableGetListHead(g_buffer_type_hash);
1728  while (b) {
1730 
1731  DetectBufferType *copy = SCCalloc(1, sizeof(*copy));
1732  BUG_ON(!copy);
1733  memcpy(copy, map, sizeof(*copy));
1734  int r = HashListTableAdd(de_ctx->buffer_type_hash_name, (void *)copy, 0);
1735  BUG_ON(r != 0);
1736  r = HashListTableAdd(de_ctx->buffer_type_hash_id, (void *)copy, 0);
1737  BUG_ON(r != 0);
1738 
1739  SCLogDebug("name %s id %d mpm %s packet %s -- %s. "
1740  "Callbacks: Setup %p Validate %p",
1741  map->name, map->id, map->mpm ? "true" : "false", map->packet ? "true" : "false",
1742  map->description, map->SetupCallback, map->ValidateCallback);
1743  b = HashListTableGetListNext(b);
1744  }
1745 
1748  DetectAppLayerInspectEngineCopyListToDetectCtx(de_ctx);
1750  DetectFrameInspectEngineCopyListToDetectCtx(de_ctx);
1752  DetectPktInspectEngineCopyListToDetectCtx(de_ctx);
1753 }
1754 
1755 static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
1756 {
1757  if (de_ctx) {
1762 
1764  while (ilist) {
1766  SCFree(ilist);
1767  ilist = next;
1768  }
1770  while (mlist) {
1771  DetectBufferMpmRegistry *next = mlist->next;
1772  SCFree(mlist);
1773  mlist = next;
1774  }
1776  while (plist) {
1778  SCFree(plist);
1779  plist = next;
1780  }
1782  while (pmlist) {
1783  DetectBufferMpmRegistry *next = pmlist->next;
1784  SCFree(pmlist);
1785  pmlist = next;
1786  }
1788  while (framelist) {
1790  SCFree(framelist);
1791  framelist = next;
1792  }
1794  while (framemlist) {
1795  DetectBufferMpmRegistry *next = framemlist->next;
1796  SCFree(framemlist);
1797  framemlist = next;
1798  }
1800  }
1801 }
1802 
1804 {
1805  BUG_ON(g_buffer_type_hash == NULL);
1806 
1807  g_buffer_type_reg_closed = 1;
1808 }
1809 
1811  DetectEngineCtx *de_ctx, const int id, TransformData *transforms, int transform_cnt)
1812 {
1813  const DetectBufferType *base_map = DetectEngineBufferTypeGetById(de_ctx, id);
1814  if (!base_map) {
1815  return -1;
1816  }
1817  if (!base_map->supports_transforms) {
1818  SCLogError("buffer '%s' does not support transformations", base_map->name);
1819  return -1;
1820  }
1821 
1822  SCLogDebug("base_map %s", base_map->name);
1823 
1825  memset(&t, 0, sizeof(t));
1826  for (int i = 0; i < transform_cnt; i++) {
1827  t.transforms[i] = transforms[i];
1828  }
1829  t.cnt = transform_cnt;
1830 
1831  DetectBufferType lookup_map;
1832  memset(&lookup_map, 0, sizeof(lookup_map));
1833  strlcpy(lookup_map.name, base_map->name, sizeof(lookup_map.name));
1834  lookup_map.transforms = t;
1836 
1837  SCLogDebug("res %p", res);
1838  if (res != NULL) {
1839  return res->id;
1840  }
1841 
1842  DetectBufferType *map = SCCalloc(1, sizeof(*map));
1843  if (map == NULL)
1844  return -1;
1845 
1846  strlcpy(map->name, base_map->name, sizeof(map->name));
1847  map->id = de_ctx->buffer_type_id++;
1848  map->parent_id = base_map->id;
1849  map->transforms = t;
1850  map->mpm = base_map->mpm;
1851  map->packet = base_map->packet;
1852  map->frame = base_map->frame;
1853  map->SetupCallback = base_map->SetupCallback;
1854  map->ValidateCallback = base_map->ValidateCallback;
1855  if (map->frame) {
1857  } else if (map->packet) {
1859  map->id, map->parent_id, &map->transforms);
1860  } else {
1862  map->id, map->parent_id, &map->transforms);
1863  }
1864 
1865  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_name, (void *)map, 0) != 0);
1866  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_id, (void *)map, 0) != 0);
1867  SCLogDebug("buffer %s registered with id %d, parent %d", map->name, map->id, map->parent_id);
1868 
1869  if (map->frame) {
1870  DetectFrameInspectEngineCopy(de_ctx, map->parent_id, map->id, &map->transforms);
1871  } else if (map->packet) {
1872  DetectPktInspectEngineCopy(de_ctx, map->parent_id, map->id, &map->transforms);
1873  } else {
1874  DetectAppLayerInspectEngineCopy(de_ctx, map->parent_id, map->id, &map->transforms);
1875  }
1876  return map->id;
1877 }
1878 
1879 /* returns false if no match, true if match */
1880 static int DetectEngineInspectRulePacketMatches(
1881  DetectEngineThreadCtx *det_ctx,
1882  const DetectEnginePktInspectionEngine *engine,
1883  const Signature *s,
1884  Packet *p, uint8_t *_alert_flags)
1885 {
1886  SCEnter();
1887 
1888  /* run the packet match functions */
1890  const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_MATCH];
1891 
1892  SCLogDebug("running match functions, sm %p", smd);
1893  while (1) {
1895  if (sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx) <= 0) {
1896  KEYWORD_PROFILING_END(det_ctx, smd->type, 0);
1897  SCLogDebug("no match");
1899  }
1900  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
1901  if (smd->is_last) {
1902  SCLogDebug("match and is_last");
1903  break;
1904  }
1905  smd++;
1906  }
1908 }
1909 
1910 static int DetectEngineInspectRulePayloadMatches(
1911  DetectEngineThreadCtx *det_ctx,
1912  const DetectEnginePktInspectionEngine *engine,
1913  const Signature *s, Packet *p, uint8_t *alert_flags)
1914 {
1915  SCEnter();
1916 
1917  DetectEngineCtx *de_ctx = det_ctx->de_ctx;
1918 
1920  /* if we have stream msgs, inspect against those first,
1921  * but not for a "dsize" signature */
1922  if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
1923  int pmatch = 0;
1924  if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
1925  pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, p->flow, p);
1926  if (pmatch) {
1927  *alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
1928  }
1929  }
1930  /* no match? then inspect packet payload */
1931  if (pmatch == 0) {
1932  SCLogDebug("no match in stream, fall back to packet payload");
1933 
1934  /* skip if we don't have to inspect the packet and segment was
1935  * added to stream */
1936  if (!(s->flags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
1938  }
1940  SCLogDebug("SIG_FLAG_REQUIRE_STREAM_ONLY, so no match");
1942  }
1943  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
1945  }
1946  }
1947  } else {
1948  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
1950  }
1951  }
1953 }
1954 
1956  DetectEngineThreadCtx *det_ctx, const Signature *s,
1957  Flow *f, Packet *p,
1958  uint8_t *alert_flags)
1959 {
1960  SCEnter();
1961 
1962  for (DetectEnginePktInspectionEngine *e = s->pkt_inspect; e != NULL; e = e->next) {
1963  if (e->v1.Callback(det_ctx, e, s, p, alert_flags) != DETECT_ENGINE_INSPECT_SIG_MATCH) {
1964  SCLogDebug("sid %u: e %p Callback returned no match", s->id, e);
1965  return false;
1966  }
1967  SCLogDebug("sid %u: e %p Callback returned true", s->id, e);
1968  }
1969 
1970  SCLogDebug("sid %u: returning true", s->id);
1971  return true;
1972 }
1973 
1974 /**
1975  * \param data pointer to SigMatchData. Allowed to be NULL.
1976  */
1977 static int DetectEnginePktInspectionAppend(Signature *s, InspectionBufferPktInspectFunc Callback,
1978  SigMatchData *data, const int list_id)
1979 {
1980  DetectEnginePktInspectionEngine *e = SCCalloc(1, sizeof(*e));
1981  if (e == NULL)
1982  return -1;
1983 
1984  e->mpm = s->init_data->mpm_sm_list == list_id;
1985  DEBUG_VALIDATE_BUG_ON(list_id < 0 || list_id > UINT16_MAX);
1986  e->sm_list = (uint16_t)list_id;
1987  e->sm_list_base = (uint16_t)list_id;
1988  e->v1.Callback = Callback;
1989  e->smd = data;
1990 
1991  if (s->pkt_inspect == NULL) {
1992  s->pkt_inspect = e;
1993  } else {
1995  while (a->next != NULL) {
1996  a = a->next;
1997  }
1998  a->next = e;
1999  }
2000  return 0;
2001 }
2002 
2004 {
2005  /* only handle PMATCH here if we're not an app inspect rule */
2007  if (DetectEnginePktInspectionAppend(
2008  s, DetectEngineInspectRulePayloadMatches, NULL, DETECT_SM_LIST_PMATCH) < 0)
2009  return -1;
2010  SCLogDebug("sid %u: DetectEngineInspectRulePayloadMatches appended", s->id);
2011  }
2012 
2013  if (s->sm_arrays[DETECT_SM_LIST_MATCH]) {
2014  if (DetectEnginePktInspectionAppend(
2015  s, DetectEngineInspectRulePacketMatches, NULL, DETECT_SM_LIST_MATCH) < 0)
2016  return -1;
2017  SCLogDebug("sid %u: DetectEngineInspectRulePacketMatches appended", s->id);
2018  }
2019 
2020  return 0;
2021 }
2022 
2023 /* code to control the main thread to do a reload */
2024 
2026  IDLE, /**< ready to start a reload */
2027  RELOAD, /**< command main thread to do the reload */
2028 };
2029 
2030 
2031 typedef struct DetectEngineSyncer_ {
2035 
2036 static DetectEngineSyncer detect_sync = { SCMUTEX_INITIALIZER, IDLE };
2037 
2038 /* tell main to start reloading */
2040 {
2041  int r = 0;
2042  SCMutexLock(&detect_sync.m);
2043  if (detect_sync.state == IDLE) {
2044  detect_sync.state = RELOAD;
2045  } else {
2046  r = -1;
2047  }
2048  SCMutexUnlock(&detect_sync.m);
2049  return r;
2050 }
2051 
2052 /* main thread checks this to see if it should start */
2054 {
2055  int r = 0;
2056  SCMutexLock(&detect_sync.m);
2057  if (detect_sync.state == RELOAD) {
2058  r = 1;
2059  }
2060  SCMutexUnlock(&detect_sync.m);
2061  return r;
2062 }
2063 
2064 /* main thread sets done when it's done */
2066 {
2067  SCMutexLock(&detect_sync.m);
2068  detect_sync.state = IDLE;
2069  SCMutexUnlock(&detect_sync.m);
2070 }
2071 
2072 /* caller loops this until it returns 1 */
2074 {
2075  int r = 0;
2076  SCMutexLock(&detect_sync.m);
2077  if (detect_sync.state == IDLE) {
2078  r = 1;
2079  }
2080  SCMutexUnlock(&detect_sync.m);
2081  return r;
2082 }
2083 
2084 /** \brief Do the content inspection & validation for a signature
2085  *
2086  * \param de_ctx Detection engine context
2087  * \param det_ctx Detection engine thread context
2088  * \param s Signature to inspect
2089  * \param sm SigMatch to inspect
2090  * \param f Flow
2091  * \param flags app layer flags
2092  * \param state App layer state
2093  *
2094  * \retval 0 no match
2095  * \retval 1 match
2096  */
2098  const struct DetectEngineAppInspectionEngine_ *engine, const Signature *s, Flow *f,
2099  uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
2100 {
2101  SigMatchData *smd = engine->smd;
2102  SCLogDebug("running match functions, sm %p", smd);
2103  if (smd != NULL) {
2104  while (1) {
2105  int match = 0;
2107  match = sigmatch_table[smd->type].
2108  AppLayerTxMatch(det_ctx, f, flags, alstate, txv, s, smd->ctx);
2109  KEYWORD_PROFILING_END(det_ctx, smd->type, (match == 1));
2110  if (match == 0)
2112  if (match == 2) {
2114  }
2115 
2116  if (smd->is_last)
2117  break;
2118  smd++;
2119  }
2120  }
2121 
2123 }
2124 
2125 
2126 /**
2127  * \brief Do the content inspection & validation for a signature
2128  *
2129  * \param de_ctx Detection engine context
2130  * \param det_ctx Detection engine thread context
2131  * \param s Signature to inspect
2132  * \param f Flow
2133  * \param flags app layer flags
2134  * \param state App layer state
2135  *
2136  * \retval 0 no match.
2137  * \retval 1 match.
2138  * \retval 2 Sig can't match.
2139  */
2141  const DetectEngineAppInspectionEngine *engine, const Signature *s, Flow *f, uint8_t flags,
2142  void *alstate, void *txv, uint64_t tx_id)
2143 {
2144  const int list_id = engine->sm_list;
2145  SCLogDebug("running inspect on %d", list_id);
2146 
2147  const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
2148 
2149  SCLogDebug("list %d mpm? %s transforms %p",
2150  engine->sm_list, engine->mpm ? "true" : "false", engine->v2.transforms);
2151 
2152  /* if prefilter didn't already run, we need to consider transformations */
2153  const DetectEngineTransforms *transforms = NULL;
2154  if (!engine->mpm) {
2155  transforms = engine->v2.transforms;
2156  }
2157 
2158  const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
2159  f, flags, txv, list_id);
2160  if (unlikely(buffer == NULL)) {
2163  }
2164 
2165  const uint32_t data_len = buffer->inspect_len;
2166  const uint8_t *data = buffer->inspect;
2167  const uint64_t offset = buffer->inspect_offset;
2168 
2169  uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0;
2170  ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0);
2171  ci_flags |= buffer->flags;
2172 
2173  /* Inspect all the uricontents fetched on each
2174  * transaction at the app layer */
2175  const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, data,
2177  if (match) {
2179  } else {
2182  }
2183 }
2184 
2185 // wrapper for both DetectAppLayerInspectEngineRegister and DetectAppLayerMpmRegister
2186 // with cast of callback function
2187 void DetectAppLayerMultiRegister(const char *name, AppProto alproto, uint32_t dir, int progress,
2188  InspectionMultiBufferGetDataPtr GetData, int priority, int tx_min_progress)
2189 {
2190  AppLayerInspectEngineRegisterInternal(
2191  name, alproto, dir, progress, DetectEngineInspectMultiBufferGeneric, NULL, GetData);
2193  alproto, tx_min_progress);
2194 }
2195 
2198  const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
2199 {
2200  uint32_t local_id = 0;
2201  const DetectEngineTransforms *transforms = NULL;
2202  if (!engine->mpm) {
2203  transforms = engine->v2.transforms;
2204  }
2205 
2206  do {
2207  InspectionBuffer *buffer = engine->v2.GetMultiData(
2208  det_ctx, transforms, f, flags, txv, engine->sm_list, local_id);
2209 
2210  if (buffer == NULL || buffer->inspect == NULL)
2211  break;
2212 
2213  // The GetData functions set buffer->flags to DETECT_CI_FLAGS_SINGLE
2214  // This is not meant for streaming buffers
2215  const bool match = DetectEngineContentInspectionBuffer(de_ctx, det_ctx, s, engine->smd,
2217  if (match) {
2219  }
2220  local_id++;
2221  } while (1);
2223 }
2224 
2225 /**
2226  * \brief Do the content inspection & validation for a signature
2227  *
2228  * \param de_ctx Detection engine context
2229  * \param det_ctx Detection engine thread context
2230  * \param s Signature to inspect
2231  * \param p Packet
2232  *
2233  * \retval 0 no match.
2234  * \retval 1 match.
2235  */
2237  DetectEngineThreadCtx *det_ctx,
2238  const DetectEnginePktInspectionEngine *engine,
2239  const Signature *s, Packet *p, uint8_t *_alert_flags)
2240 {
2241  const int list_id = engine->sm_list;
2242  SCLogDebug("running inspect on %d", list_id);
2243 
2244  SCLogDebug("list %d transforms %p",
2245  engine->sm_list, engine->v1.transforms);
2246 
2247  /* if prefilter didn't already run, we need to consider transformations */
2248  const DetectEngineTransforms *transforms = NULL;
2249  if (!engine->mpm) {
2250  transforms = engine->v1.transforms;
2251  }
2252 
2253  const InspectionBuffer *buffer = engine->v1.GetData(det_ctx, transforms, p,
2254  list_id);
2255  if (unlikely(buffer == NULL)) {
2257  }
2258 
2259  uint8_t ci_flags = DETECT_CI_FLAGS_START|DETECT_CI_FLAGS_END;
2260  ci_flags |= buffer->flags;
2261 
2262  /* Inspect all the uricontents fetched on each
2263  * transaction at the app layer */
2264  const bool match = DetectEngineContentInspection(det_ctx->de_ctx, det_ctx, s, engine->smd, p,
2265  p->flow, buffer->inspect, buffer->inspect_len, 0, ci_flags,
2267  if (match) {
2269  } else {
2271  }
2272 }
2273 
2274 /** \internal
2275  * \brief inject a pseudo packet into each detect thread that doesn't use the
2276  * new det_ctx yet
2277  */
2278 static void InjectPackets(ThreadVars **detect_tvs,
2279  DetectEngineThreadCtx **new_det_ctx,
2280  int no_of_detect_tvs)
2281 {
2282  /* inject a fake packet if the detect thread isn't using the new ctx yet,
2283  * this speeds up the process */
2284  for (int i = 0; i < no_of_detect_tvs; i++) {
2285  if (SC_ATOMIC_GET(new_det_ctx[i]->so_far_used_by_detect) != 1) {
2286  if (detect_tvs[i]->inq != NULL) {
2287  Packet *p = PacketGetFromAlloc();
2288  if (p != NULL) {
2291  PacketQueue *q = detect_tvs[i]->inq->pq;
2292  SCMutexLock(&q->mutex_q);
2293  PacketEnqueue(q, p);
2294  SCCondSignal(&q->cond_q);
2295  SCMutexUnlock(&q->mutex_q);
2296  }
2297  }
2298  }
2299  }
2300 }
2301 
2302 /** \internal
2303  * \brief Update detect threads with new detect engine
2304  *
2305  * Atomically update each detect thread with a new thread context
2306  * that is associated to the new detection engine(s).
2307  *
2308  * If called in unix socket mode, it's possible that we don't have
2309  * detect threads yet.
2310  *
2311  * \retval -1 error
2312  * \retval 0 no detection threads
2313  * \retval 1 successful reload
2314  */
2315 static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx)
2316 {
2317  SCEnter();
2318  uint32_t i = 0;
2319 
2320  /* count detect threads in use */
2321  uint32_t no_of_detect_tvs = TmThreadCountThreadsByTmmFlags(TM_FLAG_DETECT_TM);
2322  /* can be zero in unix socket mode */
2323  if (no_of_detect_tvs == 0) {
2324  return 0;
2325  }
2326 
2327  /* prepare swap structures */
2328  DetectEngineThreadCtx *old_det_ctx[no_of_detect_tvs];
2329  DetectEngineThreadCtx *new_det_ctx[no_of_detect_tvs];
2330  ThreadVars *detect_tvs[no_of_detect_tvs];
2331  memset(old_det_ctx, 0x00, (no_of_detect_tvs * sizeof(DetectEngineThreadCtx *)));
2332  memset(new_det_ctx, 0x00, (no_of_detect_tvs * sizeof(DetectEngineThreadCtx *)));
2333  memset(detect_tvs, 0x00, (no_of_detect_tvs * sizeof(ThreadVars *)));
2334 
2335  /* start the process of swapping detect threads ctxs */
2336 
2337  /* get reference to tv's and setup new_det_ctx array */
2339  for (ThreadVars *tv = tv_root[TVT_PPT]; tv != NULL; tv = tv->next) {
2340  if ((tv->tmm_flags & TM_FLAG_DETECT_TM) == 0) {
2341  continue;
2342  }
2343  for (TmSlot *s = tv->tm_slots; s != NULL; s = s->slot_next) {
2344  TmModule *tm = TmModuleGetById(s->tm_id);
2345  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
2346  continue;
2347  }
2348 
2349  if (suricata_ctl_flags != 0) {
2351  goto error;
2352  }
2353 
2354  old_det_ctx[i] = FlowWorkerGetDetectCtxPtr(SC_ATOMIC_GET(s->slot_data));
2355  detect_tvs[i] = tv;
2356 
2357  new_det_ctx[i] = DetectEngineThreadCtxInitForReload(tv, new_de_ctx, 1);
2358  if (new_det_ctx[i] == NULL) {
2359  SCLogError("Detect engine thread init "
2360  "failure in live rule swap. Let's get out of here");
2362  goto error;
2363  }
2364  SCLogDebug("live rule swap created new det_ctx - %p and de_ctx "
2365  "- %p\n", new_det_ctx[i], new_de_ctx);
2366  i++;
2367  break;
2368  }
2369  }
2370  BUG_ON(i != no_of_detect_tvs);
2371 
2372  /* atomically replace the det_ctx data */
2373  i = 0;
2374  for (ThreadVars *tv = tv_root[TVT_PPT]; tv != NULL; tv = tv->next) {
2375  if ((tv->tmm_flags & TM_FLAG_DETECT_TM) == 0) {
2376  continue;
2377  }
2378  for (TmSlot *s = tv->tm_slots; s != NULL; s = s->slot_next) {
2379  TmModule *tm = TmModuleGetById(s->tm_id);
2380  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
2381  continue;
2382  }
2383  SCLogDebug("swapping new det_ctx - %p with older one - %p",
2384  new_det_ctx[i], SC_ATOMIC_GET(s->slot_data));
2385  FlowWorkerReplaceDetectCtx(SC_ATOMIC_GET(s->slot_data), new_det_ctx[i++]);
2386  break;
2387  }
2388  }
2390 
2391  /* threads now all have new data, however they may not have started using
2392  * it and may still use the old data */
2393 
2394  SCLogDebug("Live rule swap has swapped %d old det_ctx's with new ones, "
2395  "along with the new de_ctx", no_of_detect_tvs);
2396 
2397  InjectPackets(detect_tvs, new_det_ctx, no_of_detect_tvs);
2398 
2399  /* loop waiting for detect threads to switch to the new det_ctx. Try to
2400  * wake up capture if needed (break loop). */
2401  uint32_t threads_done = 0;
2402 retry:
2403  for (i = 0; i < no_of_detect_tvs; i++) {
2404  if (suricata_ctl_flags != 0) {
2405  threads_done = no_of_detect_tvs;
2406  break;
2407  }
2408  usleep(1000);
2409  if (SC_ATOMIC_GET(new_det_ctx[i]->so_far_used_by_detect) == 1) {
2410  SCLogDebug("new_det_ctx - %p used by detect engine", new_det_ctx[i]);
2411  threads_done++;
2412  } else {
2413  TmThreadsCaptureBreakLoop(detect_tvs[i]);
2414  }
2415  }
2416  if (threads_done < no_of_detect_tvs) {
2417  threads_done = 0;
2418  SleepMsec(250);
2419  goto retry;
2420  }
2421 
2422  /* this is to make sure that if someone initiated shutdown during a live
2423  * rule swap, the live rule swap won't clean up the old det_ctx and
2424  * de_ctx, till all detect threads have stopped working and sitting
2425  * silently after setting RUNNING_DONE flag and while waiting for
2426  * THV_DEINIT flag */
2427  if (i != no_of_detect_tvs) { // not all threads we swapped
2428  for (ThreadVars *tv = tv_root[TVT_PPT]; tv != NULL; tv = tv->next) {
2429  if ((tv->tmm_flags & TM_FLAG_DETECT_TM) == 0) {
2430  continue;
2431  }
2432 
2434  usleep(100);
2435  }
2436  }
2437  }
2438 
2439  /* free all the ctxs */
2440  for (i = 0; i < no_of_detect_tvs; i++) {
2441  SCLogDebug("Freeing old_det_ctx - %p used by detect",
2442  old_det_ctx[i]);
2443  DetectEngineThreadCtxDeinit(NULL, old_det_ctx[i]);
2444  }
2445 
2447 
2448  return 1;
2449 
2450  error:
2451  for (i = 0; i < no_of_detect_tvs; i++) {
2452  if (new_det_ctx[i] != NULL)
2453  DetectEngineThreadCtxDeinit(NULL, new_det_ctx[i]);
2454  }
2455  return -1;
2456 }
2457 
2458 static DetectEngineCtx *DetectEngineCtxInitReal(
2459  enum DetectEngineType type, const char *prefix, uint32_t tenant_id)
2460 {
2462  if (unlikely(de_ctx == NULL))
2463  goto error;
2464 
2465  memset(&de_ctx->sig_stat, 0, sizeof(SigFileLoaderStat));
2466  TAILQ_INIT(&de_ctx->sig_stat.failed_sigs);
2467  de_ctx->sigerror = NULL;
2468  de_ctx->type = type;
2470  de_ctx->tenant_id = tenant_id;
2471 
2474  SCLogDebug("stub %u with version %u", type, de_ctx->version);
2475  return de_ctx;
2476  }
2477 
2478  if (prefix != NULL) {
2479  strlcpy(de_ctx->config_prefix, prefix, sizeof(de_ctx->config_prefix));
2480  }
2481 
2482  int failure_fatal = 0;
2483  if (ConfGetBool("engine.init-failure-fatal", (int *)&failure_fatal) != 1) {
2484  SCLogDebug("ConfGetBool could not load the value.");
2485  }
2486  de_ctx->failure_fatal = (failure_fatal == 1);
2487 
2490  SCLogConfig("pattern matchers: MPM: %s, SPM: %s",
2493 
2495  if (de_ctx->spm_global_thread_ctx == NULL) {
2496  SCLogDebug("Unable to alloc SpmGlobalThreadCtx.");
2497  goto error;
2498  }
2499 
2501  if (de_ctx->sm_types_prefilter == NULL) {
2502  goto error;
2503  }
2505  if (de_ctx->sm_types_silent_error == NULL) {
2506  goto error;
2507  }
2508  if (DetectEngineCtxLoadConf(de_ctx) == -1) {
2509  goto error;
2510  }
2511 
2517  DetectBufferTypeSetupDetectEngine(de_ctx);
2519 
2520  /* init iprep... ignore errors for now */
2521  (void)SRepInit(de_ctx);
2522 
2526  goto error;
2527  }
2528 
2529  if (ActionInitConfig() < 0) {
2530  goto error;
2531  }
2533  if (SCRConfLoadReferenceConfigFile(de_ctx, NULL) < 0) {
2535  goto error;
2536  }
2537 
2539  SCLogDebug("dectx with version %u", de_ctx->version);
2540  return de_ctx;
2541 error:
2542  if (de_ctx != NULL) {
2544  }
2545  return NULL;
2546 }
2547 
2549 {
2550  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_MT_STUB, NULL, 0);
2551 }
2552 
2554 {
2555  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_DD_STUB, NULL, 0);
2556 }
2557 
2559 {
2560  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_NORMAL, NULL, 0);
2561 }
2562 
2563 DetectEngineCtx *DetectEngineCtxInitWithPrefix(const char *prefix, uint32_t tenant_id)
2564 {
2565  if (prefix == NULL || strlen(prefix) == 0)
2566  return DetectEngineCtxInit();
2567  else
2568  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_NORMAL, prefix, tenant_id);
2569 }
2570 
2571 static void DetectEngineCtxFreeThreadKeywordData(DetectEngineCtx *de_ctx)
2572 {
2574 }
2575 
2576 static void DetectEngineCtxFreeFailedSigs(DetectEngineCtx *de_ctx)
2577 {
2578  SigString *item = NULL;
2579  SigString *sitem;
2580 
2581  TAILQ_FOREACH_SAFE(item, &de_ctx->sig_stat.failed_sigs, next, sitem) {
2582  SCFree(item->filename);
2583  SCFree(item->sig_str);
2584  if (item->sig_error) {
2585  SCFree(item->sig_error);
2586  }
2587  TAILQ_REMOVE(&de_ctx->sig_stat.failed_sigs, item, next);
2588  SCFree(item);
2589  }
2590 }
2591 
2592 /**
2593  * \brief Free a DetectEngineCtx::
2594  *
2595  * \param de_ctx DetectEngineCtx:: to be freed
2596  */
2598 {
2599 
2600  if (de_ctx == NULL)
2601  return;
2602 
2603 #ifdef PROFILE_RULES
2604  if (de_ctx->profile_ctx != NULL) {
2605  SCProfilingRuleDestroyCtx(de_ctx->profile_ctx);
2606  de_ctx->profile_ctx = NULL;
2607  }
2608 #endif
2609 #ifdef PROFILING
2610  if (de_ctx->profile_keyword_ctx != NULL) {
2611  SCProfilingKeywordDestroyCtx(de_ctx);//->profile_keyword_ctx);
2612 // de_ctx->profile_keyword_ctx = NULL;
2613  }
2614  if (de_ctx->profile_sgh_ctx != NULL) {
2616  }
2618 #endif
2619 
2620  /* Normally the hashes are freed elsewhere, but
2621  * to be sure look at them again here.
2622  */
2628  if (de_ctx->sig_array)
2630 
2631  if (de_ctx->filedata_config)
2633 
2637 
2639 
2643 
2645 
2646  DetectEngineCtxFreeThreadKeywordData(de_ctx);
2648  DetectEngineCtxFreeFailedSigs(de_ctx);
2649 
2652 
2653  /* if we have a config prefix, remove the config from the tree */
2654  if (strlen(de_ctx->config_prefix) > 0) {
2655  /* remove config */
2657  if (node != NULL) {
2658  ConfNodeRemove(node); /* frees node */
2659  }
2660 #if 0
2661  ConfDump();
2662 #endif
2663  }
2664 
2667 
2668  DetectBufferTypeFreeDetectEngine(de_ctx);
2671 
2672  if (de_ctx->tenant_path) {
2674  }
2675 
2676  if (de_ctx->requirements) {
2677  SCDetectRequiresStatusFree(de_ctx->requirements);
2678  }
2679 
2680  SCFree(de_ctx);
2681  //DetectAddressGroupPrintMemory();
2682  //DetectSigGroupPrintMemory();
2683  //DetectPortPrintMemory();
2684 }
2685 
2686 /** \brief Function that load DetectEngineCtx config for grouping sigs
2687  * used by the engine
2688  * \retval 0 if no config provided, 1 if config was provided
2689  * and loaded successfully
2690  */
2691 static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
2692 {
2693  uint8_t profile = ENGINE_PROFILE_MEDIUM;
2694  const char *max_uniq_toclient_groups_str = NULL;
2695  const char *max_uniq_toserver_groups_str = NULL;
2696  const char *sgh_mpm_context = NULL;
2697  const char *de_ctx_profile = NULL;
2698 
2699  (void)ConfGet("detect.profile", &de_ctx_profile);
2700  (void)ConfGet("detect.sgh-mpm-context", &sgh_mpm_context);
2701 
2702  ConfNode *de_ctx_custom = ConfGetNode("detect-engine");
2703  ConfNode *opt = NULL;
2704 
2705  if (de_ctx_custom != NULL) {
2706  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2707  if (de_ctx_profile == NULL) {
2708  if (opt->val && strcmp(opt->val, "profile") == 0) {
2709  de_ctx_profile = opt->head.tqh_first->val;
2710  }
2711  }
2712 
2713  if (sgh_mpm_context == NULL) {
2714  if (opt->val && strcmp(opt->val, "sgh-mpm-context") == 0) {
2715  sgh_mpm_context = opt->head.tqh_first->val;
2716  }
2717  }
2718  }
2719  }
2720 
2721  if (de_ctx_profile != NULL) {
2722  if (strcmp(de_ctx_profile, "low") == 0 ||
2723  strcmp(de_ctx_profile, "lowest") == 0) { // legacy
2724  profile = ENGINE_PROFILE_LOW;
2725  } else if (strcmp(de_ctx_profile, "medium") == 0) {
2726  profile = ENGINE_PROFILE_MEDIUM;
2727  } else if (strcmp(de_ctx_profile, "high") == 0 ||
2728  strcmp(de_ctx_profile, "highest") == 0) { // legacy
2729  profile = ENGINE_PROFILE_HIGH;
2730  } else if (strcmp(de_ctx_profile, "custom") == 0) {
2731  profile = ENGINE_PROFILE_CUSTOM;
2732  } else {
2733  SCLogError("invalid value for detect.profile: '%s'. "
2734  "Valid options: low, medium, high and custom.",
2735  de_ctx_profile);
2736  return -1;
2737  }
2738 
2739  SCLogDebug("Profile for detection engine groups is \"%s\"", de_ctx_profile);
2740  } else {
2741  SCLogDebug("Profile for detection engine groups not provided "
2742  "at suricata.yaml. Using default (\"medium\").");
2743  }
2744 
2745  /* detect-engine.sgh-mpm-context option parsing */
2746  if (sgh_mpm_context == NULL || strcmp(sgh_mpm_context, "auto") == 0) {
2747  /* for now, since we still haven't implemented any intelligence into
2748  * understanding the patterns and distributing mpm_ctx across sgh */
2750  de_ctx->mpm_matcher == MPM_HS) {
2752  } else {
2754  }
2755  } else {
2756  if (strcmp(sgh_mpm_context, "single") == 0) {
2758  } else if (strcmp(sgh_mpm_context, "full") == 0) {
2760  } else {
2761  SCLogError("You have supplied an "
2762  "invalid conf value for detect-engine.sgh-mpm-context-"
2763  "%s",
2764  sgh_mpm_context);
2765  exit(EXIT_FAILURE);
2766  }
2767  }
2768 
2769  if (RunmodeIsUnittests()) {
2771  }
2772 
2773  /* parse profile custom-values */
2774  opt = NULL;
2775  switch (profile) {
2776  case ENGINE_PROFILE_LOW:
2779  break;
2780 
2781  case ENGINE_PROFILE_HIGH:
2784  break;
2785 
2786  case ENGINE_PROFILE_CUSTOM:
2787  (void)ConfGet("detect.custom-values.toclient-groups",
2788  &max_uniq_toclient_groups_str);
2789  (void)ConfGet("detect.custom-values.toserver-groups",
2790  &max_uniq_toserver_groups_str);
2791 
2792  if (de_ctx_custom != NULL) {
2793  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2794  if (opt->val && strcmp(opt->val, "custom-values") == 0) {
2795  if (max_uniq_toclient_groups_str == NULL) {
2796  max_uniq_toclient_groups_str = (char *)ConfNodeLookupChildValue
2797  (opt->head.tqh_first, "toclient-sp-groups");
2798  }
2799  if (max_uniq_toclient_groups_str == NULL) {
2800  max_uniq_toclient_groups_str = (char *)ConfNodeLookupChildValue
2801  (opt->head.tqh_first, "toclient-groups");
2802  }
2803  if (max_uniq_toserver_groups_str == NULL) {
2804  max_uniq_toserver_groups_str = (char *)ConfNodeLookupChildValue
2805  (opt->head.tqh_first, "toserver-dp-groups");
2806  }
2807  if (max_uniq_toserver_groups_str == NULL) {
2808  max_uniq_toserver_groups_str = (char *)ConfNodeLookupChildValue
2809  (opt->head.tqh_first, "toserver-groups");
2810  }
2811  }
2812  }
2813  }
2814  if (max_uniq_toclient_groups_str != NULL) {
2816  (uint16_t)strlen(max_uniq_toclient_groups_str),
2817  (const char *)max_uniq_toclient_groups_str) <= 0) {
2819 
2820  SCLogWarning("parsing '%s' for "
2821  "toclient-groups failed, using %u",
2822  max_uniq_toclient_groups_str, de_ctx->max_uniq_toclient_groups);
2823  }
2824  } else {
2826  }
2827  SCLogConfig("toclient-groups %u", de_ctx->max_uniq_toclient_groups);
2828 
2829  if (max_uniq_toserver_groups_str != NULL) {
2831  (uint16_t)strlen(max_uniq_toserver_groups_str),
2832  (const char *)max_uniq_toserver_groups_str) <= 0) {
2834 
2835  SCLogWarning("parsing '%s' for "
2836  "toserver-groups failed, using %u",
2837  max_uniq_toserver_groups_str, de_ctx->max_uniq_toserver_groups);
2838  }
2839  } else {
2841  }
2842  SCLogConfig("toserver-groups %u", de_ctx->max_uniq_toserver_groups);
2843  break;
2844 
2845  /* Default (or no config provided) is profile medium */
2846  case ENGINE_PROFILE_MEDIUM:
2848  default:
2851  break;
2852  }
2853 
2854  intmax_t value = 0;
2855  if (ConfGetInt("detect.inspection-recursion-limit", &value) == 1)
2856  {
2857  if (value >= 0 && value <= INT_MAX) {
2858  de_ctx->inspection_recursion_limit = (int)value;
2859  }
2860 
2861  /* fall back to old config parsing */
2862  } else {
2863  ConfNode *insp_recursion_limit_node = NULL;
2864  char *insp_recursion_limit = NULL;
2865 
2866  if (de_ctx_custom != NULL) {
2867  opt = NULL;
2868  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2869  if (opt->val && strcmp(opt->val, "inspection-recursion-limit") != 0)
2870  continue;
2871 
2872  insp_recursion_limit_node = ConfNodeLookupChild(opt, opt->val);
2873  if (insp_recursion_limit_node == NULL) {
2874  SCLogError("Error retrieving conf "
2875  "entry for detect-engine:inspection-recursion-limit");
2876  break;
2877  }
2878  insp_recursion_limit = insp_recursion_limit_node->val;
2879  SCLogDebug("Found detect-engine.inspection-recursion-limit - %s:%s",
2880  insp_recursion_limit_node->name, insp_recursion_limit_node->val);
2881  break;
2882  }
2883 
2884  if (insp_recursion_limit != NULL) {
2886  0, (const char *)insp_recursion_limit) < 0) {
2887  SCLogWarning("Invalid value for "
2888  "detect-engine.inspection-recursion-limit: %s "
2889  "resetting to %d",
2890  insp_recursion_limit, DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT);
2893  }
2894  } else {
2897  }
2898  }
2899  }
2900 
2903 
2904  SCLogDebug("de_ctx->inspection_recursion_limit: %d",
2906 
2907  // default value is 4
2909  if (ConfGetInt("detect.stream-tx-log-limit", &value) == 1) {
2910  if (value >= 0 && value <= UINT8_MAX) {
2911  de_ctx->stream_tx_log_limit = (uint8_t)value;
2912  } else {
2913  SCLogWarning("Invalid value for detect-engine.stream-tx-log-limit: must be between 0 "
2914  "and 255, will default to 4");
2915  }
2916  }
2917 
2918  /* parse port grouping whitelisting settings */
2919 
2920  const char *ports = NULL;
2921  (void)ConfGet("detect.grouping.tcp-whitelist", &ports);
2922  if (ports) {
2923  SCLogConfig("grouping: tcp-whitelist %s", ports);
2924  } else {
2925  ports = "53, 80, 139, 443, 445, 1433, 3306, 3389, 6666, 6667, 8080";
2926  SCLogConfig("grouping: tcp-whitelist (default) %s", ports);
2927 
2928  }
2929  if (DetectPortParse(de_ctx, &de_ctx->tcp_whitelist, ports) != 0) {
2930  SCLogWarning("'%s' is not a valid value "
2931  "for detect.grouping.tcp-whitelist",
2932  ports);
2933  }
2935  for ( ; x != NULL; x = x->next) {
2936  if (x->port != x->port2) {
2937  SCLogWarning("'%s' is not a valid value "
2938  "for detect.grouping.tcp-whitelist: only single ports allowed",
2939  ports);
2941  de_ctx->tcp_whitelist = NULL;
2942  break;
2943  }
2944  }
2945 
2946  ports = NULL;
2947  (void)ConfGet("detect.grouping.udp-whitelist", &ports);
2948  if (ports) {
2949  SCLogConfig("grouping: udp-whitelist %s", ports);
2950  } else {
2951  ports = "53, 135, 5060";
2952  SCLogConfig("grouping: udp-whitelist (default) %s", ports);
2953 
2954  }
2955  if (DetectPortParse(de_ctx, &de_ctx->udp_whitelist, ports) != 0) {
2956  SCLogWarning("'%s' is not a valid value "
2957  "for detect.grouping.udp-whitelist",
2958  ports);
2959  }
2960  for (x = de_ctx->udp_whitelist; x != NULL; x = x->next) {
2961  if (x->port != x->port2) {
2962  SCLogWarning("'%s' is not a valid value "
2963  "for detect.grouping.udp-whitelist: only single ports allowed",
2964  ports);
2966  de_ctx->udp_whitelist = NULL;
2967  break;
2968  }
2969  }
2970 
2972  const char *pf_setting = NULL;
2973  if (ConfGet("detect.prefilter.default", &pf_setting) == 1 && pf_setting) {
2974  if (strcasecmp(pf_setting, "mpm") == 0) {
2976  } else if (strcasecmp(pf_setting, "auto") == 0) {
2978  }
2979  }
2980  switch (de_ctx->prefilter_setting) {
2981  case DETECT_PREFILTER_MPM:
2982  SCLogConfig("prefilter engines: MPM");
2983  break;
2984  case DETECT_PREFILTER_AUTO:
2985  SCLogConfig("prefilter engines: MPM and keywords");
2986  break;
2987  }
2988 
2989  return 0;
2990 }
2991 
2992 /*
2993  * getting & (re)setting the internal sig i
2994  */
2995 
2996 //inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *de_ctx)
2997 //{
2998 // return de_ctx->signum;
2999 //}
3000 
3002 {
3003  de_ctx->signum = 0;
3004 }
3005 
3006 static int DetectEngineThreadCtxInitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
3007 {
3008  const DetectEngineMasterCtx *master = &g_master_de_ctx;
3009 
3010  if (master->keyword_id > 0) {
3011  // coverity[suspicious_sizeof : FALSE]
3012  det_ctx->global_keyword_ctxs_array = (void **)SCCalloc(master->keyword_id, sizeof(void *));
3013  if (det_ctx->global_keyword_ctxs_array == NULL) {
3014  SCLogError("setting up thread local detect ctx");
3015  return TM_ECODE_FAILED;
3016  }
3017  det_ctx->global_keyword_ctxs_size = master->keyword_id;
3018 
3019  const DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
3020  while (item) {
3021  det_ctx->global_keyword_ctxs_array[item->id] = item->InitFunc(item->data);
3022  if (det_ctx->global_keyword_ctxs_array[item->id] == NULL) {
3023  SCLogError("setting up thread local detect ctx "
3024  "for keyword \"%s\" failed",
3025  item->name);
3026  return TM_ECODE_FAILED;
3027  }
3028  item = item->next;
3029  }
3030  }
3031  return TM_ECODE_OK;
3032 }
3033 
3034 static void DetectEngineThreadCtxDeinitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
3035 {
3036  if (det_ctx->global_keyword_ctxs_array == NULL ||
3037  det_ctx->global_keyword_ctxs_size == 0) {
3038  return;
3039  }
3040 
3041  const DetectEngineMasterCtx *master = &g_master_de_ctx;
3042  if (master->keyword_id > 0) {
3043  const DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
3044  while (item) {
3045  if (det_ctx->global_keyword_ctxs_array[item->id] != NULL)
3046  item->FreeFunc(det_ctx->global_keyword_ctxs_array[item->id]);
3047 
3048  item = item->next;
3049  }
3050  det_ctx->global_keyword_ctxs_size = 0;
3052  det_ctx->global_keyword_ctxs_array = NULL;
3053  }
3054 }
3055 
3056 static int DetectEngineThreadCtxInitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
3057 {
3058  if (de_ctx->keyword_id > 0) {
3059  // coverity[suspicious_sizeof : FALSE]
3060  det_ctx->keyword_ctxs_array = SCCalloc(de_ctx->keyword_id, sizeof(void *));
3061  if (det_ctx->keyword_ctxs_array == NULL) {
3062  SCLogError("setting up thread local detect ctx");
3063  return TM_ECODE_FAILED;
3064  }
3065 
3066  det_ctx->keyword_ctxs_size = de_ctx->keyword_id;
3067 
3069  for (; hb != NULL; hb = HashListTableGetListNext(hb)) {
3071 
3072  det_ctx->keyword_ctxs_array[item->id] = item->InitFunc(item->data);
3073  if (det_ctx->keyword_ctxs_array[item->id] == NULL) {
3074  SCLogError("setting up thread local detect ctx "
3075  "for keyword \"%s\" failed",
3076  item->name);
3077  return TM_ECODE_FAILED;
3078  }
3079  }
3080  }
3081  return TM_ECODE_OK;
3082 }
3083 
3084 static void DetectEngineThreadCtxDeinitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
3085 {
3086  if (de_ctx->keyword_id > 0) {
3088  for (; hb != NULL; hb = HashListTableGetListNext(hb)) {
3090 
3091  if (det_ctx->keyword_ctxs_array[item->id] != NULL)
3092  item->FreeFunc(det_ctx->keyword_ctxs_array[item->id]);
3093  }
3094  det_ctx->keyword_ctxs_size = 0;
3095  SCFree(det_ctx->keyword_ctxs_array);
3096  det_ctx->keyword_ctxs_array = NULL;
3097  }
3098 }
3099 
3100 /** NOTE: master MUST be locked before calling this */
3101 static TmEcode DetectEngineThreadCtxInitForMT(ThreadVars *tv, DetectEngineThreadCtx *det_ctx)
3102 {
3103  DetectEngineMasterCtx *master = &g_master_de_ctx;
3104  DetectEngineTenantMapping *map_array = NULL;
3105  uint32_t map_array_size = 0;
3106  uint32_t map_cnt = 0;
3107  uint32_t max_tenant_id = 0;
3108  DetectEngineCtx *list = master->list;
3109  HashTable *mt_det_ctxs_hash = NULL;
3110 
3111  if (master->tenant_selector == TENANT_SELECTOR_UNKNOWN) {
3112  SCLogError("no tenant selector set: "
3113  "set using multi-detect.selector");
3114  return TM_ECODE_FAILED;
3115  }
3116 
3117  uint32_t tcnt = 0;
3118  while (list) {
3119  if (list->tenant_id > max_tenant_id)
3120  max_tenant_id = list->tenant_id;
3121 
3122  list = list->next;
3123  tcnt++;
3124  }
3125 
3126  mt_det_ctxs_hash = HashTableInit(tcnt * 2, TenantIdHash, TenantIdCompare, TenantIdFree);
3127  if (mt_det_ctxs_hash == NULL) {
3128  goto error;
3129  }
3130 
3131  if (tcnt == 0) {
3132  SCLogInfo("no tenants left, or none registered yet");
3133  } else {
3134  max_tenant_id++;
3135 
3137  while (map) {
3138  map_cnt++;
3139  map = map->next;
3140  }
3141 
3142  if (map_cnt > 0) {
3143  map_array_size = map_cnt + 1;
3144 
3145  map_array = SCCalloc(map_array_size, sizeof(*map_array));
3146  if (map_array == NULL)
3147  goto error;
3148 
3149  /* fill the array */
3150  map_cnt = 0;
3151  map = master->tenant_mapping_list;
3152  while (map) {
3153  if (map_cnt >= map_array_size) {
3154  goto error;
3155  }
3156  map_array[map_cnt].traffic_id = map->traffic_id;
3157  map_array[map_cnt].tenant_id = map->tenant_id;
3158  map_cnt++;
3159  map = map->next;
3160  }
3161 
3162  }
3163 
3164  /* set up hash for tenant lookup */
3165  list = master->list;
3166  while (list) {
3167  SCLogDebug("tenant-id %u", list->tenant_id);
3168  if (list->tenant_id != 0) {
3170  if (mt_det_ctx == NULL)
3171  goto error;
3172  if (HashTableAdd(mt_det_ctxs_hash, mt_det_ctx, 0) != 0) {
3173  goto error;
3174  }
3175  }
3176  list = list->next;
3177  }
3178  }
3179 
3180  det_ctx->mt_det_ctxs_hash = mt_det_ctxs_hash;
3181  mt_det_ctxs_hash = NULL;
3182 
3183  det_ctx->mt_det_ctxs_cnt = max_tenant_id;
3184 
3185  det_ctx->tenant_array = map_array;
3186  det_ctx->tenant_array_size = map_array_size;
3187 
3188  switch (master->tenant_selector) {
3190  SCLogDebug("TENANT_SELECTOR_UNKNOWN");
3191  break;
3192  case TENANT_SELECTOR_VLAN:
3193  det_ctx->TenantGetId = DetectEngineTenantGetIdFromVlanId;
3194  SCLogDebug("TENANT_SELECTOR_VLAN");
3195  break;
3197  det_ctx->TenantGetId = DetectEngineTenantGetIdFromLivedev;
3198  SCLogDebug("TENANT_SELECTOR_LIVEDEV");
3199  break;
3201  det_ctx->TenantGetId = DetectEngineTenantGetIdFromPcap;
3202  SCLogDebug("TENANT_SELECTOR_DIRECT");
3203  break;
3204  }
3205 
3206  return TM_ECODE_OK;
3207 error:
3208  if (map_array != NULL)
3209  SCFree(map_array);
3210  if (mt_det_ctxs_hash != NULL)
3211  HashTableFree(mt_det_ctxs_hash);
3212 
3213  return TM_ECODE_FAILED;
3214 }
3215 
3216 /** \internal
3217  * \brief Helper for DetectThread setup functions
3218  */
3219 static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
3220 {
3222 
3223  PmqSetup(&det_ctx->pmq);
3224 
3226  if (det_ctx->spm_thread_ctx == NULL) {
3227  return TM_ECODE_FAILED;
3228  }
3229 
3230  /* sized to the max of our sgh settings. A max setting of 0 implies that all
3231  * sgh's have: sgh->non_pf_store_cnt == 0 */
3232  if (de_ctx->non_pf_store_cnt_max > 0) {
3234  BUG_ON(det_ctx->non_pf_id_array == NULL);
3235  }
3236 
3237  /* DeState */
3238  if (de_ctx->sig_array_len > 0) {
3239  det_ctx->match_array_len = de_ctx->sig_array_len;
3240  det_ctx->match_array = SCCalloc(det_ctx->match_array_len, sizeof(Signature *));
3241  if (det_ctx->match_array == NULL) {
3242  return TM_ECODE_FAILED;
3243  }
3244 
3246  }
3247 
3248  /* Alert processing queue */
3249  AlertQueueInit(det_ctx);
3250 
3251  /* byte_extract storage */
3252  det_ctx->byte_values = SCMalloc(sizeof(*det_ctx->byte_values) *
3254  if (det_ctx->byte_values == NULL) {
3255  return TM_ECODE_FAILED;
3256  }
3257 
3258  /* Allocate space for base64 decoded data. */
3261  if (det_ctx->base64_decoded == NULL) {
3262  return TM_ECODE_FAILED;
3263  }
3265  det_ctx->base64_decoded_len = 0;
3266  }
3267 
3269  det_ctx->inspect.buffers = SCCalloc(det_ctx->inspect.buffers_size, sizeof(InspectionBuffer));
3270  if (det_ctx->inspect.buffers == NULL) {
3271  return TM_ECODE_FAILED;
3272  }
3273  det_ctx->inspect.to_clear_queue = SCCalloc(det_ctx->inspect.buffers_size, sizeof(uint32_t));
3274  if (det_ctx->inspect.to_clear_queue == NULL) {
3275  return TM_ECODE_FAILED;
3276  }
3277  det_ctx->inspect.to_clear_idx = 0;
3278 
3281  if (det_ctx->multi_inspect.buffers == NULL) {
3282  return TM_ECODE_FAILED;
3283  }
3284  det_ctx->multi_inspect.to_clear_queue = SCCalloc(det_ctx->multi_inspect.buffers_size, sizeof(uint32_t));
3285  if (det_ctx->multi_inspect.to_clear_queue == NULL) {
3286  return TM_ECODE_FAILED;
3287  }
3288  det_ctx->multi_inspect.to_clear_idx = 0;
3289 
3290 
3291  DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
3292  DetectEngineThreadCtxInitGlobalKeywords(det_ctx);
3293 #ifdef PROFILE_RULES
3294  SCProfilingRuleThreadSetup(de_ctx->profile_ctx, det_ctx);
3295 #endif
3296 #ifdef PROFILING
3300 #endif
3301  SC_ATOMIC_INIT(det_ctx->so_far_used_by_detect);
3302 
3303  return TM_ECODE_OK;
3304 }
3305 
3306 /** \brief initialize thread specific detection engine context
3307  *
3308  * \note there is a special case when using delayed detect. In this case the
3309  * function is called twice per thread. The first time the rules are not
3310  * yet loaded. de_ctx->delayed_detect_initialized will be 0. The 2nd
3311  * time they will be loaded. de_ctx->delayed_detect_initialized will be 1.
3312  * This is needed to do the per thread counter registration before the
3313  * packet runtime starts. In delayed detect mode, the first call will
3314  * return a NULL ptr through the data ptr.
3315  *
3316  * \param tv ThreadVars for this thread
3317  * \param initdata pointer to de_ctx
3318  * \param data[out] pointer to store our thread detection ctx
3319  *
3320  * \retval TM_ECODE_OK if all went well
3321  * \retval TM_ECODE_FAILED on serious errors
3322  */
3323 TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
3324 {
3325  DetectEngineThreadCtx *det_ctx = SCCalloc(1, sizeof(DetectEngineThreadCtx));
3326  if (unlikely(det_ctx == NULL))
3327  return TM_ECODE_FAILED;
3328 
3329  det_ctx->tv = tv;
3330  det_ctx->de_ctx = DetectEngineGetCurrent();
3331  if (det_ctx->de_ctx == NULL) {
3332 #ifdef UNITTESTS
3333  if (RunmodeIsUnittests()) {
3334  det_ctx->de_ctx = (DetectEngineCtx *)initdata;
3335  } else {
3336  DetectEngineThreadCtxDeinit(tv, det_ctx);
3337  return TM_ECODE_FAILED;
3338  }
3339 #else
3340  DetectEngineThreadCtxDeinit(tv, det_ctx);
3341  return TM_ECODE_FAILED;
3342 #endif
3343  }
3344 
3345  if (det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
3346  det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_TENANT)
3347  {
3348  if (ThreadCtxDoInit(det_ctx->de_ctx, det_ctx) != TM_ECODE_OK) {
3349  DetectEngineThreadCtxDeinit(tv, det_ctx);
3350  return TM_ECODE_FAILED;
3351  }
3352  }
3353 
3354  /** alert counter setup */
3355  det_ctx->counter_alerts = StatsRegisterCounter("detect.alert", tv);
3356  det_ctx->counter_alerts_overflow = StatsRegisterCounter("detect.alert_queue_overflow", tv);
3357  det_ctx->counter_alerts_suppressed = StatsRegisterCounter("detect.alerts_suppressed", tv);
3358 
3359  /* Register counter for Lua rule errors. */
3360  det_ctx->lua_rule_errors = StatsRegisterCounter("detect.lua.errors", tv);
3361 
3362  /* Register a counter for Lua blocked function attempts. */
3363  det_ctx->lua_blocked_function_errors =
3364  StatsRegisterCounter("detect.lua.blocked_function_errors", tv);
3365 
3366  /* Register a counter for Lua instruction limit errors. */
3367  det_ctx->lua_instruction_limit_errors =
3368  StatsRegisterCounter("detect.lua.instruction_limit_errors", tv);
3369 
3370  /* Register a counter for Lua memory limit errors. */
3371  det_ctx->lua_memory_limit_errors = StatsRegisterCounter("detect.lua.memory_limit_errors", tv);
3372 
3373 #ifdef PROFILING
3374  det_ctx->counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
3375  det_ctx->counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
3376  det_ctx->counter_fnonmpm_list = StatsRegisterAvgCounter("detect.fnonmpm_list", tv);
3377  det_ctx->counter_match_list = StatsRegisterAvgCounter("detect.match_list", tv);
3378 #endif
3379 
3381  if (DetectEngineThreadCtxInitForMT(tv, det_ctx) != TM_ECODE_OK) {
3382  DetectEngineThreadCtxDeinit(tv, det_ctx);
3383  return TM_ECODE_FAILED;
3384  }
3385  }
3386 
3387  /* pass thread data back to caller */
3388  *data = (void *)det_ctx;
3389 
3390  return TM_ECODE_OK;
3391 }
3392 
3393 /**
3394  * \internal
3395  * \brief initialize a det_ctx for reload cases
3396  * \param new_de_ctx the new detection engine
3397  * \param mt flag to indicate if MT should be set up for this det_ctx
3398  * this should only be done for the 'root' det_ctx
3399  *
3400  * \retval det_ctx detection engine thread ctx or NULL in case of error
3401  */
3403  ThreadVars *tv, DetectEngineCtx *new_de_ctx, int mt)
3404 {
3405  DetectEngineThreadCtx *det_ctx = SCCalloc(1, sizeof(DetectEngineThreadCtx));
3406  if (unlikely(det_ctx == NULL))
3407  return NULL;
3408 
3409  det_ctx->tenant_id = new_de_ctx->tenant_id;
3410  det_ctx->tv = tv;
3411  det_ctx->de_ctx = DetectEngineReference(new_de_ctx);
3412  if (det_ctx->de_ctx == NULL) {
3413  SCFree(det_ctx);
3414  return NULL;
3415  }
3416 
3417  /* most of the init happens here */
3418  if (det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
3419  det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_TENANT)
3420  {
3421  if (ThreadCtxDoInit(det_ctx->de_ctx, det_ctx) != TM_ECODE_OK) {
3422  DetectEngineDeReference(&det_ctx->de_ctx);
3423  SCFree(det_ctx);
3424  return NULL;
3425  }
3426  }
3427 
3428  /** alert counter setup */
3429  det_ctx->counter_alerts = StatsRegisterCounter("detect.alert", tv);
3430  det_ctx->counter_alerts_overflow = StatsRegisterCounter("detect.alert_queue_overflow", tv);
3431  det_ctx->counter_alerts_suppressed = StatsRegisterCounter("detect.alerts_suppressed", tv);
3432 #ifdef PROFILING
3433  uint16_t counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
3434  uint16_t counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
3435  uint16_t counter_fnonmpm_list = StatsRegisterAvgCounter("detect.fnonmpm_list", tv);
3436  uint16_t counter_match_list = StatsRegisterAvgCounter("detect.match_list", tv);
3437  det_ctx->counter_mpm_list = counter_mpm_list;
3438  det_ctx->counter_nonmpm_list = counter_nonmpm_list;
3439  det_ctx->counter_fnonmpm_list = counter_fnonmpm_list;
3440  det_ctx->counter_match_list = counter_match_list;
3441 #endif
3442 
3443  if (mt && DetectEngineMultiTenantEnabled()) {
3444  if (DetectEngineThreadCtxInitForMT(tv, det_ctx) != TM_ECODE_OK) {
3445  DetectEngineDeReference(&det_ctx->de_ctx);
3446  SCFree(det_ctx);
3447  return NULL;
3448  }
3449  }
3450 
3451  return det_ctx;
3452 }
3453 
3454 static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
3455 {
3456 #if DEBUG
3457  SCLogDebug("PACKET PKT_STREAM_ADD: %"PRIu64, det_ctx->pkt_stream_add_cnt);
3458 
3459  SCLogDebug("PAYLOAD MPM %"PRIu64"/%"PRIu64, det_ctx->payload_mpm_cnt, det_ctx->payload_mpm_size);
3460  SCLogDebug("STREAM MPM %"PRIu64"/%"PRIu64, det_ctx->stream_mpm_cnt, det_ctx->stream_mpm_size);
3461 
3462  SCLogDebug("PAYLOAD SIG %"PRIu64"/%"PRIu64, det_ctx->payload_persig_cnt, det_ctx->payload_persig_size);
3463  SCLogDebug("STREAM SIG %"PRIu64"/%"PRIu64, det_ctx->stream_persig_cnt, det_ctx->stream_persig_size);
3464 #endif
3465 
3466  if (det_ctx->tenant_array != NULL) {
3467  SCFree(det_ctx->tenant_array);
3468  det_ctx->tenant_array = NULL;
3469  }
3470 
3471 #ifdef PROFILE_RULES
3473 #endif
3474 #ifdef PROFILING
3477  SCProfilingSghThreadCleanup(det_ctx);
3478 #endif
3479 
3480  /** \todo get rid of this static */
3481  if (det_ctx->de_ctx != NULL) {
3482  PatternMatchThreadDestroy(&det_ctx->mtc, det_ctx->de_ctx->mpm_matcher);
3483  }
3484 
3485  PmqFree(&det_ctx->pmq);
3486 
3487  if (det_ctx->spm_thread_ctx != NULL) {
3489  }
3490 
3491  if (det_ctx->non_pf_id_array != NULL)
3492  SCFree(det_ctx->non_pf_id_array);
3493 
3494  if (det_ctx->match_array != NULL)
3495  SCFree(det_ctx->match_array);
3496 
3498 
3499  AlertQueueFree(det_ctx);
3500 
3501  if (det_ctx->byte_values != NULL)
3502  SCFree(det_ctx->byte_values);
3503 
3504  /* Decoded base64 data. */
3505  if (det_ctx->base64_decoded != NULL) {
3506  SCFree(det_ctx->base64_decoded);
3507  }
3508 
3509  if (det_ctx->inspect.buffers) {
3510  for (uint32_t i = 0; i < det_ctx->inspect.buffers_size; i++) {
3511  InspectionBufferFree(&det_ctx->inspect.buffers[i]);
3512  }
3513  SCFree(det_ctx->inspect.buffers);
3514  }
3515  if (det_ctx->inspect.to_clear_queue) {
3516  SCFree(det_ctx->inspect.to_clear_queue);
3517  }
3518  if (det_ctx->multi_inspect.buffers) {
3519  for (uint32_t i = 0; i < det_ctx->multi_inspect.buffers_size; i++) {
3521  for (uint32_t x = 0; x < fb->size; x++) {
3523  }
3525  }
3526  SCFree(det_ctx->multi_inspect.buffers);
3527  }
3528  if (det_ctx->multi_inspect.to_clear_queue) {
3530  }
3531 
3532  DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx);
3533  if (det_ctx->de_ctx != NULL) {
3534  DetectEngineThreadCtxDeinitKeywords(det_ctx->de_ctx, det_ctx);
3535 #ifdef UNITTESTS
3536  if (!RunmodeIsUnittests() || det_ctx->de_ctx->ref_cnt > 0)
3537  DetectEngineDeReference(&det_ctx->de_ctx);
3538 #else
3539  DetectEngineDeReference(&det_ctx->de_ctx);
3540 #endif
3541  }
3542 
3544 
3545  SCFree(det_ctx);
3546 
3548 }
3549 
3551 {
3552  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
3553 
3554  if (det_ctx == NULL) {
3555  SCLogWarning("argument \"data\" NULL");
3556  return TM_ECODE_OK;
3557  }
3558 
3559  if (det_ctx->mt_det_ctxs_hash != NULL) {
3560  HashTableFree(det_ctx->mt_det_ctxs_hash);
3561  det_ctx->mt_det_ctxs_hash = NULL;
3562  }
3563  DetectEngineThreadCtxFree(det_ctx);
3564 
3565  return TM_ECODE_OK;
3566 }
3567 
3568 static uint32_t DetectKeywordCtxHashFunc(HashListTable *ht, void *data, uint16_t datalen)
3569 {
3571  const char *name = ctx->name;
3572  uint64_t hash = StringHashDjb2((const uint8_t *)name, strlen(name)) + (ptrdiff_t)ctx->data;
3573  hash %= ht->array_size;
3574  return hash;
3575 }
3576 
3577 static char DetectKeywordCtxCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
3578 {
3579  DetectEngineThreadKeywordCtxItem *ctx1 = data1;
3580  DetectEngineThreadKeywordCtxItem *ctx2 = data2;
3581  const char *name1 = ctx1->name;
3582  const char *name2 = ctx2->name;
3583  return (strcmp(name1, name2) == 0 && ctx1->data == ctx2->data);
3584 }
3585 
3586 static void DetectKeywordCtxFreeFunc(void *ptr)
3587 {
3588  SCFree(ptr);
3589 }
3590 
3591 /** \brief Register Thread keyword context Funcs
3592  *
3593  * \param de_ctx detection engine to register in
3594  * \param name keyword name for error printing
3595  * \param InitFunc function ptr
3596  * \param data keyword init data to pass to Func. Can be NULL.
3597  * \param FreeFunc function ptr
3598  * \param mode 0 normal (ctx per keyword instance) 1 shared (one ctx per det_ct)
3599  *
3600  * \retval id for retrieval of ctx at runtime
3601  * \retval -1 on error
3602  *
3603  * \note make sure "data" remains valid and it free'd elsewhere. It's
3604  * recommended to store it in the keywords global ctx so that
3605  * it's freed when the de_ctx is freed.
3606  */
3607 int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int mode)
3608 {
3609  BUG_ON(de_ctx == NULL || InitFunc == NULL || FreeFunc == NULL);
3610 
3611  if (de_ctx->keyword_hash == NULL) {
3612  de_ctx->keyword_hash = HashListTableInit(4096, // TODO
3613  DetectKeywordCtxHashFunc, DetectKeywordCtxCompareFunc, DetectKeywordCtxFreeFunc);
3614  BUG_ON(de_ctx->keyword_hash == NULL);
3615  }
3616 
3617  if (mode) {
3618  DetectEngineThreadKeywordCtxItem search = { .data = data, .name = name };
3619 
3621  HashListTableLookup(de_ctx->keyword_hash, (void *)&search, 0);
3622  if (item)
3623  return item->id;
3624 
3625  /* fall through */
3626  }
3627 
3629  if (unlikely(item == NULL))
3630  return -1;
3631 
3632  item->InitFunc = InitFunc;
3633  item->FreeFunc = FreeFunc;
3634  item->data = data;
3635  item->name = name;
3636  item->id = de_ctx->keyword_id++;
3637 
3638  if (HashListTableAdd(de_ctx->keyword_hash, (void *)item, 0) < 0) {
3639  SCFree(item);
3640  return -1;
3641  }
3642  return item->id;
3643 }
3644 
3645 /** \brief Remove Thread keyword context registration
3646  *
3647  * \param de_ctx detection engine to deregister from
3648  * \param det_ctx detection engine thread context to deregister from
3649  * \param data keyword init data to pass to Func. Can be NULL.
3650  * \param name keyword name for error printing
3651  *
3652  * \retval 1 Item unregistered
3653  * \retval 0 otherwise
3654  *
3655  * \note make sure "data" remains valid and it free'd elsewhere. It's
3656  * recommended to store it in the keywords global ctx so that
3657  * it's freed when the de_ctx is freed.
3658  */
3659 int DetectUnregisterThreadCtxFuncs(DetectEngineCtx *de_ctx, void *data, const char *name)
3660 {
3661  /* might happen if we call this before a call to *Register* */
3662  if (de_ctx->keyword_hash == NULL)
3663  return 1;
3664  DetectEngineThreadKeywordCtxItem remove = { .data = data, .name = name };
3665  if (HashListTableRemove(de_ctx->keyword_hash, (void *)&remove, 0) == 0)
3666  return 1;
3667  return 0;
3668 }
3669 /** \brief Retrieve thread local keyword ctx by id
3670  *
3671  * \param det_ctx detection engine thread ctx to retrieve the ctx from
3672  * \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
3673  * keyword init.
3674  *
3675  * \retval ctx or NULL on error
3676  */
3678 {
3679  if (id < 0 || id > det_ctx->keyword_ctxs_size || det_ctx->keyword_ctxs_array == NULL)
3680  return NULL;
3681 
3682  return det_ctx->keyword_ctxs_array[id];
3683 }
3684 
3685 
3686 /** \brief Register Thread keyword context Funcs (Global)
3687  *
3688  * IDs stay static over reloads and between tenants
3689  *
3690  * \param name keyword name for error printing
3691  * \param InitFunc function ptr
3692  * \param FreeFunc function ptr
3693  *
3694  * \retval id for retrieval of ctx at runtime
3695  * \retval -1 on error
3696  */
3698  void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *))
3699 {
3700  int id;
3701  BUG_ON(InitFunc == NULL || FreeFunc == NULL);
3702 
3703  DetectEngineMasterCtx *master = &g_master_de_ctx;
3704 
3705  /* if already registered, return existing id */
3707  while (item != NULL) {
3708  if (strcmp(name, item->name) == 0) {
3709  id = item->id;
3710  return id;
3711  }
3712 
3713  item = item->next;
3714  }
3715 
3716  item = SCCalloc(1, sizeof(*item));
3717  if (unlikely(item == NULL)) {
3718  return -1;
3719  }
3720  item->InitFunc = InitFunc;
3721  item->FreeFunc = FreeFunc;
3722  item->name = name;
3723  item->data = data;
3724 
3725  item->next = master->keyword_list;
3726  master->keyword_list = item;
3727  item->id = master->keyword_id++;
3728 
3729  id = item->id;
3730  return id;
3731 }
3732 
3733 /** \brief Retrieve thread local keyword ctx by id
3734  *
3735  * \param det_ctx detection engine thread ctx to retrieve the ctx from
3736  * \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
3737  * keyword init.
3738  *
3739  * \retval ctx or NULL on error
3740  */
3742 {
3743  if (id < 0 || id > det_ctx->global_keyword_ctxs_size ||
3744  det_ctx->global_keyword_ctxs_array == NULL) {
3745  return NULL;
3746  }
3747 
3748  return det_ctx->global_keyword_ctxs_array[id];
3749 }
3750 
3751 /** \brief Check if detection is enabled
3752  * \retval bool true or false */
3754 {
3755  DetectEngineMasterCtx *master = &g_master_de_ctx;
3756  SCMutexLock(&master->lock);
3757 
3758  if (master->list == NULL) {
3759  SCMutexUnlock(&master->lock);
3760  return 0;
3761  }
3762 
3763  SCMutexUnlock(&master->lock);
3764  return 1;
3765 }
3766 
3768 {
3769  uint32_t version;
3770  DetectEngineMasterCtx *master = &g_master_de_ctx;
3771  SCMutexLock(&master->lock);
3772  version = master->version;
3773  SCMutexUnlock(&master->lock);
3774  return version;
3775 }
3776 
3778 {
3779  DetectEngineMasterCtx *master = &g_master_de_ctx;
3780  SCMutexLock(&master->lock);
3781  master->version++;
3782  SCLogDebug("master version now %u", master->version);
3783  SCMutexUnlock(&master->lock);
3784 }
3785 
3787 {
3788  DetectEngineMasterCtx *master = &g_master_de_ctx;
3789  SCMutexLock(&master->lock);
3790 
3791  DetectEngineCtx *de_ctx = master->list;
3792  while (de_ctx) {
3796  {
3797  de_ctx->ref_cnt++;
3798  SCLogDebug("de_ctx %p ref_cnt %u", de_ctx, de_ctx->ref_cnt);
3799  SCMutexUnlock(&master->lock);
3800  return de_ctx;
3801  }
3802  de_ctx = de_ctx->next;
3803  }
3804 
3805  SCMutexUnlock(&master->lock);
3806  return NULL;
3807 }
3808 
3810 {
3811  if (de_ctx == NULL)
3812  return NULL;
3813  de_ctx->ref_cnt++;
3814  return de_ctx;
3815 }
3816 
3817 /** TODO locking? Not needed if this is a one time setting at startup */
3819 {
3820  DetectEngineMasterCtx *master = &g_master_de_ctx;
3821  return (master->multi_tenant_enabled);
3822 }
3823 
3824 /** \internal
3825  * \brief load a tenant from a yaml file
3826  *
3827  * \param tenant_id the tenant id by which the config is known
3828  * \param filename full path of a yaml file
3829  * \param loader_id id of loader thread or -1
3830  *
3831  * \retval 0 ok
3832  * \retval -1 failed
3833  */
3834 static int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename, int loader_id)
3835 {
3836  DetectEngineCtx *de_ctx = NULL;
3837  char prefix[64];
3838 
3839  snprintf(prefix, sizeof(prefix), "multi-detect.%u", tenant_id);
3840 
3841  SCStat st;
3842  if (SCStatFn(filename, &st) != 0) {
3843  SCLogError("failed to stat file %s", filename);
3844  goto error;
3845  }
3846 
3847  de_ctx = DetectEngineGetByTenantId(tenant_id);
3848  if (de_ctx != NULL) {
3849  SCLogError("tenant %u already registered", tenant_id);
3851  goto error;
3852  }
3853 
3854  ConfNode *node = ConfGetNode(prefix);
3855  if (node == NULL) {
3856  SCLogError("failed to properly setup yaml %s", filename);
3857  goto error;
3858  }
3859 
3860  de_ctx = DetectEngineCtxInitWithPrefix(prefix, tenant_id);
3861  if (de_ctx == NULL) {
3862  SCLogError("initializing detection engine "
3863  "context failed.");
3864  goto error;
3865  }
3866  SCLogDebug("de_ctx %p with prefix %s", de_ctx, de_ctx->config_prefix);
3867 
3869  de_ctx->tenant_id = tenant_id;
3870  de_ctx->loader_id = loader_id;
3871  de_ctx->tenant_path = SCStrdup(filename);
3872  if (de_ctx->tenant_path == NULL) {
3873  SCLogError("Failed to duplicate path");
3874  goto error;
3875  }
3876 
3877  if (SigLoadSignatures(de_ctx, NULL, false) < 0) {
3878  SCLogError("Loading signatures failed.");
3879  goto error;
3880  }
3881 
3883 
3884  return 0;
3885 
3886 error:
3887  if (de_ctx != NULL) {
3889  }
3890  return -1;
3891 }
3892 
3893 static int DetectEngineMultiTenantReloadTenant(uint32_t tenant_id, const char *filename, int reload_cnt)
3894 {
3895  DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
3896  if (old_de_ctx == NULL) {
3897  SCLogError("tenant detect engine not found");
3898  return -1;
3899  }
3900 
3901  if (filename == NULL)
3902  filename = old_de_ctx->tenant_path;
3903 
3904  char prefix[64];
3905  snprintf(prefix, sizeof(prefix), "multi-detect.%u.reload.%d", tenant_id, reload_cnt);
3906  reload_cnt++;
3907  SCLogDebug("prefix %s", prefix);
3908 
3909  if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) {
3910  SCLogError("failed to load yaml");
3911  goto error;
3912  }
3913 
3914  ConfNode *node = ConfGetNode(prefix);
3915  if (node == NULL) {
3916  SCLogError("failed to properly setup yaml %s", filename);
3917  goto error;
3918  }
3919 
3920  DetectEngineCtx *new_de_ctx = DetectEngineCtxInitWithPrefix(prefix, tenant_id);
3921  if (new_de_ctx == NULL) {
3922  SCLogError("initializing detection engine "
3923  "context failed.");
3924  goto error;
3925  }
3926  SCLogDebug("de_ctx %p with prefix %s", new_de_ctx, new_de_ctx->config_prefix);
3927 
3928  new_de_ctx->type = DETECT_ENGINE_TYPE_TENANT;
3929  new_de_ctx->tenant_id = tenant_id;
3930  new_de_ctx->loader_id = old_de_ctx->loader_id;
3931  new_de_ctx->tenant_path = SCStrdup(filename);
3932  if (new_de_ctx->tenant_path == NULL) {
3933  SCLogError("Failed to duplicate path");
3934  goto new_de_ctx_error;
3935  }
3936 
3937  if (SigLoadSignatures(new_de_ctx, NULL, false) < 0) {
3938  SCLogError("Loading signatures failed.");
3939  goto new_de_ctx_error;
3940  }
3941 
3942  DetectEngineAddToMaster(new_de_ctx);
3943 
3944  /* move to free list */
3945  DetectEngineMoveToFreeList(old_de_ctx);
3946  DetectEngineDeReference(&old_de_ctx);
3947  return 0;
3948 
3949 new_de_ctx_error:
3950  DetectEngineCtxFree(new_de_ctx);
3951 
3952 error:
3953  DetectEngineDeReference(&old_de_ctx);
3954  return -1;
3955 }
3956 
3957 
3958 typedef struct TenantLoaderCtx_ {
3959  uint32_t tenant_id;
3960  int reload_cnt; /**< used by reload */
3961  char *yaml; /**< heap alloc'd copy of file path for the yaml */
3963 
3964 static void DetectLoaderFreeTenant(void *ctx)
3965 {
3967  if (t->yaml != NULL) {
3968  SCFree(t->yaml);
3969  }
3970  SCFree(t);
3971 }
3972 
3973 static int DetectLoaderFuncLoadTenant(void *vctx, int loader_id)
3974 {
3975  TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
3976 
3977  SCLogDebug("loader %d", loader_id);
3978  if (DetectEngineMultiTenantLoadTenant(ctx->tenant_id, ctx->yaml, loader_id) != 0) {
3979  return -1;
3980  }
3981  return 0;
3982 }
3983 
3984 static int DetectLoaderSetupLoadTenant(uint32_t tenant_id, const char *yaml)
3985 {
3986  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
3987  if (t == NULL)
3988  return -ENOMEM;
3989 
3990  t->tenant_id = tenant_id;
3991  t->yaml = SCStrdup(yaml);
3992  if (t->yaml == NULL) {
3993  SCFree(t);
3994  return -ENOMEM;
3995  }
3996 
3997  return DetectLoaderQueueTask(-1, DetectLoaderFuncLoadTenant, t, DetectLoaderFreeTenant);
3998 }
3999 
4000 static int DetectLoaderFuncReloadTenant(void *vctx, int loader_id)
4001 {
4002  TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
4003 
4004  SCLogDebug("loader_id %d", loader_id);
4005 
4006  if (DetectEngineMultiTenantReloadTenant(ctx->tenant_id, ctx->yaml, ctx->reload_cnt) != 0) {
4007  return -1;
4008  }
4009  return 0;
4010 }
4011 
4012 static int DetectLoaderSetupReloadTenants(const int reload_cnt)
4013 {
4014  int ret = 0;
4015  DetectEngineMasterCtx *master = &g_master_de_ctx;
4016  SCMutexLock(&master->lock);
4017 
4018  DetectEngineCtx *de_ctx = master->list;
4019  while (de_ctx) {
4021  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
4022  if (t == NULL) {
4023  ret = -1;
4024  goto error;
4025  }
4026  t->tenant_id = de_ctx->tenant_id;
4027  t->reload_cnt = reload_cnt;
4028  int loader_id = de_ctx->loader_id;
4029 
4030  int r = DetectLoaderQueueTask(
4031  loader_id, DetectLoaderFuncReloadTenant, t, DetectLoaderFreeTenant);
4032  if (r < 0) {
4033  ret = -2;
4034  goto error;
4035  }
4036  }
4037 
4038  de_ctx = de_ctx->next;
4039  }
4040 error:
4041  SCMutexUnlock(&master->lock);
4042  return ret;
4043 }
4044 
4045 static int DetectLoaderSetupReloadTenant(uint32_t tenant_id, const char *yaml, int reload_cnt)
4046 {
4047  DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
4048  if (old_de_ctx == NULL)
4049  return -ENOENT;
4050  int loader_id = old_de_ctx->loader_id;
4051  DetectEngineDeReference(&old_de_ctx);
4052 
4053  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
4054  if (t == NULL)
4055  return -ENOMEM;
4056 
4057  t->tenant_id = tenant_id;
4058  if (yaml != NULL) {
4059  t->yaml = SCStrdup(yaml);
4060  if (t->yaml == NULL) {
4061  SCFree(t);
4062  return -ENOMEM;
4063  }
4064  }
4065  t->reload_cnt = reload_cnt;
4066 
4067  SCLogDebug("loader_id %d", loader_id);
4068 
4069  return DetectLoaderQueueTask(
4070  loader_id, DetectLoaderFuncReloadTenant, t, DetectLoaderFreeTenant);
4071 }
4072 
4073 /** \brief Load a tenant and wait for loading to complete
4074  */
4075 int DetectEngineLoadTenantBlocking(uint32_t tenant_id, const char *yaml)
4076 {
4077  int r = DetectLoaderSetupLoadTenant(tenant_id, yaml);
4078  if (r < 0)
4079  return r;
4080 
4081  if (DetectLoadersSync() != 0)
4082  return -1;
4083 
4084  return 0;
4085 }
4086 
4087 /** \brief Reload a tenant and wait for loading to complete
4088  */
4089 int DetectEngineReloadTenantBlocking(uint32_t tenant_id, const char *yaml, int reload_cnt)
4090 {
4091  int r = DetectLoaderSetupReloadTenant(tenant_id, yaml, reload_cnt);
4092  if (r < 0)
4093  return r;
4094 
4095  if (DetectLoadersSync() != 0)
4096  return -1;
4097 
4098  return 0;
4099 }
4100 
4101 /** \brief Reload all tenants and wait for loading to complete
4102  */
4103 int DetectEngineReloadTenantsBlocking(const int reload_cnt)
4104 {
4105  int r = DetectLoaderSetupReloadTenants(reload_cnt);
4106  if (r < 0)
4107  return r;
4108 
4109  if (DetectLoadersSync() != 0)
4110  return -1;
4111 
4112  return 0;
4113 }
4114 
4115 static int DetectEngineMultiTenantSetupLoadLivedevMappings(const ConfNode *mappings_root_node,
4116  bool failure_fatal)
4117 {
4118  ConfNode *mapping_node = NULL;
4119 
4120  int mapping_cnt = 0;
4121  if (mappings_root_node != NULL) {
4122  TAILQ_FOREACH(mapping_node, &mappings_root_node->head, next) {
4123  ConfNode *tenant_id_node = ConfNodeLookupChild(mapping_node, "tenant-id");
4124  if (tenant_id_node == NULL)
4125  goto bad_mapping;
4126  ConfNode *device_node = ConfNodeLookupChild(mapping_node, "device");
4127  if (device_node == NULL)
4128  goto bad_mapping;
4129 
4130  uint32_t tenant_id = 0;
4131  if (StringParseUint32(&tenant_id, 10, (uint16_t)strlen(tenant_id_node->val),
4132  tenant_id_node->val) < 0) {
4133  SCLogError("tenant-id "
4134  "of %s is invalid",
4135  tenant_id_node->val);
4136  goto bad_mapping;
4137  }
4138 
4139  const char *dev = device_node->val;
4140  LiveDevice *ld = LiveGetDevice(dev);
4141  if (ld == NULL) {
4142  SCLogWarning("device %s not found", dev);
4143  goto bad_mapping;
4144  }
4145 
4146  if (ld->tenant_id_set) {
4147  SCLogWarning("device %s already mapped to tenant-id %u", dev, ld->tenant_id);
4148  goto bad_mapping;
4149  }
4150 
4151  ld->tenant_id = tenant_id;
4152  ld->tenant_id_set = true;
4153 
4154  if (DetectEngineTenantRegisterLivedev(tenant_id, ld->id) != 0) {
4155  goto error;
4156  }
4157 
4158  SCLogConfig("device %s connected to tenant-id %u", dev, tenant_id);
4159  mapping_cnt++;
4160  continue;
4161 
4162  bad_mapping:
4163  if (failure_fatal)
4164  goto error;
4165  }
4166  }
4167  SCLogConfig("%d device - tenant-id mappings defined", mapping_cnt);
4168  return mapping_cnt;
4169 
4170 error:
4171  return 0;
4172 }
4173 
4174 static int DetectEngineMultiTenantSetupLoadVlanMappings(const ConfNode *mappings_root_node,
4175  bool failure_fatal)
4176 {
4177  ConfNode *mapping_node = NULL;
4178 
4179  int mapping_cnt = 0;
4180  if (mappings_root_node != NULL) {
4181  TAILQ_FOREACH(mapping_node, &mappings_root_node->head, next) {
4182  ConfNode *tenant_id_node = ConfNodeLookupChild(mapping_node, "tenant-id");
4183  if (tenant_id_node == NULL)
4184  goto bad_mapping;
4185  ConfNode *vlan_id_node = ConfNodeLookupChild(mapping_node, "vlan-id");
4186  if (vlan_id_node == NULL)
4187  goto bad_mapping;
4188 
4189  uint32_t tenant_id = 0;
4190  if (StringParseUint32(&tenant_id, 10, (uint16_t)strlen(tenant_id_node->val),
4191  tenant_id_node->val) < 0) {
4192  SCLogError("tenant-id "
4193  "of %s is invalid",
4194  tenant_id_node->val);
4195  goto bad_mapping;
4196  }
4197 
4198  uint16_t vlan_id = 0;
4199  if (StringParseUint16(
4200  &vlan_id, 10, (uint16_t)strlen(vlan_id_node->val), vlan_id_node->val) < 0) {
4201  SCLogError("vlan-id "
4202  "of %s is invalid",
4203  vlan_id_node->val);
4204  goto bad_mapping;
4205  }
4206  if (vlan_id == 0 || vlan_id >= 4095) {
4207  SCLogError("vlan-id "
4208  "of %s is invalid. Valid range 1-4094.",
4209  vlan_id_node->val);
4210  goto bad_mapping;
4211  }
4212 
4213  if (DetectEngineTenantRegisterVlanId(tenant_id, vlan_id) != 0) {
4214  goto error;
4215  }
4216  SCLogConfig("vlan %u connected to tenant-id %u", vlan_id, tenant_id);
4217  mapping_cnt++;
4218  continue;
4219 
4220  bad_mapping:
4221  if (failure_fatal)
4222  goto error;
4223  }
4224  }
4225  return mapping_cnt;
4226 
4227 error:
4228  return 0;
4229 }
4230 
4231 /**
4232  * \brief setup multi-detect / multi-tenancy
4233  *
4234  * See if MT is enabled. If so, setup the selector, tenants and mappings.
4235  * Tenants and mappings are optional, and can also dynamically be added
4236  * and removed from the unix socket.
4237  */
4238 int DetectEngineMultiTenantSetup(const bool unix_socket)
4239 {
4241  DetectEngineMasterCtx *master = &g_master_de_ctx;
4242  int failure_fatal = 0;
4243  (void)ConfGetBool("engine.init-failure-fatal", &failure_fatal);
4244 
4245  int enabled = 0;
4246  (void)ConfGetBool("multi-detect.enabled", &enabled);
4247  if (enabled == 1) {
4252 
4253  SCMutexLock(&master->lock);
4254  master->multi_tenant_enabled = 1;
4255 
4256  const char *handler = NULL;
4257  if (ConfGet("multi-detect.selector", &handler) == 1) {
4258  SCLogConfig("multi-tenant selector type %s", handler);
4259 
4260  if (strcmp(handler, "vlan") == 0) {
4261  tenant_selector = master->tenant_selector = TENANT_SELECTOR_VLAN;
4262 
4263  int vlanbool = 0;
4264  if ((ConfGetBool("vlan.use-for-tracking", &vlanbool)) == 1 && vlanbool == 0) {
4265  SCLogError("vlan tracking is disabled, "
4266  "can't use multi-detect selector 'vlan'");
4267  SCMutexUnlock(&master->lock);
4268  goto error;
4269  }
4270 
4271  } else if (strcmp(handler, "direct") == 0) {
4272  tenant_selector = master->tenant_selector = TENANT_SELECTOR_DIRECT;
4273  } else if (strcmp(handler, "device") == 0) {
4274  tenant_selector = master->tenant_selector = TENANT_SELECTOR_LIVEDEV;
4275  if (EngineModeIsIPS()) {
4276  SCLogWarning("multi-tenant 'device' mode not supported for IPS");
4277  SCMutexUnlock(&master->lock);
4278  goto error;
4279  }
4280 
4281  } else {
4282  SCLogError("unknown value %s "
4283  "multi-detect.selector",
4284  handler);
4285  SCMutexUnlock(&master->lock);
4286  goto error;
4287  }
4288  }
4289  SCMutexUnlock(&master->lock);
4290  SCLogConfig("multi-detect is enabled (multi tenancy). Selector: %s", handler);
4291 
4292  /* traffic -- tenant mappings */
4293  ConfNode *mappings_root_node = ConfGetNode("multi-detect.mappings");
4294 
4295  if (tenant_selector == TENANT_SELECTOR_VLAN) {
4296  int mapping_cnt = DetectEngineMultiTenantSetupLoadVlanMappings(mappings_root_node,
4297  failure_fatal);
4298  if (mapping_cnt == 0) {
4299  /* no mappings are valid when we're in unix socket mode,
4300  * they can be added on the fly. Otherwise warn/error
4301  * depending on failure_fatal */
4302 
4303  if (unix_socket) {
4304  SCLogNotice("no tenant traffic mappings defined, "
4305  "tenants won't be used until mappings are added");
4306  } else {
4307  if (failure_fatal) {
4308  SCLogError("no multi-detect mappings defined");
4309  goto error;
4310  } else {
4311  SCLogWarning("no multi-detect mappings defined");
4312  }
4313  }
4314  }
4315  } else if (tenant_selector == TENANT_SELECTOR_LIVEDEV) {
4316  int mapping_cnt = DetectEngineMultiTenantSetupLoadLivedevMappings(mappings_root_node,
4317  failure_fatal);
4318  if (mapping_cnt == 0) {
4319  if (failure_fatal) {
4320  SCLogError("no multi-detect mappings defined");
4321  goto error;
4322  } else {
4323  SCLogWarning("no multi-detect mappings defined");
4324  }
4325  }
4326  }
4327 
4328  /* tenants */
4329  ConfNode *tenants_root_node = ConfGetNode("multi-detect.tenants");
4330  ConfNode *tenant_node = NULL;
4331 
4332  if (tenants_root_node != NULL) {
4333  const char *path = NULL;
4334  ConfNode *path_node = ConfGetNode("multi-detect.config-path");
4335  if (path_node) {
4336  path = path_node->val;
4337  SCLogConfig("tenants config path: %s", path);
4338  }
4339 
4340  TAILQ_FOREACH(tenant_node, &tenants_root_node->head, next) {
4341  ConfNode *id_node = ConfNodeLookupChild(tenant_node, "id");
4342  if (id_node == NULL) {
4343  goto bad_tenant;
4344  }
4345  ConfNode *yaml_node = ConfNodeLookupChild(tenant_node, "yaml");
4346  if (yaml_node == NULL) {
4347  goto bad_tenant;
4348  }
4349 
4350  uint32_t tenant_id = 0;
4351  if (StringParseUint32(
4352  &tenant_id, 10, (uint16_t)strlen(id_node->val), id_node->val) < 0) {
4353  SCLogError("tenant_id "
4354  "of %s is invalid",
4355  id_node->val);
4356  goto bad_tenant;
4357  }
4358  SCLogDebug("tenant id: %u, %s", tenant_id, yaml_node->val);
4359 
4360  char yaml_path[PATH_MAX] = "";
4361  if (path) {
4362  PathMerge(yaml_path, PATH_MAX, path, yaml_node->val);
4363  } else {
4364  strlcpy(yaml_path, yaml_node->val, sizeof(yaml_path));
4365  }
4366  SCLogDebug("tenant path: %s", yaml_path);
4367 
4368  /* setup the yaml in this loop so that it's not done by the loader
4369  * threads. ConfYamlLoadFileWithPrefix is not thread safe. */
4370  char prefix[64];
4371  snprintf(prefix, sizeof(prefix), "multi-detect.%u", tenant_id);
4372  if (ConfYamlLoadFileWithPrefix(yaml_path, prefix) != 0) {
4373  SCLogError("failed to load yaml %s", yaml_path);
4374  goto bad_tenant;
4375  }
4376 
4377  int r = DetectLoaderSetupLoadTenant(tenant_id, yaml_path);
4378  if (r < 0) {
4379  /* error logged already */
4380  goto bad_tenant;
4381  }
4382  continue;
4383 
4384  bad_tenant:
4385  if (failure_fatal)
4386  goto error;
4387  }
4388  }
4389 
4390  /* wait for our loaders to complete their tasks */
4391  if (DetectLoadersSync() != 0) {
4392  goto error;
4393  }
4394 
4396 
4397  } else {
4398  SCLogDebug("multi-detect not enabled (multi tenancy)");
4399  }
4400  return 0;
4401 error:
4402  return -1;
4403 }
4404 
4405 static uint32_t DetectEngineTenantGetIdFromVlanId(const void *ctx, const Packet *p)
4406 {
4407  const DetectEngineThreadCtx *det_ctx = ctx;
4408  uint32_t x = 0;
4409  uint32_t vlan_id = 0;
4410 
4411  if (p->vlan_idx == 0)
4412  return 0;
4413 
4414  vlan_id = p->vlan_id[0];
4415 
4416  if (det_ctx == NULL || det_ctx->tenant_array == NULL || det_ctx->tenant_array_size == 0)
4417  return 0;
4418 
4419  /* not very efficient, but for now we're targeting only limited amounts.
4420  * Can use hash/tree approach later. */
4421  for (x = 0; x < det_ctx->tenant_array_size; x++) {
4422  if (det_ctx->tenant_array[x].traffic_id == vlan_id)
4423  return det_ctx->tenant_array[x].tenant_id;
4424  }
4425 
4426  return 0;
4427 }
4428 
4429 static uint32_t DetectEngineTenantGetIdFromLivedev(const void *ctx, const Packet *p)
4430 {
4431  const DetectEngineThreadCtx *det_ctx = ctx;
4432  const LiveDevice *ld = p->livedev;
4433 
4434  if (ld == NULL || det_ctx == NULL)
4435  return 0;
4436 
4437  SCLogDebug("using tenant-id %u for packet on device %s", ld->tenant_id, ld->dev);
4438  return ld->tenant_id;
4439 }
4440 
4441 static int DetectEngineTenantRegisterSelector(
4442  enum DetectEngineTenantSelectors selector, uint32_t tenant_id, uint32_t traffic_id)
4443 {
4444  DetectEngineMasterCtx *master = &g_master_de_ctx;
4445  SCMutexLock(&master->lock);
4446 
4447  if (!(master->tenant_selector == TENANT_SELECTOR_UNKNOWN || master->tenant_selector == selector)) {
4448  SCLogInfo("conflicting selector already set");
4449  SCMutexUnlock(&master->lock);
4450  return -1;
4451  }
4452 
4454  while (m) {
4455  if (m->traffic_id == traffic_id) {
4456  SCLogInfo("traffic id already registered");
4457  SCMutexUnlock(&master->lock);
4458  return -1;
4459  }
4460  m = m->next;
4461  }
4462 
4463  DetectEngineTenantMapping *map = SCCalloc(1, sizeof(*map));
4464  if (map == NULL) {
4465  SCLogInfo("memory fail");
4466  SCMutexUnlock(&master->lock);
4467  return -1;
4468  }
4469  map->traffic_id = traffic_id;
4470  map->tenant_id = tenant_id;
4471 
4472  map->next = master->tenant_mapping_list;
4473  master->tenant_mapping_list = map;
4474 
4475  master->tenant_selector = selector;
4476 
4477  SCLogDebug("tenant handler %u %u %u registered", selector, tenant_id, traffic_id);
4478  SCMutexUnlock(&master->lock);
4479  return 0;
4480 }
4481 
4482 static int DetectEngineTenantUnregisterSelector(
4483  enum DetectEngineTenantSelectors selector, uint32_t tenant_id, uint32_t traffic_id)
4484 {
4485  DetectEngineMasterCtx *master = &g_master_de_ctx;
4486  SCMutexLock(&master->lock);
4487 
4488  if (master->tenant_mapping_list == NULL) {
4489  SCMutexUnlock(&master->lock);
4490  return -1;
4491  }
4492 
4493  DetectEngineTenantMapping *prev = NULL;
4495  while (map) {
4496  if (map->traffic_id == traffic_id &&
4497  map->tenant_id == tenant_id)
4498  {
4499  if (prev != NULL)
4500  prev->next = map->next;
4501  else
4502  master->tenant_mapping_list = map->next;
4503 
4504  map->next = NULL;
4505  SCFree(map);
4506  SCLogInfo("tenant handler %u %u %u unregistered", selector, tenant_id, traffic_id);
4507  SCMutexUnlock(&master->lock);
4508  return 0;
4509  }
4510  prev = map;
4511  map = map->next;
4512  }
4513 
4514  SCMutexUnlock(&master->lock);
4515  return -1;
4516 }
4517 
4518 int DetectEngineTenantRegisterLivedev(uint32_t tenant_id, int device_id)
4519 {
4520  return DetectEngineTenantRegisterSelector(
4521  TENANT_SELECTOR_LIVEDEV, tenant_id, (uint32_t)device_id);
4522 }
4523 
4524 int DetectEngineTenantRegisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
4525 {
4526  return DetectEngineTenantRegisterSelector(TENANT_SELECTOR_VLAN, tenant_id, (uint32_t)vlan_id);
4527 }
4528 
4529 int DetectEngineTenantUnregisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
4530 {
4531  return DetectEngineTenantUnregisterSelector(TENANT_SELECTOR_VLAN, tenant_id, (uint32_t)vlan_id);
4532 }
4533 
4534 int DetectEngineTenantRegisterPcapFile(uint32_t tenant_id)
4535 {
4536  SCLogInfo("registering %u %d 0", TENANT_SELECTOR_DIRECT, tenant_id);
4537  return DetectEngineTenantRegisterSelector(TENANT_SELECTOR_DIRECT, tenant_id, 0);
4538 }
4539 
4541 {
4542  SCLogInfo("unregistering %u %d 0", TENANT_SELECTOR_DIRECT, tenant_id);
4543  return DetectEngineTenantUnregisterSelector(TENANT_SELECTOR_DIRECT, tenant_id, 0);
4544 }
4545 
4546 static uint32_t DetectEngineTenantGetIdFromPcap(const void *ctx, const Packet *p)
4547 {
4548  return p->pcap_v.tenant_id;
4549 }
4550 
4552 {
4553  DetectEngineMasterCtx *master = &g_master_de_ctx;
4554  SCMutexLock(&master->lock);
4555 
4556  if (master->list == NULL) {
4557  SCMutexUnlock(&master->lock);
4558  return NULL;
4559  }
4560 
4561  DetectEngineCtx *de_ctx = master->list;
4562  while (de_ctx) {
4564  de_ctx->tenant_id == tenant_id)
4565  {
4566  de_ctx->ref_cnt++;
4567  break;
4568  }
4569 
4570  de_ctx = de_ctx->next;
4571  }
4572 
4573  SCMutexUnlock(&master->lock);
4574  return de_ctx;
4575 }
4576 
4578 {
4579  BUG_ON((*de_ctx)->ref_cnt == 0);
4580  (*de_ctx)->ref_cnt--;
4581  *de_ctx = NULL;
4582 }
4583 
4584 static int DetectEngineAddToList(DetectEngineCtx *instance)
4585 {
4586  DetectEngineMasterCtx *master = &g_master_de_ctx;
4587 
4588  if (instance == NULL)
4589  return -1;
4590 
4591  if (master->list == NULL) {
4592  master->list = instance;
4593  } else {
4594  instance->next = master->list;
4595  master->list = instance;
4596  }
4597 
4598  return 0;
4599 }
4600 
4602 {
4603  int r;
4604 
4605  if (de_ctx == NULL)
4606  return -1;
4607 
4608  SCLogDebug("adding de_ctx %p to master", de_ctx);
4609 
4610  DetectEngineMasterCtx *master = &g_master_de_ctx;
4611  SCMutexLock(&master->lock);
4612  r = DetectEngineAddToList(de_ctx);
4613  SCMutexUnlock(&master->lock);
4614  return r;
4615 }
4616 
4617 static int DetectEngineMoveToFreeListNoLock(DetectEngineMasterCtx *master, DetectEngineCtx *de_ctx)
4618 {
4619  DetectEngineCtx *instance = master->list;
4620  if (instance == NULL) {
4621  return -1;
4622  }
4623 
4624  /* remove from active list */
4625  if (instance == de_ctx) {
4626  master->list = instance->next;
4627  } else {
4628  DetectEngineCtx *prev = instance;
4629  instance = instance->next; /* already checked first element */
4630 
4631  while (instance) {
4632  DetectEngineCtx *next = instance->next;
4633 
4634  if (instance == de_ctx) {
4635  prev->next = instance->next;
4636  break;
4637  }
4638 
4639  prev = instance;
4640  instance = next;
4641  }
4642  if (instance == NULL) {
4643  return -1;
4644  }
4645  }
4646 
4647  /* instance is now detached from list */
4648  instance->next = NULL;
4649 
4650  /* add to free list */
4651  if (master->free_list == NULL) {
4652  master->free_list = instance;
4653  } else {
4654  instance->next = master->free_list;
4655  master->free_list = instance;
4656  }
4657  SCLogDebug("detect engine %p moved to free list (%u refs)", de_ctx, de_ctx->ref_cnt);
4658  return 0;
4659 }
4660 
4662 {
4663  int ret = 0;
4664  DetectEngineMasterCtx *master = &g_master_de_ctx;
4665  SCMutexLock(&master->lock);
4666  ret = DetectEngineMoveToFreeListNoLock(master, de_ctx);
4667  SCMutexUnlock(&master->lock);
4668  return ret;
4669 }
4670 
4672 {
4673  DetectEngineMasterCtx *master = &g_master_de_ctx;
4674  SCMutexLock(&master->lock);
4675 
4676  DetectEngineCtx *prev = NULL;
4677  DetectEngineCtx *instance = master->free_list;
4678  while (instance) {
4679  DetectEngineCtx *next = instance->next;
4680 
4681  SCLogDebug("detect engine %p has %u ref(s)", instance, instance->ref_cnt);
4682 
4683  if (instance->ref_cnt == 0) {
4684  if (prev == NULL) {
4685  master->free_list = next;
4686  } else {
4687  prev->next = next;
4688  }
4689 
4690  SCLogDebug("freeing detect engine %p", instance);
4691  DetectEngineCtxFree(instance);
4692  instance = NULL;
4693  }
4694 
4695  prev = instance;
4696  instance = next;
4697  }
4698  SCMutexUnlock(&master->lock);
4699 }
4700 
4702 {
4703  DetectEngineMasterCtx *master = &g_master_de_ctx;
4704  SCMutexLock(&master->lock);
4705 
4706  DetectEngineCtx *instance = master->list;
4707  while (instance) {
4708  DetectEngineCtx *next = instance->next;
4709  DEBUG_VALIDATE_BUG_ON(instance->ref_cnt);
4710  SCLogDebug("detect engine %p has %u ref(s)", instance, instance->ref_cnt);
4711  instance->ref_cnt = 0;
4712  DetectEngineMoveToFreeListNoLock(master, instance);
4713  instance = next;
4714  }
4715  SCMutexUnlock(&master->lock);
4717 }
4718 
4719 static int reloads = 0;
4720 
4721 /** \brief Reload the detection engine
4722  *
4723  * \param filename YAML file to load for the detect config
4724  *
4725  * \retval -1 error
4726  * \retval 0 ok
4727  */
4729 {
4730  DetectEngineCtx *new_de_ctx = NULL;
4731  DetectEngineCtx *old_de_ctx = NULL;
4732 
4733  char prefix[128];
4734  memset(prefix, 0, sizeof(prefix));
4735 
4736  SCLogNotice("rule reload starting");
4737 
4738  if (suri->conf_filename != NULL) {
4739  snprintf(prefix, sizeof(prefix), "detect-engine-reloads.%d", reloads++);
4740  SCLogConfig("Reloading %s", suri->conf_filename);
4741  if (ConfYamlLoadFileWithPrefix(suri->conf_filename, prefix) != 0) {
4742  SCLogError("failed to load yaml %s", suri->conf_filename);
4743  return -1;
4744  }
4745 
4746  ConfNode *node = ConfGetNode(prefix);
4747  if (node == NULL) {
4748  SCLogError("failed to properly setup yaml %s", suri->conf_filename);
4749  return -1;
4750  }
4751 
4752  if (suri->additional_configs) {
4753  for (int i = 0; suri->additional_configs[i] != NULL; i++) {
4754  SCLogConfig("Reloading %s", suri->additional_configs[i]);
4755  ConfYamlHandleInclude(node, suri->additional_configs[i]);
4756  }
4757  }
4758 
4759 #if 0
4760  ConfDump();
4761 #endif
4762  }
4763 
4764  /* get a reference to the current de_ctx */
4765  old_de_ctx = DetectEngineGetCurrent();
4766  if (old_de_ctx == NULL)
4767  return -1;
4768  SCLogDebug("get ref to old_de_ctx %p", old_de_ctx);
4769  DatasetReload();
4770 
4771  /* only reload a regular 'normal' and 'delayed detect stub' detect engines */
4772  if (!(old_de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
4773  old_de_ctx->type == DETECT_ENGINE_TYPE_DD_STUB))
4774  {
4775  DetectEngineDeReference(&old_de_ctx);
4776  SCLogNotice("rule reload complete");
4777  return -1;
4778  }
4779 
4780  /* get new detection engine */
4781  new_de_ctx = DetectEngineCtxInitWithPrefix(prefix, old_de_ctx->tenant_id);
4782  if (new_de_ctx == NULL) {
4783  SCLogError("initializing detection engine "
4784  "context failed.");
4785  DetectEngineDeReference(&old_de_ctx);
4786  return -1;
4787  }
4788  if (SigLoadSignatures(new_de_ctx,
4789  suri->sig_file, suri->sig_file_exclusive) != 0) {
4790  DetectEngineCtxFree(new_de_ctx);
4791  DetectEngineDeReference(&old_de_ctx);
4792  return -1;
4793  }
4794  SCLogDebug("set up new_de_ctx %p", new_de_ctx);
4795 
4796  /* add to master */
4797  DetectEngineAddToMaster(new_de_ctx);
4798 
4799  /* move to old free list */
4800  DetectEngineMoveToFreeList(old_de_ctx);
4801  DetectEngineDeReference(&old_de_ctx);
4802 
4803  SCLogDebug("going to reload the threads to use new_de_ctx %p", new_de_ctx);
4804  /* update the threads */
4805  DetectEngineReloadThreads(new_de_ctx);
4806  SCLogDebug("threads now run new_de_ctx %p", new_de_ctx);
4807 
4808  /* walk free list, freeing the old_de_ctx */
4810 
4812 
4814 
4815  SCLogDebug("old_de_ctx should have been freed");
4816 
4817  SCLogNotice("rule reload complete");
4818 
4819 #ifdef HAVE_MALLOC_TRIM
4820  /* The reload process potentially frees up large amounts of memory.
4821  * Encourage the memory management system to reclaim as much as it
4822  * can.
4823  */
4824  malloc_trim(0);
4825 #endif
4826 
4827  return 0;
4828 }
4829 
4830 static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len)
4831 {
4832  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
4833  return det_ctx->tenant_id % h->array_size;
4834 }
4835 
4836 static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len)
4837 {
4840  return (det1->tenant_id == det2->tenant_id);
4841 }
4842 
4843 static void TenantIdFree(void *d)
4844 {
4845  DetectEngineThreadCtxFree(d);
4846 }
4847 
4849 {
4850  DetectEngineMasterCtx *master = &g_master_de_ctx;
4851  SCMutexLock(&master->lock);
4852 
4853  if (master->tenant_selector == TENANT_SELECTOR_UNKNOWN) {
4854  SCLogInfo("error, no tenant selector");
4855  SCMutexUnlock(&master->lock);
4856  return -1;
4857  }
4858 
4859  DetectEngineCtx *stub_de_ctx = NULL;
4860  DetectEngineCtx *list = master->list;
4861  for ( ; list != NULL; list = list->next) {
4862  SCLogDebug("list %p tenant %u", list, list->tenant_id);
4863 
4864  if (list->type == DETECT_ENGINE_TYPE_NORMAL ||
4865  list->type == DETECT_ENGINE_TYPE_MT_STUB ||
4867  {
4868  stub_de_ctx = list;
4869  break;
4870  }
4871  }
4872  if (stub_de_ctx == NULL) {
4873  stub_de_ctx = DetectEngineCtxInitStubForMT();
4874  if (stub_de_ctx == NULL) {
4875  SCMutexUnlock(&master->lock);
4876  return -1;
4877  }
4878 
4879  if (master->list == NULL) {
4880  master->list = stub_de_ctx;
4881  } else {
4882  stub_de_ctx->next = master->list;
4883  master->list = stub_de_ctx;
4884  }
4885  }
4886 
4887  /* update the threads */
4888  SCLogDebug("MT reload starting");
4889  DetectEngineReloadThreads(stub_de_ctx);
4890  SCLogDebug("MT reload done");
4891 
4892  SCMutexUnlock(&master->lock);
4893 
4894  /* walk free list, freeing the old_de_ctx */
4896  // needed for VarNameStoreFree
4898 
4899  SCLogDebug("old_de_ctx should have been freed");
4900  return 0;
4901 }
4902 
4903 static int g_parse_metadata = 0;
4904 
4906 {
4907  g_parse_metadata = 1;
4908 }
4909 
4911 {
4912  g_parse_metadata = 0;
4913 }
4914 
4916 {
4917  return g_parse_metadata;
4918 }
4919 
4921 {
4922  switch (type) {
4923  case DETECT_SM_LIST_MATCH:
4924  return "packet";
4925  case DETECT_SM_LIST_PMATCH:
4926  return "packet/stream payload";
4927 
4928  case DETECT_SM_LIST_TMATCH:
4929  return "tag";
4930 
4932  return "base64_data";
4933 
4935  return "post-match";
4936 
4938  return "suppress";
4940  return "threshold";
4941 
4942  case DETECT_SM_LIST_MAX:
4943  return "max (internal)";
4944  }
4945  return "error";
4946 }
4947 
4948 /* events api */
4950 {
4952  det_ctx->events++;
4953 }
4954 
4955 /*************************************Unittest*********************************/
4956 
4957 #ifdef UNITTESTS
4958 
4959 static int DetectEngineInitYamlConf(const char *conf)
4960 {
4962  ConfInit();
4963  return ConfYamlLoadString(conf, strlen(conf));
4964 }
4965 
4966 static void DetectEngineDeInitYamlConf(void)
4967 {
4968  ConfDeInit();
4970 }
4971 
4972 static int DetectEngineTest01(void)
4973 {
4974  const char *conf =
4975  "%YAML 1.1\n"
4976  "---\n"
4977  "detect-engine:\n"
4978  " - profile: medium\n"
4979  " - custom-values:\n"
4980  " toclient_src_groups: 2\n"
4981  " toclient_dst_groups: 2\n"
4982  " toclient_sp_groups: 2\n"
4983  " toclient_dp_groups: 3\n"
4984  " toserver_src_groups: 2\n"
4985  " toserver_dst_groups: 4\n"
4986  " toserver_sp_groups: 2\n"
4987  " toserver_dp_groups: 25\n"
4988  " - inspection-recursion-limit: 0\n";
4989 
4990  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
4991 
4994 
4996 
4998 
4999  DetectEngineDeInitYamlConf();
5000 
5001  PASS;
5002 }
5003 
5004 static int DetectEngineTest02(void)
5005 {
5006  const char *conf =
5007  "%YAML 1.1\n"
5008  "---\n"
5009  "detect-engine:\n"
5010  " - profile: medium\n"
5011  " - custom-values:\n"
5012  " toclient_src_groups: 2\n"
5013  " toclient_dst_groups: 2\n"
5014  " toclient_sp_groups: 2\n"
5015  " toclient_dp_groups: 3\n"
5016  " toserver_src_groups: 2\n"
5017  " toserver_dst_groups: 4\n"
5018  " toserver_sp_groups: 2\n"
5019  " toserver_dp_groups: 25\n"
5020  " - inspection-recursion-limit:\n";
5021 
5022  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5023 
5026 
5027  FAIL_IF_NOT(
5029 
5031 
5032  DetectEngineDeInitYamlConf();
5033 
5034  PASS;
5035 }
5036 
5037 static int DetectEngineTest03(void)
5038 {
5039  const char *conf =
5040  "%YAML 1.1\n"
5041  "---\n"
5042  "detect-engine:\n"
5043  " - profile: medium\n"
5044  " - custom-values:\n"
5045  " toclient_src_groups: 2\n"
5046  " toclient_dst_groups: 2\n"
5047  " toclient_sp_groups: 2\n"
5048  " toclient_dp_groups: 3\n"
5049  " toserver_src_groups: 2\n"
5050  " toserver_dst_groups: 4\n"
5051  " toserver_sp_groups: 2\n"
5052  " toserver_dp_groups: 25\n";
5053 
5054  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5055 
5058 
5059  FAIL_IF_NOT(
5061 
5063 
5064  DetectEngineDeInitYamlConf();
5065 
5066  PASS;
5067 }
5068 
5069 static int DetectEngineTest04(void)
5070 {
5071  const char *conf =
5072  "%YAML 1.1\n"
5073  "---\n"
5074  "detect-engine:\n"
5075  " - profile: medium\n"
5076  " - custom-values:\n"
5077  " toclient_src_groups: 2\n"
5078  " toclient_dst_groups: 2\n"
5079  " toclient_sp_groups: 2\n"
5080  " toclient_dp_groups: 3\n"
5081  " toserver_src_groups: 2\n"
5082  " toserver_dst_groups: 4\n"
5083  " toserver_sp_groups: 2\n"
5084  " toserver_dp_groups: 25\n"
5085  " - inspection-recursion-limit: 10\n";
5086 
5087  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5088 
5091 
5093 
5095 
5096  DetectEngineDeInitYamlConf();
5097 
5098  PASS;
5099 }
5100 
5101 static int DetectEngineTest08(void)
5102 {
5103  const char *conf =
5104  "%YAML 1.1\n"
5105  "---\n"
5106  "detect-engine:\n"
5107  " - profile: custom\n"
5108  " - custom-values:\n"
5109  " toclient-groups: 23\n"
5110  " toserver-groups: 27\n";
5111 
5112  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5113 
5115  FAIL_IF_NULL(