suricata
detect-engine.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "detect.h"
27 #include "flow.h"
28 #include "flow-private.h"
29 #include "flow-util.h"
30 #include "flow-worker.h"
31 #include "conf.h"
32 #include "conf-yaml-loader.h"
33 #include "datasets.h"
34 
35 #include "app-layer-parser.h"
36 #include "app-layer-htp.h"
37 
38 #include "detect-parse.h"
39 #include "detect-engine-sigorder.h"
40 
41 #include "detect-engine-build.h"
42 #include "detect-engine-siggroup.h"
43 #include "detect-engine-address.h"
44 #include "detect-engine-port.h"
46 #include "detect-engine-mpm.h"
47 #include "detect-engine-iponly.h"
48 #include "detect-engine-tag.h"
49 #include "detect-engine-frame.h"
50 
51 #include "detect-engine-file.h"
52 
53 #include "detect-engine.h"
54 #include "detect-engine-state.h"
55 #include "detect-engine-payload.h"
56 #include "detect-fast-pattern.h"
57 #include "detect-byte-extract.h"
58 #include "detect-content.h"
59 #include "detect-uricontent.h"
60 #include "detect-tcphdr.h"
63 
64 #include "detect-engine-loader.h"
65 
67 #include "util-reference-config.h"
68 #include "util-threshold-config.h"
69 #include "util-error.h"
70 #include "util-hash.h"
71 #include "util-byte.h"
72 #include "util-debug.h"
73 #include "util-unittest.h"
74 #include "util-action.h"
75 #include "util-magic.h"
76 #include "util-signal.h"
77 #include "util-spm.h"
78 #include "util-device.h"
79 #include "util-var-name.h"
80 #include "util-path.h"
81 #include "util-profiling.h"
82 #include "util-validate.h"
83 #include "util-hash-string.h"
84 #include "util-enum.h"
85 #include "util-conf.h"
86 
87 #include "tm-threads.h"
88 #include "runmodes.h"
89 
90 #include "reputation.h"
91 
92 #define DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT 3000
93 
94 static int DetectEngineCtxLoadConf(DetectEngineCtx *);
95 
96 static DetectEngineMasterCtx g_master_de_ctx = { SCMUTEX_INITIALIZER,
97  0, 99, NULL, NULL, TENANT_SELECTOR_UNKNOWN, NULL, NULL, 0};
98 
99 static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len);
100 static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len);
101 static void TenantIdFree(void *d);
102 static uint32_t DetectEngineTenantGetIdFromLivedev(const void *ctx, const Packet *p);
103 static uint32_t DetectEngineTenantGetIdFromVlanId(const void *ctx, const Packet *p);
104 static uint32_t DetectEngineTenantGetIdFromPcap(const void *ctx, const Packet *p);
105 
106 static DetectEngineAppInspectionEngine *g_app_inspect_engines = NULL;
107 static DetectEnginePktInspectionEngine *g_pkt_inspect_engines = NULL;
108 static DetectEngineFrameInspectionEngine *g_frame_inspect_engines = NULL;
109 
110 // clang-format off
112  /* SIG_TYPE_NOT_SET */ { SIG_PROP_FLOW_ACTION_PACKET, },
113  /* SIG_TYPE_IPONLY */ { SIG_PROP_FLOW_ACTION_FLOW, },
114  /* SIG_TYPE_LIKE_IPONLY */ { SIG_PROP_FLOW_ACTION_FLOW, },
115  /* SIG_TYPE_PDONLY */ { SIG_PROP_FLOW_ACTION_FLOW, },
116  /* SIG_TYPE_DEONLY */ { SIG_PROP_FLOW_ACTION_PACKET, },
117  /* SIG_TYPE_PKT */ { SIG_PROP_FLOW_ACTION_PACKET, },
118  /* SIG_TYPE_PKT_STREAM */ { SIG_PROP_FLOW_ACTION_FLOW_IF_STATEFUL, },
119  /* SIG_TYPE_STREAM */ { SIG_PROP_FLOW_ACTION_FLOW_IF_STATEFUL, },
120  /* SIG_TYPE_APPLAYER */ { SIG_PROP_FLOW_ACTION_FLOW, },
121  /* SIG_TYPE_APP_TX */ { SIG_PROP_FLOW_ACTION_FLOW, },
122 };
123 // clang-format on
124 
125 /** \brief register inspect engine at start up time
126  *
127  * \note errors are fatal */
128 void DetectPktInspectEngineRegister(const char *name,
131 {
133  const int sm_list = DetectBufferTypeGetByName(name);
134  if (sm_list == -1) {
135  FatalError("failed to register inspect engine %s", name);
136  }
137 
138  if ((sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
139  (Callback == NULL))
140  {
141  SCLogError("Invalid arguments");
142  BUG_ON(1);
143  }
144 
145  DetectEnginePktInspectionEngine *new_engine = SCCalloc(1, sizeof(*new_engine));
146  if (unlikely(new_engine == NULL)) {
147  FatalError("failed to register inspect engine %s: %s", name, strerror(errno));
148  }
149  new_engine->sm_list = (uint16_t)sm_list;
150  new_engine->sm_list_base = (uint16_t)sm_list;
151  new_engine->v1.Callback = Callback;
152  new_engine->v1.GetData = GetPktData;
153 
154  if (g_pkt_inspect_engines == NULL) {
155  g_pkt_inspect_engines = new_engine;
156  } else {
157  DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
158  while (t->next != NULL) {
159  t = t->next;
160  }
161 
162  t->next = new_engine;
163  }
164 }
165 
166 /** \brief register inspect engine at start up time
167  *
168  * \note errors are fatal */
169 static void AppLayerInspectEngineRegisterInternal(const char *name, AppProto alproto, uint32_t dir,
170  int progress, InspectEngineFuncPtr Callback, InspectionBufferGetDataPtr GetData,
171  InspectionMultiBufferGetDataPtr GetMultiData)
172 {
173  BUG_ON(progress >= 48);
174 
176  const int sm_list = DetectBufferTypeGetByName(name);
177  if (sm_list == -1) {
178  FatalError("failed to register inspect engine %s", name);
179  }
180  SCLogDebug("name %s id %d", name, sm_list);
181 
182  if ((alproto >= ALPROTO_FAILED) || (!(dir == SIG_FLAG_TOSERVER || dir == SIG_FLAG_TOCLIENT)) ||
183  (sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
184  (progress < 0 || progress >= SHRT_MAX) || (Callback == NULL)) {
185  SCLogError("Invalid arguments");
186  BUG_ON(1);
187  } else if (Callback == DetectEngineInspectBufferGeneric && GetData == NULL) {
188  SCLogError("Invalid arguments: must register "
189  "GetData with DetectEngineInspectBufferGeneric");
190  BUG_ON(1);
191  } else if (Callback == DetectEngineInspectMultiBufferGeneric && GetMultiData == NULL) {
192  SCLogError("Invalid arguments: must register "
193  "GetData with DetectEngineInspectMultiBufferGeneric");
194  BUG_ON(1);
195  }
196 
197  uint8_t direction;
198  if (dir == SIG_FLAG_TOSERVER) {
199  direction = 0;
200  } else {
201  direction = 1;
202  }
203 
204  DetectEngineAppInspectionEngine *new_engine =
206  if (unlikely(new_engine == NULL)) {
207  exit(EXIT_FAILURE);
208  }
209  new_engine->alproto = alproto;
210  new_engine->dir = direction;
211  new_engine->sm_list = (uint16_t)sm_list;
212  new_engine->sm_list_base = (uint16_t)sm_list;
213  new_engine->progress = (int16_t)progress;
214  new_engine->v2.Callback = Callback;
215  if (Callback == DetectEngineInspectBufferGeneric) {
216  new_engine->v2.GetData = GetData;
217  } else if (Callback == DetectEngineInspectMultiBufferGeneric) {
218  new_engine->v2.GetMultiData = GetMultiData;
219  }
220 
221  if (g_app_inspect_engines == NULL) {
222  g_app_inspect_engines = new_engine;
223  } else {
224  DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
225  while (t->next != NULL) {
226  t = t->next;
227  }
228 
229  t->next = new_engine;
230  }
231 }
232 
233 void DetectAppLayerInspectEngineRegister(const char *name, AppProto alproto, uint32_t dir,
234  int progress, InspectEngineFuncPtr Callback, InspectionBufferGetDataPtr GetData)
235 {
236  AppLayerInspectEngineRegisterInternal(name, alproto, dir, progress, Callback, GetData, NULL);
237 }
238 
239 /* copy an inspect engine with transforms to a new list id. */
240 static void DetectAppLayerInspectEngineCopy(
242  int sm_list, int new_list,
243  const DetectEngineTransforms *transforms)
244 {
245  const DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
246  while (t) {
247  if (t->sm_list == sm_list) {
249  if (unlikely(new_engine == NULL)) {
250  exit(EXIT_FAILURE);
251  }
252  new_engine->alproto = t->alproto;
253  new_engine->dir = t->dir;
254  DEBUG_VALIDATE_BUG_ON(new_list < 0 || new_list > UINT16_MAX);
255  new_engine->sm_list = (uint16_t)new_list; /* use new list id */
256  DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > UINT16_MAX);
257  new_engine->sm_list_base = (uint16_t)sm_list;
258  new_engine->progress = t->progress;
259  new_engine->v2 = t->v2;
260  new_engine->v2.transforms = transforms; /* assign transforms */
261 
262  if (de_ctx->app_inspect_engines == NULL) {
263  de_ctx->app_inspect_engines = new_engine;
264  } else {
266  while (list->next != NULL) {
267  list = list->next;
268  }
269 
270  list->next = new_engine;
271  }
272  }
273  t = t->next;
274  }
275 }
276 
277 /* copy inspect engines from global registrations to de_ctx list */
278 static void DetectAppLayerInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
279 {
280  const DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
282  while (t) {
284  if (unlikely(new_engine == NULL)) {
285  exit(EXIT_FAILURE);
286  }
287  new_engine->alproto = t->alproto;
288  new_engine->dir = t->dir;
289  new_engine->sm_list = t->sm_list;
290  new_engine->sm_list_base = t->sm_list;
291  new_engine->progress = t->progress;
292  new_engine->v2 = t->v2;
293 
294  if (list == NULL) {
295  de_ctx->app_inspect_engines = new_engine;
296  } else {
297  list->next = new_engine;
298  }
299  list = new_engine;
300 
301  t = t->next;
302  }
303 }
304 
305 /* copy an inspect engine with transforms to a new list id. */
306 static void DetectPktInspectEngineCopy(
308  int sm_list, int new_list,
309  const DetectEngineTransforms *transforms)
310 {
311  const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
312  while (t) {
313  if (t->sm_list == sm_list) {
315  if (unlikely(new_engine == NULL)) {
316  exit(EXIT_FAILURE);
317  }
318  DEBUG_VALIDATE_BUG_ON(new_list < 0 || new_list > UINT16_MAX);
319  new_engine->sm_list = (uint16_t)new_list; /* use new list id */
320  DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > UINT16_MAX);
321  new_engine->sm_list_base = (uint16_t)sm_list;
322  new_engine->v1 = t->v1;
323  new_engine->v1.transforms = transforms; /* assign transforms */
324 
325  if (de_ctx->pkt_inspect_engines == NULL) {
326  de_ctx->pkt_inspect_engines = new_engine;
327  } else {
329  while (list->next != NULL) {
330  list = list->next;
331  }
332 
333  list->next = new_engine;
334  }
335  }
336  t = t->next;
337  }
338 }
339 
340 /* copy inspect engines from global registrations to de_ctx list */
341 static void DetectPktInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
342 {
343  const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
344  while (t) {
345  SCLogDebug("engine %p", t);
347  if (unlikely(new_engine == NULL)) {
348  exit(EXIT_FAILURE);
349  }
350  new_engine->sm_list = t->sm_list;
351  new_engine->sm_list_base = t->sm_list;
352  new_engine->v1 = t->v1;
353 
354  if (de_ctx->pkt_inspect_engines == NULL) {
355  de_ctx->pkt_inspect_engines = new_engine;
356  } else {
358  while (list->next != NULL) {
359  list = list->next;
360  }
361 
362  list->next = new_engine;
363  }
364 
365  t = t->next;
366  }
367 }
368 
369 /** \brief register inspect engine at start up time
370  *
371  * \note errors are fatal */
373  InspectionBufferFrameInspectFunc Callback, AppProto alproto, uint8_t type)
374 {
375  const int sm_list = DetectEngineBufferTypeRegister(de_ctx, name);
376  if (sm_list < 0) {
377  FatalError("failed to register inspect engine %s", name);
378  }
379 
380  if ((sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) || (Callback == NULL)) {
381  SCLogError("Invalid arguments");
382  BUG_ON(1);
383  }
384 
385  uint8_t direction;
386  if (dir == SIG_FLAG_TOSERVER) {
387  direction = 0;
388  } else {
389  direction = 1;
390  }
391 
392  DetectEngineFrameInspectionEngine *new_engine = SCCalloc(1, sizeof(*new_engine));
393  if (unlikely(new_engine == NULL)) {
394  FatalError("failed to register inspect engine %s: %s", name, strerror(errno));
395  }
396  new_engine->sm_list = (uint16_t)sm_list;
397  new_engine->sm_list_base = (uint16_t)sm_list;
398  new_engine->dir = direction;
399  new_engine->v1.Callback = Callback;
400  new_engine->alproto = alproto;
401  new_engine->type = type;
402 
403  if (de_ctx->frame_inspect_engines == NULL) {
404  de_ctx->frame_inspect_engines = new_engine;
405  } else {
407  while (list->next != NULL) {
408  list = list->next;
409  }
410 
411  list->next = new_engine;
412  }
413 }
414 
415 /* copy an inspect engine with transforms to a new list id. */
416 static void DetectFrameInspectEngineCopy(DetectEngineCtx *de_ctx, int sm_list, int new_list,
417  const DetectEngineTransforms *transforms)
418 {
419  /* take the list from the detect engine as the buffers can be registered
420  * dynamically. */
422  while (t) {
423  if (t->sm_list == sm_list) {
426  if (unlikely(new_engine == NULL)) {
427  exit(EXIT_FAILURE);
428  }
429  DEBUG_VALIDATE_BUG_ON(new_list < 0 || new_list > UINT16_MAX);
430  new_engine->sm_list = (uint16_t)new_list; /* use new list id */
431  DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > UINT16_MAX);
432  new_engine->sm_list_base = (uint16_t)sm_list;
433  new_engine->dir = t->dir;
434  new_engine->alproto = t->alproto;
435  new_engine->type = t->type;
436  new_engine->v1 = t->v1;
437  new_engine->v1.transforms = transforms; /* assign transforms */
438 
439  /* append to the list */
441  while (list->next != NULL) {
442  list = list->next;
443  }
444 
445  list->next = new_engine;
446  }
447  t = t->next;
448  }
449 }
450 
451 /* copy inspect engines from global registrations to de_ctx list */
452 static void DetectFrameInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
453 {
454  const DetectEngineFrameInspectionEngine *t = g_frame_inspect_engines;
455  while (t) {
456  SCLogDebug("engine %p", t);
459  if (unlikely(new_engine == NULL)) {
460  exit(EXIT_FAILURE);
461  }
462  new_engine->sm_list = t->sm_list;
463  new_engine->sm_list_base = t->sm_list;
464  new_engine->dir = t->dir;
465  new_engine->alproto = t->alproto;
466  new_engine->type = t->type;
467  new_engine->v1 = t->v1;
468 
469  if (de_ctx->frame_inspect_engines == NULL) {
470  de_ctx->frame_inspect_engines = new_engine;
471  } else {
473  while (list->next != NULL) {
474  list = list->next;
475  }
476 
477  list->next = new_engine;
478  }
479 
480  t = t->next;
481  }
482 }
483 
484 /** \internal
485  * \brief append the stream inspection
486  *
487  * If stream inspection is MPM, then prepend it.
488  */
489 static void AppendStreamInspectEngine(
490  Signature *s, SigMatchData *stream, uint8_t direction, uint8_t id)
491 {
492  bool prepend = false;
493 
495  if (unlikely(new_engine == NULL)) {
496  exit(EXIT_FAILURE);
497  }
499  SCLogDebug("stream is mpm");
500  prepend = true;
501  new_engine->mpm = true;
502  }
503  new_engine->alproto = ALPROTO_UNKNOWN; /* all */
504  new_engine->dir = direction;
505  new_engine->stream = true;
506  new_engine->sm_list = DETECT_SM_LIST_PMATCH;
507  new_engine->sm_list_base = DETECT_SM_LIST_PMATCH;
508  new_engine->smd = stream;
509  new_engine->v2.Callback = DetectEngineInspectStream;
510  new_engine->progress = 0;
511 
512  /* append */
513  if (s->app_inspect == NULL) {
514  s->app_inspect = new_engine;
515  new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
516  } else if (prepend) {
517  new_engine->next = s->app_inspect;
518  s->app_inspect = new_engine;
519  new_engine->id = id;
520 
521  } else {
523  while (a->next != NULL) {
524  a = a->next;
525  }
526 
527  a->next = new_engine;
528  new_engine->id = id;
529  }
530  SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
531 }
532 
533 static void AppendFrameInspectEngine(DetectEngineCtx *de_ctx,
535  const int mpm_list)
536 {
537  bool prepend = false;
538 
539  if (u->alproto == ALPROTO_UNKNOWN) {
540  /* special case, inspect engine applies to all protocols */
541  } else if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, u->alproto))
542  return;
543 
544  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
545  if (u->dir == 1)
546  return;
547  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
548  if (u->dir == 0)
549  return;
550  }
551 
554  if (unlikely(new_engine == NULL)) {
555  exit(EXIT_FAILURE);
556  }
557  if (mpm_list == u->sm_list) {
559  prepend = true;
560  new_engine->mpm = true;
561  }
562 
563  new_engine->type = u->type;
564  new_engine->sm_list = u->sm_list;
565  new_engine->sm_list_base = u->sm_list_base;
566  new_engine->smd = smd;
567  new_engine->v1 = u->v1;
568  SCLogDebug("sm_list %d new_engine->v1 %p/%p", new_engine->sm_list, new_engine->v1.Callback,
569  new_engine->v1.transforms);
570 
571  if (s->frame_inspect == NULL) {
572  s->frame_inspect = new_engine;
573  } else if (prepend) {
574  new_engine->next = s->frame_inspect;
575  s->frame_inspect = new_engine;
576  } else {
578  while (a->next != NULL) {
579  a = a->next;
580  }
581  new_engine->next = a->next;
582  a->next = new_engine;
583  }
584 }
585 
586 static void AppendPacketInspectEngine(DetectEngineCtx *de_ctx,
588  const int mpm_list)
589 {
590  bool prepend = false;
591 
592  DetectEnginePktInspectionEngine *new_engine =
594  if (unlikely(new_engine == NULL)) {
595  exit(EXIT_FAILURE);
596  }
597  if (mpm_list == e->sm_list) {
599  prepend = true;
600  new_engine->mpm = true;
601  }
602 
603  new_engine->sm_list = e->sm_list;
604  new_engine->sm_list_base = e->sm_list_base;
605  new_engine->smd = smd;
606  new_engine->v1 = e->v1;
607  SCLogDebug("sm_list %d new_engine->v1 %p/%p/%p", new_engine->sm_list, new_engine->v1.Callback,
608  new_engine->v1.GetData, new_engine->v1.transforms);
609 
610  if (s->pkt_inspect == NULL) {
611  s->pkt_inspect = new_engine;
612  } else if (prepend) {
613  new_engine->next = s->pkt_inspect;
614  s->pkt_inspect = new_engine;
615  } else {
617  while (a->next != NULL) {
618  a = a->next;
619  }
620  new_engine->next = a->next;
621  a->next = new_engine;
622  }
623 }
624 
625 static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
627  const int mpm_list, const int files_id, uint8_t *last_id, bool *head_is_mpm)
628 {
629  if (t->alproto == ALPROTO_UNKNOWN) {
630  /* special case, inspect engine applies to all protocols */
631  } else if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, t->alproto))
632  return;
633 
634  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
635  if (t->dir == 1)
636  return;
637  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
638  if (t->dir == 0)
639  return;
640  }
641  SCLogDebug("app engine: t %p t->id %u => alproto:%s files:%s", t, t->id,
642  AppProtoToString(t->alproto), BOOL2STR(t->sm_list == files_id));
643 
644  DetectEngineAppInspectionEngine *new_engine =
646  if (unlikely(new_engine == NULL)) {
647  exit(EXIT_FAILURE);
648  }
649  bool prepend = false;
650  if (mpm_list == t->sm_list) {
652  prepend = true;
653  *head_is_mpm = true;
654  new_engine->mpm = true;
655  }
656 
657  new_engine->alproto = t->alproto;
658  new_engine->dir = t->dir;
659  new_engine->sm_list = t->sm_list;
660  new_engine->sm_list_base = t->sm_list_base;
661  new_engine->smd = smd;
662  new_engine->progress = t->progress;
663  new_engine->v2 = t->v2;
664  SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
665  new_engine->v2.GetData, new_engine->v2.transforms);
666 
667  if (s->app_inspect == NULL) {
668  s->app_inspect = new_engine;
669  if (new_engine->sm_list == files_id) {
670  new_engine->id = DE_STATE_ID_FILE_INSPECT;
671  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
672  } else {
673  new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
674  SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
676  }
677 
678  /* prepend engine if forced or if our engine has a lower progress. */
679  } else if (prepend || (!(*head_is_mpm) && s->app_inspect->progress > new_engine->progress)) {
680  new_engine->next = s->app_inspect;
681  s->app_inspect = new_engine;
682  if (new_engine->sm_list == files_id) {
683  new_engine->id = DE_STATE_ID_FILE_INSPECT;
684  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
685  } else {
686  new_engine->id = ++(*last_id);
687  SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
689  }
690 
691  } else {
693  while (a->next != NULL) {
694  if (a->next && a->next->progress > new_engine->progress) {
695  break;
696  }
697  a = a->next;
698  }
699 
700  new_engine->next = a->next;
701  a->next = new_engine;
702  if (new_engine->sm_list == files_id) {
703  new_engine->id = DE_STATE_ID_FILE_INSPECT;
704  SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
705  } else {
706  new_engine->id = ++(*last_id);
707  SCLogDebug("sid %u: engine %p/%u %s", s->id, new_engine, new_engine->id,
709  }
710  }
711 
712  SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
713 
715 }
716 
717 /**
718  * \note for the file inspect engine, the id DE_STATE_ID_FILE_INSPECT
719  * is assigned.
720  */
722 {
723  const int mpm_list = s->init_data->mpm_sm ? s->init_data->mpm_sm_list : -1;
724  const int files_id = DetectBufferTypeGetByName("files");
725  bool head_is_mpm = false;
726  uint8_t last_id = DE_STATE_FLAG_BASE;
727 
728  for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
730  SCLogDebug("smd %p, id %u", smd, s->init_data->buffers[x].id);
731 
732  const DetectBufferType *b =
734  if (b == NULL)
735  FatalError("unknown buffer");
736 
737  if (b->frame) {
739  u != NULL; u = u->next) {
740  if (u->sm_list == s->init_data->buffers[x].id) {
741  AppendFrameInspectEngine(de_ctx, u, s, smd, mpm_list);
742  }
743  }
744  } else if (b->packet) {
745  /* set up pkt inspect engines */
746  for (const DetectEnginePktInspectionEngine *e = de_ctx->pkt_inspect_engines; e != NULL;
747  e = e->next) {
748  SCLogDebug("e %p sm_list %u", e, e->sm_list);
749  if (e->sm_list == s->init_data->buffers[x].id) {
750  AppendPacketInspectEngine(de_ctx, e, s, smd, mpm_list);
751  }
752  }
753  } else {
754  SCLogDebug("app %s id %u parent %u rule %u xforms %u", b->name, b->id, b->parent_id,
755  s->init_data->buffers[x].id, b->transforms.cnt);
756  for (const DetectEngineAppInspectionEngine *t = de_ctx->app_inspect_engines; t != NULL;
757  t = t->next) {
758  if (t->sm_list == s->init_data->buffers[x].id) {
759  AppendAppInspectEngine(
760  de_ctx, t, s, smd, mpm_list, files_id, &last_id, &head_is_mpm);
761  }
762  }
763  }
764  }
765 
768  {
769  /* if engine is added multiple times, we pass it the same list */
771  BUG_ON(stream == NULL);
772  if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
773  AppendStreamInspectEngine(s, stream, 0, last_id + 1);
774  } else if (s->flags & SIG_FLAG_TOCLIENT && !(s->flags & SIG_FLAG_TOSERVER)) {
775  AppendStreamInspectEngine(s, stream, 1, last_id + 1);
776  } else {
777  AppendStreamInspectEngine(s, stream, 0, last_id + 1);
778  AppendStreamInspectEngine(s, stream, 1, last_id + 1);
779  }
780 
782  SCLogDebug("set SIG_FLAG_FLUSH on %u", s->id);
783  s->flags |= SIG_FLAG_FLUSH;
784  }
785  }
786 
787 #ifdef DEBUG
789  while (iter) {
790  SCLogDebug("%u: engine %s id %u progress %d %s", s->id,
792  iter->sm_list == mpm_list ? "MPM" : "");
793  iter = iter->next;
794  }
795 #endif
796  return 0;
797 }
798 
799 /** \brief free app inspect engines for a signature
800  *
801  * For lists that are registered multiple times, like http_header and
802  * http_cookie, making the engines owner of the lists is complicated.
803  * Multiple engines in a sig may be pointing to the same list. To
804  * address this the 'free' code needs to be extra careful about not
805  * double freeing, so it takes an approach to first fill an array
806  * of the to-free pointers before freeing them.
807  */
809 {
810  int engines = 0;
811 
813  while (ie) {
814  ie = ie->next;
815  engines++;
816  }
818  while (e) {
819  e = e->next;
820  engines++;
821  }
823  while (u) {
824  u = u->next;
825  engines++;
826  }
827  if (engines == 0) {
828  BUG_ON(s->pkt_inspect);
829  BUG_ON(s->frame_inspect);
830  return;
831  }
832 
833  SigMatchData *bufs[engines];
834  memset(&bufs, 0, (engines * sizeof(SigMatchData *)));
835  int arrays = 0;
836 
837  /* free engines and put smd in the array */
838  ie = s->app_inspect;
839  while (ie) {
841 
842  bool skip = false;
843  for (int i = 0; i < arrays; i++) {
844  if (bufs[i] == ie->smd) {
845  skip = true;
846  break;
847  }
848  }
849  if (!skip) {
850  bufs[arrays++] = ie->smd;
851  }
852  SCFree(ie);
853  ie = next;
854  }
855  e = s->pkt_inspect;
856  while (e) {
858 
859  bool skip = false;
860  for (int i = 0; i < arrays; i++) {
861  if (bufs[i] == e->smd) {
862  skip = true;
863  break;
864  }
865  }
866  if (!skip) {
867  bufs[arrays++] = e->smd;
868  }
869  SCFree(e);
870  e = next;
871  }
872  u = s->frame_inspect;
873  while (u) {
875 
876  bool skip = false;
877  for (int i = 0; i < arrays; i++) {
878  if (bufs[i] == u->smd) {
879  skip = true;
880  break;
881  }
882  }
883  if (!skip) {
884  bufs[arrays++] = u->smd;
885  }
886  SCFree(u);
887  u = next;
888  }
889 
890  for (int i = 0; i < engines; i++) {
891  if (bufs[i] == NULL)
892  continue;
893  SigMatchData *smd = bufs[i];
894  while (1) {
895  if (sigmatch_table[smd->type].Free != NULL) {
896  sigmatch_table[smd->type].Free(de_ctx, smd->ctx);
897  }
898  if (smd->is_last)
899  break;
900  smd++;
901  }
902  SCFree(bufs[i]);
903  }
904 }
905 
906 /* code for registering buffers */
907 
908 #include "util-hash-lookup3.h"
909 
910 static HashListTable *g_buffer_type_hash = NULL;
911 static int g_buffer_type_id = DETECT_SM_LIST_DYNAMIC_START;
912 static int g_buffer_type_reg_closed = 0;
913 
915 {
916  return g_buffer_type_id;
917 }
918 
919 static uint32_t DetectBufferTypeHashNameFunc(HashListTable *ht, void *data, uint16_t datalen)
920 {
921  const DetectBufferType *map = (DetectBufferType *)data;
922  uint32_t hash = hashlittle_safe(map->name, strlen(map->name), 0);
923  hash += hashlittle_safe((uint8_t *)&map->transforms, sizeof(map->transforms), 0);
924  hash %= ht->array_size;
925  return hash;
926 }
927 
928 static uint32_t DetectBufferTypeHashIdFunc(HashListTable *ht, void *data, uint16_t datalen)
929 {
930  const DetectBufferType *map = (DetectBufferType *)data;
931  uint32_t hash = map->id;
932  hash %= ht->array_size;
933  return hash;
934 }
935 
936 static char DetectBufferTypeCompareNameFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
937 {
938  DetectBufferType *map1 = (DetectBufferType *)data1;
939  DetectBufferType *map2 = (DetectBufferType *)data2;
940 
941  char r = (strcmp(map1->name, map2->name) == 0);
942  r &= (memcmp((uint8_t *)&map1->transforms, (uint8_t *)&map2->transforms, sizeof(map2->transforms)) == 0);
943  return r;
944 }
945 
946 static char DetectBufferTypeCompareIdFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
947 {
948  DetectBufferType *map1 = (DetectBufferType *)data1;
949  DetectBufferType *map2 = (DetectBufferType *)data2;
950  return map1->id == map2->id;
951 }
952 
953 static void DetectBufferTypeFreeFunc(void *data)
954 {
955  DetectBufferType *map = (DetectBufferType *)data;
956 
957  if (map == NULL) {
958  return;
959  }
960 
961  /* Release transformation option memory, if any */
962  for (int i = 0; i < map->transforms.cnt; i++) {
963  if (map->transforms.transforms[i].options == NULL)
964  continue;
965  if (sigmatch_table[map->transforms.transforms[i].transform].Free == NULL) {
966  SCLogError("%s allocates transform option memory but has no free routine",
968  continue;
969  }
971  }
972 
973  SCFree(map);
974 }
975 
976 static int DetectBufferTypeInit(void)
977 {
978  BUG_ON(g_buffer_type_hash);
979  g_buffer_type_hash = HashListTableInit(256, DetectBufferTypeHashNameFunc,
980  DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFunc);
981  if (g_buffer_type_hash == NULL)
982  return -1;
983 
984  return 0;
985 }
986 #if 0
987 static void DetectBufferTypeFree(void)
988 {
989  if (g_buffer_type_hash == NULL)
990  return;
991 
992  HashListTableFree(g_buffer_type_hash);
993  g_buffer_type_hash = NULL;
994 }
995 #endif
996 static int DetectBufferTypeAdd(const char *string)
997 {
998  BUG_ON(string == NULL || strlen(string) >= 32);
999 
1000  DetectBufferType *map = SCCalloc(1, sizeof(*map));
1001  if (map == NULL)
1002  return -1;
1003 
1004  strlcpy(map->name, string, sizeof(map->name));
1005  map->id = g_buffer_type_id++;
1006 
1007  BUG_ON(HashListTableAdd(g_buffer_type_hash, (void *)map, 0) != 0);
1008  SCLogDebug("buffer %s registered with id %d", map->name, map->id);
1009  return map->id;
1010 }
1011 
1012 static DetectBufferType *DetectBufferTypeLookupByName(const char *string)
1013 {
1014  DetectBufferType map;
1015  memset(&map, 0, sizeof(map));
1016  strlcpy(map.name, string, sizeof(map.name));
1017 
1018  DetectBufferType *res = HashListTableLookup(g_buffer_type_hash, &map, 0);
1019  return res;
1020 }
1021 
1022 int DetectBufferTypeRegister(const char *name)
1023 {
1024  BUG_ON(g_buffer_type_reg_closed);
1025  if (g_buffer_type_hash == NULL)
1026  DetectBufferTypeInit();
1027 
1028  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1029  if (!exists) {
1030  return DetectBufferTypeAdd(name);
1031  } else {
1032  return exists->id;
1033  }
1034 }
1035 
1037 {
1038  BUG_ON(g_buffer_type_reg_closed);
1040  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1041  BUG_ON(!exists);
1042  exists->multi_instance = true;
1043  SCLogDebug("%p %s -- %d supports multi instance", exists, name, exists->id);
1044 }
1045 
1046 void DetectBufferTypeSupportsFrames(const char *name)
1047 {
1048  BUG_ON(g_buffer_type_reg_closed);
1050  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1051  BUG_ON(!exists);
1052  exists->frame = true;
1053  SCLogDebug("%p %s -- %d supports frame inspection", exists, name, exists->id);
1054 }
1055 
1056 void DetectBufferTypeSupportsPacket(const char *name)
1057 {
1058  BUG_ON(g_buffer_type_reg_closed);
1060  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1061  BUG_ON(!exists);
1062  exists->packet = true;
1063  SCLogDebug("%p %s -- %d supports packet inspection", exists, name, exists->id);
1064 }
1065 
1066 void DetectBufferTypeSupportsMpm(const char *name)
1067 {
1068  BUG_ON(g_buffer_type_reg_closed);
1070  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1071  BUG_ON(!exists);
1072  exists->mpm = true;
1073  SCLogDebug("%p %s -- %d supports mpm", exists, name, exists->id);
1074 }
1075 
1077 {
1078  BUG_ON(g_buffer_type_reg_closed);
1080  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1081  BUG_ON(!exists);
1082  exists->supports_transforms = true;
1083  SCLogDebug("%p %s -- %d supports transformations", exists, name, exists->id);
1084 }
1085 
1086 int DetectBufferTypeGetByName(const char *name)
1087 {
1088  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1089  if (!exists) {
1090  return -1;
1091  }
1092  return exists->id;
1093 }
1094 
1095 static DetectBufferType *DetectEngineBufferTypeLookupByName(
1096  const DetectEngineCtx *de_ctx, const char *string)
1097 {
1098  DetectBufferType map;
1099  memset(&map, 0, sizeof(map));
1100  strlcpy(map.name, string, sizeof(map.name));
1101 
1103  return res;
1104 }
1105 
1107 {
1108  DetectBufferType lookup;
1109  memset(&lookup, 0, sizeof(lookup));
1110  lookup.id = id;
1111  const DetectBufferType *res =
1112  HashListTableLookup(de_ctx->buffer_type_hash_id, (void *)&lookup, 0);
1113  return res;
1114 }
1115 
1117 {
1119  return res ? res->name : NULL;
1120 }
1121 
1122 static int DetectEngineBufferTypeAdd(DetectEngineCtx *de_ctx, const char *string)
1123 {
1124  BUG_ON(string == NULL || strlen(string) >= 32);
1125 
1126  DetectBufferType *map = SCCalloc(1, sizeof(*map));
1127  if (map == NULL)
1128  return -1;
1129 
1130  strlcpy(map->name, string, sizeof(map->name));
1131  map->id = de_ctx->buffer_type_id++;
1132 
1133  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_name, (void *)map, 0) != 0);
1134  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_id, (void *)map, 0) != 0);
1135  SCLogDebug("buffer %s registered with id %d", map->name, map->id);
1136  return map->id;
1137 }
1138 
1140  const int direction, const AppProto alproto, const uint8_t frame_type)
1141 {
1142  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1143  if (exists) {
1144  return exists->id;
1145  }
1146 
1147  const int buffer_id = DetectEngineBufferTypeAdd(de_ctx, name);
1148  if (buffer_id < 0) {
1149  return -1;
1150  }
1151 
1152  /* TODO hack we need the map to get the name. Should we return the map at reg? */
1153  const DetectBufferType *map = DetectEngineBufferTypeGetById(de_ctx, buffer_id);
1154  BUG_ON(!map);
1155 
1156  /* register MPM/inspect engines */
1157  if (direction & SIG_FLAG_TOSERVER) {
1159  PrefilterGenericMpmFrameRegister, alproto, frame_type);
1161  DetectEngineInspectFrameBufferGeneric, alproto, frame_type);
1162  }
1163  if (direction & SIG_FLAG_TOCLIENT) {
1165  PrefilterGenericMpmFrameRegister, alproto, frame_type);
1167  DetectEngineInspectFrameBufferGeneric, alproto, frame_type);
1168  }
1169 
1170  return buffer_id;
1171 }
1172 
1174 {
1175  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1176  if (!exists) {
1177  return DetectEngineBufferTypeAdd(de_ctx, name);
1178  } else {
1179  return exists->id;
1180  }
1181 }
1182 
1183 void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc)
1184 {
1185  BUG_ON(desc == NULL || strlen(desc) >= 128);
1186 
1187  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1188  if (!exists) {
1189  return;
1190  }
1191  strlcpy(exists->description, desc, sizeof(exists->description));
1192 }
1193 
1195 {
1197  if (!exists) {
1198  return NULL;
1199  }
1200  return exists->description;
1201 }
1202 
1203 const char *DetectBufferTypeGetDescriptionByName(const char *name)
1204 {
1205  const DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1206  if (!exists) {
1207  return NULL;
1208  }
1209  return exists->description;
1210 }
1211 
1213 {
1214  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1215  BUG_ON(!exists);
1216  exists->frame = true;
1217  SCLogDebug("%p %s -- %d supports frame inspection", exists, name, exists->id);
1218 }
1219 
1221 {
1222  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1223  BUG_ON(!exists);
1224  exists->packet = true;
1225  SCLogDebug("%p %s -- %d supports packet inspection", exists, name, exists->id);
1226 }
1227 
1229 {
1230  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1231  BUG_ON(!exists);
1232  exists->mpm = true;
1233  SCLogDebug("%p %s -- %d supports mpm", exists, name, exists->id);
1234 }
1235 
1237 {
1238  DetectBufferType *exists = DetectEngineBufferTypeLookupByName(de_ctx, name);
1239  BUG_ON(!exists);
1240  exists->supports_transforms = true;
1241  SCLogDebug("%p %s -- %d supports transformations", exists, name, exists->id);
1242 }
1243 
1245 {
1247  if (map == NULL)
1248  return false;
1249  SCLogDebug("map %p id %d multi_instance? %s", map, id, BOOL2STR(map->multi_instance));
1250  return map->multi_instance;
1251 }
1252 
1254 {
1256  if (map == NULL)
1257  return false;
1258  SCLogDebug("map %p id %d packet? %d", map, id, map->packet);
1259  return map->packet;
1260 }
1261 
1263 {
1265  if (map == NULL)
1266  return false;
1267  SCLogDebug("map %p id %d mpm? %d", map, id, map->mpm);
1268  return map->mpm;
1269 }
1270 
1272 {
1274  if (map == NULL)
1275  return false;
1276  SCLogDebug("map %p id %d frame? %d", map, id, map->frame);
1277  return map->frame;
1278 }
1279 
1281  void (*SetupCallback)(const DetectEngineCtx *, Signature *))
1282 {
1283  BUG_ON(g_buffer_type_reg_closed);
1285  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1286  BUG_ON(!exists);
1287  exists->SetupCallback = SetupCallback;
1288 }
1289 
1291 {
1293  if (map && map->SetupCallback) {
1294  map->SetupCallback(de_ctx, s);
1295  }
1296 }
1297 
1299  bool (*ValidateCallback)(const Signature *, const char **sigerror))
1300 {
1301  BUG_ON(g_buffer_type_reg_closed);
1303  DetectBufferType *exists = DetectBufferTypeLookupByName(name);
1304  BUG_ON(!exists);
1305  exists->ValidateCallback = ValidateCallback;
1306 }
1307 
1309  const DetectEngineCtx *de_ctx, const int id, const Signature *s, const char **sigerror)
1310 {
1312  if (map && map->ValidateCallback) {
1313  return map->ValidateCallback(s, sigerror);
1314  }
1315  return true;
1316 }
1317 
1318 SigMatch *DetectBufferGetFirstSigMatch(const Signature *s, const uint32_t buf_id)
1319 {
1320  for (uint32_t i = 0; i < s->init_data->buffer_index; i++) {
1321  if (buf_id == s->init_data->buffers[i].id) {
1322  return s->init_data->buffers[i].head;
1323  }
1324  }
1325  return NULL;
1326 }
1327 
1328 SigMatch *DetectBufferGetLastSigMatch(const Signature *s, const uint32_t buf_id)
1329 {
1330  SigMatch *last = NULL;
1331  for (uint32_t i = 0; i < s->init_data->buffer_index; i++) {
1332  if (buf_id == s->init_data->buffers[i].id) {
1333  last = s->init_data->buffers[i].tail;
1334  }
1335  }
1336  return last;
1337 }
1338 
1339 bool DetectBufferIsPresent(const Signature *s, const uint32_t buf_id)
1340 {
1341  for (uint32_t i = 0; i < s->init_data->buffer_index; i++) {
1342  if (buf_id == s->init_data->buffers[i].id) {
1343  return true;
1344  }
1345  }
1346  return false;
1347 }
1348 
1350 {
1351  BUG_ON(s->init_data == NULL);
1352 
1354  SCLogError("Rule buffer cannot be reset after base64_data.");
1355  return -1;
1356  }
1357 
1358  if (s->init_data->list && s->init_data->transforms.cnt) {
1359  SCLogError("no matches following transform(s)");
1360  return -1;
1361  }
1362  s->init_data->list = list;
1363  s->init_data->list_set = true;
1364 
1365  // check if last has matches -> if no, error
1366  if (s->init_data->curbuf && s->init_data->curbuf->head == NULL) {
1367  SCLogError("previous sticky buffer has no matches");
1368  return -1;
1369  }
1370 
1371  for (uint32_t x = 0; x < s->init_data->buffers_size; x++) {
1373  for (SigMatch *sm = b->head; sm != NULL; sm = sm->next) {
1374  SCLogDebug(
1375  "buf:%p: id:%u: '%s' pos %u", b, b->id, sigmatch_table[sm->type].name, sm->idx);
1376  }
1377  if ((uint32_t)list == b->id) {
1378  SCLogDebug("found buffer %p for list %d", b, list);
1379  if (s->init_data->buffers[x].sm_init) {
1380  s->init_data->buffers[x].sm_init = false;
1381  SCLogDebug("sm_init was true for %p list %d", b, list);
1382  s->init_data->curbuf = b;
1383  return 0;
1384 
1386  // fall through
1387  } else {
1388  SCLogWarning("duplicate instance for %s in '%s'",
1390  s->init_data->curbuf = b;
1391  return 0;
1392  }
1393  }
1394  }
1395 
1396  if (list < DETECT_SM_LIST_MAX)
1397  return 0;
1398 
1400  SCLogError("failed to expand rule buffer array");
1401  return -1;
1402  }
1403 
1404  /* initialize new buffer */
1406  s->init_data->curbuf->id = list;
1407  s->init_data->curbuf->head = NULL;
1408  s->init_data->curbuf->tail = NULL;
1411  SCLogDebug("new: idx %u list %d set up curbuf %p", s->init_data->buffer_index - 1, list,
1412  s->init_data->curbuf);
1413 
1414  return 0;
1415 }
1416 
1418 {
1419  BUG_ON(s->init_data == NULL);
1420 
1421  if (s->init_data->list && s->init_data->transforms.cnt) {
1422  if (s->init_data->list == DETECT_SM_LIST_NOTSET ||
1424  SCLogError("previous transforms not consumed "
1425  "(list: %u, transform_cnt %u)",
1427  SCReturnInt(-1);
1428  }
1429 
1430  SCLogDebug("buffer %d has transform(s) registered: %d",
1434  if (new_list == -1) {
1435  SCReturnInt(-1);
1436  }
1437  int base_list = s->init_data->list;
1438  SCLogDebug("new_list %d", new_list);
1439  s->init_data->list = new_list;
1440  s->init_data->list_set = false;
1441  // reset transforms now that we've set up the list
1442  s->init_data->transforms.cnt = 0;
1443 
1444  if (s->init_data->curbuf && s->init_data->curbuf->head != NULL) {
1446  SCLogError("failed to expand rule buffer array");
1447  return -1;
1448  }
1452  }
1453  if (s->init_data->curbuf == NULL) {
1454  SCLogError("failed to setup buffer");
1456  SCReturnInt(-1);
1457  }
1458  s->init_data->curbuf->id = new_list;
1459  SCLogDebug("new list after applying transforms: %u", new_list);
1460  }
1461 
1462  SCReturnInt(0);
1463 }
1464 
1466 {
1467  /* single buffers */
1468  for (uint32_t i = 0; i < det_ctx->inspect.to_clear_idx; i++)
1469  {
1470  const uint32_t idx = det_ctx->inspect.to_clear_queue[i];
1471  InspectionBuffer *buffer = &det_ctx->inspect.buffers[idx];
1472  buffer->inspect = NULL;
1473  buffer->initialized = false;
1474  }
1475  det_ctx->inspect.to_clear_idx = 0;
1476 
1477  /* multi buffers */
1478  for (uint32_t i = 0; i < det_ctx->multi_inspect.to_clear_idx; i++)
1479  {
1480  const uint32_t idx = det_ctx->multi_inspect.to_clear_queue[i];
1481  InspectionBufferMultipleForList *mbuffer = &det_ctx->multi_inspect.buffers[idx];
1482  for (uint32_t x = 0; x <= mbuffer->max; x++) {
1483  InspectionBuffer *buffer = &mbuffer->inspection_buffers[x];
1484  buffer->inspect = NULL;
1485  buffer->initialized = false;
1486  }
1487  mbuffer->init = 0;
1488  mbuffer->max = 0;
1489  }
1490  det_ctx->multi_inspect.to_clear_idx = 0;
1491 }
1492 
1494 {
1495  return &det_ctx->inspect.buffers[list_id];
1496 }
1497 
1498 static InspectionBufferMultipleForList *InspectionBufferGetMulti(
1499  DetectEngineThreadCtx *det_ctx, const int list_id)
1500 {
1501  InspectionBufferMultipleForList *buffer = &det_ctx->multi_inspect.buffers[list_id];
1502  if (!buffer->init) {
1503  det_ctx->multi_inspect.to_clear_queue[det_ctx->multi_inspect.to_clear_idx++] = list_id;
1504  buffer->init = 1;
1505  }
1506  return buffer;
1507 }
1508 
1509 /** \brief for a InspectionBufferMultipleForList get a InspectionBuffer
1510  * \param fb the multiple buffer array
1511  * \param local_id the index to get a buffer
1512  * \param buffer the inspect buffer or NULL in case of error */
1514  DetectEngineThreadCtx *det_ctx, const int list_id, const uint32_t local_id)
1515 {
1516  if (unlikely(local_id >= 1024)) {
1518  return NULL;
1519  }
1520 
1521  InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
1522 
1523  if (local_id >= fb->size) {
1524  uint32_t old_size = fb->size;
1525  uint32_t new_size = local_id + 1;
1526  uint32_t grow_by = new_size - old_size;
1527  SCLogDebug("size is %u, need %u, so growing by %u", old_size, new_size, grow_by);
1528 
1529  SCLogDebug("fb->inspection_buffers %p", fb->inspection_buffers);
1530  void *ptr = SCRealloc(fb->inspection_buffers, (local_id + 1) * sizeof(InspectionBuffer));
1531  if (ptr == NULL)
1532  return NULL;
1533 
1534  InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
1535  SCLogDebug("ptr %p to_zero %p", ptr, to_zero);
1536  memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
1537  fb->inspection_buffers = ptr;
1538  fb->size = new_size;
1539  }
1540 
1541  fb->max = MAX(fb->max, local_id);
1542  InspectionBuffer *buffer = &fb->inspection_buffers[local_id];
1543  SCLogDebug("using buffer %p", buffer);
1544 #ifdef DEBUG_VALIDATION
1545  buffer->multi = true;
1546 #endif
1547  return buffer;
1548 }
1549 
1550 void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
1551 {
1552  memset(buffer, 0, sizeof(*buffer));
1553  buffer->buf = SCCalloc(initial_size, sizeof(uint8_t));
1554  if (buffer->buf != NULL) {
1555  buffer->size = initial_size;
1556  }
1557 }
1558 
1559 /** \brief setup the buffer empty */
1561 {
1562 #ifdef DEBUG_VALIDATION
1564  DEBUG_VALIDATE_BUG_ON(!buffer->multi);
1565 #endif
1566  buffer->inspect = NULL;
1567  buffer->inspect_len = 0;
1568  buffer->len = 0;
1569  buffer->initialized = true;
1570 }
1571 
1572 /** \brief setup the buffer with our initial data */
1574  const uint8_t *data, const uint32_t data_len)
1575 {
1576 #ifdef DEBUG_VALIDATION
1577  DEBUG_VALIDATE_BUG_ON(!buffer->multi);
1578 #endif
1579  buffer->inspect = buffer->orig = data;
1580  buffer->inspect_len = buffer->orig_len = data_len;
1581  buffer->len = 0;
1582  buffer->initialized = true;
1583 
1584  InspectionBufferApplyTransforms(buffer, transforms);
1585 }
1586 
1587 /** \brief setup the buffer with our initial data */
1588 void InspectionBufferSetup(DetectEngineThreadCtx *det_ctx, const int list_id,
1589  InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
1590 {
1591 #ifdef DEBUG_VALIDATION
1592  DEBUG_VALIDATE_BUG_ON(buffer->multi);
1593  DEBUG_VALIDATE_BUG_ON(buffer != InspectionBufferGet(det_ctx, list_id));
1594 #endif
1595  if (buffer->inspect == NULL) {
1596 #ifdef UNITTESTS
1597  if (det_ctx && list_id != -1)
1598 #endif
1599  det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id;
1600  }
1601  buffer->inspect = buffer->orig = data;
1602  buffer->inspect_len = buffer->orig_len = data_len;
1603  buffer->len = 0;
1604  buffer->initialized = true;
1605 }
1606 
1608 {
1609  if (buffer->buf != NULL) {
1610  SCFree(buffer->buf);
1611  }
1612  memset(buffer, 0, sizeof(*buffer));
1613 }
1614 
1615 /**
1616  * \brief make sure that the buffer has at least 'min_size' bytes
1617  * Expand the buffer if necessary
1618  */
1619 void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
1620 {
1621  if (likely(buffer->size >= min_size))
1622  return;
1623 
1624  uint32_t new_size = (buffer->size == 0) ? 4096 : buffer->size;
1625  while (new_size < min_size) {
1626  new_size *= 2;
1627  }
1628 
1629  void *ptr = SCRealloc(buffer->buf, new_size);
1630  if (ptr != NULL) {
1631  buffer->buf = ptr;
1632  buffer->size = new_size;
1633  }
1634 }
1635 
1636 void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len)
1637 {
1638  InspectionBufferCheckAndExpand(buffer, buf_len);
1639 
1640  if (buffer->size) {
1641  uint32_t copy_size = MIN(buf_len, buffer->size);
1642  memcpy(buffer->buf, buf, copy_size);
1643  buffer->inspect = buffer->buf;
1644  buffer->inspect_len = copy_size;
1645  buffer->initialized = true;
1646  }
1647 }
1648 
1649 /** \brief Check content byte array compatibility with transforms
1650  *
1651  * The "content" array is presented to the transforms so that each
1652  * transform may validate that it's compatible with the transform.
1653  *
1654  * When a transform indicates the byte array is incompatible, none of the
1655  * subsequent transforms, if any, are invoked. This means the first validation
1656  * failure terminates the loop.
1657  *
1658  * \param de_ctx Detection engine context.
1659  * \param sm_list The SM list id.
1660  * \param content The byte array being validated
1661  * \param namestr returns the name of the transform that is incompatible with
1662  * content.
1663  *
1664  * \retval true (false) If any of the transforms indicate the byte array is
1665  * (is not) compatible.
1666  **/
1668  const uint8_t *content, uint16_t content_len, const char **namestr)
1669 {
1670  const DetectBufferType *dbt = DetectEngineBufferTypeGetById(de_ctx, sm_list);
1671  BUG_ON(dbt == NULL);
1672 
1673  for (int i = 0; i < dbt->transforms.cnt; i++) {
1674  const TransformData *t = &dbt->transforms.transforms[i];
1676  continue;
1677 
1678  if (sigmatch_table[t->transform].TransformValidate(content, content_len, t->options)) {
1679  continue;
1680  }
1681 
1682  if (namestr) {
1683  *namestr = sigmatch_table[t->transform].name;
1684  }
1685 
1686  return false;
1687  }
1688 
1689  return true;
1690 }
1691 
1693  const DetectEngineTransforms *transforms)
1694 {
1695  if (transforms) {
1696  for (int i = 0; i < DETECT_TRANSFORMS_MAX; i++) {
1697  const int id = transforms->transforms[i].transform;
1698  if (id == 0)
1699  break;
1700  BUG_ON(sigmatch_table[id].Transform == NULL);
1701  sigmatch_table[id].Transform(buffer, transforms->transforms[i].options);
1702  SCLogDebug("applied transform %s", sigmatch_table[id].name);
1703  }
1704  }
1705 }
1706 
1707 static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx)
1708 {
1709  const int size = g_buffer_type_id;
1710  BUG_ON(!(size > 0));
1711 
1712  de_ctx->buffer_type_hash_name = HashListTableInit(256, DetectBufferTypeHashNameFunc,
1713  DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFunc);
1716  HashListTableInit(256, DetectBufferTypeHashIdFunc, DetectBufferTypeCompareIdFunc,
1717  NULL); // entries owned by buffer_type_hash_name
1718  BUG_ON(de_ctx->buffer_type_hash_id == NULL);
1719  de_ctx->buffer_type_id = g_buffer_type_id;
1720 
1721  SCLogDebug("DETECT_SM_LIST_DYNAMIC_START %u", DETECT_SM_LIST_DYNAMIC_START);
1722  HashListTableBucket *b = HashListTableGetListHead(g_buffer_type_hash);
1723  while (b) {
1725 
1726  DetectBufferType *copy = SCCalloc(1, sizeof(*copy));
1727  BUG_ON(!copy);
1728  memcpy(copy, map, sizeof(*copy));
1729  int r = HashListTableAdd(de_ctx->buffer_type_hash_name, (void *)copy, 0);
1730  BUG_ON(r != 0);
1731  r = HashListTableAdd(de_ctx->buffer_type_hash_id, (void *)copy, 0);
1732  BUG_ON(r != 0);
1733 
1734  SCLogDebug("name %s id %d mpm %s packet %s -- %s. "
1735  "Callbacks: Setup %p Validate %p",
1736  map->name, map->id, map->mpm ? "true" : "false", map->packet ? "true" : "false",
1737  map->description, map->SetupCallback, map->ValidateCallback);
1738  b = HashListTableGetListNext(b);
1739  }
1740 
1743  DetectAppLayerInspectEngineCopyListToDetectCtx(de_ctx);
1745  DetectFrameInspectEngineCopyListToDetectCtx(de_ctx);
1747  DetectPktInspectEngineCopyListToDetectCtx(de_ctx);
1748 }
1749 
1750 static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
1751 {
1752  if (de_ctx) {
1757 
1759  while (ilist) {
1761  SCFree(ilist);
1762  ilist = next;
1763  }
1765  while (mlist) {
1766  DetectBufferMpmRegistry *next = mlist->next;
1767  SCFree(mlist);
1768  mlist = next;
1769  }
1771  while (plist) {
1773  SCFree(plist);
1774  plist = next;
1775  }
1777  while (pmlist) {
1778  DetectBufferMpmRegistry *next = pmlist->next;
1779  SCFree(pmlist);
1780  pmlist = next;
1781  }
1783  while (framelist) {
1785  SCFree(framelist);
1786  framelist = next;
1787  }
1789  while (framemlist) {
1790  DetectBufferMpmRegistry *next = framemlist->next;
1791  SCFree(framemlist);
1792  framemlist = next;
1793  }
1795  }
1796 }
1797 
1799 {
1800  BUG_ON(g_buffer_type_hash == NULL);
1801 
1802  g_buffer_type_reg_closed = 1;
1803 }
1804 
1806  DetectEngineCtx *de_ctx, const int id, TransformData *transforms, int transform_cnt)
1807 {
1808  const DetectBufferType *base_map = DetectEngineBufferTypeGetById(de_ctx, id);
1809  if (!base_map) {
1810  return -1;
1811  }
1812  if (!base_map->supports_transforms) {
1813  SCLogError("buffer '%s' does not support transformations", base_map->name);
1814  return -1;
1815  }
1816 
1817  SCLogDebug("base_map %s", base_map->name);
1818 
1820  memset(&t, 0, sizeof(t));
1821  for (int i = 0; i < transform_cnt; i++) {
1822  t.transforms[i] = transforms[i];
1823  }
1824  t.cnt = transform_cnt;
1825 
1826  DetectBufferType lookup_map;
1827  memset(&lookup_map, 0, sizeof(lookup_map));
1828  strlcpy(lookup_map.name, base_map->name, sizeof(lookup_map.name));
1829  lookup_map.transforms = t;
1831 
1832  SCLogDebug("res %p", res);
1833  if (res != NULL) {
1834  return res->id;
1835  }
1836 
1837  DetectBufferType *map = SCCalloc(1, sizeof(*map));
1838  if (map == NULL)
1839  return -1;
1840 
1841  strlcpy(map->name, base_map->name, sizeof(map->name));
1842  map->id = de_ctx->buffer_type_id++;
1843  map->parent_id = base_map->id;
1844  map->transforms = t;
1845  map->mpm = base_map->mpm;
1846  map->packet = base_map->packet;
1847  map->frame = base_map->frame;
1848  map->SetupCallback = base_map->SetupCallback;
1849  map->ValidateCallback = base_map->ValidateCallback;
1850  if (map->frame) {
1852  } else if (map->packet) {
1854  map->id, map->parent_id, &map->transforms);
1855  } else {
1857  map->id, map->parent_id, &map->transforms);
1858  }
1859 
1860  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_name, (void *)map, 0) != 0);
1861  BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash_id, (void *)map, 0) != 0);
1862  SCLogDebug("buffer %s registered with id %d, parent %d", map->name, map->id, map->parent_id);
1863 
1864  if (map->frame) {
1865  DetectFrameInspectEngineCopy(de_ctx, map->parent_id, map->id, &map->transforms);
1866  } else if (map->packet) {
1867  DetectPktInspectEngineCopy(de_ctx, map->parent_id, map->id, &map->transforms);
1868  } else {
1869  DetectAppLayerInspectEngineCopy(de_ctx, map->parent_id, map->id, &map->transforms);
1870  }
1871  return map->id;
1872 }
1873 
1874 /* returns false if no match, true if match */
1875 static int DetectEngineInspectRulePacketMatches(
1876  DetectEngineThreadCtx *det_ctx,
1877  const DetectEnginePktInspectionEngine *engine,
1878  const Signature *s,
1879  Packet *p, uint8_t *_alert_flags)
1880 {
1881  SCEnter();
1882 
1883  /* run the packet match functions */
1885  const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_MATCH];
1886 
1887  SCLogDebug("running match functions, sm %p", smd);
1888  while (1) {
1890  if (sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx) <= 0) {
1891  KEYWORD_PROFILING_END(det_ctx, smd->type, 0);
1892  SCLogDebug("no match");
1894  }
1895  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
1896  if (smd->is_last) {
1897  SCLogDebug("match and is_last");
1898  break;
1899  }
1900  smd++;
1901  }
1903 }
1904 
1905 static int DetectEngineInspectRulePayloadMatches(
1906  DetectEngineThreadCtx *det_ctx,
1907  const DetectEnginePktInspectionEngine *engine,
1908  const Signature *s, Packet *p, uint8_t *alert_flags)
1909 {
1910  SCEnter();
1911 
1912  DetectEngineCtx *de_ctx = det_ctx->de_ctx;
1913 
1915  /* if we have stream msgs, inspect against those first,
1916  * but not for a "dsize" signature */
1917  if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
1918  int pmatch = 0;
1919  if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
1920  pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, p->flow, p);
1921  if (pmatch) {
1923  *alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
1924  }
1925  }
1926  /* no match? then inspect packet payload */
1927  if (pmatch == 0) {
1928  SCLogDebug("no match in stream, fall back to packet payload");
1929 
1930  /* skip if we don't have to inspect the packet and segment was
1931  * added to stream */
1932  if (!(s->flags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
1934  }
1936  SCLogDebug("SIG_FLAG_REQUIRE_STREAM_ONLY, so no match");
1938  }
1939  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
1941  }
1942  }
1943  } else {
1944  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
1946  }
1947  }
1949 }
1950 
1952  DetectEngineThreadCtx *det_ctx, const Signature *s,
1953  Flow *f, Packet *p,
1954  uint8_t *alert_flags)
1955 {
1956  SCEnter();
1957 
1958  for (DetectEnginePktInspectionEngine *e = s->pkt_inspect; e != NULL; e = e->next) {
1959  if (e->v1.Callback(det_ctx, e, s, p, alert_flags) != DETECT_ENGINE_INSPECT_SIG_MATCH) {
1960  SCLogDebug("sid %u: e %p Callback returned no match", s->id, e);
1961  return false;
1962  }
1963  SCLogDebug("sid %u: e %p Callback returned true", s->id, e);
1964  }
1965 
1966  SCLogDebug("sid %u: returning true", s->id);
1967  return true;
1968 }
1969 
1970 /**
1971  * \param data pointer to SigMatchData. Allowed to be NULL.
1972  */
1973 static int DetectEnginePktInspectionAppend(Signature *s, InspectionBufferPktInspectFunc Callback,
1974  SigMatchData *data, const int list_id)
1975 {
1976  DetectEnginePktInspectionEngine *e = SCCalloc(1, sizeof(*e));
1977  if (e == NULL)
1978  return -1;
1979 
1980  e->mpm = s->init_data->mpm_sm_list == list_id;
1981  DEBUG_VALIDATE_BUG_ON(list_id < 0 || list_id > UINT16_MAX);
1982  e->sm_list = (uint16_t)list_id;
1983  e->sm_list_base = (uint16_t)list_id;
1984  e->v1.Callback = Callback;
1985  e->smd = data;
1986 
1987  if (s->pkt_inspect == NULL) {
1988  s->pkt_inspect = e;
1989  } else {
1991  while (a->next != NULL) {
1992  a = a->next;
1993  }
1994  a->next = e;
1995  }
1996  return 0;
1997 }
1998 
2000 {
2001  /* only handle PMATCH here if we're not an app inspect rule */
2003  if (DetectEnginePktInspectionAppend(
2004  s, DetectEngineInspectRulePayloadMatches, NULL, DETECT_SM_LIST_PMATCH) < 0)
2005  return -1;
2006  SCLogDebug("sid %u: DetectEngineInspectRulePayloadMatches appended", s->id);
2007  }
2008 
2009  if (s->sm_arrays[DETECT_SM_LIST_MATCH]) {
2010  if (DetectEnginePktInspectionAppend(
2011  s, DetectEngineInspectRulePacketMatches, NULL, DETECT_SM_LIST_MATCH) < 0)
2012  return -1;
2013  SCLogDebug("sid %u: DetectEngineInspectRulePacketMatches appended", s->id);
2014  }
2015 
2016  return 0;
2017 }
2018 
2019 /* code to control the main thread to do a reload */
2020 
2022  IDLE, /**< ready to start a reload */
2023  RELOAD, /**< command main thread to do the reload */
2024 };
2025 
2026 
2027 typedef struct DetectEngineSyncer_ {
2031 
2032 static DetectEngineSyncer detect_sync = { SCMUTEX_INITIALIZER, IDLE };
2033 
2034 /* tell main to start reloading */
2036 {
2037  int r = 0;
2038  SCMutexLock(&detect_sync.m);
2039  if (detect_sync.state == IDLE) {
2040  detect_sync.state = RELOAD;
2041  } else {
2042  r = -1;
2043  }
2044  SCMutexUnlock(&detect_sync.m);
2045  return r;
2046 }
2047 
2048 /* main thread checks this to see if it should start */
2050 {
2051  int r = 0;
2052  SCMutexLock(&detect_sync.m);
2053  if (detect_sync.state == RELOAD) {
2054  r = 1;
2055  }
2056  SCMutexUnlock(&detect_sync.m);
2057  return r;
2058 }
2059 
2060 /* main thread sets done when it's done */
2062 {
2063  SCMutexLock(&detect_sync.m);
2064  detect_sync.state = IDLE;
2065  SCMutexUnlock(&detect_sync.m);
2066 }
2067 
2068 /* caller loops this until it returns 1 */
2070 {
2071  int r = 0;
2072  SCMutexLock(&detect_sync.m);
2073  if (detect_sync.state == IDLE) {
2074  r = 1;
2075  }
2076  SCMutexUnlock(&detect_sync.m);
2077  return r;
2078 }
2079 
2080 /** \brief Do the content inspection & validation for a signature
2081  *
2082  * \param de_ctx Detection engine context
2083  * \param det_ctx Detection engine thread context
2084  * \param s Signature to inspect
2085  * \param sm SigMatch to inspect
2086  * \param f Flow
2087  * \param flags app layer flags
2088  * \param state App layer state
2089  *
2090  * \retval 0 no match
2091  * \retval 1 match
2092  */
2094  const struct DetectEngineAppInspectionEngine_ *engine, const Signature *s, Flow *f,
2095  uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
2096 {
2097  SigMatchData *smd = engine->smd;
2098  SCLogDebug("running match functions, sm %p", smd);
2099  if (smd != NULL) {
2100  while (1) {
2101  int match = 0;
2103  match = sigmatch_table[smd->type].
2104  AppLayerTxMatch(det_ctx, f, flags, alstate, txv, s, smd->ctx);
2105  KEYWORD_PROFILING_END(det_ctx, smd->type, (match == 1));
2106  if (match == 0)
2108  if (match == 2) {
2110  }
2111 
2112  if (smd->is_last)
2113  break;
2114  smd++;
2115  }
2116  }
2117 
2119 }
2120 
2121 
2122 /**
2123  * \brief Do the content inspection & validation for a signature
2124  *
2125  * \param de_ctx Detection engine context
2126  * \param det_ctx Detection engine thread context
2127  * \param s Signature to inspect
2128  * \param f Flow
2129  * \param flags app layer flags
2130  * \param state App layer state
2131  *
2132  * \retval 0 no match.
2133  * \retval 1 match.
2134  * \retval 2 Sig can't match.
2135  */
2137  const DetectEngineAppInspectionEngine *engine, const Signature *s, Flow *f, uint8_t flags,
2138  void *alstate, void *txv, uint64_t tx_id)
2139 {
2140  const int list_id = engine->sm_list;
2141  SCLogDebug("running inspect on %d", list_id);
2142 
2143  const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
2144 
2145  SCLogDebug("list %d mpm? %s transforms %p",
2146  engine->sm_list, engine->mpm ? "true" : "false", engine->v2.transforms);
2147 
2148  /* if prefilter didn't already run, we need to consider transformations */
2149  const DetectEngineTransforms *transforms = NULL;
2150  if (!engine->mpm) {
2151  transforms = engine->v2.transforms;
2152  }
2153 
2154  const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
2155  f, flags, txv, list_id);
2156  if (unlikely(buffer == NULL)) {
2159  }
2160 
2161  const uint32_t data_len = buffer->inspect_len;
2162  const uint8_t *data = buffer->inspect;
2163  const uint64_t offset = buffer->inspect_offset;
2164 
2165  uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0;
2166  ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0);
2167  ci_flags |= buffer->flags;
2168 
2169  /* Inspect all the uricontents fetched on each
2170  * transaction at the app layer */
2171  const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, data,
2173  if (match) {
2175  } else {
2178  }
2179 }
2180 
2181 // wrapper for both DetectAppLayerInspectEngineRegister and DetectAppLayerMpmRegister
2182 // with cast of callback function
2183 void DetectAppLayerMultiRegister(const char *name, AppProto alproto, uint32_t dir, int progress,
2184  InspectionMultiBufferGetDataPtr GetData, int priority, int tx_min_progress)
2185 {
2186  AppLayerInspectEngineRegisterInternal(
2187  name, alproto, dir, progress, DetectEngineInspectMultiBufferGeneric, NULL, GetData);
2189  alproto, tx_min_progress);
2190 }
2191 
2194  const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
2195 {
2196  uint32_t local_id = 0;
2197  const DetectEngineTransforms *transforms = NULL;
2198  if (!engine->mpm) {
2199  transforms = engine->v2.transforms;
2200  }
2201 
2202  do {
2203  InspectionBuffer *buffer = engine->v2.GetMultiData(
2204  det_ctx, transforms, f, flags, txv, engine->sm_list, local_id);
2205 
2206  if (buffer == NULL || buffer->inspect == NULL)
2207  break;
2208 
2209  // The GetData functions set buffer->flags to DETECT_CI_FLAGS_SINGLE
2210  // This is not meant for streaming buffers
2211  const bool match = DetectEngineContentInspectionBuffer(de_ctx, det_ctx, s, engine->smd,
2213  if (match) {
2215  }
2216  local_id++;
2217  } while (1);
2219 }
2220 
2221 /**
2222  * \brief Do the content inspection & validation for a signature
2223  *
2224  * \param de_ctx Detection engine context
2225  * \param det_ctx Detection engine thread context
2226  * \param s Signature to inspect
2227  * \param p Packet
2228  *
2229  * \retval 0 no match.
2230  * \retval 1 match.
2231  */
2233  DetectEngineThreadCtx *det_ctx,
2234  const DetectEnginePktInspectionEngine *engine,
2235  const Signature *s, Packet *p, uint8_t *_alert_flags)
2236 {
2237  const int list_id = engine->sm_list;
2238  SCLogDebug("running inspect on %d", list_id);
2239 
2240  SCLogDebug("list %d transforms %p",
2241  engine->sm_list, engine->v1.transforms);
2242 
2243  /* if prefilter didn't already run, we need to consider transformations */
2244  const DetectEngineTransforms *transforms = NULL;
2245  if (!engine->mpm) {
2246  transforms = engine->v1.transforms;
2247  }
2248 
2249  const InspectionBuffer *buffer = engine->v1.GetData(det_ctx, transforms, p,
2250  list_id);
2251  if (unlikely(buffer == NULL)) {
2253  }
2254 
2255  uint8_t ci_flags = DETECT_CI_FLAGS_START|DETECT_CI_FLAGS_END;
2256  ci_flags |= buffer->flags;
2257 
2258  /* Inspect all the uricontents fetched on each
2259  * transaction at the app layer */
2260  const bool match = DetectEngineContentInspection(det_ctx->de_ctx, det_ctx, s, engine->smd, p,
2261  p->flow, buffer->inspect, buffer->inspect_len, 0, ci_flags,
2263  if (match) {
2265  } else {
2267  }
2268 }
2269 
2270 /** \internal
2271  * \brief inject a pseudo packet into each detect thread that doesn't use the
2272  * new det_ctx yet
2273  */
2274 static void InjectPackets(ThreadVars **detect_tvs,
2275  DetectEngineThreadCtx **new_det_ctx,
2276  int no_of_detect_tvs)
2277 {
2278  /* inject a fake packet if the detect thread isn't using the new ctx yet,
2279  * this speeds up the process */
2280  for (int i = 0; i < no_of_detect_tvs; i++) {
2281  if (SC_ATOMIC_GET(new_det_ctx[i]->so_far_used_by_detect) != 1) {
2282  if (detect_tvs[i]->inq != NULL) {
2283  Packet *p = PacketGetFromAlloc();
2284  if (p != NULL) {
2287  PacketQueue *q = detect_tvs[i]->inq->pq;
2288  SCMutexLock(&q->mutex_q);
2289  PacketEnqueue(q, p);
2290  SCCondSignal(&q->cond_q);
2291  SCMutexUnlock(&q->mutex_q);
2292  }
2293  }
2294  }
2295  }
2296 }
2297 
2298 /** \internal
2299  * \brief Update detect threads with new detect engine
2300  *
2301  * Atomically update each detect thread with a new thread context
2302  * that is associated to the new detection engine(s).
2303  *
2304  * If called in unix socket mode, it's possible that we don't have
2305  * detect threads yet.
2306  *
2307  * \retval -1 error
2308  * \retval 0 no detection threads
2309  * \retval 1 successful reload
2310  */
2311 static int DetectEngineReloadThreads(DetectEngineCtx *new_de_ctx)
2312 {
2313  SCEnter();
2314  uint32_t i = 0;
2315 
2316  /* count detect threads in use */
2317  uint32_t no_of_detect_tvs = TmThreadCountThreadsByTmmFlags(TM_FLAG_DETECT_TM);
2318  /* can be zero in unix socket mode */
2319  if (no_of_detect_tvs == 0) {
2320  return 0;
2321  }
2322 
2323  /* prepare swap structures */
2324  DetectEngineThreadCtx *old_det_ctx[no_of_detect_tvs];
2325  DetectEngineThreadCtx *new_det_ctx[no_of_detect_tvs];
2326  ThreadVars *detect_tvs[no_of_detect_tvs];
2327  memset(old_det_ctx, 0x00, (no_of_detect_tvs * sizeof(DetectEngineThreadCtx *)));
2328  memset(new_det_ctx, 0x00, (no_of_detect_tvs * sizeof(DetectEngineThreadCtx *)));
2329  memset(detect_tvs, 0x00, (no_of_detect_tvs * sizeof(ThreadVars *)));
2330 
2331  /* start the process of swapping detect threads ctxs */
2332 
2333  /* get reference to tv's and setup new_det_ctx array */
2335  for (ThreadVars *tv = tv_root[TVT_PPT]; tv != NULL; tv = tv->next) {
2336  if ((tv->tmm_flags & TM_FLAG_DETECT_TM) == 0) {
2337  continue;
2338  }
2339  for (TmSlot *s = tv->tm_slots; s != NULL; s = s->slot_next) {
2340  TmModule *tm = TmModuleGetById(s->tm_id);
2341  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
2342  continue;
2343  }
2344 
2345  if (suricata_ctl_flags != 0) {
2347  goto error;
2348  }
2349 
2350  old_det_ctx[i] = FlowWorkerGetDetectCtxPtr(SC_ATOMIC_GET(s->slot_data));
2351  detect_tvs[i] = tv;
2352 
2353  new_det_ctx[i] = DetectEngineThreadCtxInitForReload(tv, new_de_ctx, 1);
2354  if (new_det_ctx[i] == NULL) {
2355  SCLogError("Detect engine thread init "
2356  "failure in live rule swap. Let's get out of here");
2358  goto error;
2359  }
2360  SCLogDebug("live rule swap created new det_ctx - %p and de_ctx "
2361  "- %p\n", new_det_ctx[i], new_de_ctx);
2362  i++;
2363  break;
2364  }
2365  }
2366  BUG_ON(i != no_of_detect_tvs);
2367 
2368  /* atomically replace the det_ctx data */
2369  i = 0;
2370  for (ThreadVars *tv = tv_root[TVT_PPT]; tv != NULL; tv = tv->next) {
2371  if ((tv->tmm_flags & TM_FLAG_DETECT_TM) == 0) {
2372  continue;
2373  }
2374  for (TmSlot *s = tv->tm_slots; s != NULL; s = s->slot_next) {
2375  TmModule *tm = TmModuleGetById(s->tm_id);
2376  if (!(tm->flags & TM_FLAG_DETECT_TM)) {
2377  continue;
2378  }
2379  SCLogDebug("swapping new det_ctx - %p with older one - %p",
2380  new_det_ctx[i], SC_ATOMIC_GET(s->slot_data));
2381  FlowWorkerReplaceDetectCtx(SC_ATOMIC_GET(s->slot_data), new_det_ctx[i++]);
2382  break;
2383  }
2384  }
2386 
2387  /* threads now all have new data, however they may not have started using
2388  * it and may still use the old data */
2389 
2390  SCLogDebug("Live rule swap has swapped %d old det_ctx's with new ones, "
2391  "along with the new de_ctx", no_of_detect_tvs);
2392 
2393  InjectPackets(detect_tvs, new_det_ctx, no_of_detect_tvs);
2394 
2395  /* loop waiting for detect threads to switch to the new det_ctx. Try to
2396  * wake up capture if needed (break loop). */
2397  uint32_t threads_done = 0;
2398 retry:
2399  for (i = 0; i < no_of_detect_tvs; i++) {
2400  if (suricata_ctl_flags != 0) {
2401  threads_done = no_of_detect_tvs;
2402  break;
2403  }
2404  usleep(1000);
2405  if (SC_ATOMIC_GET(new_det_ctx[i]->so_far_used_by_detect) == 1) {
2406  SCLogDebug("new_det_ctx - %p used by detect engine", new_det_ctx[i]);
2407  threads_done++;
2408  } else {
2409  TmThreadsCaptureBreakLoop(detect_tvs[i]);
2410  }
2411  }
2412  if (threads_done < no_of_detect_tvs) {
2413  threads_done = 0;
2414  SleepMsec(250);
2415  goto retry;
2416  }
2417 
2418  /* this is to make sure that if someone initiated shutdown during a live
2419  * rule swap, the live rule swap won't clean up the old det_ctx and
2420  * de_ctx, till all detect threads have stopped working and sitting
2421  * silently after setting RUNNING_DONE flag and while waiting for
2422  * THV_DEINIT flag */
2423  if (i != no_of_detect_tvs) { // not all threads we swapped
2424  for (ThreadVars *tv = tv_root[TVT_PPT]; tv != NULL; tv = tv->next) {
2425  if ((tv->tmm_flags & TM_FLAG_DETECT_TM) == 0) {
2426  continue;
2427  }
2428 
2430  usleep(100);
2431  }
2432  }
2433  }
2434 
2435  /* free all the ctxs */
2436  for (i = 0; i < no_of_detect_tvs; i++) {
2437  SCLogDebug("Freeing old_det_ctx - %p used by detect",
2438  old_det_ctx[i]);
2439  DetectEngineThreadCtxDeinit(NULL, old_det_ctx[i]);
2440  }
2441 
2443 
2444  return 1;
2445 
2446  error:
2447  for (i = 0; i < no_of_detect_tvs; i++) {
2448  if (new_det_ctx[i] != NULL)
2449  DetectEngineThreadCtxDeinit(NULL, new_det_ctx[i]);
2450  }
2451  return -1;
2452 }
2453 
2454 static DetectEngineCtx *DetectEngineCtxInitReal(
2455  enum DetectEngineType type, const char *prefix, uint32_t tenant_id)
2456 {
2458  if (unlikely(de_ctx == NULL))
2459  goto error;
2460 
2461  memset(&de_ctx->sig_stat, 0, sizeof(SigFileLoaderStat));
2462  TAILQ_INIT(&de_ctx->sig_stat.failed_sigs);
2463  de_ctx->sigerror = NULL;
2464  de_ctx->type = type;
2466  de_ctx->tenant_id = tenant_id;
2467 
2470  SCLogDebug("stub %u with version %u", type, de_ctx->version);
2471  return de_ctx;
2472  }
2473 
2474  if (prefix != NULL) {
2475  strlcpy(de_ctx->config_prefix, prefix, sizeof(de_ctx->config_prefix));
2476  }
2477 
2478  int failure_fatal = 0;
2479  if (ConfGetBool("engine.init-failure-fatal", (int *)&failure_fatal) != 1) {
2480  SCLogDebug("ConfGetBool could not load the value.");
2481  }
2482  de_ctx->failure_fatal = (failure_fatal == 1);
2483 
2486  SCLogConfig("pattern matchers: MPM: %s, SPM: %s",
2489 
2491  if (de_ctx->spm_global_thread_ctx == NULL) {
2492  SCLogDebug("Unable to alloc SpmGlobalThreadCtx.");
2493  goto error;
2494  }
2495 
2497  if (de_ctx->sm_types_prefilter == NULL) {
2498  goto error;
2499  }
2501  if (de_ctx->sm_types_silent_error == NULL) {
2502  goto error;
2503  }
2504  if (DetectEngineCtxLoadConf(de_ctx) == -1) {
2505  goto error;
2506  }
2507 
2514  DetectBufferTypeSetupDetectEngine(de_ctx);
2516 
2517  /* init iprep... ignore errors for now */
2518  (void)SRepInit(de_ctx);
2519 
2523  goto error;
2524  }
2525 
2526  if (ActionInitConfig() < 0) {
2527  goto error;
2528  }
2530  if (SCRConfLoadReferenceConfigFile(de_ctx, NULL) < 0) {
2532  goto error;
2533  }
2534 
2536  SCLogDebug("dectx with version %u", de_ctx->version);
2537  return de_ctx;
2538 error:
2539  if (de_ctx != NULL) {
2541  }
2542  return NULL;
2543 }
2544 
2546 {
2547  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_MT_STUB, NULL, 0);
2548 }
2549 
2551 {
2552  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_DD_STUB, NULL, 0);
2553 }
2554 
2556 {
2557  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_NORMAL, NULL, 0);
2558 }
2559 
2560 DetectEngineCtx *DetectEngineCtxInitWithPrefix(const char *prefix, uint32_t tenant_id)
2561 {
2562  if (prefix == NULL || strlen(prefix) == 0)
2563  return DetectEngineCtxInit();
2564  else
2565  return DetectEngineCtxInitReal(DETECT_ENGINE_TYPE_NORMAL, prefix, tenant_id);
2566 }
2567 
2568 static void DetectEngineCtxFreeThreadKeywordData(DetectEngineCtx *de_ctx)
2569 {
2571 }
2572 
2573 static void DetectEngineCtxFreeFailedSigs(DetectEngineCtx *de_ctx)
2574 {
2575  SigString *item = NULL;
2576  SigString *sitem;
2577 
2578  TAILQ_FOREACH_SAFE(item, &de_ctx->sig_stat.failed_sigs, next, sitem) {
2579  SCFree(item->filename);
2580  SCFree(item->sig_str);
2581  if (item->sig_error) {
2582  SCFree(item->sig_error);
2583  }
2584  TAILQ_REMOVE(&de_ctx->sig_stat.failed_sigs, item, next);
2585  SCFree(item);
2586  }
2587 }
2588 
2589 /**
2590  * \brief Free a DetectEngineCtx::
2591  *
2592  * \param de_ctx DetectEngineCtx:: to be freed
2593  */
2595 {
2596 
2597  if (de_ctx == NULL)
2598  return;
2599 
2600 #ifdef PROFILE_RULES
2601  if (de_ctx->profile_ctx != NULL) {
2602  SCProfilingRuleDestroyCtx(de_ctx->profile_ctx);
2603  de_ctx->profile_ctx = NULL;
2604  }
2605 #endif
2606 #ifdef PROFILING
2607  if (de_ctx->profile_keyword_ctx != NULL) {
2608  SCProfilingKeywordDestroyCtx(de_ctx);//->profile_keyword_ctx);
2609 // de_ctx->profile_keyword_ctx = NULL;
2610  }
2611  if (de_ctx->profile_sgh_ctx != NULL) {
2613  }
2615 #endif
2616 
2617  /* Normally the hashes are freed elsewhere, but
2618  * to be sure look at them again here.
2619  */
2626  if (de_ctx->sig_array)
2628 
2632 
2634 
2638 
2640 
2641  DetectEngineCtxFreeThreadKeywordData(de_ctx);
2643  DetectEngineCtxFreeFailedSigs(de_ctx);
2644 
2647 
2648  /* if we have a config prefix, remove the config from the tree */
2649  if (strlen(de_ctx->config_prefix) > 0) {
2650  /* remove config */
2652  if (node != NULL) {
2653  ConfNodeRemove(node); /* frees node */
2654  }
2655 #if 0
2656  ConfDump();
2657 #endif
2658  }
2659 
2662 
2663  DetectBufferTypeFreeDetectEngine(de_ctx);
2666 
2667  if (de_ctx->tenant_path) {
2669  }
2670 
2671  if (de_ctx->requirements) {
2672  SCDetectRequiresStatusFree(de_ctx->requirements);
2673  }
2674 
2675  SCFree(de_ctx);
2676  //DetectAddressGroupPrintMemory();
2677  //DetectSigGroupPrintMemory();
2678  //DetectPortPrintMemory();
2679 }
2680 
2681 /** \brief Function that load DetectEngineCtx config for grouping sigs
2682  * used by the engine
2683  * \retval 0 if no config provided, 1 if config was provided
2684  * and loaded successfully
2685  */
2686 static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
2687 {
2688  uint8_t profile = ENGINE_PROFILE_MEDIUM;
2689  const char *max_uniq_toclient_groups_str = NULL;
2690  const char *max_uniq_toserver_groups_str = NULL;
2691  const char *sgh_mpm_context = NULL;
2692  const char *de_ctx_profile = NULL;
2693 
2694  (void)ConfGet("detect.profile", &de_ctx_profile);
2695  (void)ConfGet("detect.sgh-mpm-context", &sgh_mpm_context);
2696 
2697  ConfNode *de_ctx_custom = ConfGetNode("detect-engine");
2698  ConfNode *opt = NULL;
2699 
2700  if (de_ctx_custom != NULL) {
2701  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2702  if (de_ctx_profile == NULL) {
2703  if (opt->val && strcmp(opt->val, "profile") == 0) {
2704  de_ctx_profile = opt->head.tqh_first->val;
2705  }
2706  }
2707 
2708  if (sgh_mpm_context == NULL) {
2709  if (opt->val && strcmp(opt->val, "sgh-mpm-context") == 0) {
2710  sgh_mpm_context = opt->head.tqh_first->val;
2711  }
2712  }
2713  }
2714  }
2715 
2716  if (de_ctx_profile != NULL) {
2717  if (strcmp(de_ctx_profile, "low") == 0 ||
2718  strcmp(de_ctx_profile, "lowest") == 0) { // legacy
2719  profile = ENGINE_PROFILE_LOW;
2720  } else if (strcmp(de_ctx_profile, "medium") == 0) {
2721  profile = ENGINE_PROFILE_MEDIUM;
2722  } else if (strcmp(de_ctx_profile, "high") == 0 ||
2723  strcmp(de_ctx_profile, "highest") == 0) { // legacy
2724  profile = ENGINE_PROFILE_HIGH;
2725  } else if (strcmp(de_ctx_profile, "custom") == 0) {
2726  profile = ENGINE_PROFILE_CUSTOM;
2727  } else {
2728  SCLogError("invalid value for detect.profile: '%s'. "
2729  "Valid options: low, medium, high and custom.",
2730  de_ctx_profile);
2731  return -1;
2732  }
2733 
2734  SCLogDebug("Profile for detection engine groups is \"%s\"", de_ctx_profile);
2735  } else {
2736  SCLogDebug("Profile for detection engine groups not provided "
2737  "at suricata.yaml. Using default (\"medium\").");
2738  }
2739 
2740  /* detect-engine.sgh-mpm-context option parsing */
2741  if (sgh_mpm_context == NULL || strcmp(sgh_mpm_context, "auto") == 0) {
2742  /* for now, since we still haven't implemented any intelligence into
2743  * understanding the patterns and distributing mpm_ctx across sgh */
2745  de_ctx->mpm_matcher == MPM_HS) {
2747  } else {
2749  }
2750  } else {
2751  if (strcmp(sgh_mpm_context, "single") == 0) {
2753  } else if (strcmp(sgh_mpm_context, "full") == 0) {
2755  } else {
2756  SCLogError("You have supplied an "
2757  "invalid conf value for detect-engine.sgh-mpm-context-"
2758  "%s",
2759  sgh_mpm_context);
2760  exit(EXIT_FAILURE);
2761  }
2762  }
2763 
2764  if (RunmodeIsUnittests()) {
2766  }
2767 
2768  /* parse profile custom-values */
2769  opt = NULL;
2770  switch (profile) {
2771  case ENGINE_PROFILE_LOW:
2774  break;
2775 
2776  case ENGINE_PROFILE_HIGH:
2779  break;
2780 
2781  case ENGINE_PROFILE_CUSTOM:
2782  (void)ConfGet("detect.custom-values.toclient-groups",
2783  &max_uniq_toclient_groups_str);
2784  (void)ConfGet("detect.custom-values.toserver-groups",
2785  &max_uniq_toserver_groups_str);
2786 
2787  if (de_ctx_custom != NULL) {
2788  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2789  if (opt->val && strcmp(opt->val, "custom-values") == 0) {
2790  if (max_uniq_toclient_groups_str == NULL) {
2791  max_uniq_toclient_groups_str = (char *)ConfNodeLookupChildValue
2792  (opt->head.tqh_first, "toclient-sp-groups");
2793  }
2794  if (max_uniq_toclient_groups_str == NULL) {
2795  max_uniq_toclient_groups_str = (char *)ConfNodeLookupChildValue
2796  (opt->head.tqh_first, "toclient-groups");
2797  }
2798  if (max_uniq_toserver_groups_str == NULL) {
2799  max_uniq_toserver_groups_str = (char *)ConfNodeLookupChildValue
2800  (opt->head.tqh_first, "toserver-dp-groups");
2801  }
2802  if (max_uniq_toserver_groups_str == NULL) {
2803  max_uniq_toserver_groups_str = (char *)ConfNodeLookupChildValue
2804  (opt->head.tqh_first, "toserver-groups");
2805  }
2806  }
2807  }
2808  }
2809  if (max_uniq_toclient_groups_str != NULL) {
2811  (uint16_t)strlen(max_uniq_toclient_groups_str),
2812  (const char *)max_uniq_toclient_groups_str) <= 0) {
2814 
2815  SCLogWarning("parsing '%s' for "
2816  "toclient-groups failed, using %u",
2817  max_uniq_toclient_groups_str, de_ctx->max_uniq_toclient_groups);
2818  }
2819  } else {
2821  }
2822  SCLogConfig("toclient-groups %u", de_ctx->max_uniq_toclient_groups);
2823 
2824  if (max_uniq_toserver_groups_str != NULL) {
2826  (uint16_t)strlen(max_uniq_toserver_groups_str),
2827  (const char *)max_uniq_toserver_groups_str) <= 0) {
2829 
2830  SCLogWarning("parsing '%s' for "
2831  "toserver-groups failed, using %u",
2832  max_uniq_toserver_groups_str, de_ctx->max_uniq_toserver_groups);
2833  }
2834  } else {
2836  }
2837  SCLogConfig("toserver-groups %u", de_ctx->max_uniq_toserver_groups);
2838  break;
2839 
2840  /* Default (or no config provided) is profile medium */
2841  case ENGINE_PROFILE_MEDIUM:
2843  default:
2846  break;
2847  }
2848 
2849  intmax_t value = 0;
2850  if (ConfGetInt("detect.inspection-recursion-limit", &value) == 1)
2851  {
2852  if (value >= 0 && value <= INT_MAX) {
2853  de_ctx->inspection_recursion_limit = (int)value;
2854  }
2855 
2856  /* fall back to old config parsing */
2857  } else {
2858  ConfNode *insp_recursion_limit_node = NULL;
2859  char *insp_recursion_limit = NULL;
2860 
2861  if (de_ctx_custom != NULL) {
2862  opt = NULL;
2863  TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
2864  if (opt->val && strcmp(opt->val, "inspection-recursion-limit") != 0)
2865  continue;
2866 
2867  insp_recursion_limit_node = ConfNodeLookupChild(opt, opt->val);
2868  if (insp_recursion_limit_node == NULL) {
2869  SCLogError("Error retrieving conf "
2870  "entry for detect-engine:inspection-recursion-limit");
2871  break;
2872  }
2873  insp_recursion_limit = insp_recursion_limit_node->val;
2874  SCLogDebug("Found detect-engine.inspection-recursion-limit - %s:%s",
2875  insp_recursion_limit_node->name, insp_recursion_limit_node->val);
2876  break;
2877  }
2878 
2879  if (insp_recursion_limit != NULL) {
2881  0, (const char *)insp_recursion_limit) < 0) {
2882  SCLogWarning("Invalid value for "
2883  "detect-engine.inspection-recursion-limit: %s "
2884  "resetting to %d",
2885  insp_recursion_limit, DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT);
2888  }
2889  } else {
2892  }
2893  }
2894  }
2895 
2898 
2899  SCLogDebug("de_ctx->inspection_recursion_limit: %d",
2901 
2902  /* parse port grouping whitelisting settings */
2903 
2904  const char *ports = NULL;
2905  (void)ConfGet("detect.grouping.tcp-whitelist", &ports);
2906  if (ports) {
2907  SCLogConfig("grouping: tcp-whitelist %s", ports);
2908  } else {
2909  ports = "53, 80, 139, 443, 445, 1433, 3306, 3389, 6666, 6667, 8080";
2910  SCLogConfig("grouping: tcp-whitelist (default) %s", ports);
2911 
2912  }
2913  if (DetectPortParse(de_ctx, &de_ctx->tcp_whitelist, ports) != 0) {
2914  SCLogWarning("'%s' is not a valid value "
2915  "for detect.grouping.tcp-whitelist",
2916  ports);
2917  }
2919  for ( ; x != NULL; x = x->next) {
2920  if (x->port != x->port2) {
2921  SCLogWarning("'%s' is not a valid value "
2922  "for detect.grouping.tcp-whitelist: only single ports allowed",
2923  ports);
2925  de_ctx->tcp_whitelist = NULL;
2926  break;
2927  }
2928  }
2929 
2930  ports = NULL;
2931  (void)ConfGet("detect.grouping.udp-whitelist", &ports);
2932  if (ports) {
2933  SCLogConfig("grouping: udp-whitelist %s", ports);
2934  } else {
2935  ports = "53, 135, 5060";
2936  SCLogConfig("grouping: udp-whitelist (default) %s", ports);
2937 
2938  }
2939  if (DetectPortParse(de_ctx, &de_ctx->udp_whitelist, ports) != 0) {
2940  SCLogWarning("'%s' is not a valid value "
2941  "for detect.grouping.udp-whitelist",
2942  ports);
2943  }
2944  for (x = de_ctx->udp_whitelist; x != NULL; x = x->next) {
2945  if (x->port != x->port2) {
2946  SCLogWarning("'%s' is not a valid value "
2947  "for detect.grouping.udp-whitelist: only single ports allowed",
2948  ports);
2950  de_ctx->udp_whitelist = NULL;
2951  break;
2952  }
2953  }
2954 
2956  const char *pf_setting = NULL;
2957  if (ConfGet("detect.prefilter.default", &pf_setting) == 1 && pf_setting) {
2958  if (strcasecmp(pf_setting, "mpm") == 0) {
2960  } else if (strcasecmp(pf_setting, "auto") == 0) {
2962  }
2963  }
2964  switch (de_ctx->prefilter_setting) {
2965  case DETECT_PREFILTER_MPM:
2966  SCLogConfig("prefilter engines: MPM");
2967  break;
2968  case DETECT_PREFILTER_AUTO:
2969  SCLogConfig("prefilter engines: MPM and keywords");
2970  break;
2971  }
2972 
2973  return 0;
2974 }
2975 
2976 /*
2977  * getting & (re)setting the internal sig i
2978  */
2979 
2980 //inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *de_ctx)
2981 //{
2982 // return de_ctx->signum;
2983 //}
2984 
2986 {
2987  de_ctx->signum = 0;
2988 }
2989 
2990 static int DetectEngineThreadCtxInitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
2991 {
2992  const DetectEngineMasterCtx *master = &g_master_de_ctx;
2993 
2994  if (master->keyword_id > 0) {
2995  // coverity[suspicious_sizeof : FALSE]
2996  det_ctx->global_keyword_ctxs_array = (void **)SCCalloc(master->keyword_id, sizeof(void *));
2997  if (det_ctx->global_keyword_ctxs_array == NULL) {
2998  SCLogError("setting up thread local detect ctx");
2999  return TM_ECODE_FAILED;
3000  }
3001  det_ctx->global_keyword_ctxs_size = master->keyword_id;
3002 
3003  const DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
3004  while (item) {
3005  det_ctx->global_keyword_ctxs_array[item->id] = item->InitFunc(item->data);
3006  if (det_ctx->global_keyword_ctxs_array[item->id] == NULL) {
3007  SCLogError("setting up thread local detect ctx "
3008  "for keyword \"%s\" failed",
3009  item->name);
3010  return TM_ECODE_FAILED;
3011  }
3012  item = item->next;
3013  }
3014  }
3015  return TM_ECODE_OK;
3016 }
3017 
3018 static void DetectEngineThreadCtxDeinitGlobalKeywords(DetectEngineThreadCtx *det_ctx)
3019 {
3020  if (det_ctx->global_keyword_ctxs_array == NULL ||
3021  det_ctx->global_keyword_ctxs_size == 0) {
3022  return;
3023  }
3024 
3025  const DetectEngineMasterCtx *master = &g_master_de_ctx;
3026  if (master->keyword_id > 0) {
3027  const DetectEngineThreadKeywordCtxItem *item = master->keyword_list;
3028  while (item) {
3029  if (det_ctx->global_keyword_ctxs_array[item->id] != NULL)
3030  item->FreeFunc(det_ctx->global_keyword_ctxs_array[item->id]);
3031 
3032  item = item->next;
3033  }
3034  det_ctx->global_keyword_ctxs_size = 0;
3036  det_ctx->global_keyword_ctxs_array = NULL;
3037  }
3038 }
3039 
3040 static int DetectEngineThreadCtxInitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
3041 {
3042  if (de_ctx->keyword_id > 0) {
3043  // coverity[suspicious_sizeof : FALSE]
3044  det_ctx->keyword_ctxs_array = SCCalloc(de_ctx->keyword_id, sizeof(void *));
3045  if (det_ctx->keyword_ctxs_array == NULL) {
3046  SCLogError("setting up thread local detect ctx");
3047  return TM_ECODE_FAILED;
3048  }
3049 
3050  det_ctx->keyword_ctxs_size = de_ctx->keyword_id;
3051 
3053  for (; hb != NULL; hb = HashListTableGetListNext(hb)) {
3055 
3056  det_ctx->keyword_ctxs_array[item->id] = item->InitFunc(item->data);
3057  if (det_ctx->keyword_ctxs_array[item->id] == NULL) {
3058  SCLogError("setting up thread local detect ctx "
3059  "for keyword \"%s\" failed",
3060  item->name);
3061  return TM_ECODE_FAILED;
3062  }
3063  }
3064  }
3065  return TM_ECODE_OK;
3066 }
3067 
3068 static void DetectEngineThreadCtxDeinitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
3069 {
3070  if (de_ctx->keyword_id > 0) {
3072  for (; hb != NULL; hb = HashListTableGetListNext(hb)) {
3074 
3075  if (det_ctx->keyword_ctxs_array[item->id] != NULL)
3076  item->FreeFunc(det_ctx->keyword_ctxs_array[item->id]);
3077  }
3078  det_ctx->keyword_ctxs_size = 0;
3079  SCFree(det_ctx->keyword_ctxs_array);
3080  det_ctx->keyword_ctxs_array = NULL;
3081  }
3082 }
3083 
3084 /** NOTE: master MUST be locked before calling this */
3085 static TmEcode DetectEngineThreadCtxInitForMT(ThreadVars *tv, DetectEngineThreadCtx *det_ctx)
3086 {
3087  DetectEngineMasterCtx *master = &g_master_de_ctx;
3088  DetectEngineTenantMapping *map_array = NULL;
3089  uint32_t map_array_size = 0;
3090  uint32_t map_cnt = 0;
3091  uint32_t max_tenant_id = 0;
3092  DetectEngineCtx *list = master->list;
3093  HashTable *mt_det_ctxs_hash = NULL;
3094 
3095  if (master->tenant_selector == TENANT_SELECTOR_UNKNOWN) {
3096  SCLogError("no tenant selector set: "
3097  "set using multi-detect.selector");
3098  return TM_ECODE_FAILED;
3099  }
3100 
3101  uint32_t tcnt = 0;
3102  while (list) {
3103  if (list->tenant_id > max_tenant_id)
3104  max_tenant_id = list->tenant_id;
3105 
3106  list = list->next;
3107  tcnt++;
3108  }
3109 
3110  mt_det_ctxs_hash = HashTableInit(tcnt * 2, TenantIdHash, TenantIdCompare, TenantIdFree);
3111  if (mt_det_ctxs_hash == NULL) {
3112  goto error;
3113  }
3114 
3115  if (tcnt == 0) {
3116  SCLogInfo("no tenants left, or none registered yet");
3117  } else {
3118  max_tenant_id++;
3119 
3121  while (map) {
3122  map_cnt++;
3123  map = map->next;
3124  }
3125 
3126  if (map_cnt > 0) {
3127  map_array_size = map_cnt + 1;
3128 
3129  map_array = SCCalloc(map_array_size, sizeof(*map_array));
3130  if (map_array == NULL)
3131  goto error;
3132 
3133  /* fill the array */
3134  map_cnt = 0;
3135  map = master->tenant_mapping_list;
3136  while (map) {
3137  if (map_cnt >= map_array_size) {
3138  goto error;
3139  }
3140  map_array[map_cnt].traffic_id = map->traffic_id;
3141  map_array[map_cnt].tenant_id = map->tenant_id;
3142  map_cnt++;
3143  map = map->next;
3144  }
3145 
3146  }
3147 
3148  /* set up hash for tenant lookup */
3149  list = master->list;
3150  while (list) {
3151  SCLogDebug("tenant-id %u", list->tenant_id);
3152  if (list->tenant_id != 0) {
3154  if (mt_det_ctx == NULL)
3155  goto error;
3156  if (HashTableAdd(mt_det_ctxs_hash, mt_det_ctx, 0) != 0) {
3157  goto error;
3158  }
3159  }
3160  list = list->next;
3161  }
3162  }
3163 
3164  det_ctx->mt_det_ctxs_hash = mt_det_ctxs_hash;
3165  mt_det_ctxs_hash = NULL;
3166 
3167  det_ctx->mt_det_ctxs_cnt = max_tenant_id;
3168 
3169  det_ctx->tenant_array = map_array;
3170  det_ctx->tenant_array_size = map_array_size;
3171 
3172  switch (master->tenant_selector) {
3174  SCLogDebug("TENANT_SELECTOR_UNKNOWN");
3175  break;
3176  case TENANT_SELECTOR_VLAN:
3177  det_ctx->TenantGetId = DetectEngineTenantGetIdFromVlanId;
3178  SCLogDebug("TENANT_SELECTOR_VLAN");
3179  break;
3181  det_ctx->TenantGetId = DetectEngineTenantGetIdFromLivedev;
3182  SCLogDebug("TENANT_SELECTOR_LIVEDEV");
3183  break;
3185  det_ctx->TenantGetId = DetectEngineTenantGetIdFromPcap;
3186  SCLogDebug("TENANT_SELECTOR_DIRECT");
3187  break;
3188  }
3189 
3190  return TM_ECODE_OK;
3191 error:
3192  if (map_array != NULL)
3193  SCFree(map_array);
3194  if (mt_det_ctxs_hash != NULL)
3195  HashTableFree(mt_det_ctxs_hash);
3196 
3197  return TM_ECODE_FAILED;
3198 }
3199 
3200 /** \internal
3201  * \brief Helper for DetectThread setup functions
3202  */
3203 static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
3204 {
3206 
3207  PmqSetup(&det_ctx->pmq);
3208 
3210  if (det_ctx->spm_thread_ctx == NULL) {
3211  return TM_ECODE_FAILED;
3212  }
3213 
3214  /* sized to the max of our sgh settings. A max setting of 0 implies that all
3215  * sgh's have: sgh->non_pf_store_cnt == 0 */
3216  if (de_ctx->non_pf_store_cnt_max > 0) {
3218  BUG_ON(det_ctx->non_pf_id_array == NULL);
3219  }
3220 
3221  /* DeState */
3222  if (de_ctx->sig_array_len > 0) {
3223  det_ctx->match_array_len = de_ctx->sig_array_len;
3224  det_ctx->match_array = SCCalloc(det_ctx->match_array_len, sizeof(Signature *));
3225  if (det_ctx->match_array == NULL) {
3226  return TM_ECODE_FAILED;
3227  }
3228 
3230  }
3231 
3232  /* Alert processing queue */
3233  AlertQueueInit(det_ctx);
3234 
3235  /* byte_extract storage */
3236  det_ctx->byte_values = SCMalloc(sizeof(*det_ctx->byte_values) *
3238  if (det_ctx->byte_values == NULL) {
3239  return TM_ECODE_FAILED;
3240  }
3241 
3242  /* Allocate space for base64 decoded data. */
3245  if (det_ctx->base64_decoded == NULL) {
3246  return TM_ECODE_FAILED;
3247  }
3249  det_ctx->base64_decoded_len = 0;
3250  }
3251 
3253  det_ctx->inspect.buffers = SCCalloc(det_ctx->inspect.buffers_size, sizeof(InspectionBuffer));
3254  if (det_ctx->inspect.buffers == NULL) {
3255  return TM_ECODE_FAILED;
3256  }
3257  det_ctx->inspect.to_clear_queue = SCCalloc(det_ctx->inspect.buffers_size, sizeof(uint32_t));
3258  if (det_ctx->inspect.to_clear_queue == NULL) {
3259  return TM_ECODE_FAILED;
3260  }
3261  det_ctx->inspect.to_clear_idx = 0;
3262 
3265  if (det_ctx->multi_inspect.buffers == NULL) {
3266  return TM_ECODE_FAILED;
3267  }
3268  det_ctx->multi_inspect.to_clear_queue = SCCalloc(det_ctx->multi_inspect.buffers_size, sizeof(uint32_t));
3269  if (det_ctx->multi_inspect.to_clear_queue == NULL) {
3270  return TM_ECODE_FAILED;
3271  }
3272  det_ctx->multi_inspect.to_clear_idx = 0;
3273 
3274 
3275  DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
3276  DetectEngineThreadCtxInitGlobalKeywords(det_ctx);
3277 #ifdef PROFILE_RULES
3278  SCProfilingRuleThreadSetup(de_ctx->profile_ctx, det_ctx);
3279 #endif
3280 #ifdef PROFILING
3284 #endif
3285  SC_ATOMIC_INIT(det_ctx->so_far_used_by_detect);
3286 
3287  return TM_ECODE_OK;
3288 }
3289 
3290 /** \brief initialize thread specific detection engine context
3291  *
3292  * \note there is a special case when using delayed detect. In this case the
3293  * function is called twice per thread. The first time the rules are not
3294  * yet loaded. de_ctx->delayed_detect_initialized will be 0. The 2nd
3295  * time they will be loaded. de_ctx->delayed_detect_initialized will be 1.
3296  * This is needed to do the per thread counter registration before the
3297  * packet runtime starts. In delayed detect mode, the first call will
3298  * return a NULL ptr through the data ptr.
3299  *
3300  * \param tv ThreadVars for this thread
3301  * \param initdata pointer to de_ctx
3302  * \param data[out] pointer to store our thread detection ctx
3303  *
3304  * \retval TM_ECODE_OK if all went well
3305  * \retval TM_ECODE_FAILED on serious errors
3306  */
3307 TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
3308 {
3309  DetectEngineThreadCtx *det_ctx = SCCalloc(1, sizeof(DetectEngineThreadCtx));
3310  if (unlikely(det_ctx == NULL))
3311  return TM_ECODE_FAILED;
3312 
3313  det_ctx->tv = tv;
3314  det_ctx->de_ctx = DetectEngineGetCurrent();
3315  if (det_ctx->de_ctx == NULL) {
3316 #ifdef UNITTESTS
3317  if (RunmodeIsUnittests()) {
3318  det_ctx->de_ctx = (DetectEngineCtx *)initdata;
3319  } else {
3320  DetectEngineThreadCtxDeinit(tv, det_ctx);
3321  return TM_ECODE_FAILED;
3322  }
3323 #else
3324  DetectEngineThreadCtxDeinit(tv, det_ctx);
3325  return TM_ECODE_FAILED;
3326 #endif
3327  }
3328 
3329  if (det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
3330  det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_TENANT)
3331  {
3332  if (ThreadCtxDoInit(det_ctx->de_ctx, det_ctx) != TM_ECODE_OK) {
3333  DetectEngineThreadCtxDeinit(tv, det_ctx);
3334  return TM_ECODE_FAILED;
3335  }
3336  }
3337 
3338  /** alert counter setup */
3339  det_ctx->counter_alerts = StatsRegisterCounter("detect.alert", tv);
3340  det_ctx->counter_alerts_overflow = StatsRegisterCounter("detect.alert_queue_overflow", tv);
3341  det_ctx->counter_alerts_suppressed = StatsRegisterCounter("detect.alerts_suppressed", tv);
3342 
3343  /* Register counter for Lua rule errors. */
3344  det_ctx->lua_rule_errors = StatsRegisterCounter("detect.lua.errors", tv);
3345 
3346  /* Register a counter for Lua blocked function attempts. */
3347  det_ctx->lua_blocked_function_errors =
3348  StatsRegisterCounter("detect.lua.blocked_function_errors", tv);
3349 
3350  /* Register a counter for Lua instruction limit errors. */
3351  det_ctx->lua_instruction_limit_errors =
3352  StatsRegisterCounter("detect.lua.instruction_limit_errors", tv);
3353 
3354  /* Register a counter for Lua memory limit errors. */
3355  det_ctx->lua_memory_limit_errors = StatsRegisterCounter("detect.lua.memory_limit_errors", tv);
3356 
3357 #ifdef PROFILING
3358  det_ctx->counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
3359  det_ctx->counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
3360  det_ctx->counter_fnonmpm_list = StatsRegisterAvgCounter("detect.fnonmpm_list", tv);
3361  det_ctx->counter_match_list = StatsRegisterAvgCounter("detect.match_list", tv);
3362 #endif
3363 
3365  if (DetectEngineThreadCtxInitForMT(tv, det_ctx) != TM_ECODE_OK) {
3366  DetectEngineThreadCtxDeinit(tv, det_ctx);
3367  return TM_ECODE_FAILED;
3368  }
3369  }
3370 
3371  /* pass thread data back to caller */
3372  *data = (void *)det_ctx;
3373 
3374  return TM_ECODE_OK;
3375 }
3376 
3377 /**
3378  * \internal
3379  * \brief initialize a det_ctx for reload cases
3380  * \param new_de_ctx the new detection engine
3381  * \param mt flag to indicate if MT should be set up for this det_ctx
3382  * this should only be done for the 'root' det_ctx
3383  *
3384  * \retval det_ctx detection engine thread ctx or NULL in case of error
3385  */
3387  ThreadVars *tv, DetectEngineCtx *new_de_ctx, int mt)
3388 {
3389  DetectEngineThreadCtx *det_ctx = SCCalloc(1, sizeof(DetectEngineThreadCtx));
3390  if (unlikely(det_ctx == NULL))
3391  return NULL;
3392 
3393  det_ctx->tenant_id = new_de_ctx->tenant_id;
3394  det_ctx->tv = tv;
3395  det_ctx->de_ctx = DetectEngineReference(new_de_ctx);
3396  if (det_ctx->de_ctx == NULL) {
3397  SCFree(det_ctx);
3398  return NULL;
3399  }
3400 
3401  /* most of the init happens here */
3402  if (det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
3403  det_ctx->de_ctx->type == DETECT_ENGINE_TYPE_TENANT)
3404  {
3405  if (ThreadCtxDoInit(det_ctx->de_ctx, det_ctx) != TM_ECODE_OK) {
3406  DetectEngineDeReference(&det_ctx->de_ctx);
3407  SCFree(det_ctx);
3408  return NULL;
3409  }
3410  }
3411 
3412  /** alert counter setup */
3413  det_ctx->counter_alerts = StatsRegisterCounter("detect.alert", tv);
3414  det_ctx->counter_alerts_overflow = StatsRegisterCounter("detect.alert_queue_overflow", tv);
3415  det_ctx->counter_alerts_suppressed = StatsRegisterCounter("detect.alerts_suppressed", tv);
3416 #ifdef PROFILING
3417  uint16_t counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
3418  uint16_t counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
3419  uint16_t counter_fnonmpm_list = StatsRegisterAvgCounter("detect.fnonmpm_list", tv);
3420  uint16_t counter_match_list = StatsRegisterAvgCounter("detect.match_list", tv);
3421  det_ctx->counter_mpm_list = counter_mpm_list;
3422  det_ctx->counter_nonmpm_list = counter_nonmpm_list;
3423  det_ctx->counter_fnonmpm_list = counter_fnonmpm_list;
3424  det_ctx->counter_match_list = counter_match_list;
3425 #endif
3426 
3427  if (mt && DetectEngineMultiTenantEnabled()) {
3428  if (DetectEngineThreadCtxInitForMT(tv, det_ctx) != TM_ECODE_OK) {
3429  DetectEngineDeReference(&det_ctx->de_ctx);
3430  SCFree(det_ctx);
3431  return NULL;
3432  }
3433  }
3434 
3435  return det_ctx;
3436 }
3437 
3438 static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
3439 {
3440 #if DEBUG
3441  SCLogDebug("PACKET PKT_STREAM_ADD: %"PRIu64, det_ctx->pkt_stream_add_cnt);
3442 
3443  SCLogDebug("PAYLOAD MPM %"PRIu64"/%"PRIu64, det_ctx->payload_mpm_cnt, det_ctx->payload_mpm_size);
3444  SCLogDebug("STREAM MPM %"PRIu64"/%"PRIu64, det_ctx->stream_mpm_cnt, det_ctx->stream_mpm_size);
3445 
3446  SCLogDebug("PAYLOAD SIG %"PRIu64"/%"PRIu64, det_ctx->payload_persig_cnt, det_ctx->payload_persig_size);
3447  SCLogDebug("STREAM SIG %"PRIu64"/%"PRIu64, det_ctx->stream_persig_cnt, det_ctx->stream_persig_size);
3448 #endif
3449 
3450  if (det_ctx->tenant_array != NULL) {
3451  SCFree(det_ctx->tenant_array);
3452  det_ctx->tenant_array = NULL;
3453  }
3454 
3455 #ifdef PROFILE_RULES
3457 #endif
3458 #ifdef PROFILING
3461  SCProfilingSghThreadCleanup(det_ctx);
3462 #endif
3463 
3464  /** \todo get rid of this static */
3465  if (det_ctx->de_ctx != NULL) {
3466  PatternMatchThreadDestroy(&det_ctx->mtc, det_ctx->de_ctx->mpm_matcher);
3467  }
3468 
3469  PmqFree(&det_ctx->pmq);
3470 
3471  if (det_ctx->spm_thread_ctx != NULL) {
3473  }
3474 
3475  if (det_ctx->non_pf_id_array != NULL)
3476  SCFree(det_ctx->non_pf_id_array);
3477 
3478  if (det_ctx->match_array != NULL)
3479  SCFree(det_ctx->match_array);
3480 
3482 
3483  AlertQueueFree(det_ctx);
3484 
3485  if (det_ctx->byte_values != NULL)
3486  SCFree(det_ctx->byte_values);
3487 
3488  /* Decoded base64 data. */
3489  if (det_ctx->base64_decoded != NULL) {
3490  SCFree(det_ctx->base64_decoded);
3491  }
3492 
3493  if (det_ctx->inspect.buffers) {
3494  for (uint32_t i = 0; i < det_ctx->inspect.buffers_size; i++) {
3495  InspectionBufferFree(&det_ctx->inspect.buffers[i]);
3496  }
3497  SCFree(det_ctx->inspect.buffers);
3498  }
3499  if (det_ctx->inspect.to_clear_queue) {
3500  SCFree(det_ctx->inspect.to_clear_queue);
3501  }
3502  if (det_ctx->multi_inspect.buffers) {
3503  for (uint32_t i = 0; i < det_ctx->multi_inspect.buffers_size; i++) {
3505  for (uint32_t x = 0; x < fb->size; x++) {
3507  }
3509  }
3510  SCFree(det_ctx->multi_inspect.buffers);
3511  }
3512  if (det_ctx->multi_inspect.to_clear_queue) {
3514  }
3515 
3516  DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx);
3517  if (det_ctx->de_ctx != NULL) {
3518  DetectEngineThreadCtxDeinitKeywords(det_ctx->de_ctx, det_ctx);
3519 #ifdef UNITTESTS
3520  if (!RunmodeIsUnittests() || det_ctx->de_ctx->ref_cnt > 0)
3521  DetectEngineDeReference(&det_ctx->de_ctx);
3522 #else
3523  DetectEngineDeReference(&det_ctx->de_ctx);
3524 #endif
3525  }
3526 
3528 
3529  SCFree(det_ctx);
3530 }
3531 
3533 {
3534  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
3535 
3536  if (det_ctx == NULL) {
3537  SCLogWarning("argument \"data\" NULL");
3538  return TM_ECODE_OK;
3539  }
3540 
3541  if (det_ctx->mt_det_ctxs_hash != NULL) {
3542  HashTableFree(det_ctx->mt_det_ctxs_hash);
3543  det_ctx->mt_det_ctxs_hash = NULL;
3544  }
3545  DetectEngineThreadCtxFree(det_ctx);
3546 
3547  return TM_ECODE_OK;
3548 }
3549 
3550 static uint32_t DetectKeywordCtxHashFunc(HashListTable *ht, void *data, uint16_t datalen)
3551 {
3553  const char *name = ctx->name;
3554  uint64_t hash = StringHashDjb2((const uint8_t *)name, strlen(name)) + (ptrdiff_t)ctx->data;
3555  hash %= ht->array_size;
3556  return hash;
3557 }
3558 
3559 static char DetectKeywordCtxCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
3560 {
3561  DetectEngineThreadKeywordCtxItem *ctx1 = data1;
3562  DetectEngineThreadKeywordCtxItem *ctx2 = data2;
3563  const char *name1 = ctx1->name;
3564  const char *name2 = ctx2->name;
3565  return (strcmp(name1, name2) == 0 && ctx1->data == ctx2->data);
3566 }
3567 
3568 static void DetectKeywordCtxFreeFunc(void *ptr)
3569 {
3570  SCFree(ptr);
3571 }
3572 
3573 /** \brief Register Thread keyword context Funcs
3574  *
3575  * \param de_ctx detection engine to register in
3576  * \param name keyword name for error printing
3577  * \param InitFunc function ptr
3578  * \param data keyword init data to pass to Func. Can be NULL.
3579  * \param FreeFunc function ptr
3580  * \param mode 0 normal (ctx per keyword instance) 1 shared (one ctx per det_ct)
3581  *
3582  * \retval id for retrieval of ctx at runtime
3583  * \retval -1 on error
3584  *
3585  * \note make sure "data" remains valid and it free'd elsewhere. It's
3586  * recommended to store it in the keywords global ctx so that
3587  * it's freed when the de_ctx is freed.
3588  */
3589 int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int mode)
3590 {
3591  BUG_ON(de_ctx == NULL || InitFunc == NULL || FreeFunc == NULL);
3592 
3593  if (de_ctx->keyword_hash == NULL) {
3594  de_ctx->keyword_hash = HashListTableInit(4096, // TODO
3595  DetectKeywordCtxHashFunc, DetectKeywordCtxCompareFunc, DetectKeywordCtxFreeFunc);
3596  BUG_ON(de_ctx->keyword_hash == NULL);
3597  }
3598 
3599  if (mode) {
3600  DetectEngineThreadKeywordCtxItem search = { .data = data, .name = name };
3601 
3603  HashListTableLookup(de_ctx->keyword_hash, (void *)&search, 0);
3604  if (item)
3605  return item->id;
3606 
3607  /* fall through */
3608  }
3609 
3611  if (unlikely(item == NULL))
3612  return -1;
3613 
3614  item->InitFunc = InitFunc;
3615  item->FreeFunc = FreeFunc;
3616  item->data = data;
3617  item->name = name;
3618  item->id = de_ctx->keyword_id++;
3619 
3620  if (HashListTableAdd(de_ctx->keyword_hash, (void *)item, 0) < 0) {
3621  SCFree(item);
3622  return -1;
3623  }
3624  return item->id;
3625 }
3626 
3627 /** \brief Remove Thread keyword context registration
3628  *
3629  * \param de_ctx detection engine to deregister from
3630  * \param det_ctx detection engine thread context to deregister from
3631  * \param data keyword init data to pass to Func. Can be NULL.
3632  * \param name keyword name for error printing
3633  *
3634  * \retval 1 Item unregistered
3635  * \retval 0 otherwise
3636  *
3637  * \note make sure "data" remains valid and it free'd elsewhere. It's
3638  * recommended to store it in the keywords global ctx so that
3639  * it's freed when the de_ctx is freed.
3640  */
3641 int DetectUnregisterThreadCtxFuncs(DetectEngineCtx *de_ctx, void *data, const char *name)
3642 {
3643  /* might happen if we call this before a call to *Register* */
3644  if (de_ctx->keyword_hash == NULL)
3645  return 1;
3646  DetectEngineThreadKeywordCtxItem remove = { .data = data, .name = name };
3647  if (HashListTableRemove(de_ctx->keyword_hash, (void *)&remove, 0) == 0)
3648  return 1;
3649  return 0;
3650 }
3651 /** \brief Retrieve thread local keyword ctx by id
3652  *
3653  * \param det_ctx detection engine thread ctx to retrieve the ctx from
3654  * \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
3655  * keyword init.
3656  *
3657  * \retval ctx or NULL on error
3658  */
3660 {
3661  if (id < 0 || id > det_ctx->keyword_ctxs_size || det_ctx->keyword_ctxs_array == NULL)
3662  return NULL;
3663 
3664  return det_ctx->keyword_ctxs_array[id];
3665 }
3666 
3667 
3668 /** \brief Register Thread keyword context Funcs (Global)
3669  *
3670  * IDs stay static over reloads and between tenants
3671  *
3672  * \param name keyword name for error printing
3673  * \param InitFunc function ptr
3674  * \param FreeFunc function ptr
3675  *
3676  * \retval id for retrieval of ctx at runtime
3677  * \retval -1 on error
3678  */
3680  void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *))
3681 {
3682  int id;
3683  BUG_ON(InitFunc == NULL || FreeFunc == NULL);
3684 
3685  DetectEngineMasterCtx *master = &g_master_de_ctx;
3686 
3687  /* if already registered, return existing id */
3689  while (item != NULL) {
3690  if (strcmp(name, item->name) == 0) {
3691  id = item->id;
3692  return id;
3693  }
3694 
3695  item = item->next;
3696  }
3697 
3698  item = SCCalloc(1, sizeof(*item));
3699  if (unlikely(item == NULL)) {
3700  return -1;
3701  }
3702  item->InitFunc = InitFunc;
3703  item->FreeFunc = FreeFunc;
3704  item->name = name;
3705  item->data = data;
3706 
3707  item->next = master->keyword_list;
3708  master->keyword_list = item;
3709  item->id = master->keyword_id++;
3710 
3711  id = item->id;
3712  return id;
3713 }
3714 
3715 /** \brief Retrieve thread local keyword ctx by id
3716  *
3717  * \param det_ctx detection engine thread ctx to retrieve the ctx from
3718  * \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
3719  * keyword init.
3720  *
3721  * \retval ctx or NULL on error
3722  */
3724 {
3725  if (id < 0 || id > det_ctx->global_keyword_ctxs_size ||
3726  det_ctx->global_keyword_ctxs_array == NULL) {
3727  return NULL;
3728  }
3729 
3730  return det_ctx->global_keyword_ctxs_array[id];
3731 }
3732 
3733 /** \brief Check if detection is enabled
3734  * \retval bool true or false */
3736 {
3737  DetectEngineMasterCtx *master = &g_master_de_ctx;
3738  SCMutexLock(&master->lock);
3739 
3740  if (master->list == NULL) {
3741  SCMutexUnlock(&master->lock);
3742  return 0;
3743  }
3744 
3745  SCMutexUnlock(&master->lock);
3746  return 1;
3747 }
3748 
3750 {
3751  uint32_t version;
3752  DetectEngineMasterCtx *master = &g_master_de_ctx;
3753  SCMutexLock(&master->lock);
3754  version = master->version;
3755  SCMutexUnlock(&master->lock);
3756  return version;
3757 }
3758 
3760 {
3761  DetectEngineMasterCtx *master = &g_master_de_ctx;
3762  SCMutexLock(&master->lock);
3763  master->version++;
3764  SCLogDebug("master version now %u", master->version);
3765  SCMutexUnlock(&master->lock);
3766 }
3767 
3769 {
3770  DetectEngineMasterCtx *master = &g_master_de_ctx;
3771  SCMutexLock(&master->lock);
3772 
3773  DetectEngineCtx *de_ctx = master->list;
3774  while (de_ctx) {
3778  {
3779  de_ctx->ref_cnt++;
3780  SCLogDebug("de_ctx %p ref_cnt %u", de_ctx, de_ctx->ref_cnt);
3781  SCMutexUnlock(&master->lock);
3782  return de_ctx;
3783  }
3784  de_ctx = de_ctx->next;
3785  }
3786 
3787  SCMutexUnlock(&master->lock);
3788  return NULL;
3789 }
3790 
3792 {
3793  if (de_ctx == NULL)
3794  return NULL;
3795  de_ctx->ref_cnt++;
3796  return de_ctx;
3797 }
3798 
3799 /** TODO locking? Not needed if this is a one time setting at startup */
3801 {
3802  DetectEngineMasterCtx *master = &g_master_de_ctx;
3803  return (master->multi_tenant_enabled);
3804 }
3805 
3806 /** \internal
3807  * \brief load a tenant from a yaml file
3808  *
3809  * \param tenant_id the tenant id by which the config is known
3810  * \param filename full path of a yaml file
3811  * \param loader_id id of loader thread or -1
3812  *
3813  * \retval 0 ok
3814  * \retval -1 failed
3815  */
3816 static int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename, int loader_id)
3817 {
3818  DetectEngineCtx *de_ctx = NULL;
3819  char prefix[64];
3820 
3821  snprintf(prefix, sizeof(prefix), "multi-detect.%u", tenant_id);
3822 
3823  SCStat st;
3824  if (SCStatFn(filename, &st) != 0) {
3825  SCLogError("failed to stat file %s", filename);
3826  goto error;
3827  }
3828 
3829  de_ctx = DetectEngineGetByTenantId(tenant_id);
3830  if (de_ctx != NULL) {
3831  SCLogError("tenant %u already registered", tenant_id);
3833  goto error;
3834  }
3835 
3836  ConfNode *node = ConfGetNode(prefix);
3837  if (node == NULL) {
3838  SCLogError("failed to properly setup yaml %s", filename);
3839  goto error;
3840  }
3841 
3842  de_ctx = DetectEngineCtxInitWithPrefix(prefix, tenant_id);
3843  if (de_ctx == NULL) {
3844  SCLogError("initializing detection engine "
3845  "context failed.");
3846  goto error;
3847  }
3848  SCLogDebug("de_ctx %p with prefix %s", de_ctx, de_ctx->config_prefix);
3849 
3851  de_ctx->tenant_id = tenant_id;
3852  de_ctx->loader_id = loader_id;
3853  de_ctx->tenant_path = SCStrdup(filename);
3854  if (de_ctx->tenant_path == NULL) {
3855  SCLogError("Failed to duplicate path");
3856  goto error;
3857  }
3858 
3859  if (SigLoadSignatures(de_ctx, NULL, false) < 0) {
3860  SCLogError("Loading signatures failed.");
3861  goto error;
3862  }
3863 
3865 
3866  return 0;
3867 
3868 error:
3869  if (de_ctx != NULL) {
3871  }
3872  return -1;
3873 }
3874 
3875 static int DetectEngineMultiTenantReloadTenant(uint32_t tenant_id, const char *filename, int reload_cnt)
3876 {
3877  DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
3878  if (old_de_ctx == NULL) {
3879  SCLogError("tenant detect engine not found");
3880  return -1;
3881  }
3882 
3883  if (filename == NULL)
3884  filename = old_de_ctx->tenant_path;
3885 
3886  char prefix[64];
3887  snprintf(prefix, sizeof(prefix), "multi-detect.%u.reload.%d", tenant_id, reload_cnt);
3888  reload_cnt++;
3889  SCLogDebug("prefix %s", prefix);
3890 
3891  if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) {
3892  SCLogError("failed to load yaml");
3893  goto error;
3894  }
3895 
3896  ConfNode *node = ConfGetNode(prefix);
3897  if (node == NULL) {
3898  SCLogError("failed to properly setup yaml %s", filename);
3899  goto error;
3900  }
3901 
3902  DetectEngineCtx *new_de_ctx = DetectEngineCtxInitWithPrefix(prefix, tenant_id);
3903  if (new_de_ctx == NULL) {
3904  SCLogError("initializing detection engine "
3905  "context failed.");
3906  goto error;
3907  }
3908  SCLogDebug("de_ctx %p with prefix %s", new_de_ctx, new_de_ctx->config_prefix);
3909 
3910  new_de_ctx->type = DETECT_ENGINE_TYPE_TENANT;
3911  new_de_ctx->tenant_id = tenant_id;
3912  new_de_ctx->loader_id = old_de_ctx->loader_id;
3913  new_de_ctx->tenant_path = SCStrdup(filename);
3914  if (new_de_ctx->tenant_path == NULL) {
3915  SCLogError("Failed to duplicate path");
3916  goto error;
3917  }
3918 
3919  if (SigLoadSignatures(new_de_ctx, NULL, false) < 0) {
3920  SCLogError("Loading signatures failed.");
3921  goto error;
3922  }
3923 
3924  DetectEngineAddToMaster(new_de_ctx);
3925 
3926  /* move to free list */
3927  DetectEngineMoveToFreeList(old_de_ctx);
3928  DetectEngineDeReference(&old_de_ctx);
3929  return 0;
3930 
3931 error:
3932  DetectEngineDeReference(&old_de_ctx);
3933  return -1;
3934 }
3935 
3936 
3937 typedef struct TenantLoaderCtx_ {
3938  uint32_t tenant_id;
3939  int reload_cnt; /**< used by reload */
3940  char *yaml; /**< heap alloc'd copy of file path for the yaml */
3942 
3943 static void DetectLoaderFreeTenant(void *ctx)
3944 {
3945  TenantLoaderCtx *t = (TenantLoaderCtx *)ctx;
3946  if (t->yaml != NULL) {
3947  SCFree(t->yaml);
3948  }
3949  SCFree(t);
3950 }
3951 
3952 static int DetectLoaderFuncLoadTenant(void *vctx, int loader_id)
3953 {
3954  TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
3955 
3956  SCLogDebug("loader %d", loader_id);
3957  if (DetectEngineMultiTenantLoadTenant(ctx->tenant_id, ctx->yaml, loader_id) != 0) {
3958  return -1;
3959  }
3960  return 0;
3961 }
3962 
3963 static int DetectLoaderSetupLoadTenant(uint32_t tenant_id, const char *yaml)
3964 {
3965  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
3966  if (t == NULL)
3967  return -ENOMEM;
3968 
3969  t->tenant_id = tenant_id;
3970  t->yaml = SCStrdup(yaml);
3971  if (t->yaml == NULL) {
3972  SCFree(t);
3973  return -ENOMEM;
3974  }
3975 
3976  return DetectLoaderQueueTask(-1, DetectLoaderFuncLoadTenant, t, DetectLoaderFreeTenant);
3977 }
3978 
3979 static int DetectLoaderFuncReloadTenant(void *vctx, int loader_id)
3980 {
3981  TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
3982 
3983  SCLogDebug("loader_id %d", loader_id);
3984 
3985  if (DetectEngineMultiTenantReloadTenant(ctx->tenant_id, ctx->yaml, ctx->reload_cnt) != 0) {
3986  return -1;
3987  }
3988  return 0;
3989 }
3990 
3991 static int DetectLoaderSetupReloadTenants(const int reload_cnt)
3992 {
3993  int ret = 0;
3994  DetectEngineMasterCtx *master = &g_master_de_ctx;
3995  SCMutexLock(&master->lock);
3996 
3997  DetectEngineCtx *de_ctx = master->list;
3998  while (de_ctx) {
4000  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
4001  if (t == NULL) {
4002  ret = -1;
4003  goto error;
4004  }
4005  t->tenant_id = de_ctx->tenant_id;
4006  t->reload_cnt = reload_cnt;
4007  int loader_id = de_ctx->loader_id;
4008 
4009  int r = DetectLoaderQueueTask(
4010  loader_id, DetectLoaderFuncReloadTenant, t, DetectLoaderFreeTenant);
4011  if (r < 0) {
4012  ret = -2;
4013  goto error;
4014  }
4015  }
4016 
4017  de_ctx = de_ctx->next;
4018  }
4019 error:
4020  SCMutexUnlock(&master->lock);
4021  return ret;
4022 }
4023 
4024 static int DetectLoaderSetupReloadTenant(uint32_t tenant_id, const char *yaml, int reload_cnt)
4025 {
4026  DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
4027  if (old_de_ctx == NULL)
4028  return -ENOENT;
4029  int loader_id = old_de_ctx->loader_id;
4030  DetectEngineDeReference(&old_de_ctx);
4031 
4032  TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
4033  if (t == NULL)
4034  return -ENOMEM;
4035 
4036  t->tenant_id = tenant_id;
4037  if (yaml != NULL) {
4038  t->yaml = SCStrdup(yaml);
4039  if (t->yaml == NULL) {
4040  SCFree(t);
4041  return -ENOMEM;
4042  }
4043  }
4044  t->reload_cnt = reload_cnt;
4045 
4046  SCLogDebug("loader_id %d", loader_id);
4047 
4048  return DetectLoaderQueueTask(
4049  loader_id, DetectLoaderFuncReloadTenant, t, DetectLoaderFreeTenant);
4050 }
4051 
4052 /** \brief Load a tenant and wait for loading to complete
4053  */
4054 int DetectEngineLoadTenantBlocking(uint32_t tenant_id, const char *yaml)
4055 {
4056  int r = DetectLoaderSetupLoadTenant(tenant_id, yaml);
4057  if (r < 0)
4058  return r;
4059 
4060  if (DetectLoadersSync() != 0)
4061  return -1;
4062 
4063  return 0;
4064 }
4065 
4066 /** \brief Reload a tenant and wait for loading to complete
4067  */
4068 int DetectEngineReloadTenantBlocking(uint32_t tenant_id, const char *yaml, int reload_cnt)
4069 {
4070  int r = DetectLoaderSetupReloadTenant(tenant_id, yaml, reload_cnt);
4071  if (r < 0)
4072  return r;
4073 
4074  if (DetectLoadersSync() != 0)
4075  return -1;
4076 
4077  return 0;
4078 }
4079 
4080 /** \brief Reload all tenants and wait for loading to complete
4081  */
4082 int DetectEngineReloadTenantsBlocking(const int reload_cnt)
4083 {
4084  int r = DetectLoaderSetupReloadTenants(reload_cnt);
4085  if (r < 0)
4086  return r;
4087 
4088  if (DetectLoadersSync() != 0)
4089  return -1;
4090 
4091  return 0;
4092 }
4093 
4094 static int DetectEngineMultiTenantSetupLoadLivedevMappings(const ConfNode *mappings_root_node,
4095  bool failure_fatal)
4096 {
4097  ConfNode *mapping_node = NULL;
4098 
4099  int mapping_cnt = 0;
4100  if (mappings_root_node != NULL) {
4101  TAILQ_FOREACH(mapping_node, &mappings_root_node->head, next) {
4102  ConfNode *tenant_id_node = ConfNodeLookupChild(mapping_node, "tenant-id");
4103  if (tenant_id_node == NULL)
4104  goto bad_mapping;
4105  ConfNode *device_node = ConfNodeLookupChild(mapping_node, "device");
4106  if (device_node == NULL)
4107  goto bad_mapping;
4108 
4109  uint32_t tenant_id = 0;
4110  if (StringParseUint32(&tenant_id, 10, (uint16_t)strlen(tenant_id_node->val),
4111  tenant_id_node->val) < 0) {
4112  SCLogError("tenant-id "
4113  "of %s is invalid",
4114  tenant_id_node->val);
4115  goto bad_mapping;
4116  }
4117 
4118  const char *dev = device_node->val;
4119  LiveDevice *ld = LiveGetDevice(dev);
4120  if (ld == NULL) {
4121  SCLogWarning("device %s not found", dev);
4122  goto bad_mapping;
4123  }
4124 
4125  if (ld->tenant_id_set) {
4126  SCLogWarning("device %s already mapped to tenant-id %u", dev, ld->tenant_id);
4127  goto bad_mapping;
4128  }
4129 
4130  ld->tenant_id = tenant_id;
4131  ld->tenant_id_set = true;
4132 
4133  if (DetectEngineTenantRegisterLivedev(tenant_id, ld->id) != 0) {
4134  goto error;
4135  }
4136 
4137  SCLogConfig("device %s connected to tenant-id %u", dev, tenant_id);
4138  mapping_cnt++;
4139  continue;
4140 
4141  bad_mapping:
4142  if (failure_fatal)
4143  goto error;
4144  }
4145  }
4146  SCLogConfig("%d device - tenant-id mappings defined", mapping_cnt);
4147  return mapping_cnt;
4148 
4149 error:
4150  return 0;
4151 }
4152 
4153 static int DetectEngineMultiTenantSetupLoadVlanMappings(const ConfNode *mappings_root_node,
4154  bool failure_fatal)
4155 {
4156  ConfNode *mapping_node = NULL;
4157 
4158  int mapping_cnt = 0;
4159  if (mappings_root_node != NULL) {
4160  TAILQ_FOREACH(mapping_node, &mappings_root_node->head, next) {
4161  ConfNode *tenant_id_node = ConfNodeLookupChild(mapping_node, "tenant-id");
4162  if (tenant_id_node == NULL)
4163  goto bad_mapping;
4164  ConfNode *vlan_id_node = ConfNodeLookupChild(mapping_node, "vlan-id");
4165  if (vlan_id_node == NULL)
4166  goto bad_mapping;
4167 
4168  uint32_t tenant_id = 0;
4169  if (StringParseUint32(&tenant_id, 10, (uint16_t)strlen(tenant_id_node->val),
4170  tenant_id_node->val) < 0) {
4171  SCLogError("tenant-id "
4172  "of %s is invalid",
4173  tenant_id_node->val);
4174  goto bad_mapping;
4175  }
4176 
4177  uint16_t vlan_id = 0;
4178  if (StringParseUint16(
4179  &vlan_id, 10, (uint16_t)strlen(vlan_id_node->val), vlan_id_node->val) < 0) {
4180  SCLogError("vlan-id "
4181  "of %s is invalid",
4182  vlan_id_node->val);
4183  goto bad_mapping;
4184  }
4185  if (vlan_id == 0 || vlan_id >= 4095) {
4186  SCLogError("vlan-id "
4187  "of %s is invalid. Valid range 1-4094.",
4188  vlan_id_node->val);
4189  goto bad_mapping;
4190  }
4191 
4192  if (DetectEngineTenantRegisterVlanId(tenant_id, vlan_id) != 0) {
4193  goto error;
4194  }
4195  SCLogConfig("vlan %u connected to tenant-id %u", vlan_id, tenant_id);
4196  mapping_cnt++;
4197  continue;
4198 
4199  bad_mapping:
4200  if (failure_fatal)
4201  goto error;
4202  }
4203  }
4204  return mapping_cnt;
4205 
4206 error:
4207  return 0;
4208 }
4209 
4210 /**
4211  * \brief setup multi-detect / multi-tenancy
4212  *
4213  * See if MT is enabled. If so, setup the selector, tenants and mappings.
4214  * Tenants and mappings are optional, and can also dynamically be added
4215  * and removed from the unix socket.
4216  */
4217 int DetectEngineMultiTenantSetup(const bool unix_socket)
4218 {
4220  DetectEngineMasterCtx *master = &g_master_de_ctx;
4221  int failure_fatal = 0;
4222  (void)ConfGetBool("engine.init-failure-fatal", &failure_fatal);
4223 
4224  int enabled = 0;
4225  (void)ConfGetBool("multi-detect.enabled", &enabled);
4226  if (enabled == 1) {
4231 
4232  SCMutexLock(&master->lock);
4233  master->multi_tenant_enabled = 1;
4234 
4235  const char *handler = NULL;
4236  if (ConfGet("multi-detect.selector", &handler) == 1) {
4237  SCLogConfig("multi-tenant selector type %s", handler);
4238 
4239  if (strcmp(handler, "vlan") == 0) {
4240  tenant_selector = master->tenant_selector = TENANT_SELECTOR_VLAN;
4241 
4242  int vlanbool = 0;
4243  if ((ConfGetBool("vlan.use-for-tracking", &vlanbool)) == 1 && vlanbool == 0) {
4244  SCLogError("vlan tracking is disabled, "
4245  "can't use multi-detect selector 'vlan'");
4246  SCMutexUnlock(&master->lock);
4247  goto error;
4248  }
4249 
4250  } else if (strcmp(handler, "direct") == 0) {
4251  tenant_selector = master->tenant_selector = TENANT_SELECTOR_DIRECT;
4252  } else if (strcmp(handler, "device") == 0) {
4253  tenant_selector = master->tenant_selector = TENANT_SELECTOR_LIVEDEV;
4254  if (EngineModeIsIPS()) {
4255  SCLogWarning("multi-tenant 'device' mode not supported for IPS");
4256  SCMutexUnlock(&master->lock);
4257  goto error;
4258  }
4259 
4260  } else {
4261  SCLogError("unknown value %s "
4262  "multi-detect.selector",
4263  handler);
4264  SCMutexUnlock(&master->lock);
4265  goto error;
4266  }
4267  }
4268  SCMutexUnlock(&master->lock);
4269  SCLogConfig("multi-detect is enabled (multi tenancy). Selector: %s", handler);
4270 
4271  /* traffic -- tenant mappings */
4272  ConfNode *mappings_root_node = ConfGetNode("multi-detect.mappings");
4273 
4274  if (tenant_selector == TENANT_SELECTOR_VLAN) {
4275  int mapping_cnt = DetectEngineMultiTenantSetupLoadVlanMappings(mappings_root_node,
4276  failure_fatal);
4277  if (mapping_cnt == 0) {
4278  /* no mappings are valid when we're in unix socket mode,
4279  * they can be added on the fly. Otherwise warn/error
4280  * depending on failure_fatal */
4281 
4282  if (unix_socket) {
4283  SCLogNotice("no tenant traffic mappings defined, "
4284  "tenants won't be used until mappings are added");
4285  } else {
4286  if (failure_fatal) {
4287  SCLogError("no multi-detect mappings defined");
4288  goto error;
4289  } else {
4290  SCLogWarning("no multi-detect mappings defined");
4291  }
4292  }
4293  }
4294  } else if (tenant_selector == TENANT_SELECTOR_LIVEDEV) {
4295  int mapping_cnt = DetectEngineMultiTenantSetupLoadLivedevMappings(mappings_root_node,
4296  failure_fatal);
4297  if (mapping_cnt == 0) {
4298  if (failure_fatal) {
4299  SCLogError("no multi-detect mappings defined");
4300  goto error;
4301  } else {
4302  SCLogWarning("no multi-detect mappings defined");
4303  }
4304  }
4305  }
4306 
4307  /* tenants */
4308  ConfNode *tenants_root_node = ConfGetNode("multi-detect.tenants");
4309  ConfNode *tenant_node = NULL;
4310 
4311  if (tenants_root_node != NULL) {
4312  const char *path = NULL;
4313  ConfNode *path_node = ConfGetNode("multi-detect.config-path");
4314  if (path_node) {
4315  path = path_node->val;
4316  SCLogConfig("tenants config path: %s", path);
4317  }
4318 
4319  TAILQ_FOREACH(tenant_node, &tenants_root_node->head, next) {
4320  ConfNode *id_node = ConfNodeLookupChild(tenant_node, "id");
4321  if (id_node == NULL) {
4322  goto bad_tenant;
4323  }
4324  ConfNode *yaml_node = ConfNodeLookupChild(tenant_node, "yaml");
4325  if (yaml_node == NULL) {
4326  goto bad_tenant;
4327  }
4328 
4329  uint32_t tenant_id = 0;
4330  if (StringParseUint32(
4331  &tenant_id, 10, (uint16_t)strlen(id_node->val), id_node->val) < 0) {
4332  SCLogError("tenant_id "
4333  "of %s is invalid",
4334  id_node->val);
4335  goto bad_tenant;
4336  }
4337  SCLogDebug("tenant id: %u, %s", tenant_id, yaml_node->val);
4338 
4339  char yaml_path[PATH_MAX] = "";
4340  if (path) {
4341  PathMerge(yaml_path, PATH_MAX, path, yaml_node->val);
4342  } else {
4343  strlcpy(yaml_path, yaml_node->val, sizeof(yaml_path));
4344  }
4345  SCLogDebug("tenant path: %s", yaml_path);
4346 
4347  /* setup the yaml in this loop so that it's not done by the loader
4348  * threads. ConfYamlLoadFileWithPrefix is not thread safe. */
4349  char prefix[64];
4350  snprintf(prefix, sizeof(prefix), "multi-detect.%u", tenant_id);
4351  if (ConfYamlLoadFileWithPrefix(yaml_path, prefix) != 0) {
4352  SCLogError("failed to load yaml %s", yaml_path);
4353  goto bad_tenant;
4354  }
4355 
4356  int r = DetectLoaderSetupLoadTenant(tenant_id, yaml_path);
4357  if (r < 0) {
4358  /* error logged already */
4359  goto bad_tenant;
4360  }
4361  continue;
4362 
4363  bad_tenant:
4364  if (failure_fatal)
4365  goto error;
4366  }
4367  }
4368 
4369  /* wait for our loaders to complete their tasks */
4370  if (DetectLoadersSync() != 0) {
4371  goto error;
4372  }
4373 
4375 
4376  } else {
4377  SCLogDebug("multi-detect not enabled (multi tenancy)");
4378  }
4379  return 0;
4380 error:
4381  return -1;
4382 }
4383 
4384 static uint32_t DetectEngineTenantGetIdFromVlanId(const void *ctx, const Packet *p)
4385 {
4386  const DetectEngineThreadCtx *det_ctx = ctx;
4387  uint32_t x = 0;
4388  uint32_t vlan_id = 0;
4389 
4390  if (p->vlan_idx == 0)
4391  return 0;
4392 
4393  vlan_id = p->vlan_id[0];
4394 
4395  if (det_ctx == NULL || det_ctx->tenant_array == NULL || det_ctx->tenant_array_size == 0)
4396  return 0;
4397 
4398  /* not very efficient, but for now we're targeting only limited amounts.
4399  * Can use hash/tree approach later. */
4400  for (x = 0; x < det_ctx->tenant_array_size; x++) {
4401  if (det_ctx->tenant_array[x].traffic_id == vlan_id)
4402  return det_ctx->tenant_array[x].tenant_id;
4403  }
4404 
4405  return 0;
4406 }
4407 
4408 static uint32_t DetectEngineTenantGetIdFromLivedev(const void *ctx, const Packet *p)
4409 {
4410  const DetectEngineThreadCtx *det_ctx = ctx;
4411  const LiveDevice *ld = p->livedev;
4412 
4413  if (ld == NULL || det_ctx == NULL)
4414  return 0;
4415 
4416  SCLogDebug("using tenant-id %u for packet on device %s", ld->tenant_id, ld->dev);
4417  return ld->tenant_id;
4418 }
4419 
4420 static int DetectEngineTenantRegisterSelector(
4421  enum DetectEngineTenantSelectors selector, uint32_t tenant_id, uint32_t traffic_id)
4422 {
4423  DetectEngineMasterCtx *master = &g_master_de_ctx;
4424  SCMutexLock(&master->lock);
4425 
4426  if (!(master->tenant_selector == TENANT_SELECTOR_UNKNOWN || master->tenant_selector == selector)) {
4427  SCLogInfo("conflicting selector already set");
4428  SCMutexUnlock(&master->lock);
4429  return -1;
4430  }
4431 
4433  while (m) {
4434  if (m->traffic_id == traffic_id) {
4435  SCLogInfo("traffic id already registered");
4436  SCMutexUnlock(&master->lock);
4437  return -1;
4438  }
4439  m = m->next;
4440  }
4441 
4442  DetectEngineTenantMapping *map = SCCalloc(1, sizeof(*map));
4443  if (map == NULL) {
4444  SCLogInfo("memory fail");
4445  SCMutexUnlock(&master->lock);
4446  return -1;
4447  }
4448  map->traffic_id = traffic_id;
4449  map->tenant_id = tenant_id;
4450 
4451  map->next = master->tenant_mapping_list;
4452  master->tenant_mapping_list = map;
4453 
4454  master->tenant_selector = selector;
4455 
4456  SCLogDebug("tenant handler %u %u %u registered", selector, tenant_id, traffic_id);
4457  SCMutexUnlock(&master->lock);
4458  return 0;
4459 }
4460 
4461 static int DetectEngineTenantUnregisterSelector(
4462  enum DetectEngineTenantSelectors selector, uint32_t tenant_id, uint32_t traffic_id)
4463 {
4464  DetectEngineMasterCtx *master = &g_master_de_ctx;
4465  SCMutexLock(&master->lock);
4466 
4467  if (master->tenant_mapping_list == NULL) {
4468  SCMutexUnlock(&master->lock);
4469  return -1;
4470  }
4471 
4472  DetectEngineTenantMapping *prev = NULL;
4474  while (map) {
4475  if (map->traffic_id == traffic_id &&
4476  map->tenant_id == tenant_id)
4477  {
4478  if (prev != NULL)
4479  prev->next = map->next;
4480  else
4481  master->tenant_mapping_list = map->next;
4482 
4483  map->next = NULL;
4484  SCFree(map);
4485  SCLogInfo("tenant handler %u %u %u unregistered", selector, tenant_id, traffic_id);
4486  SCMutexUnlock(&master->lock);
4487  return 0;
4488  }
4489  prev = map;
4490  map = map->next;
4491  }
4492 
4493  SCMutexUnlock(&master->lock);
4494  return -1;
4495 }
4496 
4497 int DetectEngineTenantRegisterLivedev(uint32_t tenant_id, int device_id)
4498 {
4499  return DetectEngineTenantRegisterSelector(
4500  TENANT_SELECTOR_LIVEDEV, tenant_id, (uint32_t)device_id);
4501 }
4502 
4503 int DetectEngineTenantRegisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
4504 {
4505  return DetectEngineTenantRegisterSelector(TENANT_SELECTOR_VLAN, tenant_id, (uint32_t)vlan_id);
4506 }
4507 
4508 int DetectEngineTenantUnregisterVlanId(uint32_t tenant_id, uint16_t vlan_id)
4509 {
4510  return DetectEngineTenantUnregisterSelector(TENANT_SELECTOR_VLAN, tenant_id, (uint32_t)vlan_id);
4511 }
4512 
4513 int DetectEngineTenantRegisterPcapFile(uint32_t tenant_id)
4514 {
4515  SCLogInfo("registering %u %d 0", TENANT_SELECTOR_DIRECT, tenant_id);
4516  return DetectEngineTenantRegisterSelector(TENANT_SELECTOR_DIRECT, tenant_id, 0);
4517 }
4518 
4520 {
4521  SCLogInfo("unregistering %u %d 0", TENANT_SELECTOR_DIRECT, tenant_id);
4522  return DetectEngineTenantUnregisterSelector(TENANT_SELECTOR_DIRECT, tenant_id, 0);
4523 }
4524 
4525 static uint32_t DetectEngineTenantGetIdFromPcap(const void *ctx, const Packet *p)
4526 {
4527  return p->pcap_v.tenant_id;
4528 }
4529 
4531 {
4532  DetectEngineMasterCtx *master = &g_master_de_ctx;
4533  SCMutexLock(&master->lock);
4534 
4535  if (master->list == NULL) {
4536  SCMutexUnlock(&master->lock);
4537  return NULL;
4538  }
4539 
4540  DetectEngineCtx *de_ctx = master->list;
4541  while (de_ctx) {
4543  de_ctx->tenant_id == tenant_id)
4544  {
4545  de_ctx->ref_cnt++;
4546  break;
4547  }
4548 
4549  de_ctx = de_ctx->next;
4550  }
4551 
4552  SCMutexUnlock(&master->lock);
4553  return de_ctx;
4554 }
4555 
4557 {
4558  BUG_ON((*de_ctx)->ref_cnt == 0);
4559  (*de_ctx)->ref_cnt--;
4560  *de_ctx = NULL;
4561 }
4562 
4563 static int DetectEngineAddToList(DetectEngineCtx *instance)
4564 {
4565  DetectEngineMasterCtx *master = &g_master_de_ctx;
4566 
4567  if (instance == NULL)
4568  return -1;
4569 
4570  if (master->list == NULL) {
4571  master->list = instance;
4572  } else {
4573  instance->next = master->list;
4574  master->list = instance;
4575  }
4576 
4577  return 0;
4578 }
4579 
4581 {
4582  int r;
4583 
4584  if (de_ctx == NULL)
4585  return -1;
4586 
4587  SCLogDebug("adding de_ctx %p to master", de_ctx);
4588 
4589  DetectEngineMasterCtx *master = &g_master_de_ctx;
4590  SCMutexLock(&master->lock);
4591  r = DetectEngineAddToList(de_ctx);
4592  SCMutexUnlock(&master->lock);
4593  return r;
4594 }
4595 
4596 static int DetectEngineMoveToFreeListNoLock(DetectEngineMasterCtx *master, DetectEngineCtx *de_ctx)
4597 {
4598  DetectEngineCtx *instance = master->list;
4599  if (instance == NULL) {
4600  return -1;
4601  }
4602 
4603  /* remove from active list */
4604  if (instance == de_ctx) {
4605  master->list = instance->next;
4606  } else {
4607  DetectEngineCtx *prev = instance;
4608  instance = instance->next; /* already checked first element */
4609 
4610  while (instance) {
4611  DetectEngineCtx *next = instance->next;
4612 
4613  if (instance == de_ctx) {
4614  prev->next = instance->next;
4615  break;
4616  }
4617 
4618  prev = instance;
4619  instance = next;
4620  }
4621  if (instance == NULL) {
4622  return -1;
4623  }
4624  }
4625 
4626  /* instance is now detached from list */
4627  instance->next = NULL;
4628 
4629  /* add to free list */
4630  if (master->free_list == NULL) {
4631  master->free_list = instance;
4632  } else {
4633  instance->next = master->free_list;
4634  master->free_list = instance;
4635  }
4636  SCLogDebug("detect engine %p moved to free list (%u refs)", de_ctx, de_ctx->ref_cnt);
4637  return 0;
4638 }
4639 
4641 {
4642  int ret = 0;
4643  DetectEngineMasterCtx *master = &g_master_de_ctx;
4644  SCMutexLock(&master->lock);
4645  ret = DetectEngineMoveToFreeListNoLock(master, de_ctx);
4646  SCMutexUnlock(&master->lock);
4647  return ret;
4648 }
4649 
4651 {
4652  DetectEngineMasterCtx *master = &g_master_de_ctx;
4653  SCMutexLock(&master->lock);
4654 
4655  DetectEngineCtx *prev = NULL;
4656  DetectEngineCtx *instance = master->free_list;
4657  while (instance) {
4658  DetectEngineCtx *next = instance->next;
4659 
4660  SCLogDebug("detect engine %p has %u ref(s)", instance, instance->ref_cnt);
4661 
4662  if (instance->ref_cnt == 0) {
4663  if (prev == NULL) {
4664  master->free_list = next;
4665  } else {
4666  prev->next = next;
4667  }
4668 
4669  SCLogDebug("freeing detect engine %p", instance);
4670  DetectEngineCtxFree(instance);
4671  instance = NULL;
4672  }
4673 
4674  prev = instance;
4675  instance = next;
4676  }
4677  SCMutexUnlock(&master->lock);
4678 }
4679 
4681 {
4682  DetectEngineMasterCtx *master = &g_master_de_ctx;
4683  SCMutexLock(&master->lock);
4684 
4685  DetectEngineCtx *instance = master->list;
4686  while (instance) {
4687  DetectEngineCtx *next = instance->next;
4688  DEBUG_VALIDATE_BUG_ON(instance->ref_cnt);
4689  SCLogDebug("detect engine %p has %u ref(s)", instance, instance->ref_cnt);
4690  instance->ref_cnt = 0;
4691  DetectEngineMoveToFreeListNoLock(master, instance);
4692  instance = next;
4693  }
4694  SCMutexUnlock(&master->lock);
4696 }
4697 
4698 static int reloads = 0;
4699 
4700 /** \brief Reload the detection engine
4701  *
4702  * \param filename YAML file to load for the detect config
4703  *
4704  * \retval -1 error
4705  * \retval 0 ok
4706  */
4708 {
4709  DetectEngineCtx *new_de_ctx = NULL;
4710  DetectEngineCtx *old_de_ctx = NULL;
4711 
4712  char prefix[128];
4713  memset(prefix, 0, sizeof(prefix));
4714 
4715  SCLogNotice("rule reload starting");
4716 
4717  if (suri->conf_filename != NULL) {
4718  snprintf(prefix, sizeof(prefix), "detect-engine-reloads.%d", reloads++);
4719  SCLogConfig("Reloading %s", suri->conf_filename);
4720  if (ConfYamlLoadFileWithPrefix(suri->conf_filename, prefix) != 0) {
4721  SCLogError("failed to load yaml %s", suri->conf_filename);
4722  return -1;
4723  }
4724 
4725  ConfNode *node = ConfGetNode(prefix);
4726  if (node == NULL) {
4727  SCLogError("failed to properly setup yaml %s", suri->conf_filename);
4728  return -1;
4729  }
4730 
4731  if (suri->additional_configs) {
4732  for (int i = 0; suri->additional_configs[i] != NULL; i++) {
4733  SCLogConfig("Reloading %s", suri->additional_configs[i]);
4734  ConfYamlHandleInclude(node, suri->additional_configs[i]);
4735  }
4736  }
4737 
4738 #if 0
4739  ConfDump();
4740 #endif
4741  }
4742 
4743  /* get a reference to the current de_ctx */
4744  old_de_ctx = DetectEngineGetCurrent();
4745  if (old_de_ctx == NULL)
4746  return -1;
4747  SCLogDebug("get ref to old_de_ctx %p", old_de_ctx);
4748  DatasetReload();
4749 
4750  /* only reload a regular 'normal' and 'delayed detect stub' detect engines */
4751  if (!(old_de_ctx->type == DETECT_ENGINE_TYPE_NORMAL ||
4752  old_de_ctx->type == DETECT_ENGINE_TYPE_DD_STUB))
4753  {
4754  DetectEngineDeReference(&old_de_ctx);
4755  SCLogNotice("rule reload complete");
4756  return -1;
4757  }
4758 
4759  /* get new detection engine */
4760  new_de_ctx = DetectEngineCtxInitWithPrefix(prefix, old_de_ctx->tenant_id);
4761  if (new_de_ctx == NULL) {
4762  SCLogError("initializing detection engine "
4763  "context failed.");
4764  DetectEngineDeReference(&old_de_ctx);
4765  return -1;
4766  }
4767  if (SigLoadSignatures(new_de_ctx,
4768  suri->sig_file, suri->sig_file_exclusive) != 0) {
4769  DetectEngineCtxFree(new_de_ctx);
4770  DetectEngineDeReference(&old_de_ctx);
4771  return -1;
4772  }
4773  SCLogDebug("set up new_de_ctx %p", new_de_ctx);
4774 
4775  /* add to master */
4776  DetectEngineAddToMaster(new_de_ctx);
4777 
4778  /* move to old free list */
4779  DetectEngineMoveToFreeList(old_de_ctx);
4780  DetectEngineDeReference(&old_de_ctx);
4781 
4782  SCLogDebug("going to reload the threads to use new_de_ctx %p", new_de_ctx);
4783  /* update the threads */
4784  DetectEngineReloadThreads(new_de_ctx);
4785  SCLogDebug("threads now run new_de_ctx %p", new_de_ctx);
4786 
4787  /* walk free list, freeing the old_de_ctx */
4789 
4791 
4793 
4794  SCLogDebug("old_de_ctx should have been freed");
4795 
4796  SCLogNotice("rule reload complete");
4797 
4798 #ifdef HAVE_MALLOC_TRIM
4799  /* The reload process potentially frees up large amounts of memory.
4800  * Encourage the memory management system to reclaim as much as it
4801  * can.
4802  */
4803  malloc_trim(0);
4804 #endif
4805 
4806  return 0;
4807 }
4808 
4809 static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len)
4810 {
4811  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
4812  return det_ctx->tenant_id % h->array_size;
4813 }
4814 
4815 static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len)
4816 {
4819  return (det1->tenant_id == det2->tenant_id);
4820 }
4821 
4822 static void TenantIdFree(void *d)
4823 {
4824  DetectEngineThreadCtxFree(d);
4825 }
4826 
4828 {
4829  DetectEngineMasterCtx *master = &g_master_de_ctx;
4830  SCMutexLock(&master->lock);
4831 
4832  if (master->tenant_selector == TENANT_SELECTOR_UNKNOWN) {
4833  SCLogInfo("error, no tenant selector");
4834  SCMutexUnlock(&master->lock);
4835  return -1;
4836  }
4837 
4838  DetectEngineCtx *stub_de_ctx = NULL;
4839  DetectEngineCtx *list = master->list;
4840  for ( ; list != NULL; list = list->next) {
4841  SCLogDebug("list %p tenant %u", list, list->tenant_id);
4842 
4843  if (list->type == DETECT_ENGINE_TYPE_NORMAL ||
4844  list->type == DETECT_ENGINE_TYPE_MT_STUB ||
4846  {
4847  stub_de_ctx = list;
4848  break;
4849  }
4850  }
4851  if (stub_de_ctx == NULL) {
4852  stub_de_ctx = DetectEngineCtxInitStubForMT();
4853  if (stub_de_ctx == NULL) {
4854  SCMutexUnlock(&master->lock);
4855  return -1;
4856  }
4857 
4858  if (master->list == NULL) {
4859  master->list = stub_de_ctx;
4860  } else {
4861  stub_de_ctx->next = master->list;
4862  master->list = stub_de_ctx;
4863  }
4864  }
4865 
4866  /* update the threads */
4867  SCLogDebug("MT reload starting");
4868  DetectEngineReloadThreads(stub_de_ctx);
4869  SCLogDebug("MT reload done");
4870 
4871  SCMutexUnlock(&master->lock);
4872 
4873  /* walk free list, freeing the old_de_ctx */
4875  // needed for VarNameStoreFree
4877 
4878  SCLogDebug("old_de_ctx should have been freed");
4879  return 0;
4880 }
4881 
4882 static int g_parse_metadata = 0;
4883 
4885 {
4886  g_parse_metadata = 1;
4887 }
4888 
4890 {
4891  g_parse_metadata = 0;
4892 }
4893 
4895 {
4896  return g_parse_metadata;
4897 }
4898 
4900 {
4901  switch (type) {
4902  case DETECT_SM_LIST_MATCH:
4903  return "packet";
4904  case DETECT_SM_LIST_PMATCH:
4905  return "packet/stream payload";
4906 
4907  case DETECT_SM_LIST_TMATCH:
4908  return "tag";
4909 
4911  return "base64_data";
4912 
4914  return "post-match";
4915 
4917  return "suppress";
4919  return "threshold";
4920 
4921  case DETECT_SM_LIST_MAX:
4922  return "max (internal)";
4923  }
4924  return "error";
4925 }
4926 
4927 /* events api */
4929 {
4931  det_ctx->events++;
4932 }
4933 
4934 /*************************************Unittest*********************************/
4935 
4936 #ifdef UNITTESTS
4937 
4938 static int DetectEngineInitYamlConf(const char *conf)
4939 {
4941  ConfInit();
4942  return ConfYamlLoadString(conf, strlen(conf));
4943 }
4944 
4945 static void DetectEngineDeInitYamlConf(void)
4946 {
4947  ConfDeInit();
4949 }
4950 
4951 static int DetectEngineTest01(void)
4952 {
4953  const char *conf =
4954  "%YAML 1.1\n"
4955  "---\n"
4956  "detect-engine:\n"
4957  " - profile: medium\n"
4958  " - custom-values:\n"
4959  " toclient_src_groups: 2\n"
4960  " toclient_dst_groups: 2\n"
4961  " toclient_sp_groups: 2\n"
4962  " toclient_dp_groups: 3\n"
4963  " toserver_src_groups: 2\n"
4964  " toserver_dst_groups: 4\n"
4965  " toserver_sp_groups: 2\n"
4966  " toserver_dp_groups: 25\n"
4967  " - inspection-recursion-limit: 0\n";
4968 
4969  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
4970 
4973 
4975 
4977 
4978  DetectEngineDeInitYamlConf();
4979 
4980  PASS;
4981 }
4982 
4983 static int DetectEngineTest02(void)
4984 {
4985  const char *conf =
4986  "%YAML 1.1\n"
4987  "---\n"
4988  "detect-engine:\n"
4989  " - profile: medium\n"
4990  " - custom-values:\n"
4991  " toclient_src_groups: 2\n"
4992  " toclient_dst_groups: 2\n"
4993  " toclient_sp_groups: 2\n"
4994  " toclient_dp_groups: 3\n"
4995  " toserver_src_groups: 2\n"
4996  " toserver_dst_groups: 4\n"
4997  " toserver_sp_groups: 2\n"
4998  " toserver_dp_groups: 25\n"
4999  " - inspection-recursion-limit:\n";
5000 
5001  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5002 
5005 
5006  FAIL_IF_NOT(
5008 
5010 
5011  DetectEngineDeInitYamlConf();
5012 
5013  PASS;
5014 }
5015 
5016 static int DetectEngineTest03(void)
5017 {
5018  const char *conf =
5019  "%YAML 1.1\n"
5020  "---\n"
5021  "detect-engine:\n"
5022  " - profile: medium\n"
5023  " - custom-values:\n"
5024  " toclient_src_groups: 2\n"
5025  " toclient_dst_groups: 2\n"
5026  " toclient_sp_groups: 2\n"
5027  " toclient_dp_groups: 3\n"
5028  " toserver_src_groups: 2\n"
5029  " toserver_dst_groups: 4\n"
5030  " toserver_sp_groups: 2\n"
5031  " toserver_dp_groups: 25\n";
5032 
5033  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5034 
5037 
5038  FAIL_IF_NOT(
5040 
5042 
5043  DetectEngineDeInitYamlConf();
5044 
5045  PASS;
5046 }
5047 
5048 static int DetectEngineTest04(void)
5049 {
5050  const char *conf =
5051  "%YAML 1.1\n"
5052  "---\n"
5053  "detect-engine:\n"
5054  " - profile: medium\n"
5055  " - custom-values:\n"
5056  " toclient_src_groups: 2\n"
5057  " toclient_dst_groups: 2\n"
5058  " toclient_sp_groups: 2\n"
5059  " toclient_dp_groups: 3\n"
5060  " toserver_src_groups: 2\n"
5061  " toserver_dst_groups: 4\n"
5062  " toserver_sp_groups: 2\n"
5063  " toserver_dp_groups: 25\n"
5064  " - inspection-recursion-limit: 10\n";
5065 
5066  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5067 
5070 
5072 
5074 
5075  DetectEngineDeInitYamlConf();
5076 
5077  PASS;
5078 }
5079 
5080 static int DetectEngineTest08(void)
5081 {
5082  const char *conf =
5083  "%YAML 1.1\n"
5084  "---\n"
5085  "detect-engine:\n"
5086  " - profile: custom\n"
5087  " - custom-values:\n"
5088  " toclient-groups: 23\n"
5089  " toserver-groups: 27\n";
5090 
5091  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5092 
5095 
5098 
5100 
5101  DetectEngineDeInitYamlConf();
5102 
5103  PASS;
5104 }
5105 
5106 /** \test bug 892 bad values */
5107 static int DetectEngineTest09(void)
5108 {
5109  const char *conf =
5110  "%YAML 1.1\n"
5111  "---\n"
5112  "detect-engine:\n"
5113  " - profile: custom\n"
5114  " - custom-values:\n"
5115  " toclient-groups: BA\n"
5116  " toserver-groups: BA\n"
5117  " - inspection-recursion-limit: 10\n";
5118 
5119  FAIL_IF(DetectEngineInitYamlConf(conf) == -1);
5120