suricata
detect-engine-loader.c
Go to the documentation of this file.
1 /* Copyright (C) 2021-2023 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "conf.h"
27 #include "detect.h"
28 #include "detect-parse.h"
29 
30 #include "runmodes.h"
31 #include "threads.h"
32 #include "threadvars.h"
33 #include "tm-threads.h"
34 #include "queue.h"
35 
36 #include "detect-engine.h"
37 #include "detect-engine-loader.h"
38 #include "detect-engine-build.h"
39 #include "detect-engine-analyzer.h"
40 #include "detect-engine-mpm.h"
41 #include "detect-engine-sigorder.h"
42 
43 #include "util-detect.h"
44 #include "util-threshold-config.h"
45 #include "util-path.h"
46 
47 #include "rust.h"
48 
49 #ifdef HAVE_GLOB_H
50 #include <glob.h>
51 #endif
52 
53 #include "app-layer-parser.h"
54 
55 extern int rule_reload;
56 extern int engine_analysis;
57 static bool fp_engine_analysis_set = false;
59 
60 static char *DetectLoadCompleteSigPathWithKey(
61  const DetectEngineCtx *de_ctx, const char *default_key, const char *sig_file)
62 {
63  const char *defaultpath = NULL;
64  char *path = NULL;
65  char varname[128];
66 
67  if (sig_file == NULL) {
68  SCLogError("invalid sig_file argument - NULL");
69  return NULL;
70  }
71 
72  /* If we have a configuration prefix, only use it if the primary configuration node
73  * is not marked as final, as that means it was provided on the command line with
74  * a --set. */
75  SCConfNode *default_rule_path = SCConfGetNode(default_key);
76  if ((!default_rule_path || !default_rule_path->final) && strlen(de_ctx->config_prefix) > 0) {
77  snprintf(varname, sizeof(varname), "%s.%s", de_ctx->config_prefix, default_key);
78  default_rule_path = SCConfGetNode(varname);
79  }
80  if (default_rule_path) {
81  defaultpath = default_rule_path->val;
82  }
83 
84  /* Path not specified */
85  if (PathIsRelative(sig_file)) {
86  if (defaultpath) {
87  path = PathMergeAlloc(defaultpath, sig_file);
88  if (unlikely(path == NULL))
89  return NULL;
90  } else {
91  path = SCStrdup(sig_file);
92  if (unlikely(path == NULL))
93  return NULL;
94  }
95  } else {
96  path = SCStrdup(sig_file);
97  if (unlikely(path == NULL))
98  return NULL;
99  }
100  return path;
101 }
102 
103 /**
104  * \brief Create the path if default-rule-path was specified
105  * \param sig_file The name of the file
106  * \retval str Pointer to the string path + sig_file
107  */
108 char *DetectLoadCompleteSigPath(const DetectEngineCtx *de_ctx, const char *sig_file)
109 {
110  return DetectLoadCompleteSigPathWithKey(de_ctx, "default-rule-path", sig_file);
111 }
112 
113 /**
114  * \brief Load a file with signatures
115  * \param de_ctx Pointer to the detection engine context
116  * \param sig_file Filename to load signatures from
117  * \param goodsigs_tot Will store number of valid signatures in the file
118  * \param badsigs_tot Will store number of invalid signatures in the file
119  * \retval 0 on success, -1 on error
120  */
121 static int DetectLoadSigFile(DetectEngineCtx *de_ctx, const char *sig_file, int *goodsigs,
122  int *badsigs, int *skippedsigs, const bool firewall_rule)
123 {
124  int good = 0, bad = 0, skipped = 0;
125  char line[DETECT_MAX_RULE_SIZE] = "";
126  size_t offset = 0;
127  int lineno = 0, multiline = 0;
128 
129  (*goodsigs) = 0;
130  (*badsigs) = 0;
131  (*skippedsigs) = 0;
132 
133  FILE *fp = fopen(sig_file, "r");
134  if (fp == NULL) {
135  SCLogError("opening rule file %s:"
136  " %s.",
137  sig_file, strerror(errno));
138  return -1;
139  }
140 
141  while (1) {
142  /* help clang to understand offset can't get > sizeof(line), so the argument to
143  * fgets can't get negative. */
144  BUG_ON(offset >= sizeof(line));
145  char *res = fgets(line + offset, (int)(sizeof(line) - offset), fp);
146  if (res == NULL)
147  break;
148 
149  lineno++;
150  size_t len = strlen(line);
151 
152  /* ignore comments and empty lines */
153  if (line[0] == '\n' || line [0] == '\r' || line[0] == ' ' || line[0] == '#' || line[0] == '\t')
154  continue;
155 
156  /* Check for multiline rules. */
157  while (len > 0 && isspace((unsigned char)line[--len]));
158  if (line[len] == '\\') {
159  multiline++;
160  offset = len;
161  if (offset < sizeof(line) - 1) {
162  /* We have room for more. */
163  continue;
164  }
165  /* No more room in line buffer, continue, rule will fail
166  * to parse. */
167  }
168 
169  /* Check if we have a trailing newline, and remove it */
170  len = strlen(line);
171  DEBUG_VALIDATE_BUG_ON(len >= sizeof(line)); // help scan-build
172  if (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) {
173  line[len - 1] = '\0';
174  }
175 
176  /* Reset offset. */
177  offset = 0;
178 
179  de_ctx->rule_file = sig_file;
180  de_ctx->rule_line = lineno - multiline;
181 
182  Signature *sig = NULL;
183  if (firewall_rule)
184  sig = DetectFirewallRuleAppendNew(de_ctx, line);
185  else
186  sig = DetectEngineAppendSig(de_ctx, line);
187  if (sig != NULL) {
190  EngineAnalysisFP(de_ctx, sig, line);
191  }
193  EngineAnalysisRules(de_ctx, sig, line);
194  }
195  }
196  SCLogDebug("signature %"PRIu32" loaded", sig->id);
197  good++;
198  } else {
199  if (!de_ctx->sigerror_silent) {
200  SCLogError("error parsing signature \"%s\" from "
201  "file %s at line %" PRId32 "",
202  line, sig_file, lineno - multiline);
203 
204  if (!SigStringAppend(&de_ctx->sig_stat, sig_file, line, de_ctx->sigerror, (lineno - multiline))) {
205  SCLogError("Error adding sig \"%s\" from "
206  "file %s at line %" PRId32 "",
207  line, sig_file, lineno - multiline);
208  }
209  if (de_ctx->sigerror) {
210  de_ctx->sigerror = NULL;
211  }
212  }
214  EngineAnalysisRulesFailure(de_ctx, line, sig_file, lineno - multiline);
215  }
216  if (!de_ctx->sigerror_ok) {
217  bad++;
218  }
219  if (de_ctx->sigerror_requires) {
220  SCLogInfo("Skipping signature due to missing requirements: %s from file %s at line "
221  "%" PRId32,
222  line, sig_file, lineno - multiline);
223  skipped++;
224  }
225  }
226  multiline = 0;
227  }
228  fclose(fp);
229 
230  *goodsigs = good;
231  *badsigs = bad;
232  *skippedsigs = skipped;
233  return 0;
234 }
235 
236 /**
237  * \brief Expands wildcards and reads signatures from each matching file
238  * \param de_ctx Pointer to the detection engine context
239  * \param sig_file Filename (or pattern) holding signatures
240  * \retval -1 on error
241  */
242 static int ProcessSigFiles(DetectEngineCtx *de_ctx, char *pattern, SigFileLoaderStat *st,
243  int *good_sigs, int *bad_sigs, int *skipped_sigs)
244 {
245  int r = 0;
246 
247  if (pattern == NULL) {
248  SCLogError("opening rule file null");
249  return -1;
250  }
251 
252 #ifdef HAVE_GLOB_H
253  glob_t files;
254  r = glob(pattern, 0, NULL, &files);
255 
256  if (r == GLOB_NOMATCH) {
257  SCLogWarning("No rule files match the pattern %s", pattern);
258  ++(st->bad_files);
259  ++(st->total_files);
260  return -1;
261  } else if (r != 0) {
262  SCLogError("error expanding template %s: %s", pattern, strerror(errno));
263  return -1;
264  }
265 
266  for (size_t i = 0; i < (size_t)files.gl_pathc; i++) {
267  char *fname = files.gl_pathv[i];
268  if (strcmp("/dev/null", fname) == 0)
269  continue;
270 #else
271  char *fname = pattern;
272  if (strcmp("/dev/null", fname) == 0)
273  return 0;
274 #endif
275  if (strlen(de_ctx->config_prefix) > 0) {
276  SCLogConfig("tenant id %d: Loading rule file: %s", de_ctx->tenant_id, fname);
277  } else {
278  SCLogConfig("Loading rule file: %s", fname);
279  }
280  r = DetectLoadSigFile(de_ctx, fname, good_sigs, bad_sigs, skipped_sigs, false);
281  if (r < 0) {
282  ++(st->bad_files);
283  }
284 
285  ++(st->total_files);
286 
287  st->good_sigs_total += *good_sigs;
288  st->bad_sigs_total += *bad_sigs;
289  st->skipped_sigs_total += *skipped_sigs;
290 
291 #ifdef HAVE_GLOB_H
292  }
293  globfree(&files);
294 #endif
295  return r;
296 }
297 
298 static int LoadFirewallRuleFiles(DetectEngineCtx *de_ctx)
299 {
301  SCLogError("initializing firewall policies failed");
302  return -1;
303  }
305  SCLogError("loading firewall policies failed");
306  return -1;
307  }
308 
310  int32_t good_sigs = 0;
311  int32_t bad_sigs = 0;
312  int32_t skipped_sigs = 0;
313 
314  SCLogDebug("fw: rule file full path \"%s\"", de_ctx->firewall_rule_file_exclusive);
315 
316  int ret = DetectLoadSigFile(de_ctx, de_ctx->firewall_rule_file_exclusive, &good_sigs,
317  &bad_sigs, &skipped_sigs, true);
318 
319  /* for now be as strict as possible */
320  if (ret != 0 || bad_sigs != 0 || skipped_sigs != 0) {
321  /* Some rules failed to load, just exit as
322  * errors would have already been logged. */
323  exit(EXIT_FAILURE);
324  }
325 
326  if (good_sigs == 0) {
327  SCLogNotice("fw: No rules loaded from %s.", de_ctx->firewall_rule_file_exclusive);
328  } else {
329  SCLogNotice("fw: %d rules loaded from %s.", good_sigs,
331  de_ctx->sig_stat.good_sigs_total += good_sigs;
332  }
333 
334  return 0;
335  }
336 
337  SCConfNode *default_fw_rule_path = SCConfGetNode("firewall.rule-path");
338  if (default_fw_rule_path == NULL) {
339  SCLogNotice("fw: firewall.rule-path not defined, skip loading firewall rules");
340  return 0;
341  }
342  SCConfNode *rule_files = SCConfGetNode("firewall.rule-files");
343  if (rule_files == NULL) {
344  SCLogNotice("fw: firewall.rule-files not defined, skip loading firewall rules");
345  return 0;
346  }
347 
348  SCConfNode *file = NULL;
349  TAILQ_FOREACH (file, &rule_files->head, next) {
350  int32_t good_sigs = 0;
351  int32_t bad_sigs = 0;
352  int32_t skipped_sigs = 0;
353 
354  char *sfile = DetectLoadCompleteSigPathWithKey(de_ctx, "firewall.rule-path", file->val);
355  SCLogNotice("fw: rule file full path \"%s\"", sfile);
356 
357  int ret = DetectLoadSigFile(de_ctx, sfile, &good_sigs, &bad_sigs, &skipped_sigs, true);
358  SCFree(sfile);
359 
360  /* for now be as strict as possible */
361  if (ret != 0 || bad_sigs != 0 || skipped_sigs != 0) {
362  /* Some rules failed to load, just exit as
363  * errors would have already been logged. */
364  exit(EXIT_FAILURE);
365  }
366 
367  if (good_sigs == 0) {
368  SCLogNotice("fw: No rules loaded from %s.", file->val);
369  } else {
370  SCLogNotice("fw: %d rules loaded from %s.", good_sigs, file->val);
371  de_ctx->sig_stat.good_sigs_total += good_sigs;
372  }
373  }
374  return 0;
375 }
376 
377 /**
378  * \brief Load signatures
379  * \param de_ctx Pointer to the detection engine context
380  * \param sig_file Filename (or pattern) holding signatures
381  * \param sig_file_exclusive File passed in 'sig_file' should be loaded exclusively.
382  * \retval -1 on error
383  */
384 int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exclusive)
385 {
386  SCEnter();
387 
388  SCConfNode *rule_files;
389  SCConfNode *file = NULL;
390  SigFileLoaderStat *sig_stat = &de_ctx->sig_stat;
391  int ret = 0;
392  char *sfile = NULL;
393  char varname[128] = "rule-files";
394  int good_sigs = 0;
395  int bad_sigs = 0;
396  int skipped_sigs = 0;
397 
398  if (strlen(de_ctx->config_prefix) > 0) {
399  snprintf(varname, sizeof(varname), "%s.rule-files", de_ctx->config_prefix);
400  }
401 
404  }
405 
407  if (LoadFirewallRuleFiles(de_ctx) < 0) {
408  if (de_ctx->fw_policies == NULL || de_ctx->failure_fatal) {
409  exit(EXIT_FAILURE);
410  }
411  ret = -1;
412  goto end;
413  }
414 
415  /* skip regular rules if we used a exclusive firewall rule file */
416  if (!sig_file_exclusive && de_ctx->firewall_rule_file_exclusive) {
417  ret = 0;
418  goto skip_regular_rules;
419  }
420  }
421 
422  /* ok, let's load signature files from the general config */
423  if (!(sig_file != NULL && sig_file_exclusive)) {
424  rule_files = SCConfGetNode(varname);
425  if (rule_files != NULL) {
426  if (!SCConfNodeIsSequence(rule_files)) {
427  SCLogWarning("Invalid rule-files configuration section: "
428  "expected a list of filenames.");
429  } else {
430  TAILQ_FOREACH(file, &rule_files->head, next) {
431  sfile = DetectLoadCompleteSigPath(de_ctx, file->val);
432  good_sigs = bad_sigs = skipped_sigs = 0;
433  ret = ProcessSigFiles(
434  de_ctx, sfile, sig_stat, &good_sigs, &bad_sigs, &skipped_sigs);
435  SCFree(sfile);
436 
437  if (de_ctx->failure_fatal && ret != 0) {
438  /* Some rules failed to load, just exit as
439  * errors would have already been logged. */
440  exit(EXIT_FAILURE);
441  }
442 
443  if (good_sigs == 0) {
444  SCLogConfig("No rules loaded from %s.", file->val);
445  }
446  }
447  }
448  }
449  }
450 
451  /* If a Signature file is specified from command-line, parse it too */
452  if (sig_file != NULL) {
453  ret = ProcessSigFiles(de_ctx, sig_file, sig_stat, &good_sigs, &bad_sigs, &skipped_sigs);
454 
455  if (ret != 0) {
456  if (de_ctx->failure_fatal) {
457  exit(EXIT_FAILURE);
458  }
459  }
460 
461  if (good_sigs == 0) {
462  SCLogConfig("No rules loaded from %s", sig_file);
463  }
464  }
465 
466 skip_regular_rules:
467  /* now we should have signatures to work with */
468  if (sig_stat->good_sigs_total <= 0) {
469  if (sig_stat->total_files > 0) {
470  SCLogWarning(
471  "%d rule files specified, but no rules were loaded!", sig_stat->total_files);
472  } else {
473  SCLogInfo("No signatures supplied.");
474  goto end;
475  }
476  } else {
477  /* we report the total of files and rules successfully loaded and failed */
478  if (strlen(de_ctx->config_prefix) > 0) {
479  SCLogInfo("tenant id %d: %" PRId32 " rule files processed. %" PRId32
480  " rules successfully loaded, %" PRId32 " rules failed, %" PRId32
481  " rules skipped",
482  de_ctx->tenant_id, sig_stat->total_files, sig_stat->good_sigs_total,
483  sig_stat->bad_sigs_total, sig_stat->skipped_sigs_total);
484  } else {
485  SCLogInfo("%" PRId32 " rule files processed. %" PRId32
486  " rules successfully loaded, %" PRId32 " rules failed, %" PRId32
487  " rules skipped",
488  sig_stat->total_files, sig_stat->good_sigs_total, sig_stat->bad_sigs_total,
489  sig_stat->skipped_sigs_total);
490  }
491  if (de_ctx->requirements != NULL && sig_stat->skipped_sigs_total > 0) {
492  SCDetectRequiresStatusLog(de_ctx->requirements, PROG_VER,
493  strlen(de_ctx->config_prefix) > 0 ? de_ctx->tenant_id : 0);
494  }
495  }
496 
497  if ((sig_stat->bad_sigs_total || sig_stat->bad_files) && de_ctx->failure_fatal) {
498  ret = -1;
499  goto end;
500  }
501 
505 
507  ret = -1;
508  goto end;
509  }
510 
511  /* Setup the signature group lookup structure and pattern matchers */
512  if (SigGroupBuild(de_ctx) < 0)
513  goto end;
514 
515  ret = 0;
516 
517  end:
518  gettimeofday(&de_ctx->last_reload, NULL);
521  }
522 
524  SCReturnInt(ret);
525 }
526 
527 #define NLOADERS 4
528 static DetectLoaderControl *loaders = NULL;
529 static int cur_loader = 0;
530 static void TmThreadWakeupDetectLoaderThreads(void);
531 static int num_loaders = NLOADERS;
532 
533 /** \param loader -1 for auto select
534  * \retval loader_id or negative in case of error */
535 int DetectLoaderQueueTask(int loader_id, LoaderFunc Func, void *func_ctx, LoaderFreeFunc FreeFunc)
536 {
537  if (loader_id == -1) {
538  loader_id = cur_loader;
539  cur_loader++;
540  if (cur_loader >= num_loaders)
541  cur_loader = 0;
542  }
543  if (loader_id >= num_loaders || loader_id < 0) {
544  return -ERANGE;
545  }
546 
547  DetectLoaderControl *loader = &loaders[loader_id];
548 
549  DetectLoaderTask *t = SCCalloc(1, sizeof(*t));
550  if (t == NULL)
551  return -ENOMEM;
552 
553  t->Func = Func;
554  t->ctx = func_ctx;
555  t->FreeFunc = FreeFunc;
556 
557  SCMutexLock(&loader->m);
558  TAILQ_INSERT_TAIL(&loader->task_list, t, next);
559  SCMutexUnlock(&loader->m);
560 
561  TmThreadWakeupDetectLoaderThreads();
562 
563  SCLogDebug("%d %p %p", loader_id, Func, func_ctx);
564  return loader_id;
565 }
566 
567 /** \brief wait for loader tasks to complete
568  * \retval result 0 for ok, -1 for errors */
570 {
571  SCLogDebug("waiting");
572  int errors = 0;
573  for (int i = 0; i < num_loaders; i++) {
574  bool done = false;
575 
576  DetectLoaderControl *loader = &loaders[i];
577  while (!done) {
578  SCMutexLock(&loader->m);
579  if (TAILQ_EMPTY(&loader->task_list)) {
580  done = true;
581  }
582  SCMutexUnlock(&loader->m);
583  if (!done) {
584  /* nudge thread in case it's sleeping */
585  SCCtrlMutexLock(loader->tv->ctrl_mutex);
586  pthread_cond_broadcast(loader->tv->ctrl_cond);
587  SCCtrlMutexUnlock(loader->tv->ctrl_mutex);
588  }
589  }
590  SCMutexLock(&loader->m);
591  if (loader->result != 0) {
592  errors++;
593  loader->result = 0;
594  }
595  SCMutexUnlock(&loader->m);
596  }
597  if (errors) {
598  SCLogError("%d loaders reported errors", errors);
599  return -1;
600  }
601  SCLogDebug("done");
602  return 0;
603 }
604 
605 static void DetectLoaderInit(DetectLoaderControl *loader)
606 {
607  memset(loader, 0x00, sizeof(*loader));
608  SCMutexInit(&loader->m, NULL);
609  TAILQ_INIT(&loader->task_list);
610 }
611 
613 {
614  intmax_t setting = NLOADERS;
615  (void)SCConfGetInt("multi-detect.loaders", &setting);
616 
617  if (setting < 1 || setting > 1024) {
618  FatalError("invalid multi-detect.loaders setting %" PRIdMAX, setting);
619  }
620 
621  num_loaders = (int32_t)setting;
622  SCLogInfo("using %d detect loader threads", num_loaders);
623 
624  BUG_ON(loaders != NULL);
625  loaders = SCCalloc(num_loaders, sizeof(DetectLoaderControl));
626  BUG_ON(loaders == NULL);
627 
628  for (int i = 0; i < num_loaders; i++) {
629  DetectLoaderInit(&loaders[i]);
630  }
631 }
632 
633 /**
634  * \brief Unpauses all threads present in tv_root
635  */
636 static void TmThreadWakeupDetectLoaderThreads(void)
637 {
639  for (int i = 0; i < TVT_MAX; i++) {
640  ThreadVars *tv = tv_root[i];
641  while (tv != NULL) {
642  if (strncmp(tv->name,"DL#",3) == 0) {
643  BUG_ON(tv->ctrl_cond == NULL);
645  pthread_cond_broadcast(tv->ctrl_cond);
647  }
648  tv = tv->next;
649  }
650  }
652 }
653 
654 /**
655  * \brief Unpauses all threads present in tv_root
656  */
658 {
660  for (int i = 0; i < TVT_MAX; i++) {
661  ThreadVars *tv = tv_root[i];
662  while (tv != NULL) {
663  if (strncmp(tv->name,"DL#",3) == 0)
665 
666  tv = tv->next;
667  }
668  }
670 }
671 
672 SC_ATOMIC_DECLARE(int, detect_loader_cnt);
673 
674 typedef struct DetectLoaderThreadData_ {
675  uint32_t instance;
677 
678 static TmEcode DetectLoaderThreadInit(ThreadVars *t, const void *initdata, void **data)
679 {
681  if (ftd == NULL)
682  return TM_ECODE_FAILED;
683 
684  ftd->instance = SC_ATOMIC_ADD(detect_loader_cnt, 1); /* id's start at 0 */
685  SCLogDebug("detect loader instance %u", ftd->instance);
686 
687  /* pass thread data back to caller */
688  *data = ftd;
689 
690  DetectLoaderControl *loader = &loaders[ftd->instance];
691  loader->tv = t;
692 
693  return TM_ECODE_OK;
694 }
695 
696 static TmEcode DetectLoaderThreadDeinit(ThreadVars *t, void *data)
697 {
698  SCFree(data);
699  return TM_ECODE_OK;
700 }
701 
702 
703 static TmEcode DetectLoader(ThreadVars *th_v, void *thread_data)
704 {
705  DetectLoaderThreadData *ftd = (DetectLoaderThreadData *)thread_data;
706  BUG_ON(ftd == NULL);
707 
709  SCLogDebug("loader thread started");
710  bool run = TmThreadsWaitForUnpause(th_v);
711  while (run) {
712  /* see if we have tasks */
713 
714  DetectLoaderControl *loader = &loaders[ftd->instance];
715  SCMutexLock(&loader->m);
716 
717  DetectLoaderTask *task = NULL, *tmptask = NULL;
718  TAILQ_FOREACH_SAFE(task, &loader->task_list, next, tmptask) {
719  int r = task->Func(task->ctx, ftd->instance);
720  loader->result |= r;
721  TAILQ_REMOVE(&loader->task_list, task, next);
722  task->FreeFunc(task->ctx);
723  SCFree(task);
724  }
725 
726  SCMutexUnlock(&loader->m);
727 
728  /* just wait until someone wakes us up */
730  int rc = 0;
731  while (rc == 0) {
733  run = false;
734  break;
735  }
736  SCMutexLock(&loader->m);
737  bool has_work = loader->task_list.tqh_first != NULL;
738  SCMutexUnlock(&loader->m);
739  if (has_work)
740  break;
741 
743  }
745 
746  SCLogDebug("woke up...");
747  }
748 
752 
753  return TM_ECODE_OK;
754 }
755 
756 /** \brief spawn the detect loader manager thread */
758 {
759  for (int i = 0; i < num_loaders; i++) {
760  char name[TM_THREAD_NAME_MAX];
761  snprintf(name, sizeof(name), "%s#%02d", thread_name_detect_loader, i+1);
762 
763  ThreadVars *tv_loader = TmThreadCreateCmdThreadByName(name, "DetectLoader", 1);
764  if (tv_loader == NULL) {
765  FatalError("failed to create thread %s", name);
766  }
767  if (TmThreadSpawn(tv_loader) != TM_ECODE_OK) {
768  FatalError("failed to create spawn %s", name);
769  }
770  }
771 }
772 
774 {
775  tmm_modules[TMM_DETECTLOADER].name = "DetectLoader";
776  tmm_modules[TMM_DETECTLOADER].ThreadInit = DetectLoaderThreadInit;
777  tmm_modules[TMM_DETECTLOADER].ThreadDeinit = DetectLoaderThreadDeinit;
778  tmm_modules[TMM_DETECTLOADER].Management = DetectLoader;
781  SCLogDebug("%s registered", tmm_modules[TMM_DETECTLOADER].name);
782 
783  SC_ATOMIC_INIT(detect_loader_cnt);
784 }
DetectLoaderTask_::FreeFunc
LoaderFreeFunc FreeFunc
Definition: detect-engine-loader.h:40
TmModule_::cap_flags
uint8_t cap_flags
Definition: tm-modules.h:77
DetectLoaderControl_
Definition: detect-engine-loader.h:44
SigFileLoaderStat_::bad_files
int bad_files
Definition: detect.h:886
RUNMODE_ENGINE_ANALYSIS
@ RUNMODE_ENGINE_ANALYSIS
Definition: runmodes.h:58
tm-threads.h
SCCtrlCondWait
#define SCCtrlCondWait
Definition: threads-debug.h:387
len
uint8_t len
Definition: app-layer-dnp3.h:2
TmThreadSpawn
TmEcode TmThreadSpawn(ThreadVars *tv)
Spawns a thread associated with the ThreadVars instance tv.
Definition: tm-threads.c:1702
detect-engine.h
DetectLoaderThreadSpawn
void DetectLoaderThreadSpawn(void)
spawn the detect loader manager thread
Definition: detect-engine-loader.c:757
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
ThreadVars_::name
char name[16]
Definition: threadvars.h:65
PathMergeAlloc
char * PathMergeAlloc(const char *const dir, const char *const fname)
Definition: util-path.c:107
TAILQ_INIT
#define TAILQ_INIT(head)
Definition: queue.h:262
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
SigLoadSignatures
int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, bool sig_file_exclusive)
Load signatures.
Definition: detect-engine-loader.c:384
DetectEngineCtx_::firewall_rule_file_exclusive
const char * firewall_rule_file_exclusive
Definition: detect.h:1191
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SetupEngineAnalysis
void SetupEngineAnalysis(DetectEngineCtx *de_ctx, bool *fp_analysis, bool *rule_analysis)
Definition: detect-engine-analyzer.c:477
DetectLoaderTask_::ctx
void * ctx
Definition: detect-engine-loader.h:39
DetectEngineCtx_::sigerror_silent
bool sigerror_silent
Definition: detect.h:1073
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:282
TmThreadsSetFlag
void TmThreadsSetFlag(ThreadVars *tv, uint32_t flag)
Set a thread flag.
Definition: tm-threads.c:103
TmThreadWaitForFlag
void TmThreadWaitForFlag(ThreadVars *tv, uint32_t flags)
Waits till the specified flag(s) is(are) set. We don't bother if the kill flag has been set or not on...
Definition: tm-threads.c:1820
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(int, detect_loader_cnt)
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
name
const char * name
Definition: detect-engine-proto.c:48
THV_DEINIT
#define THV_DEINIT
Definition: threadvars.h:45
threads.h
TmThreadContinueDetectLoaderThreads
void TmThreadContinueDetectLoaderThreads(void)
Unpauses all threads present in tv_root.
Definition: detect-engine-loader.c:657
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
th_v
ThreadVars * th_v
Definition: fuzz_iprep.c:20
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:973
THV_RUNNING
#define THV_RUNNING
Definition: threadvars.h:55
TAILQ_EMPTY
#define TAILQ_EMPTY(head)
Definition: queue.h:248
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:252
LoaderFreeFunc
void(* LoaderFreeFunc)(void *ctx)
Definition: detect-engine-loader.h:35
SCSigSignatureOrderingModuleCleanup
void SCSigSignatureOrderingModuleCleanup(DetectEngineCtx *de_ctx)
De-registers all the signature ordering functions registered.
Definition: detect-engine-sigorder.c:937
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
rust.h
tv_root
ThreadVars * tv_root[TVT_MAX]
Definition: tm-threads.c:84
LoaderFunc
int(* LoaderFunc)(void *ctx, int loader_id)
Definition: detect-engine-loader.h:34
DetectEngineCtx_::sigerror_requires
bool sigerror_requires
Definition: detect.h:1077
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:294
DetectFirewallInitDefaultPolicies
int DetectFirewallInitDefaultPolicies(DetectEngineCtx *de_ctx)
allocate and initialize to default values the policies table
Definition: detect-parse.c:4018
DetectParseDupSigHashFree
void DetectParseDupSigHashFree(DetectEngineCtx *de_ctx)
Frees the hash table that is used to cull duplicate sigs.
Definition: detect-parse.c:3330
DetectEngineCtx_::sigerror_ok
bool sigerror_ok
Definition: detect.h:1074
DetectLoaderControl_::result
int result
Definition: detect-engine-loader.h:51
DetectEngineAppendSig
Signature * DetectEngineAppendSig(DetectEngineCtx *, const char *)
Parse and append a Signature into the Detection Engine Context signature list.
Definition: detect-parse.c:3580
EngineModeIsFirewall
bool EngineModeIsFirewall(void)
Definition: suricata.c:239
SCThresholdConfInitContext
int SCThresholdConfInitContext(DetectEngineCtx *de_ctx)
Inits the context to be used by the Threshold Config parsing API.
Definition: util-threshold-config.c:169
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:82
EngineAnalysisRulesFailure
void EngineAnalysisRulesFailure(const DetectEngineCtx *de_ctx, const char *line, const char *file, int lineno)
Definition: detect-engine-analyzer.c:625
TM_THREAD_NAME_MAX
#define TM_THREAD_NAME_MAX
Definition: tm-threads.h:49
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:81
SCConfNodeIsSequence
int SCConfNodeIsSequence(const SCConfNode *node)
Check if a node is a sequence or node.
Definition: conf.c:973
TmModule_::ThreadDeinit
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: tm-modules.h:53
THV_RUNNING_DONE
#define THV_RUNNING_DONE
Definition: threadvars.h:46
SigFileLoaderStat_::skipped_sigs_total
int skipped_sigs_total
Definition: detect.h:890
NLOADERS
#define NLOADERS
Definition: detect-engine-loader.c:527
TmThreadContinue
void TmThreadContinue(ThreadVars *tv)
Unpauses a thread.
Definition: tm-threads.c:1832
DetectEngineCtx_::requirements
SCDetectRequiresStatus * requirements
Definition: detect.h:1182
TAILQ_REMOVE
#define TAILQ_REMOVE(head, elm, field)
Definition: queue.h:312
SCRunmodeGet
SCRunMode SCRunmodeGet(void)
Get the current run mode.
Definition: suricata.c:301
rule_engine_analysis_set
bool rule_engine_analysis_set
Definition: detect-engine-loader.c:58
SigStringAppend
int SigStringAppend(SigFileLoaderStat *sig_stats, const char *sig_file, const char *sig_str, const char *sig_error, int line)
Append a new list member to SigString list.
Definition: util-detect.c:104
DetectEngineCtx_::fw_policies
struct DetectFirewallPolicies * fw_policies
Definition: detect.h:1003
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:22
DetectLoadersInit
void DetectLoadersInit(void)
Definition: detect-engine-loader.c:612
SCSigOrderSignatures
void SCSigOrderSignatures(DetectEngineCtx *de_ctx)
Orders the signatures.
Definition: detect-engine-sigorder.c:802
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:120
SCConfGetInt
int SCConfGetInt(const char *name, intmax_t *val)
Retrieve a configuration value as an integer.
Definition: conf.c:415
DetectEngineCtx_::last_reload
struct timeval last_reload
Definition: detect.h:1146
DetectEngineCtx_::failure_fatal
bool failure_fatal
Definition: detect.h:974
SCEnter
#define SCEnter(...)
Definition: util-debug.h:284
detect-engine-mpm.h
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
TmModule_::Management
TmEcode(* Management)(ThreadVars *, void *)
Definition: tm-modules.h:69
THV_KILL
#define THV_KILL
Definition: threadvars.h:40
SCSigRegisterSignatureOrderingFuncs
void SCSigRegisterSignatureOrderingFuncs(DetectEngineCtx *de_ctx)
Lets you register the Signature ordering functions. The order in which the functions are registered s...
Definition: detect-engine-sigorder.c:917
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:262
EngineAnalysisRules
void EngineAnalysisRules(const DetectEngineCtx *de_ctx, const Signature *s, const char *line)
Prints analysis of loaded rules.
Definition: detect-engine-analyzer.c:1704
rule_reload
int rule_reload
engine_analysis
int engine_analysis
app-layer-parser.h
ThreadVars_::next
struct ThreadVars_ * next
Definition: threadvars.h:124
DetectLoaderThreadData_::instance
uint32_t instance
Definition: detect-engine-loader.c:675
SigFileLoaderStat_::bad_sigs_total
int bad_sigs_total
Definition: detect.h:889
util-detect.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:325
tv_root_lock
SCMutex tv_root_lock
Definition: tm-threads.c:87
thread_name_detect_loader
const char * thread_name_detect_loader
Definition: runmodes.c:74
TmModuleDetectLoaderRegister
void TmModuleDetectLoaderRegister(void)
Definition: detect-engine-loader.c:773
SCCtrlMutexLock
#define SCCtrlMutexLock(mut)
Definition: threads-debug.h:377
detect-engine-build.h
tmm_modules
TmModule tmm_modules[TMM_SIZE]
Definition: tm-modules.c:29
DetectLoaderThreadData
struct DetectLoaderThreadData_ DetectLoaderThreadData
ThreadVars_::ctrl_cond
SCCtrlCondT * ctrl_cond
Definition: threadvars.h:129
conf.h
DETECT_MAX_RULE_SIZE
#define DETECT_MAX_RULE_SIZE
Definition: detect.h:46
TmEcode
TmEcode
Definition: tm-threads-common.h:80
DetectLoaderThreadData_
Definition: detect-engine-loader.c:674
queue.h
fp_engine_analysis_set
bool fp_engine_analysis_set
Definition: fuzz_siginit.c:25
TmModule_::name
const char * name
Definition: tm-modules.h:48
runmodes.h
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:232
TAILQ_FOREACH_SAFE
#define TAILQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:329
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
DetectLoaderQueueTask
int DetectLoaderQueueTask(int loader_id, LoaderFunc Func, void *func_ctx, LoaderFreeFunc FreeFunc)
Definition: detect-engine-loader.c:535
SigGroupBuild
int SigGroupBuild(DetectEngineCtx *de_ctx)
Convert the signature list into the runtime match structure.
Definition: detect-engine-build.c:2286
DetectEngineCtx_::config_prefix
char config_prefix[64]
Definition: detect.h:1098
detect-engine-analyzer.h
DetectLoaderTask_
Definition: detect-engine-loader.h:37
THV_INIT_DONE
#define THV_INIT_DONE
Definition: threadvars.h:37
SCCtrlMutexUnlock
#define SCCtrlMutexUnlock(mut)
Definition: threads-debug.h:379
DetectEngineCtx_::sig_stat
SigFileLoaderStat sig_stat
Definition: detect.h:1149
DetectEngineCtx_::rule_file
const char * rule_file
Definition: detect.h:1071
DetectFirewallLoadDefaultPolicies
int DetectFirewallLoadDefaultPolicies(DetectEngineCtx *de_ctx)
Definition: detect-parse.c:4057
suricata-common.h
TMM_DETECTLOADER
@ TMM_DETECTLOADER
Definition: tm-threads-common.h:72
util-path.h
TmThreadsWaitForUnpause
bool TmThreadsWaitForUnpause(ThreadVars *tv)
Wait for a thread to become unpaused.
Definition: tm-threads.c:365
CleanupEngineAnalysis
void CleanupEngineAnalysis(DetectEngineCtx *de_ctx)
Definition: detect-engine-analyzer.c:513
DetectLoaderControl_::m
SCMutex m
Definition: detect-engine-loader.h:50
SigFileLoaderStat_::total_files
int total_files
Definition: detect.h:887
TmModule_::ThreadInit
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:51
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
FatalError
#define FatalError(...)
Definition: util-debug.h:517
TmThreadCreateCmdThreadByName
ThreadVars * TmThreadCreateCmdThreadByName(const char *name, const char *module, int mucond)
Creates and returns the TV instance for a Command thread (CMD). This function supports only custom sl...
Definition: tm-threads.c:1156
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:33
threadvars.h
detect-engine-sigorder.h
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
EngineAnalysisFP
void EngineAnalysisFP(const DetectEngineCtx *de_ctx, const Signature *s, const char *line)
Definition: detect-engine-analyzer.c:172
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:182
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:274
SCFree
#define SCFree(p)
Definition: util-mem.h:61
SigFileLoaderStat_::good_sigs_total
int good_sigs_total
Definition: detect.h:888
Signature_::id
uint32_t id
Definition: detect.h:720
PathIsRelative
int PathIsRelative(const char *path)
Check if a path is relative.
Definition: util-path.c:69
detect-parse.h
Signature_
Signature container.
Definition: detect.h:675
SCConfNode_::final
int final
Definition: conf.h:44
DetectLoadersSync
int DetectLoadersSync(void)
wait for loader tasks to complete
Definition: detect-engine-loader.c:569
suricata.h
DetectLoaderTask_::Func
LoaderFunc Func
Definition: detect-engine-loader.h:38
DetectLoadCompleteSigPath
char * DetectLoadCompleteSigPath(const DetectEngineCtx *de_ctx, const char *sig_file)
Create the path if default-rule-path was specified.
Definition: detect-engine-loader.c:108
DetectEngineCtx_::sigerror
const char * sigerror
Definition: detect.h:1072
ThreadVars_::ctrl_mutex
SCCtrlMutex * ctrl_mutex
Definition: threadvars.h:128
DetectEngineCtx_::rule_line
int rule_line
Definition: detect.h:1070
PROG_VER
#define PROG_VER
Definition: suricata.h:76
TmThreadsCheckFlag
int TmThreadsCheckFlag(ThreadVars *tv, uint32_t flag)
Check if a thread flag is set.
Definition: tm-threads.c:95
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:250
THV_CLOSED
#define THV_CLOSED
Definition: threadvars.h:42
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:288
SCConfNode_
Definition: conf.h:37
DetectEngineCtx_::tenant_id
uint32_t tenant_id
Definition: detect.h:980
detect-engine-loader.h
SigFileLoaderStat_
Signature loader statistics.
Definition: detect.h:884
SCConfNode_::val
char * val
Definition: conf.h:39
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:109
TVT_MAX
@ TVT_MAX
Definition: tm-threads-common.h:91
TmModule_::flags
uint8_t flags
Definition: tm-modules.h:80
DetectFirewallRuleAppendNew
Signature * DetectFirewallRuleAppendNew(DetectEngineCtx *de_ctx, const char *sigstr)
Parse and append a Signature into the Detection Engine Context signature list.
Definition: detect-parse.c:3508
DetectLoaderControl_::tv
ThreadVars * tv
Definition: detect-engine-loader.h:46
util-threshold-config.h
TM_FLAG_MANAGEMENT_TM
#define TM_FLAG_MANAGEMENT_TM
Definition: tm-modules.h:36