suricata
detect-engine-analyzer.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2018 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Eileen Donlon <emdonlo@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Rule analyzers for the detection engine
25  */
26 
27 #include "suricata-common.h"
28 #include "suricata.h"
29 #include "detect.h"
30 #include "detect-parse.h"
31 #include "detect-engine.h"
32 #include "detect-engine-analyzer.h"
33 #include "detect-engine-mpm.h"
34 #include "conf.h"
35 #include "detect-content.h"
36 #include "detect-flow.h"
37 #include "detect-tcp-flags.h"
38 #include "util-print.h"
39 
40 static int rule_warnings_only = 0;
41 static FILE *rule_engine_analysis_FD = NULL;
42 static FILE *fp_engine_analysis_FD = NULL;
43 static pcre *percent_re = NULL;
44 static pcre_extra *percent_re_study = NULL;
45 static char log_path[PATH_MAX];
46 
47 typedef struct FpPatternStats_ {
48  uint16_t min;
49  uint16_t max;
50  uint32_t cnt;
51  uint64_t tot;
53 
54 static FpPatternStats fp_pattern_stats[DETECT_SM_LIST_MAX];
55 
56 static void FpPatternStatsAdd(int list, uint16_t patlen)
57 {
58  if (list < 0 || list >= DETECT_SM_LIST_MAX)
59  return;
60 
61  FpPatternStats *f = &fp_pattern_stats[list];
62 
63  if (f->min == 0)
64  f->min = patlen;
65  else if (patlen < f->min)
66  f->min = patlen;
67 
68  if (patlen > f->max)
69  f->max = patlen;
70 
71  f->cnt++;
72  f->tot += patlen;
73 }
74 
75 void EngineAnalysisFP(const DetectEngineCtx *de_ctx, const Signature *s, char *line)
76 {
77  int fast_pattern_set = 0;
78  int fast_pattern_only_set = 0;
79  int fast_pattern_chop_set = 0;
80  DetectContentData *fp_cd = NULL;
81  SigMatch *mpm_sm = s->init_data->mpm_sm;
82 
83  if (mpm_sm != NULL) {
84  fp_cd = (DetectContentData *)mpm_sm->ctx;
85  if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN) {
86  fast_pattern_set = 1;
88  fast_pattern_only_set = 1;
89  } else if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
90  fast_pattern_chop_set = 1;
91  }
92  }
93  }
94 
95  fprintf(fp_engine_analysis_FD, "== Sid: %u ==\n", s->id);
96  fprintf(fp_engine_analysis_FD, "%s\n", line);
97 
98  fprintf(fp_engine_analysis_FD, " Fast Pattern analysis:\n");
99  if (s->init_data->prefilter_sm != NULL) {
100  fprintf(fp_engine_analysis_FD, " Prefilter on: %s\n",
102  fprintf(fp_engine_analysis_FD, "\n");
103  return;
104  }
105 
106  if (fp_cd == NULL) {
107  fprintf(fp_engine_analysis_FD, " No content present\n");
108  fprintf(fp_engine_analysis_FD, "\n");
109  return;
110  }
111 
112  fprintf(fp_engine_analysis_FD, " Fast pattern matcher: ");
113  int list_type = SigMatchListSMBelongsTo(s, mpm_sm);
114  if (list_type == DETECT_SM_LIST_PMATCH)
115  fprintf(fp_engine_analysis_FD, "content\n");
116  else {
117  const char *desc = DetectBufferTypeGetDescriptionById(de_ctx, list_type);
118  const char *name = DetectBufferTypeGetNameById(de_ctx, list_type);
119  if (desc && name) {
120  fprintf(fp_engine_analysis_FD, "%s (%s)\n", desc, name);
121  }
122  }
123 
124  int flags_set = 0;
125  fprintf(fp_engine_analysis_FD, " Flags:");
126  if (fp_cd->flags & DETECT_CONTENT_OFFSET) {
127  fprintf(fp_engine_analysis_FD, " Offset");
128  flags_set = 1;
129  } if (fp_cd->flags & DETECT_CONTENT_DEPTH) {
130  fprintf(fp_engine_analysis_FD, " Depth");
131  flags_set = 1;
132  }
133  if (fp_cd->flags & DETECT_CONTENT_WITHIN) {
134  fprintf(fp_engine_analysis_FD, " Within");
135  flags_set = 1;
136  }
137  if (fp_cd->flags & DETECT_CONTENT_DISTANCE) {
138  fprintf(fp_engine_analysis_FD, " Distance");
139  flags_set = 1;
140  }
141  if (fp_cd->flags & DETECT_CONTENT_NOCASE) {
142  fprintf(fp_engine_analysis_FD, " Nocase");
143  flags_set = 1;
144  }
145  if (fp_cd->flags & DETECT_CONTENT_NEGATED) {
146  fprintf(fp_engine_analysis_FD, " Negated");
147  flags_set = 1;
148  }
149  if (flags_set == 0)
150  fprintf(fp_engine_analysis_FD, " None");
151  fprintf(fp_engine_analysis_FD, "\n");
152 
153  fprintf(fp_engine_analysis_FD, " Fast pattern set: %s\n", fast_pattern_set ? "yes" : "no");
154  fprintf(fp_engine_analysis_FD, " Fast pattern only set: %s\n",
155  fast_pattern_only_set ? "yes" : "no");
156  fprintf(fp_engine_analysis_FD, " Fast pattern chop set: %s\n",
157  fast_pattern_chop_set ? "yes" : "no");
158  if (fast_pattern_chop_set) {
159  fprintf(fp_engine_analysis_FD, " Fast pattern offset, length: %u, %u\n",
160  fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
161  }
162 
163  uint16_t patlen = fp_cd->content_len;
164  uint8_t *pat = SCMalloc(fp_cd->content_len + 1);
165  if (unlikely(pat == NULL)) {
166  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
167  exit(EXIT_FAILURE);
168  }
169  memcpy(pat, fp_cd->content, fp_cd->content_len);
170  pat[fp_cd->content_len] = '\0';
171  fprintf(fp_engine_analysis_FD, " Original content: ");
172  PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
173  fprintf(fp_engine_analysis_FD, "\n");
174 
175  if (fast_pattern_chop_set) {
176  SCFree(pat);
177  patlen = fp_cd->fp_chop_len;
178  pat = SCMalloc(fp_cd->fp_chop_len + 1);
179  if (unlikely(pat == NULL)) {
180  exit(EXIT_FAILURE);
181  }
182  memcpy(pat, fp_cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
183  pat[fp_cd->fp_chop_len] = '\0';
184  fprintf(fp_engine_analysis_FD, " Final content: ");
185  PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
186  fprintf(fp_engine_analysis_FD, "\n");
187 
188  FpPatternStatsAdd(list_type, patlen);
189  } else {
190  fprintf(fp_engine_analysis_FD, " Final content: ");
191  PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
192  fprintf(fp_engine_analysis_FD, "\n");
193 
194  FpPatternStatsAdd(list_type, patlen);
195  }
196  SCFree(pat);
197 
198  fprintf(fp_engine_analysis_FD, "\n");
199  return;
200 }
201 
202 /**
203  * \brief Sets up the fast pattern analyzer according to the config.
204  *
205  * \retval 1 If rule analyzer successfully enabled.
206  * \retval 0 If not enabled.
207  */
209 {
210  int fp_engine_analysis_set = 0;
211 
212  if ((ConfGetBool("engine-analysis.rules-fast-pattern",
213  &fp_engine_analysis_set)) == 0) {
214  return 0;
215  }
216 
217  if (fp_engine_analysis_set == 0)
218  return 0;
219 
220  const char *log_dir;
221  log_dir = ConfigGetLogDirectory();
222  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir,
223  "rules_fast_pattern.txt");
224 
225  fp_engine_analysis_FD = fopen(log_path, "w");
226  if (fp_engine_analysis_FD == NULL) {
227  SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", log_path,
228  strerror(errno));
229  return 0;
230  }
231 
232  SCLogInfo("Engine-Analysis for fast_pattern printed to file - %s",
233  log_path);
234 
235  struct timeval tval;
236  struct tm *tms;
237  gettimeofday(&tval, NULL);
238  struct tm local_tm;
239  tms = SCLocalTime(tval.tv_sec, &local_tm);
240  fprintf(fp_engine_analysis_FD, "----------------------------------------------"
241  "---------------------\n");
242  fprintf(fp_engine_analysis_FD, "Date: %" PRId32 "/%" PRId32 "/%04d -- "
243  "%02d:%02d:%02d\n",
244  tms->tm_mday, tms->tm_mon + 1, tms->tm_year + 1900, tms->tm_hour,
245  tms->tm_min, tms->tm_sec);
246  fprintf(fp_engine_analysis_FD, "----------------------------------------------"
247  "---------------------\n");
248 
249  memset(&fp_pattern_stats, 0, sizeof(fp_pattern_stats));
250  return 1;
251 }
252 
253 /**
254  * \brief Sets up the rule analyzer according to the config
255  * \retval 1 if rule analyzer successfully enabled
256  * \retval 0 if not enabled
257  */
259 {
260  ConfNode *conf = ConfGetNode("engine-analysis");
261  int enabled = 0;
262  if (conf != NULL) {
263  const char *value = ConfNodeLookupChildValue(conf, "rules");
264  if (value && ConfValIsTrue(value)) {
265  enabled = 1;
266  } else if (value && strcasecmp(value, "warnings-only") == 0) {
267  enabled = 1;
268  rule_warnings_only = 1;
269  }
270  if (enabled) {
271  const char *log_dir;
272  log_dir = ConfigGetLogDirectory();
273  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, "rules_analysis.txt");
274  rule_engine_analysis_FD = fopen(log_path, "w");
275  if (rule_engine_analysis_FD == NULL) {
276  SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", log_path, strerror(errno));
277  return 0;
278  }
279 
280  SCLogInfo("Engine-Analysis for rules printed to file - %s",
281  log_path);
282 
283  struct timeval tval;
284  struct tm *tms;
285  gettimeofday(&tval, NULL);
286  struct tm local_tm;
287  tms = SCLocalTime(tval.tv_sec, &local_tm);
288  fprintf(rule_engine_analysis_FD, "----------------------------------------------"
289  "---------------------\n");
290  fprintf(rule_engine_analysis_FD, "Date: %" PRId32 "/%" PRId32 "/%04d -- "
291  "%02d:%02d:%02d\n",
292  tms->tm_mday, tms->tm_mon + 1, tms->tm_year + 1900, tms->tm_hour,
293  tms->tm_min, tms->tm_sec);
294  fprintf(rule_engine_analysis_FD, "----------------------------------------------"
295  "---------------------\n");
296 
297  /*compile regex's for rule analysis*/
298  if (PerCentEncodingSetup()== 0) {
299  fprintf(rule_engine_analysis_FD, "Error compiling regex; can't check for percent encoding in normalized http content.\n");
300  }
301  }
302  }
303  else {
304  SCLogInfo("Conf parameter \"engine-analysis.rules\" not found. "
305  "Defaulting to not printing the rules analysis report.");
306  }
307  if (!enabled) {
308  SCLogInfo("Engine-Analysis for rules disabled in conf file.");
309  return 0;
310  }
311  return 1;
312 }
313 
315 {
316  fprintf(fp_engine_analysis_FD, "============\n"
317  "Summary:\n============\n");
318  int i;
319  for (i = 0; i < DETECT_SM_LIST_MAX; i++) {
320  FpPatternStats *f = &fp_pattern_stats[i];
321  if (f->cnt == 0)
322  continue;
323 
324  fprintf(fp_engine_analysis_FD,
325  "%s, smallest pattern %u byte(s), longest pattern %u byte(s), number of patterns %u, avg pattern len %.2f byte(s)\n",
326  DetectSigmatchListEnumToString(i), f->min, f->max, f->cnt, (float)((double)f->tot/(float)f->cnt));
327  }
328 
329  if (fp_engine_analysis_FD != NULL) {
330  fclose(fp_engine_analysis_FD);
331  fp_engine_analysis_FD = NULL;
332  }
333 
334  return;
335 }
336 
337 
339 {
340  if (rule_engine_analysis_FD != NULL) {
341  SCLogInfo("Engine-Analyis for rules printed to file - %s", log_path);
342  fclose(rule_engine_analysis_FD);
343  rule_engine_analysis_FD = NULL;
344  }
345 }
346 
347 /**
348  * \brief Compiles regex for rule analysis
349  * \retval 1 if successful
350  * \retval 0 if on error
351  */
353 {
354 #define DETECT_PERCENT_ENCODING_REGEX "%[0-9|a-f|A-F]{2}"
355  const char *eb = NULL;
356  int eo = 0;
357  int opts = 0; //PCRE_NEWLINE_ANY??
358 
359  percent_re = pcre_compile(DETECT_PERCENT_ENCODING_REGEX, opts, &eb, &eo, NULL);
360  if (percent_re == NULL) {
361  SCLogError(SC_ERR_PCRE_COMPILE, "Compile of \"%s\" failed at offset %" PRId32 ": %s",
363  return 0;
364  }
365 
366  percent_re_study = pcre_study(percent_re, 0, &eb);
367  if (eb != NULL) {
368  SCLogError(SC_ERR_PCRE_STUDY, "pcre study failed: %s", eb);
369  return 0;
370  }
371  return 1;
372 }
373 
374 /**
375  * \brief Checks for % encoding in content.
376  * \param Pointer to content
377  * \retval number of matches if content has % encoding
378  * \retval 0 if it doesn't have % encoding
379  * \retval -1 on error
380  */
381 int PerCentEncodingMatch (uint8_t *content, uint8_t content_len)
382 {
383 #define MAX_ENCODED_CHARS 240
384  int ret = 0;
385  int ov[MAX_ENCODED_CHARS];
386 
387  ret = pcre_exec(percent_re, percent_re_study, (char *)content, content_len, 0, 0, ov, MAX_ENCODED_CHARS);
388  if (ret == -1) {
389  return 0;
390  }
391  else if (ret < -1) {
392  SCLogError(SC_ERR_PCRE_MATCH, "Error parsing content - %s; error code is %d", content, ret);
393  return -1;
394  }
395  return ret;
396 }
397 
398 static void EngineAnalysisRulesPrintFP(const DetectEngineCtx *de_ctx, const Signature *s)
399 {
400  DetectContentData *fp_cd = NULL;
401  SigMatch *mpm_sm = s->init_data->mpm_sm;
402 
403  if (mpm_sm != NULL) {
404  fp_cd = (DetectContentData *)mpm_sm->ctx;
405  }
406 
407  if (fp_cd == NULL) {
408  return;
409  }
410 
411  uint16_t patlen = fp_cd->content_len;
412  uint8_t *pat = SCMalloc(fp_cd->content_len + 1);
413  if (unlikely(pat == NULL)) {
414  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
415  exit(EXIT_FAILURE);
416  }
417  memcpy(pat, fp_cd->content, fp_cd->content_len);
418  pat[fp_cd->content_len] = '\0';
419 
421  SCFree(pat);
422  patlen = fp_cd->fp_chop_len;
423  pat = SCMalloc(fp_cd->fp_chop_len + 1);
424  if (unlikely(pat == NULL)) {
425  exit(EXIT_FAILURE);
426  }
427  memcpy(pat, fp_cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
428  pat[fp_cd->fp_chop_len] = '\0';
429  fprintf(rule_engine_analysis_FD, " Fast Pattern \"");
430  PrintRawUriFp(rule_engine_analysis_FD, pat, patlen);
431  } else {
432  fprintf(rule_engine_analysis_FD, " Fast Pattern \"");
433  PrintRawUriFp(rule_engine_analysis_FD, pat, patlen);
434  }
435  SCFree(pat);
436 
437  fprintf(rule_engine_analysis_FD, "\" on \"");
438 
439  int list_type = SigMatchListSMBelongsTo(s, mpm_sm);
440  if (list_type == DETECT_SM_LIST_PMATCH) {
441  int payload = 0;
442  int stream = 0;
444  payload = 1;
446  stream = 1;
447  fprintf(rule_engine_analysis_FD, "%s",
448  payload ? (stream ? "payload and reassembled stream" : "payload") : "reassembled stream");
449  }
450  else {
451  const char *desc = DetectBufferTypeGetDescriptionById(de_ctx, list_type);
452  const char *name = DetectBufferTypeGetNameById(de_ctx, list_type);
453  if (desc && name) {
454  fprintf(rule_engine_analysis_FD, "%s (%s)", desc, name);
455  }
456  }
457 
458  fprintf(rule_engine_analysis_FD, "\" buffer.\n");
459 
460  return;
461 }
462 
463 
464 void EngineAnalysisRulesFailure(char *line, char *file, int lineno)
465 {
466  fprintf(rule_engine_analysis_FD, "== Sid: UNKNOWN ==\n");
467  fprintf(rule_engine_analysis_FD, "%s\n", line);
468  fprintf(rule_engine_analysis_FD, " FAILURE: invalid rule.\n");
469  fprintf(rule_engine_analysis_FD, " File: %s.\n", file);
470  fprintf(rule_engine_analysis_FD, " Line: %d.\n", lineno);
471  fprintf(rule_engine_analysis_FD, "\n");
472 }
473 
474 #include "util-buffer.h"
475 #include "output-json.h"
476 
477 typedef struct RuleAnalyzer {
478  json_t *js; /* document root */
479 
480  json_t *js_warnings;
481  json_t *js_notes;
482 } RuleAnalyzer;
483 
484 static void __attribute__ ((format (printf, 2, 3)))
485 AnalyzerNote(RuleAnalyzer *ctx, char *fmt, ...)
486 {
487  va_list ap;
488  char str[1024];
489 
490  va_start(ap, fmt);
491  vsnprintf(str, sizeof(str), fmt, ap);
492  va_end(ap);
493 
494  if (!ctx->js_notes)
495  ctx->js_notes = json_array();
496  if (ctx->js_notes)
497  json_array_append_new(ctx->js_notes, json_string(str));
498 }
499 
500 static void __attribute__ ((format (printf, 2, 3)))
501 AnalyzerWarning(RuleAnalyzer *ctx, char *fmt, ...)
502 {
503  va_list ap;
504  char str[1024];
505 
506  va_start(ap, fmt);
507  vsnprintf(str, sizeof(str), fmt, ap);
508  va_end(ap);
509 
510  if (!ctx->js_warnings)
511  ctx->js_warnings = json_array();
512  if (ctx->js_warnings)
513  json_array_append_new(ctx->js_warnings, json_string(str));
514 }
515 
516 #define CHECK(pat) if (strlen((pat)) <= len && memcmp((pat), buf, MIN(len, strlen((pat)))) == 0) return true;
517 
518 static bool LooksLikeHTTPMethod(const uint8_t *buf, uint16_t len)
519 {
520  CHECK("GET /");
521  CHECK("POST /");
522  CHECK("HEAD /");
523  CHECK("PUT /");
524  return false;
525 }
526 
527 static bool LooksLikeHTTPUA(const uint8_t *buf, uint16_t len)
528 {
529  CHECK("User-Agent: ");
530  CHECK("\nUser-Agent: ");
531  return false;
532 }
533 
534 static void DumpMatches(RuleAnalyzer *ctx, json_t *js, const SigMatchData *smd)
535 {
536  if (smd == NULL)
537  return;
538 
539  json_t *js_matches = json_array();
540  if (js_matches == NULL) {
541  return;
542  }
543  do {
544  json_t *js_match = json_object();
545  if (js_match != NULL) {
546  const char *mname = sigmatch_table[smd->type].name;
547  json_object_set_new(js_match, "name", json_string(mname));
548 
549  switch (smd->type) {
550  case DETECT_CONTENT: {
551  const DetectContentData *cd = (const DetectContentData *)smd->ctx;
552  uint8_t *pat = SCMalloc(cd->content_len + 1);
553  if (unlikely(pat == NULL)) {
554  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
555  exit(EXIT_FAILURE);
556  }
557  memcpy(pat, cd->content, cd->content_len);
558  pat[cd->content_len] = '\0';
559 
560  json_t *js_match_content = json_object();
561  if (js_match_content != NULL) {
562  json_object_set_new(js_match_content, "pattern", SCJsonString((const char *)pat));
563  json_object_set_new(js_match_content, "nocase", json_boolean(cd->flags & DETECT_CONTENT_NOCASE));
564  json_object_set_new(js_match_content, "negated", json_boolean(cd->flags & DETECT_CONTENT_NEGATED));
565  json_object_set_new(js_match_content, "starts_with", json_boolean(cd->flags & DETECT_CONTENT_STARTS_WITH));
566  json_object_set_new(js_match_content, "ends_with", json_boolean(cd->flags & DETECT_CONTENT_ENDS_WITH));
567  json_object_set_new(js_match_content, "is_mpm", json_boolean(cd->flags & DETECT_CONTENT_MPM));
568  if (cd->flags & DETECT_CONTENT_OFFSET) {
569  json_object_set_new(js_match_content, "offset", json_integer(cd->offset));
570  }
571  if (cd->flags & DETECT_CONTENT_DEPTH) {
572  json_object_set_new(js_match_content, "depth", json_integer(cd->depth));
573  }
574  if (cd->flags & DETECT_CONTENT_DISTANCE) {
575  json_object_set_new(js_match_content, "distance", json_integer(cd->distance));
576  }
577  if (cd->flags & DETECT_CONTENT_WITHIN) {
578  json_object_set_new(js_match_content, "within", json_integer(cd->within));
579  }
580 
581  json_object_set_new(js_match_content, "fast_pattern", json_boolean(cd->flags & DETECT_CONTENT_FAST_PATTERN));
583  AnalyzerNote(ctx, (char *)"'fast_pattern:only' option is silently ignored and is interpreted as regular 'fast_pattern'");
584  }
585 
586  json_object_set_new(js_match, "content", js_match_content);
587  }
588 
589  if (LooksLikeHTTPMethod(cd->content, cd->content_len)) {
590  AnalyzerWarning(ctx, (char *)"pattern looks like it inspects HTTP, use http_request_line or http_method and http_uri instead for improved performance");
591  }
592  if (LooksLikeHTTPUA(cd->content, cd->content_len)) {
593  AnalyzerWarning(ctx, (char *)"pattern looks like it inspects HTTP, use http_user_agent or http_header for improved performance");
594  }
595 
596  SCFree(pat);
597  break;
598  }
599  }
600  }
601  json_array_append_new(js_matches, js_match);
602 
603  if (smd->is_last)
604  break;
605  smd++;
606  } while (1);
607  json_object_set_new(js, "matches", js_matches);
608 }
609 
611 void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
612 {
613  SCEnter();
614 
615  RuleAnalyzer ctx = { NULL, NULL, NULL };
616 
617  ctx.js = json_object();
618  if (ctx.js == NULL)
619  SCReturn;
620 
621  json_object_set_new(ctx.js, "raw", json_string(s->sig_str));
622  json_object_set_new(ctx.js, "id", json_integer(s->id));
623  json_object_set_new(ctx.js, "gid", json_integer(s->gid));
624  json_object_set_new(ctx.js, "rev", json_integer(s->rev));
625  json_object_set_new(ctx.js, "msg", json_string(s->msg));
626 
627  const char *alproto = AppProtoToString(s->alproto);
628  json_object_set_new(ctx.js, "app_proto", json_string(alproto));
629 
630  json_t *js_flags = json_array();
631  if (js_flags != NULL) {
632  if (s->mask & SIG_MASK_REQUIRE_PAYLOAD) {
633  json_array_append_new(js_flags, json_string("payload"));
634  }
636  json_array_append_new(js_flags, json_string("no_payload"));
637  }
638  if (s->mask & SIG_MASK_REQUIRE_FLOW) {
639  json_array_append_new(js_flags, json_string("flow"));
640  }
642  json_array_append_new(js_flags, json_string("tcp_flags_init_deinit"));
643  }
645  json_array_append_new(js_flags, json_string("tcp_flags_unusual"));
646  }
647  if (s->mask & SIG_MASK_REQUIRE_DCERPC) {
648  json_array_append_new(js_flags, json_string("dcerpc"));
649  }
651  json_array_append_new(js_flags, json_string("engine_event"));
652  }
653  json_object_set_new(ctx.js, "requirements", js_flags);
654  }
655 
656  js_flags = json_array();
657  if (js_flags != NULL) {
658  if (s->flags & SIG_FLAG_SRC_ANY) {
659  json_array_append_new(js_flags, json_string("src_any"));
660  }
661  if (s->flags & SIG_FLAG_DST_ANY) {
662  json_array_append_new(js_flags, json_string("dst_any"));
663  }
664  if (s->flags & SIG_FLAG_SP_ANY) {
665  json_array_append_new(js_flags, json_string("sp_any"));
666  }
667  if (s->flags & SIG_FLAG_DP_ANY) {
668  json_array_append_new(js_flags, json_string("dp_any"));
669  }
670  if (s->flags & SIG_FLAG_NOALERT) {
671  json_array_append_new(js_flags, json_string("noalert"));
672  }
673  if (s->flags & SIG_FLAG_DSIZE) {
674  json_array_append_new(js_flags, json_string("dsize"));
675  }
676  if (s->flags & SIG_FLAG_APPLAYER) {
677  json_array_append_new(js_flags, json_string("applayer"));
678  }
679  if (s->flags & SIG_FLAG_IPONLY) {
680  json_array_append_new(js_flags, json_string("ip_only"));
681  }
682  if (s->flags & SIG_FLAG_REQUIRE_PACKET) {
683  json_array_append_new(js_flags, json_string("need_packet"));
684  }
685  if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
686  json_array_append_new(js_flags, json_string("need_stream"));
687  }
688  if (s->flags & SIG_FLAG_MPM_NEG) {
689  json_array_append_new(js_flags, json_string("negated_mpm"));
690  }
691  if (s->flags & SIG_FLAG_FLUSH) {
692  json_array_append_new(js_flags, json_string("flush"));
693  }
694  if (s->flags & SIG_FLAG_REQUIRE_FLOWVAR) {
695  json_array_append_new(js_flags, json_string("need_flowvar"));
696  }
697  if (s->flags & SIG_FLAG_FILESTORE) {
698  json_array_append_new(js_flags, json_string("filestore"));
699  }
700  if (s->flags & SIG_FLAG_TOSERVER) {
701  json_array_append_new(js_flags, json_string("toserver"));
702  }
703  if (s->flags & SIG_FLAG_TOCLIENT) {
704  json_array_append_new(js_flags, json_string("toclient"));
705  }
706  if (s->flags & SIG_FLAG_TLSSTORE) {
707  json_array_append_new(js_flags, json_string("tlsstore"));
708  }
709  if (s->flags & SIG_FLAG_BYPASS) {
710  json_array_append_new(js_flags, json_string("bypass"));
711  }
712  if (s->flags & SIG_FLAG_PREFILTER) {
713  json_array_append_new(js_flags, json_string("prefilter"));
714  }
715  if (s->flags & SIG_FLAG_PDONLY) {
716  json_array_append_new(js_flags, json_string("proto_detect_only"));
717  }
718  if (s->flags & SIG_FLAG_SRC_IS_TARGET) {
719  json_array_append_new(js_flags, json_string("src_is_target"));
720  }
721  if (s->flags & SIG_FLAG_DEST_IS_TARGET) {
722  json_array_append_new(js_flags, json_string("dst_is_target"));
723  }
724  json_object_set_new(ctx.js, "flags", js_flags);
725  }
726 
727  json_t *js_pkt_array = json_array();
729  for ( ; pkt != NULL; pkt = pkt->next) {
730  const char *name = DetectBufferTypeGetNameById(de_ctx, pkt->sm_list);
731  if (name == NULL) {
732  switch (pkt->sm_list) {
734  name = "payload";
735  break;
737  name = "packet";
738  break;
739  default:
740  name = "unknown";
741  break;
742  }
743  }
744  json_t *js_engine = json_object();
745  if (js_engine != NULL) {
746  json_object_set_new(js_engine, "name", json_string(name));
747 
748  json_object_set_new(js_engine, "is_mpm", json_boolean(pkt->mpm));
749 
750  DumpMatches(&ctx, js_engine, pkt->smd);
751 
752  json_array_append_new(js_pkt_array, js_engine);
753  }
754  }
755  json_object_set_new(ctx.js, "pkt_engines", js_pkt_array);
756 
758  bool has_stream = false;
759  bool has_client_body_mpm = false;
760  bool has_file_data_mpm = false;
761 
762  json_t *js_array = json_array();
764  for ( ; app != NULL; app = app->next) {
765  const char *name = DetectBufferTypeGetNameById(de_ctx, app->sm_list);
766  if (name == NULL) {
767  switch (app->sm_list) {
769  name = "stream";
770  break;
771  default:
772  name = "unknown";
773  break;
774  }
775  }
776 
777  if (app->sm_list == DETECT_SM_LIST_PMATCH && !app->mpm) {
778  has_stream = true;
779  } else if (app->mpm && strcmp(name, "http_client_body") == 0) {
780  has_client_body_mpm = true;
781  } else if (app->mpm && strcmp(name, "file_data") == 0) {
782  has_file_data_mpm = true;
783  }
784 
785  json_t *js_engine = json_object();
786  if (js_engine != NULL) {
787  json_object_set_new(js_engine, "name", json_string(name));
788 
789  const char *direction = app->dir == 0 ? "toserver" : "toclient";
790  json_object_set_new(js_engine, "direction", json_string(direction));
791  json_object_set_new(js_engine, "is_mpm", json_boolean(app->mpm));
792  json_object_set_new(js_engine, "app_proto", json_string(AppProtoToString(app->alproto)));
793  json_object_set_new(js_engine, "progress", json_integer(app->progress));
794 
795  DumpMatches(&ctx, js_engine, app->smd);
796 
797  json_array_append_new(js_array, js_engine);
798  }
799  }
800  json_object_set_new(ctx.js, "engines", js_array);
801 
802  if (has_stream && has_client_body_mpm)
803  AnalyzerNote(&ctx, (char *)"mpm in http_client_body combined with stream match leads to stream buffering");
804  if (has_stream && has_file_data_mpm)
805  AnalyzerNote(&ctx, (char *)"mpm in file_data combined with stream match leads to stream buffering");
806  }
807 
808  json_t *js_lists = json_object();
809  for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
810  if (s->sm_arrays[i] != NULL) {
811  json_t *js_list = json_object();
812  if (js_list != NULL) {
813  DumpMatches(&ctx, js_list, s->sm_arrays[i]);
814  json_object_set_new(js_lists, DetectSigmatchListEnumToString(i), js_list);
815  }
816  }
817  }
818  json_object_set_new(ctx.js, "lists", js_lists);
819 
820  if (ctx.js_warnings) {
821  json_object_set_new(ctx.js, "warnings", ctx.js_warnings);
822  }
823  if (ctx.js_notes) {
824  json_object_set_new(ctx.js, "notes", ctx.js_notes);
825  }
826 
827  const char *filename = "rules.json";
828  const char *log_dir = ConfigGetLogDirectory();
829  char json_path[PATH_MAX] = "";
830  snprintf(json_path, sizeof(json_path), "%s/%s", log_dir, filename);
831 
832  MemBuffer *mbuf = MemBufferCreateNew(4096);
833  if (mbuf != NULL) {
834  OutputJSONMemBufferWrapper wrapper = {
835  .buffer = &mbuf,
836  .expand_by = 4096,
837  };
838 
839  int r = json_dump_callback(ctx.js, OutputJSONMemBufferCallback, &wrapper,
840  JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
841  JSON_ESCAPE_SLASH);
842  if (r != 0) {
843  SCLogWarning(SC_ERR_SOCKET, "unable to serialize JSON object");
844  } else {
845  MemBufferWriteString(mbuf, "\n");
846  SCMutexLock(&g_rules_analyzer_write_m);
847  FILE *fp = fopen(json_path, "a");
848  if (fp != NULL) {
849  MemBufferPrintToFPAsString(mbuf, fp);
850  fclose(fp);
851  }
852  SCMutexUnlock(&g_rules_analyzer_write_m);
853  }
854 
855  MemBufferFree(mbuf);
856  }
857  json_object_clear(ctx.js);
858  json_decref(ctx.js);
859  SCReturn;
860 }
861 
862 /**
863  * \brief Prints analysis of loaded rules.
864  *
865  * Warns if potential rule issues are detected. For example,
866  * warns if a rule uses a construct that may perform poorly,
867  * e.g. pcre without content or with http_method content only;
868  * warns if a rule uses a construct that may not be consistent with intent,
869  * e.g. client side ports only, http and content without any http_* modifiers, etc.
870  *
871  * \param s Pointer to the signature.
872  */
874  const Signature *s, const char *line)
875 {
876  uint32_t rule_bidirectional = 0;
877  uint32_t rule_pcre = 0;
878  uint32_t rule_pcre_http = 0;
879  uint32_t rule_content = 0;
880  uint32_t rule_flow = 0;
881  uint32_t rule_flags = 0;
882  uint32_t rule_flow_toserver = 0;
883  uint32_t rule_flow_toclient = 0;
884  uint32_t rule_flow_nostream = 0;
885  uint32_t rule_ipv4_only = 0;
886  uint32_t rule_ipv6_only = 0;
887  uint32_t rule_flowbits = 0;
888  uint32_t rule_flowint = 0;
889  //uint32_t rule_flowvar = 0;
890  uint32_t rule_content_http = 0;
891  uint32_t rule_content_offset_depth = 0;
892  int32_t list_id = 0;
893  uint32_t rule_warning = 0;
894  uint32_t raw_http_buf = 0;
895  uint32_t norm_http_buf = 0;
896  uint32_t stream_buf = 0;
897  uint32_t packet_buf = 0;
898  uint32_t http_header_buf = 0;
899  uint32_t http_uri_buf = 0;
900  uint32_t http_method_buf = 0;
901  uint32_t http_cookie_buf = 0;
902  uint32_t http_content_type_buf = 0;
903  uint32_t http_client_body_buf = 0;
904  uint32_t http_server_body_buf = 0;
905  uint32_t http_stat_code_buf = 0;
906  uint32_t http_stat_msg_buf = 0;
907  uint32_t http_raw_header_buf = 0;
908  uint32_t http_raw_uri_buf = 0;
909  uint32_t http_ua_buf = 0;
910  uint32_t http_host_buf = 0;
911  uint32_t http_rawhost_buf = 0;
912  uint32_t http_headernames_buf = 0;
913  uint32_t http_referer_buf = 0;
914  uint32_t warn_pcre_no_content = 0;
915  uint32_t warn_pcre_http_content = 0;
916  uint32_t warn_pcre_http = 0;
917  uint32_t warn_content_http_content = 0;
918  uint32_t warn_content_http = 0;
919  uint32_t warn_tcp_no_flow = 0;
920  uint32_t warn_client_ports = 0;
921  uint32_t warn_direction = 0;
922  uint32_t warn_method_toclient = 0;
923  uint32_t warn_method_serverbody = 0;
924  uint32_t warn_pcre_method = 0;
925  uint32_t warn_encoding_norm_http_buf = 0;
926  uint32_t warn_offset_depth_pkt_stream = 0;
927  uint32_t warn_offset_depth_alproto = 0;
928  uint32_t warn_non_alproto_fp_for_alproto_sig = 0;
929  uint32_t warn_no_direction = 0;
930  uint32_t warn_both_direction = 0;
931 
932  const int filedata_id = DetectBufferTypeGetByName("file_data");
933  const int httpmethod_id = DetectBufferTypeGetByName("http_method");
934  const int httpuri_id = DetectBufferTypeGetByName("http_uri");
935  const int httpuseragent_id = DetectBufferTypeGetByName("http_user_agent");
936  const int httpcontenttype_id = DetectBufferTypeGetByName("http_content_type");
937  const int httpcookie_id = DetectBufferTypeGetByName("http_cookie");
938  const int httpstatcode_id = DetectBufferTypeGetByName("http_stat_code");
939  const int httpstatmsg_id = DetectBufferTypeGetByName("http_stat_msg");
940  const int httpheader_id = DetectBufferTypeGetByName("http_header");
941  const int httprawheader_id = DetectBufferTypeGetByName("http_raw_header");
942  const int httpclientbody_id = DetectBufferTypeGetByName("http_client_body");
943  const int httprawuri_id = DetectBufferTypeGetByName("http_raw_uri");
944  const int httphost_id = DetectBufferTypeGetByName("http_host");
945  const int httprawhost_id = DetectBufferTypeGetByName("http_raw_host");
946  const int httpreferer_id = DetectBufferTypeGetByName("http_referer");
947  const int httpheadernames_id = DetectBufferTypeGetByName("http_header_names");
948 
950  rule_bidirectional = 1;
951  }
952 
953  if (s->flags & SIG_FLAG_REQUIRE_PACKET) {
954  packet_buf += 1;
955  }
956  if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
957  stream_buf += 1;
958  }
959 
960  if (s->proto.flags & DETECT_PROTO_IPV4) {
961  rule_ipv4_only += 1;
962  }
963  if (s->proto.flags & DETECT_PROTO_IPV6) {
964  rule_ipv6_only += 1;
965  }
966 
967  for (list_id = 0; list_id < (int)s->init_data->smlists_array_size; list_id++) {
968  SigMatch *sm = NULL;
969  for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) {
970  if (sm->type == DETECT_PCRE) {
971  if (list_id == httpclientbody_id) {
972  rule_pcre_http += 1;
973  http_client_body_buf += 1;
974  raw_http_buf += 1;
975  }
976  else if (list_id == httpuri_id) {
977  rule_pcre_http += 1;
978  norm_http_buf += 1;
979  http_uri_buf += 1;
980  }
981  else if (list_id == httpheader_id) {
982  rule_pcre_http += 1;
983  norm_http_buf += 1;
984  http_header_buf += 1;
985  }
986  else if (list_id == httpcookie_id) {
987  rule_pcre_http += 1;
988  norm_http_buf += 1;
989  http_cookie_buf += 1;
990  }
991  else if (list_id == httpcontenttype_id) {
992  rule_pcre_http += 1;
993  norm_http_buf += 1;
994  http_content_type_buf += 1;
995  }
996  else if (list_id == filedata_id) {
997  rule_pcre_http += 1;
998  http_server_body_buf += 1;
999  raw_http_buf += 1;
1000  }
1001  else if (list_id == httprawheader_id) {
1002  rule_pcre_http += 1;
1003  raw_http_buf += 1;
1004  http_raw_header_buf += 1;
1005  }
1006  else if (list_id == httpmethod_id) {
1007  rule_pcre_http += 1;
1008  raw_http_buf += 1;
1009  http_method_buf += 1;
1010  }
1011  else if (list_id == httprawuri_id) {
1012  rule_pcre_http += 1;
1013  raw_http_buf += 1;
1014  http_raw_uri_buf += 1;
1015  }
1016  else if (list_id == httpstatmsg_id) {
1017  rule_pcre_http += 1;
1018  raw_http_buf += 1;
1019  http_stat_msg_buf += 1;
1020  }
1021  else if (list_id == httpstatcode_id) {
1022  rule_pcre_http += 1;
1023  raw_http_buf += 1;
1024  http_stat_code_buf += 1;
1025  }
1026  else if (list_id == httpuseragent_id) {
1027  rule_pcre_http += 1;
1028  norm_http_buf += 1;
1029  http_ua_buf += 1;
1030  }
1031  else if (list_id == httphost_id) {
1032  rule_pcre_http += 1;
1033  norm_http_buf += 1;
1034  http_host_buf += 1;
1035  }
1036  else if (list_id == httprawhost_id) {
1037  rule_pcre_http += 1;
1038  raw_http_buf += 1;
1039  http_rawhost_buf += 1;
1040  }
1041  else if (list_id == httpheadernames_id) {
1042  rule_pcre_http += 1;
1043  raw_http_buf += 1;
1044  http_headernames_buf += 1;
1045  }
1046  else if (list_id == httpreferer_id) {
1047  rule_pcre_http += 1;
1048  raw_http_buf += 1;
1049  http_referer_buf += 1;
1050  }
1051  else {
1052  rule_pcre += 1;
1053  }
1054  }
1055  else if (sm->type == DETECT_CONTENT) {
1056 
1057  if (list_id == httpuri_id
1058  || list_id == httpheader_id
1059  || list_id == httpcookie_id) {
1060  rule_content_http += 1;
1061  norm_http_buf += 1;
1063  if (cd != NULL && PerCentEncodingMatch(cd->content, cd->content_len) > 0) {
1064  warn_encoding_norm_http_buf += 1;
1065  rule_warning += 1;
1066  }
1067  if (list_id == httpuri_id) {
1068  http_uri_buf += 1;
1069  }
1070  else if (list_id == httpheader_id) {
1071  http_header_buf += 1;
1072  }
1073  else if (list_id == httpcookie_id) {
1074  http_cookie_buf += 1;
1075  }
1076  }
1077  else if (list_id == httpclientbody_id) {
1078  rule_content_http += 1;
1079  raw_http_buf += 1;
1080  http_client_body_buf += 1;
1081  }
1082  else if (list_id == httpcontenttype_id) {
1083  rule_content_http += 1;
1084  raw_http_buf += 1;
1085  http_content_type_buf += 1;
1086  }
1087  else if (list_id == filedata_id) {
1088  rule_content_http += 1;
1089  raw_http_buf += 1;
1090  http_server_body_buf += 1;
1091  }
1092  else if (list_id == httprawheader_id) {
1093  rule_content_http += 1;
1094  raw_http_buf += 1;
1095  http_raw_header_buf += 1;
1096  }
1097  else if (list_id == httprawuri_id) {
1098  rule_content_http += 1;
1099  raw_http_buf += 1;
1100  http_raw_uri_buf += 1;
1101  }
1102  else if (list_id == httpstatmsg_id) {
1103  rule_content_http += 1;
1104  raw_http_buf += 1;
1105  http_stat_msg_buf += 1;
1106  }
1107  else if (list_id == httpstatcode_id) {
1108  rule_content_http += 1;
1109  raw_http_buf += 1;
1110  http_stat_code_buf += 1;
1111  }
1112  else if (list_id == httpmethod_id) {
1113  rule_content_http += 1;
1114  raw_http_buf += 1;
1115  http_method_buf += 1;
1116  }
1117  else if (list_id == httphost_id) {
1118  rule_content_http += 1;
1119  raw_http_buf += 1;
1120  http_host_buf += 1;
1121  }
1122  else if (list_id == httpuseragent_id) {
1123  rule_content_http += 1;
1124  norm_http_buf += 1;
1125  http_ua_buf += 1;
1126  }
1127  else if (list_id == httprawhost_id) {
1128  rule_content_http += 1;
1129  raw_http_buf += 1;
1130  http_rawhost_buf += 1;
1131  }
1132  else if (list_id == httpheadernames_id) {
1133  rule_content_http += 1;
1134  raw_http_buf += 1;
1135  http_headernames_buf += 1;
1136  }
1137  else if (list_id == httpreferer_id) {
1138  rule_content_http += 1;
1139  raw_http_buf += 1;
1140  http_referer_buf += 1;
1141  }
1142  else if (list_id == DETECT_SM_LIST_PMATCH) {
1143  rule_content += 1;
1145  if (cd->flags &
1147  rule_content_offset_depth++;
1148  }
1149  } else {
1150  rule_content += 1;
1151  }
1152  }
1153  else if (sm->type == DETECT_FLOW) {
1154  rule_flow += 1;
1155  if ((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_TOCLIENT)) {
1156  rule_flow_toserver = 1;
1157  }
1158  else if ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_TOSERVER)) {
1159  rule_flow_toclient = 1;
1160  }
1161  DetectFlowData *fd = (DetectFlowData *)sm->ctx;
1162  if (fd != NULL) {
1163  if (fd->flags & DETECT_FLOW_FLAG_NOSTREAM)
1164  rule_flow_nostream = 1;
1165  }
1166  }
1167  else if (sm->type == DETECT_FLOWBITS) {
1168  if (list_id == DETECT_SM_LIST_MATCH) {
1169  rule_flowbits += 1;
1170  }
1171  }
1172  else if (sm->type == DETECT_FLOWINT) {
1173  if (list_id == DETECT_SM_LIST_MATCH) {
1174  rule_flowint += 1;
1175  }
1176  }
1177  else if (sm->type == DETECT_FLAGS) {
1178  DetectFlagsData *fd = (DetectFlagsData *)sm->ctx;
1179  if (fd != NULL) {
1180  rule_flags = 1;
1181  }
1182  }
1183  } /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
1184 
1185  } /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
1186 
1187 
1188  if (rule_pcre > 0 && rule_content == 0 && rule_content_http == 0) {
1189  rule_warning += 1;
1190  warn_pcre_no_content = 1;
1191  }
1192 
1193  if (rule_content_http > 0 && rule_pcre > 0 && rule_pcre_http == 0) {
1194  rule_warning += 1;
1195  warn_pcre_http_content = 1;
1196  }
1197  else if (s->alproto == ALPROTO_HTTP && rule_pcre > 0 && rule_pcre_http == 0) {
1198  rule_warning += 1;
1199  warn_pcre_http = 1;
1200  }
1201 
1202  if (rule_content > 0 && rule_content_http > 0) {
1203  rule_warning += 1;
1204  warn_content_http_content = 1;
1205  }
1206  if (s->alproto == ALPROTO_HTTP && rule_content > 0 && rule_content_http == 0) {
1207  rule_warning += 1;
1208  warn_content_http = 1;
1209  }
1210  if (rule_content == 1) {
1211  //todo: warning if content is weak, separate warning for pcre + weak content
1212  }
1213  if (rule_flow == 0 && rule_flags == 0
1214  && !(s->proto.flags & DETECT_PROTO_ANY) && DetectProtoContainsProto(&s->proto, IPPROTO_TCP)
1215  && (rule_content || rule_content_http || rule_pcre || rule_pcre_http || rule_flowbits)) {
1216  rule_warning += 1;
1217  warn_tcp_no_flow = 1;
1218  }
1219  if (rule_flow && !rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)
1220  && !((s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))) {
1221  if (((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))
1222  || ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_DP_ANY) && (s->flags & SIG_FLAG_SP_ANY))) {
1223  rule_warning += 1;
1224  warn_client_ports = 1;
1225  }
1226  }
1227  if (rule_flow && rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)) {
1228  rule_warning += 1;
1229  warn_direction = 1;
1230  }
1231  if (http_method_buf) {
1232  if (rule_flow && rule_flow_toclient) {
1233  rule_warning += 1;
1234  warn_method_toclient = 1;
1235  }
1236  if (http_server_body_buf) {
1237  rule_warning += 1;
1238  warn_method_serverbody = 1;
1239  }
1240  if (rule_content == 0 && rule_content_http == 0 && (rule_pcre > 0 || rule_pcre_http > 0)) {
1241  rule_warning += 1;
1242  warn_pcre_method = 1;
1243  }
1244  }
1245  if (rule_content_offset_depth > 0 && stream_buf && packet_buf) {
1246  rule_warning += 1;
1247  warn_offset_depth_pkt_stream = 1;
1248  }
1249  if (rule_content_offset_depth > 0 && !stream_buf && packet_buf && s->alproto != ALPROTO_UNKNOWN) {
1250  rule_warning += 1;
1251  warn_offset_depth_alproto = 1;
1252  }
1253  if (s->init_data->mpm_sm != NULL && s->alproto == ALPROTO_HTTP &&
1255  rule_warning += 1;
1256  warn_non_alproto_fp_for_alproto_sig = 1;
1257  }
1258 
1259  if ((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) {
1260  warn_no_direction += 1;
1261  rule_warning += 1;
1262  }
1264  warn_both_direction += 1;
1265  rule_warning += 1;
1266  }
1267 
1268  if (!rule_warnings_only || (rule_warnings_only && rule_warning > 0)) {
1269  fprintf(rule_engine_analysis_FD, "== Sid: %u ==\n", s->id);
1270  fprintf(rule_engine_analysis_FD, "%s\n", line);
1271 
1272  if (s->flags & SIG_FLAG_IPONLY) fprintf(rule_engine_analysis_FD, " Rule is ip only.\n");
1273  if (s->flags & SIG_FLAG_PDONLY) fprintf(rule_engine_analysis_FD, " Rule is PD only.\n");
1274  if (rule_ipv6_only) fprintf(rule_engine_analysis_FD, " Rule is IPv6 only.\n");
1275  if (rule_ipv4_only) fprintf(rule_engine_analysis_FD, " Rule is IPv4 only.\n");
1276  if (packet_buf) fprintf(rule_engine_analysis_FD, " Rule matches on packets.\n");
1277  if (!rule_flow_nostream && stream_buf && (rule_flow || rule_flowbits || rule_content || rule_pcre)) {
1278  fprintf(rule_engine_analysis_FD, " Rule matches on reassembled stream.\n");
1279  }
1280  if (http_uri_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http uri buffer.\n");
1281  if (http_header_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http header buffer.\n");
1282  if (http_cookie_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http cookie buffer.\n");
1283  if (http_content_type_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http content type buffer.\n");
1284  if (http_raw_uri_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http raw uri buffer.\n");
1285  if (http_raw_header_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http raw header buffer.\n");
1286  if (http_method_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http method buffer.\n");
1287  if (http_server_body_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http server body buffer.\n");
1288  if (http_client_body_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http client body buffer.\n");
1289  if (http_stat_msg_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http stat msg buffer.\n");
1290  if (http_stat_code_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http stat code buffer.\n");
1291  if (http_ua_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http user agent buffer.\n");
1292  if (http_host_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http host buffer.\n");
1293  if (http_rawhost_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http rawhost buffer.\n");
1294  if (http_headernames_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http header names buffer.\n");
1295  if (http_referer_buf) fprintf(rule_engine_analysis_FD, " Rule matches on http header referer buffer.\n");
1296  if (s->alproto != ALPROTO_UNKNOWN) {
1297  fprintf(rule_engine_analysis_FD, " App layer protocol is %s.\n", AppProtoToString(s->alproto));
1298  }
1299  if (rule_content || rule_content_http || rule_pcre || rule_pcre_http) {
1300  fprintf(rule_engine_analysis_FD, " Rule contains %d content options, %d http content options, %d pcre options, and %d pcre options with http modifiers.\n", rule_content, rule_content_http, rule_pcre, rule_pcre_http);
1301  }
1302 
1303  /* print fast pattern info */
1304  if (s->init_data->prefilter_sm) {
1305  fprintf(rule_engine_analysis_FD, " Prefilter on: %s.\n",
1307  } else {
1308  EngineAnalysisRulesPrintFP(de_ctx, s);
1309  }
1310 
1311  /* this is where the warnings start */
1312  if (warn_pcre_no_content /*rule_pcre > 0 && rule_content == 0 && rule_content_http == 0*/) {
1313  fprintf(rule_engine_analysis_FD, " Warning: Rule uses pcre without a content option present.\n"
1314  " -Consider adding a content to improve performance of this rule.\n");
1315  }
1316  if (warn_pcre_http_content /*rule_content_http > 0 && rule_pcre > 0 && rule_pcre_http == 0*/) {
1317  fprintf(rule_engine_analysis_FD, " Warning: Rule uses content options with http_* and pcre options without http modifiers.\n"
1318  " -Consider adding http pcre modifier.\n");
1319  }
1320  else if (warn_pcre_http /*s->alproto == ALPROTO_HTTP && rule_pcre > 0 && rule_pcre_http == 0*/) {
1321  fprintf(rule_engine_analysis_FD, " Warning: Rule app layer protocol is http, but pcre options do not have http modifiers.\n"
1322  " -Consider adding http pcre modifiers.\n");
1323  }
1324  if (warn_content_http_content /*rule_content > 0 && rule_content_http > 0*/) {
1325  fprintf(rule_engine_analysis_FD, " Warning: Rule contains content with http_* and content without http_*.\n"
1326  " -Consider adding http content modifiers.\n");
1327  }
1328  if (warn_content_http /*s->alproto == ALPROTO_HTTP && rule_content > 0 && rule_content_http == 0*/) {
1329  fprintf(rule_engine_analysis_FD, " Warning: Rule app layer protocol is http, but content options do not have http_* modifiers.\n"
1330  " -Consider adding http content modifiers.\n");
1331  }
1332  if (rule_content == 1) {
1333  //todo: warning if content is weak, separate warning for pcre + weak content
1334  }
1335  if (warn_encoding_norm_http_buf) {
1336  fprintf(rule_engine_analysis_FD, " Warning: Rule may contain percent encoded content for a normalized http buffer match.\n");
1337  }
1338  if (warn_tcp_no_flow /*rule_flow == 0 && rule_flags == 0
1339  && !(s->proto.flags & DETECT_PROTO_ANY) && DetectProtoContainsProto(&s->proto, IPPROTO_TCP)*/) {
1340  fprintf(rule_engine_analysis_FD, " Warning: TCP rule without a flow or flags option.\n"
1341  " -Consider adding flow or flags to improve performance of this rule.\n");
1342  }
1343  if (warn_client_ports /*rule_flow && !rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)
1344  && !((s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY)))
1345  if (((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))
1346  || ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_DP_ANY) && (s->flags & SIG_FLAG_SP_ANY))*/) {
1347  fprintf(rule_engine_analysis_FD, " Warning: Rule contains ports or port variables only on the client side.\n"
1348  " -Flow direction possibly inconsistent with rule.\n");
1349  }
1350  if (warn_direction /*rule_flow && rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)*/) {
1351  fprintf(rule_engine_analysis_FD, " Warning: Rule is bidirectional and has a flow option with a specific direction.\n");
1352  }
1353  if (warn_method_toclient /*http_method_buf && rule_flow && rule_flow_toclient*/) {
1354  fprintf(rule_engine_analysis_FD, " Warning: Rule uses content or pcre for http_method with flow:to_client or from_server\n");
1355  }
1356  if (warn_method_serverbody /*http_method_buf && http_server_body_buf*/) {
1357  fprintf(rule_engine_analysis_FD, " Warning: Rule uses content or pcre for http_method with content or pcre for http_server_body.\n");
1358  }
1359  if (warn_pcre_method /*http_method_buf && rule_content == 0 && rule_content_http == 0
1360  && (rule_pcre > 0 || rule_pcre_http > 0)*/) {
1361  fprintf(rule_engine_analysis_FD, " Warning: Rule uses pcre with only a http_method content; possible performance issue.\n");
1362  }
1363  if (warn_offset_depth_pkt_stream) {
1364  fprintf(rule_engine_analysis_FD, " Warning: Rule has depth"
1365  "/offset with raw content keywords. Please note the "
1366  "offset/depth will be checked against both packet "
1367  "payloads and stream. If you meant to have the offset/"
1368  "depth checked against just the payload, you can update "
1369  "the signature as \"alert tcp-pkt...\"\n");
1370  }
1371  if (warn_offset_depth_alproto) {
1372  fprintf(rule_engine_analysis_FD, " Warning: Rule has "
1373  "offset/depth set along with a match on a specific "
1374  "app layer protocol - %d. This can lead to FNs if we "
1375  "have a offset/depth content match on a packet payload "
1376  "before we can detect the app layer protocol for the "
1377  "flow.\n", s->alproto);
1378  }
1379  if (warn_non_alproto_fp_for_alproto_sig) {
1380  fprintf(rule_engine_analysis_FD, " Warning: Rule app layer "
1381  "protocol is http, but the fast_pattern is set on the raw "
1382  "stream. Consider adding fast_pattern over a http "
1383  "buffer for increased performance.");
1384  }
1385  if (warn_no_direction) {
1386  fprintf(rule_engine_analysis_FD, " Warning: Rule has no direction indicator.\n");
1387  }
1388  if (warn_both_direction) {
1389  fprintf(rule_engine_analysis_FD, " Warning: Rule is inspecting both the request and the response.\n");
1390  }
1391  if (rule_warning == 0) {
1392  fprintf(rule_engine_analysis_FD, " No warnings for this rule.\n");
1393  }
1394  fprintf(rule_engine_analysis_FD, "\n");
1395  }
1396  return;
1397 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:230
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
void CleanupFPAnalyzer(void)
#define SCMutex
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1439
DetectProto proto
Definition: detect.h:534
SignatureInitData * init_data
Definition: detect.h:586
#define DETECT_PROTO_IPV6
#define SIG_FLAG_PDONLY
Definition: detect.h:243
int PerCentEncodingMatch(uint8_t *content, uint8_t content_len)
Checks for % encoding in content.
struct RuleAnalyzer RuleAnalyzer
#define SIG_FLAG_PREFILTER
Definition: detect.h:239
#define MemBufferWriteString(dst,...)
Write a string buffer to the Membuffer dst.
Definition: util-buffer.h:162
SigMatch * prefilter_sm
Definition: detect.h:490
uint32_t flags
Definition: detect.h:518
char * msg
Definition: detect.h:575
uint32_t id
Definition: detect.h:550
#define DETECT_PROTO_ANY
#define DETECT_CONTENT_FAST_PATTERN
#define unlikely(expr)
Definition: util-optimize.h:35
uint8_t is_last
Definition: detect.h:324
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
SCMutex g_rules_analyzer_write_m
#define SIG_FLAG_REQUIRE_STREAM
Definition: detect.h:220
#define DETECT_PERCENT_ENCODING_REGEX
Data needed for Match()
Definition: detect.h:322
uint16_t flags
Definition: detect-flow.h:38
#define DETECT_CONTENT_DISTANCE
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:228
#define SIG_FLAG_REQUIRE_PACKET
Definition: detect.h:219
#define SIG_FLAG_TLSSTORE
Definition: detect.h:235
const char * name
Definition: detect.h:1193
#define SIG_FLAG_DEST_IS_TARGET
Definition: detect.h:247
Signature container.
Definition: detect.h:517
#define SIG_MASK_REQUIRE_ENGINE_EVENT
Definition: detect.h:270
#define DETECT_CONTENT_DEPTH
#define SCMutexLock(mut)
#define SIG_FLAG_APPLAYER
Definition: detect.h:214
enum @32 __attribute__
DNP3 application header.
#define SIG_FLAG_SRC_IS_TARGET
Definition: detect.h:245
struct SigMatch_ * next
Definition: detect.h:317
main detection engine ctx
Definition: detect.h:756
struct DetectEnginePktInspectionEngine * next
Definition: detect.h:459
#define SIG_MASK_REQUIRE_FLAGS_INITDEINIT
Definition: detect.h:265
#define SIG_FLAG_DST_ANY
Definition: detect.h:208
#define SIG_FLAG_MPM_NEG
Definition: detect.h:222
#define SIG_FLAG_INIT_BIDIREC
Definition: detect.h:255
int DetectBufferTypeGetByName(const char *name)
#define str(s)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define SCMutexUnlock(mut)
#define SIG_FLAG_TOCLIENT
Definition: detect.h:233
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:240
#define SIG_FLAG_SP_ANY
Definition: detect.h:209
#define SIG_FLAG_IPONLY
Definition: detect.h:215
#define SCMUTEX_INITIALIZER
#define SIG_MASK_REQUIRE_FLOW
Definition: detect.h:264
void CleanupRuleAnalyzer(void)
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
struct FpPatternStats_ FpPatternStats
struct DetectEngineAppInspectionEngine_ * next
Definition: detect.h:416
#define SIG_FLAG_FLUSH
Definition: detect.h:224
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void EngineAnalysisRules(const DetectEngineCtx *de_ctx, const Signature *s, const char *line)
Prints analysis of loaded rules.
#define MemBufferPrintToFPAsString(mem_buffer, fp)
Write a buffer to the file pointer as a printable char string.
Definition: util-buffer.h:93
#define SIG_MASK_REQUIRE_FLAGS_UNUSUAL
Definition: detect.h:266
#define SIG_FLAG_TOSERVER
Definition: detect.h:232
#define DETECT_CONTENT_STARTS_WITH
#define SCEnter(...)
Definition: util-debug.h:337
#define MAX_ENCODED_CHARS
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:570
void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
int SignatureHasPacketContent(const Signature *s)
check if a signature has patterns that are to be inspected against a packets payload (as opposed to t...
AppProto alproto
Definition: detect.h:521
void PrintRawUriFp(FILE *fp, uint8_t *buf, uint32_t buflen)
Definition: util-print.c:93
int PerCentEncodingSetup()
Compiles regex for rule analysis.
const char * DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
#define CHECK(pat)
uint32_t gid
Definition: detect.h:551
uint8_t type
Definition: detect.h:314
#define DETECT_CONTENT_FAST_PATTERN_ONLY
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:258
#define DETECT_PROTO_IPV4
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
#define DETECT_CONTENT_ENDS_WITH
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
struct SigMatch_ ** smlists
Definition: detect.h:511
void EngineAnalysisRulesFailure(char *line, char *file, int lineno)
#define SIG_MASK_REQUIRE_PAYLOAD
Definition: detect.h:263
uint32_t smlists_array_size
Definition: detect.h:509
#define DETECT_CONTENT_MPM
Definition: conf.h:32
void EngineAnalysisFP(const DetectEngineCtx *de_ctx, const Signature *s, char *line)
SigMatchCtx * ctx
Definition: detect.h:316
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:222
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
uint8_t type
Definition: detect.h:323
#define DETECT_CONTENT_NEGATED
char * sig_str
Definition: detect.h:584
#define SCFree(a)
Definition: util-mem.h:322
#define SIG_FLAG_NOALERT
Definition: detect.h:212
DetectEnginePktInspectionEngine * pkt_inspect
Definition: detect.h:566
#define SIG_FLAG_DSIZE
Definition: detect.h:213
uint32_t init_flags
Definition: detect.h:481
DetectEngineAppInspectionEngine * app_inspect
Definition: detect.h:565
uint32_t rev
Definition: detect.h:552
const char * DetectBufferTypeGetDescriptionById(const DetectEngineCtx *de_ctx, const int id)
#define SIG_FLAG_BYPASS
Definition: detect.h:237
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define DETECT_CONTENT_WITHIN
int SignatureHasStreamContent(const Signature *s)
check if a signature has patterns that are to be inspected against the stream payload (as opposed to ...
int SetupFPAnalyzer(void)
Sets up the fast pattern analyzer according to the config.
#define SIG_FLAG_DP_ANY
Definition: detect.h:210
#define DETECT_CONTENT_FAST_PATTERN_CHOP
#define DETECT_CONTENT_NOCASE
#define SCReturn
Definition: util-debug.h:339
uint8_t len
SignatureMask mask
Definition: detect.h:526
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:561
#define DETECT_FLOW_FLAG_NOSTREAM
Definition: detect-flow.h:33
#define SIG_FLAG_SRC_ANY
Definition: detect.h:207
#define SIG_MASK_REQUIRE_DCERPC
Definition: detect.h:268
const char * DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
#define SIG_MASK_REQUIRE_NO_PAYLOAD
Definition: detect.h:267
SigMatch * mpm_sm
Definition: detect.h:488
#define DETECT_CONTENT_OFFSET
int DetectProtoContainsProto(const DetectProto *dp, int proto)
see if a DetectProto contains a certain proto
a single match condition for a signature
Definition: detect.h:313
int SetupRuleAnalyzer(void)
Sets up the rule analyzer according to the config.
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
SigMatchCtx * ctx
Definition: detect.h:325