suricata
detect-engine-analyzer.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2018 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Eileen Donlon <emdonlo@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Rule analyzers for the detection engine
25  */
26 
27 #include "suricata-common.h"
28 #include "suricata.h"
29 #include "detect.h"
30 #include "detect-parse.h"
31 #include "detect-engine.h"
32 #include "detect-engine-analyzer.h"
33 #include "detect-engine-mpm.h"
34 #include "conf.h"
35 #include "detect-content.h"
36 #include "detect-flow.h"
37 #include "detect-tcp-flags.h"
38 #include "util-print.h"
39 
40 static int rule_warnings_only = 0;
41 static FILE *rule_engine_analysis_FD = NULL;
42 static FILE *fp_engine_analysis_FD = NULL;
43 static pcre *percent_re = NULL;
44 static pcre_extra *percent_re_study = NULL;
45 static char log_path[PATH_MAX];
46 
47 typedef struct FpPatternStats_ {
48  uint16_t min;
49  uint16_t max;
50  uint32_t cnt;
51  uint64_t tot;
53 
54 /* Details for each buffer being tracked */
55 typedef struct DetectEngineAnalyzerItems {
56  int16_t item_id;
57  bool item_seen;
60  const char *item_name;
61  const char *display_name;
63 
64 /* Track which items require the item_seen value to be exposed */
66  const char *bufname;
68 };
69 
71  /* request keywords */
72  { 0, false, false, true, "http_uri", "http uri" },
73  { 0, false, false, false, "http_raw_uri", "http raw uri" },
74  { 0, false, true, false, "http_method", "http method" },
75  { 0, false, false, false, "http_request_line", "http request line" },
76  { 0, false, false, false, "http_client_body", "http client body" },
77  { 0, false, false, true, "http_header", "http header" },
78  { 0, false, false, false, "http_raw_header", "http raw header" },
79  { 0, false, false, true, "http_cookie", "http cookie" },
80  { 0, false, false, false, "http_user_agent", "http user agent" },
81  { 0, false, false, false, "http_host", "http host" },
82  { 0, false, false, false, "http_raw_host", "http raw host" },
83  { 0, false, false, false, "http_accept_enc", "http accept enc" },
84  { 0, false, false, false, "http_referer", "http referer" },
85  { 0, false, false, false, "http_content_type", "http content type" },
86  { 0, false, false, false, "http_header_names", "http header names" },
87 
88  /* response keywords not listed above */
89  { 0, false, false, false, "http_stat_msg", "http stat msg" },
90  { 0, false, false, false, "http_stat_code", "http stat code" },
91  { 0, false, true, false, "file_data", "http server body"},
92 
93  /* missing request keywords */
94  { 0, false, false, false, "http_request_line", "http request line" },
95  { 0, false, false, false, "http_accept", "http accept" },
96  { 0, false, false, false, "http_accept_lang", "http accept lang" },
97  { 0, false, false, false, "http_connection", "http connection" },
98  { 0, false, false, false, "http_content_len", "http content len" },
99  { 0, false, false, false, "http_protocol", "http protocol" },
100  { 0, false, false, false, "http_start", "http start" },
101 
102  /* missing response keywords; some of the missing are listed above*/
103  { 0, false, false, false, "http_response_line", "http response line" },
104  { 0, false, false, false, "http.server", "http server" },
105  { 0, false, false, false, "http.location", "http location" },
106 };
107 
108 /*
109  * This array contains the map between the `analyzer_items` array listed above and
110  * the item ids returned by DetectBufferTypeGetByName. Iterating signature's sigmatch
111  * array provides list_ids. The map converts those ids into elements of the
112  * analyzer items array.
113  *
114  * Ultimately, the g_buffer_type_hash is searched for each buffer name. The size of that
115  * hashlist is 256, so that's the value we use here.
116  */
117 int16_t analyzer_item_map[256];
118 
119 /*
120  * Certain values must be directly accessible. This array contains items that are directly
121  * accessed when checking if they've been seen or not.
122  */
124  { .bufname = "http_method"},
125  { .bufname = "file_data"}
126 };
127 
128 static FpPatternStats fp_pattern_stats[DETECT_SM_LIST_MAX];
129 
130 static void FpPatternStatsAdd(int list, uint16_t patlen)
131 {
132  if (list < 0 || list >= DETECT_SM_LIST_MAX)
133  return;
134 
135  FpPatternStats *f = &fp_pattern_stats[list];
136 
137  if (f->min == 0)
138  f->min = patlen;
139  else if (patlen < f->min)
140  f->min = patlen;
141 
142  if (patlen > f->max)
143  f->max = patlen;
144 
145  f->cnt++;
146  f->tot += patlen;
147 }
148 
149 void EngineAnalysisFP(const DetectEngineCtx *de_ctx, const Signature *s, char *line)
150 {
151  int fast_pattern_set = 0;
152  int fast_pattern_only_set = 0;
153  int fast_pattern_chop_set = 0;
154  DetectContentData *fp_cd = NULL;
155  SigMatch *mpm_sm = s->init_data->mpm_sm;
156 
157  if (mpm_sm != NULL) {
158  fp_cd = (DetectContentData *)mpm_sm->ctx;
159  if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN) {
160  fast_pattern_set = 1;
162  fast_pattern_only_set = 1;
163  } else if (fp_cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
164  fast_pattern_chop_set = 1;
165  }
166  }
167  }
168 
169  fprintf(fp_engine_analysis_FD, "== Sid: %u ==\n", s->id);
170  fprintf(fp_engine_analysis_FD, "%s\n", line);
171 
172  fprintf(fp_engine_analysis_FD, " Fast Pattern analysis:\n");
173  if (s->init_data->prefilter_sm != NULL) {
174  fprintf(fp_engine_analysis_FD, " Prefilter on: %s\n",
176  fprintf(fp_engine_analysis_FD, "\n");
177  return;
178  }
179 
180  if (fp_cd == NULL) {
181  fprintf(fp_engine_analysis_FD, " No content present\n");
182  fprintf(fp_engine_analysis_FD, "\n");
183  return;
184  }
185 
186  fprintf(fp_engine_analysis_FD, " Fast pattern matcher: ");
187  int list_type = SigMatchListSMBelongsTo(s, mpm_sm);
188  if (list_type == DETECT_SM_LIST_PMATCH)
189  fprintf(fp_engine_analysis_FD, "content\n");
190  else {
191  const char *desc = DetectBufferTypeGetDescriptionById(de_ctx, list_type);
192  const char *name = DetectBufferTypeGetNameById(de_ctx, list_type);
193  if (desc && name) {
194  fprintf(fp_engine_analysis_FD, "%s (%s)\n", desc, name);
195  }
196  }
197 
198  int flags_set = 0;
199  fprintf(fp_engine_analysis_FD, " Flags:");
200  if (fp_cd->flags & DETECT_CONTENT_OFFSET) {
201  fprintf(fp_engine_analysis_FD, " Offset");
202  flags_set = 1;
203  } if (fp_cd->flags & DETECT_CONTENT_DEPTH) {
204  fprintf(fp_engine_analysis_FD, " Depth");
205  flags_set = 1;
206  }
207  if (fp_cd->flags & DETECT_CONTENT_WITHIN) {
208  fprintf(fp_engine_analysis_FD, " Within");
209  flags_set = 1;
210  }
211  if (fp_cd->flags & DETECT_CONTENT_DISTANCE) {
212  fprintf(fp_engine_analysis_FD, " Distance");
213  flags_set = 1;
214  }
215  if (fp_cd->flags & DETECT_CONTENT_NOCASE) {
216  fprintf(fp_engine_analysis_FD, " Nocase");
217  flags_set = 1;
218  }
219  if (fp_cd->flags & DETECT_CONTENT_NEGATED) {
220  fprintf(fp_engine_analysis_FD, " Negated");
221  flags_set = 1;
222  }
223  if (flags_set == 0)
224  fprintf(fp_engine_analysis_FD, " None");
225  fprintf(fp_engine_analysis_FD, "\n");
226 
227  fprintf(fp_engine_analysis_FD, " Fast pattern set: %s\n", fast_pattern_set ? "yes" : "no");
228  fprintf(fp_engine_analysis_FD, " Fast pattern only set: %s\n",
229  fast_pattern_only_set ? "yes" : "no");
230  fprintf(fp_engine_analysis_FD, " Fast pattern chop set: %s\n",
231  fast_pattern_chop_set ? "yes" : "no");
232  if (fast_pattern_chop_set) {
233  fprintf(fp_engine_analysis_FD, " Fast pattern offset, length: %u, %u\n",
234  fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
235  }
236 
237  uint16_t patlen = fp_cd->content_len;
238  uint8_t *pat = SCMalloc(fp_cd->content_len + 1);
239  if (unlikely(pat == NULL)) {
240  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
241  exit(EXIT_FAILURE);
242  }
243  memcpy(pat, fp_cd->content, fp_cd->content_len);
244  pat[fp_cd->content_len] = '\0';
245  fprintf(fp_engine_analysis_FD, " Original content: ");
246  PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
247  fprintf(fp_engine_analysis_FD, "\n");
248 
249  if (fast_pattern_chop_set) {
250  SCFree(pat);
251  patlen = fp_cd->fp_chop_len;
252  pat = SCMalloc(fp_cd->fp_chop_len + 1);
253  if (unlikely(pat == NULL)) {
254  exit(EXIT_FAILURE);
255  }
256  memcpy(pat, fp_cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
257  pat[fp_cd->fp_chop_len] = '\0';
258  fprintf(fp_engine_analysis_FD, " Final content: ");
259  PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
260  fprintf(fp_engine_analysis_FD, "\n");
261 
262  FpPatternStatsAdd(list_type, patlen);
263  } else {
264  fprintf(fp_engine_analysis_FD, " Final content: ");
265  PrintRawUriFp(fp_engine_analysis_FD, pat, patlen);
266  fprintf(fp_engine_analysis_FD, "\n");
267 
268  FpPatternStatsAdd(list_type, patlen);
269  }
270  SCFree(pat);
271 
272  fprintf(fp_engine_analysis_FD, "\n");
273  return;
274 }
275 
276 /**
277  * \brief Sets up the fast pattern analyzer according to the config.
278  *
279  * \retval 1 If rule analyzer successfully enabled.
280  * \retval 0 If not enabled.
281  */
283 {
284  int fp_engine_analysis_set = 0;
285 
286  if ((ConfGetBool("engine-analysis.rules-fast-pattern",
287  &fp_engine_analysis_set)) == 0) {
288  return 0;
289  }
290 
291  if (fp_engine_analysis_set == 0)
292  return 0;
293 
294  const char *log_dir;
295  log_dir = ConfigGetLogDirectory();
296  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir,
297  "rules_fast_pattern.txt");
298 
299  fp_engine_analysis_FD = fopen(log_path, "w");
300  if (fp_engine_analysis_FD == NULL) {
301  SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", log_path,
302  strerror(errno));
303  return 0;
304  }
305 
306  SCLogInfo("Engine-Analysis for fast_pattern printed to file - %s",
307  log_path);
308 
309  struct timeval tval;
310  struct tm *tms;
311  gettimeofday(&tval, NULL);
312  struct tm local_tm;
313  tms = SCLocalTime(tval.tv_sec, &local_tm);
314  fprintf(fp_engine_analysis_FD, "----------------------------------------------"
315  "---------------------\n");
316  fprintf(fp_engine_analysis_FD, "Date: %" PRId32 "/%" PRId32 "/%04d -- "
317  "%02d:%02d:%02d\n",
318  tms->tm_mday, tms->tm_mon + 1, tms->tm_year + 1900, tms->tm_hour,
319  tms->tm_min, tms->tm_sec);
320  fprintf(fp_engine_analysis_FD, "----------------------------------------------"
321  "---------------------\n");
322 
323  memset(&fp_pattern_stats, 0, sizeof(fp_pattern_stats));
324  return 1;
325 }
326 
327 /**
328  * \brief Sets up the rule analyzer according to the config
329  * \retval 1 if rule analyzer successfully enabled
330  * \retval 0 if not enabled
331  */
333 {
334  ConfNode *conf = ConfGetNode("engine-analysis");
335  int enabled = 0;
336  if (conf != NULL) {
337  const char *value = ConfNodeLookupChildValue(conf, "rules");
338  if (value && ConfValIsTrue(value)) {
339  enabled = 1;
340  } else if (value && strcasecmp(value, "warnings-only") == 0) {
341  enabled = 1;
342  rule_warnings_only = 1;
343  }
344  if (enabled) {
345  const char *log_dir;
346  log_dir = ConfigGetLogDirectory();
347  snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, "rules_analysis.txt");
348  rule_engine_analysis_FD = fopen(log_path, "w");
349  if (rule_engine_analysis_FD == NULL) {
350  SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", log_path, strerror(errno));
351  return 0;
352  }
353 
354  SCLogInfo("Engine-Analysis for rules printed to file - %s",
355  log_path);
356 
357  struct timeval tval;
358  struct tm *tms;
359  gettimeofday(&tval, NULL);
360  struct tm local_tm;
361  tms = SCLocalTime(tval.tv_sec, &local_tm);
362  fprintf(rule_engine_analysis_FD, "----------------------------------------------"
363  "---------------------\n");
364  fprintf(rule_engine_analysis_FD, "Date: %" PRId32 "/%" PRId32 "/%04d -- "
365  "%02d:%02d:%02d\n",
366  tms->tm_mday, tms->tm_mon + 1, tms->tm_year + 1900, tms->tm_hour,
367  tms->tm_min, tms->tm_sec);
368  fprintf(rule_engine_analysis_FD, "----------------------------------------------"
369  "---------------------\n");
370 
371  /*compile regex's for rule analysis*/
372  if (PerCentEncodingSetup()== 0) {
373  fprintf(rule_engine_analysis_FD, "Error compiling regex; can't check for percent encoding in normalized http content.\n");
374  }
375  }
376  }
377  else {
378  SCLogInfo("Conf parameter \"engine-analysis.rules\" not found. "
379  "Defaulting to not printing the rules analysis report.");
380  }
381  if (!enabled) {
382  SCLogInfo("Engine-Analysis for rules disabled in conf file.");
383  return 0;
384  }
385  return 1;
386 }
387 
389 {
390  fprintf(fp_engine_analysis_FD, "============\n"
391  "Summary:\n============\n");
392  int i;
393  for (i = 0; i < DETECT_SM_LIST_MAX; i++) {
394  FpPatternStats *f = &fp_pattern_stats[i];
395  if (f->cnt == 0)
396  continue;
397 
398  fprintf(fp_engine_analysis_FD,
399  "%s, smallest pattern %u byte(s), longest pattern %u byte(s), number of patterns %u, avg pattern len %.2f byte(s)\n",
400  DetectSigmatchListEnumToString(i), f->min, f->max, f->cnt, (float)((double)f->tot/(float)f->cnt));
401  }
402 
403  if (fp_engine_analysis_FD != NULL) {
404  fclose(fp_engine_analysis_FD);
405  fp_engine_analysis_FD = NULL;
406  }
407 
408  return;
409 }
410 
411 
413 {
414  if (rule_engine_analysis_FD != NULL) {
415  SCLogInfo("Engine-Analyis for rules printed to file - %s", log_path);
416  fclose(rule_engine_analysis_FD);
417  rule_engine_analysis_FD = NULL;
418  }
419 }
420 
421 /**
422  * \brief Compiles regex for rule analysis
423  * \retval 1 if successful
424  * \retval 0 if on error
425  */
427 {
428 #define DETECT_PERCENT_ENCODING_REGEX "%[0-9|a-f|A-F]{2}"
429  const char *eb = NULL;
430  int eo = 0;
431  int opts = 0; //PCRE_NEWLINE_ANY??
432 
433  percent_re = pcre_compile(DETECT_PERCENT_ENCODING_REGEX, opts, &eb, &eo, NULL);
434  if (percent_re == NULL) {
435  SCLogError(SC_ERR_PCRE_COMPILE, "Compile of \"%s\" failed at offset %" PRId32 ": %s",
437  return 0;
438  }
439 
440  percent_re_study = pcre_study(percent_re, 0, &eb);
441  if (eb != NULL) {
442  SCLogError(SC_ERR_PCRE_STUDY, "pcre study failed: %s", eb);
443  return 0;
444  }
445  return 1;
446 }
447 
448 /**
449  * \brief Checks for % encoding in content.
450  * \param Pointer to content
451  * \retval number of matches if content has % encoding
452  * \retval 0 if it doesn't have % encoding
453  * \retval -1 on error
454  */
455 int PerCentEncodingMatch (uint8_t *content, uint8_t content_len)
456 {
457 #define MAX_ENCODED_CHARS 240
458  int ret = 0;
459  int ov[MAX_ENCODED_CHARS];
460 
461  ret = pcre_exec(percent_re, percent_re_study, (char *)content, content_len, 0, 0, ov, MAX_ENCODED_CHARS);
462  if (ret == -1) {
463  return 0;
464  }
465  else if (ret < -1) {
466  SCLogError(SC_ERR_PCRE_MATCH, "Error parsing content - %s; error code is %d", content, ret);
467  return -1;
468  }
469  return ret;
470 }
471 
472 static void EngineAnalysisRulesPrintFP(const DetectEngineCtx *de_ctx, const Signature *s)
473 {
474  DetectContentData *fp_cd = NULL;
475  SigMatch *mpm_sm = s->init_data->mpm_sm;
476 
477  if (mpm_sm != NULL) {
478  fp_cd = (DetectContentData *)mpm_sm->ctx;
479  }
480 
481  if (fp_cd == NULL) {
482  return;
483  }
484 
485  uint16_t patlen = fp_cd->content_len;
486  uint8_t *pat = SCMalloc(fp_cd->content_len + 1);
487  if (unlikely(pat == NULL)) {
488  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
489  exit(EXIT_FAILURE);
490  }
491  memcpy(pat, fp_cd->content, fp_cd->content_len);
492  pat[fp_cd->content_len] = '\0';
493 
495  SCFree(pat);
496  patlen = fp_cd->fp_chop_len;
497  pat = SCMalloc(fp_cd->fp_chop_len + 1);
498  if (unlikely(pat == NULL)) {
499  exit(EXIT_FAILURE);
500  }
501  memcpy(pat, fp_cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
502  pat[fp_cd->fp_chop_len] = '\0';
503  fprintf(rule_engine_analysis_FD, " Fast Pattern \"");
504  PrintRawUriFp(rule_engine_analysis_FD, pat, patlen);
505  } else {
506  fprintf(rule_engine_analysis_FD, " Fast Pattern \"");
507  PrintRawUriFp(rule_engine_analysis_FD, pat, patlen);
508  }
509  SCFree(pat);
510 
511  fprintf(rule_engine_analysis_FD, "\" on \"");
512 
513  int list_type = SigMatchListSMBelongsTo(s, mpm_sm);
514  if (list_type == DETECT_SM_LIST_PMATCH) {
515  int payload = 0;
516  int stream = 0;
518  payload = 1;
520  stream = 1;
521  fprintf(rule_engine_analysis_FD, "%s",
522  payload ? (stream ? "payload and reassembled stream" : "payload") : "reassembled stream");
523  }
524  else {
525  const char *desc = DetectBufferTypeGetDescriptionById(de_ctx, list_type);
526  const char *name = DetectBufferTypeGetNameById(de_ctx, list_type);
527  if (desc && name) {
528  fprintf(rule_engine_analysis_FD, "%s (%s)", desc, name);
529  } else if (desc || name) {
530  fprintf(rule_engine_analysis_FD, "%s", desc ? desc : name);
531  }
532 
533  }
534 
535  fprintf(rule_engine_analysis_FD, "\" ");
536  if (de_ctx->buffer_type_map[list_type] && de_ctx->buffer_type_map[list_type]->transforms.cnt) {
537  fprintf(rule_engine_analysis_FD, "(with %d transform(s)) ",
538  de_ctx->buffer_type_map[list_type]->transforms.cnt);
539  }
540  fprintf(rule_engine_analysis_FD, "buffer.\n");
541 
542  return;
543 }
544 
545 
546 void EngineAnalysisRulesFailure(char *line, char *file, int lineno)
547 {
548  fprintf(rule_engine_analysis_FD, "== Sid: UNKNOWN ==\n");
549  fprintf(rule_engine_analysis_FD, "%s\n", line);
550  fprintf(rule_engine_analysis_FD, " FAILURE: invalid rule.\n");
551  fprintf(rule_engine_analysis_FD, " File: %s.\n", file);
552  fprintf(rule_engine_analysis_FD, " Line: %d.\n", lineno);
553  fprintf(rule_engine_analysis_FD, "\n");
554 }
555 
556 #include "util-buffer.h"
557 #include "output-json.h"
558 
559 typedef struct RuleAnalyzer {
560  json_t *js; /* document root */
561 
562  json_t *js_warnings;
563  json_t *js_notes;
564 } RuleAnalyzer;
565 
566 static void __attribute__ ((format (printf, 2, 3)))
567 AnalyzerNote(RuleAnalyzer *ctx, char *fmt, ...)
568 {
569  va_list ap;
570  char str[1024];
571 
572  va_start(ap, fmt);
573  vsnprintf(str, sizeof(str), fmt, ap);
574  va_end(ap);
575 
576  if (!ctx->js_notes)
577  ctx->js_notes = json_array();
578  if (ctx->js_notes)
579  json_array_append_new(ctx->js_notes, json_string(str));
580 }
581 
582 static void __attribute__ ((format (printf, 2, 3)))
583 AnalyzerWarning(RuleAnalyzer *ctx, char *fmt, ...)
584 {
585  va_list ap;
586  char str[1024];
587 
588  va_start(ap, fmt);
589  vsnprintf(str, sizeof(str), fmt, ap);
590  va_end(ap);
591 
592  if (!ctx->js_warnings)
593  ctx->js_warnings = json_array();
594  if (ctx->js_warnings)
595  json_array_append_new(ctx->js_warnings, json_string(str));
596 }
597 
598 #define CHECK(pat) if (strlen((pat)) <= len && memcmp((pat), buf, MIN(len, strlen((pat)))) == 0) return true;
599 
600 static bool LooksLikeHTTPMethod(const uint8_t *buf, uint16_t len)
601 {
602  CHECK("GET /");
603  CHECK("POST /");
604  CHECK("HEAD /");
605  CHECK("PUT /");
606  return false;
607 }
608 
609 static bool LooksLikeHTTPUA(const uint8_t *buf, uint16_t len)
610 {
611  CHECK("User-Agent: ");
612  CHECK("\nUser-Agent: ");
613  return false;
614 }
615 
616 static void DumpMatches(RuleAnalyzer *ctx, json_t *js, const SigMatchData *smd)
617 {
618  if (smd == NULL)
619  return;
620 
621  json_t *js_matches = json_array();
622  if (js_matches == NULL) {
623  return;
624  }
625  do {
626  json_t *js_match = json_object();
627  if (js_match != NULL) {
628  const char *mname = sigmatch_table[smd->type].name;
629  json_object_set_new(js_match, "name", json_string(mname));
630 
631  switch (smd->type) {
632  case DETECT_CONTENT: {
633  const DetectContentData *cd = (const DetectContentData *)smd->ctx;
634  uint8_t *pat = SCMalloc(cd->content_len + 1);
635  if (unlikely(pat == NULL)) {
636  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
637  exit(EXIT_FAILURE);
638  }
639  memcpy(pat, cd->content, cd->content_len);
640  pat[cd->content_len] = '\0';
641 
642  json_t *js_match_content = json_object();
643  if (js_match_content != NULL) {
644  json_object_set_new(js_match_content, "pattern", SCJsonString((const char *)pat));
645  json_object_set_new(js_match_content, "nocase", json_boolean(cd->flags & DETECT_CONTENT_NOCASE));
646  json_object_set_new(js_match_content, "negated", json_boolean(cd->flags & DETECT_CONTENT_NEGATED));
647  json_object_set_new(js_match_content, "starts_with", json_boolean(cd->flags & DETECT_CONTENT_STARTS_WITH));
648  json_object_set_new(js_match_content, "ends_with", json_boolean(cd->flags & DETECT_CONTENT_ENDS_WITH));
649  json_object_set_new(js_match_content, "is_mpm", json_boolean(cd->flags & DETECT_CONTENT_MPM));
650  if (cd->flags & DETECT_CONTENT_OFFSET) {
651  json_object_set_new(js_match_content, "offset", json_integer(cd->offset));
652  }
653  if (cd->flags & DETECT_CONTENT_DEPTH) {
654  json_object_set_new(js_match_content, "depth", json_integer(cd->depth));
655  }
656  if (cd->flags & DETECT_CONTENT_DISTANCE) {
657  json_object_set_new(js_match_content, "distance", json_integer(cd->distance));
658  }
659  if (cd->flags & DETECT_CONTENT_WITHIN) {
660  json_object_set_new(js_match_content, "within", json_integer(cd->within));
661  }
662 
663  json_object_set_new(js_match_content, "fast_pattern", json_boolean(cd->flags & DETECT_CONTENT_FAST_PATTERN));
665  AnalyzerNote(ctx, (char *)"'fast_pattern:only' option is silently ignored and is interpreted as regular 'fast_pattern'");
666  }
667 
668  json_object_set_new(js_match, "content", js_match_content);
669  }
670 
671  if (LooksLikeHTTPMethod(cd->content, cd->content_len)) {
672  AnalyzerWarning(ctx, (char *)"pattern looks like it inspects HTTP, use http_request_line or http_method and http_uri instead for improved performance");
673  }
674  if (LooksLikeHTTPUA(cd->content, cd->content_len)) {
675  AnalyzerWarning(ctx, (char *)"pattern looks like it inspects HTTP, use http_user_agent or http_header for improved performance");
676  }
677 
678  SCFree(pat);
679  break;
680  }
681  }
682  }
683  json_array_append_new(js_matches, js_match);
684 
685  if (smd->is_last)
686  break;
687  smd++;
688  } while (1);
689  json_object_set_new(js, "matches", js_matches);
690 }
691 
693 void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
694 {
695  SCEnter();
696 
697  RuleAnalyzer ctx = { NULL, NULL, NULL };
698 
699  ctx.js = json_object();
700  if (ctx.js == NULL)
701  SCReturn;
702 
703  json_object_set_new(ctx.js, "raw", json_string(s->sig_str));
704  json_object_set_new(ctx.js, "id", json_integer(s->id));
705  json_object_set_new(ctx.js, "gid", json_integer(s->gid));
706  json_object_set_new(ctx.js, "rev", json_integer(s->rev));
707  json_object_set_new(ctx.js, "msg", json_string(s->msg));
708 
709  const char *alproto = AppProtoToString(s->alproto);
710  json_object_set_new(ctx.js, "app_proto", json_string(alproto));
711 
712  json_t *js_flags = json_array();
713  if (js_flags != NULL) {
714  if (s->mask & SIG_MASK_REQUIRE_PAYLOAD) {
715  json_array_append_new(js_flags, json_string("payload"));
716  }
718  json_array_append_new(js_flags, json_string("no_payload"));
719  }
720  if (s->mask & SIG_MASK_REQUIRE_FLOW) {
721  json_array_append_new(js_flags, json_string("flow"));
722  }
724  json_array_append_new(js_flags, json_string("tcp_flags_init_deinit"));
725  }
727  json_array_append_new(js_flags, json_string("tcp_flags_unusual"));
728  }
729  if (s->mask & SIG_MASK_REQUIRE_DCERPC) {
730  json_array_append_new(js_flags, json_string("dcerpc"));
731  }
733  json_array_append_new(js_flags, json_string("engine_event"));
734  }
735  json_object_set_new(ctx.js, "requirements", js_flags);
736  }
737 
738  js_flags = json_array();
739  if (js_flags != NULL) {
740  if (s->flags & SIG_FLAG_SRC_ANY) {
741  json_array_append_new(js_flags, json_string("src_any"));
742  }
743  if (s->flags & SIG_FLAG_DST_ANY) {
744  json_array_append_new(js_flags, json_string("dst_any"));
745  }
746  if (s->flags & SIG_FLAG_SP_ANY) {
747  json_array_append_new(js_flags, json_string("sp_any"));
748  }
749  if (s->flags & SIG_FLAG_DP_ANY) {
750  json_array_append_new(js_flags, json_string("dp_any"));
751  }
752  if (s->flags & SIG_FLAG_NOALERT) {
753  json_array_append_new(js_flags, json_string("noalert"));
754  }
755  if (s->flags & SIG_FLAG_DSIZE) {
756  json_array_append_new(js_flags, json_string("dsize"));
757  }
758  if (s->flags & SIG_FLAG_APPLAYER) {
759  json_array_append_new(js_flags, json_string("applayer"));
760  }
761  if (s->flags & SIG_FLAG_IPONLY) {
762  json_array_append_new(js_flags, json_string("ip_only"));
763  }
764  if (s->flags & SIG_FLAG_REQUIRE_PACKET) {
765  json_array_append_new(js_flags, json_string("need_packet"));
766  }
767  if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
768  json_array_append_new(js_flags, json_string("need_stream"));
769  }
770  if (s->flags & SIG_FLAG_MPM_NEG) {
771  json_array_append_new(js_flags, json_string("negated_mpm"));
772  }
773  if (s->flags & SIG_FLAG_FLUSH) {
774  json_array_append_new(js_flags, json_string("flush"));
775  }
776  if (s->flags & SIG_FLAG_REQUIRE_FLOWVAR) {
777  json_array_append_new(js_flags, json_string("need_flowvar"));
778  }
779  if (s->flags & SIG_FLAG_FILESTORE) {
780  json_array_append_new(js_flags, json_string("filestore"));
781  }
782  if (s->flags & SIG_FLAG_TOSERVER) {
783  json_array_append_new(js_flags, json_string("toserver"));
784  }
785  if (s->flags & SIG_FLAG_TOCLIENT) {
786  json_array_append_new(js_flags, json_string("toclient"));
787  }
788  if (s->flags & SIG_FLAG_TLSSTORE) {
789  json_array_append_new(js_flags, json_string("tlsstore"));
790  }
791  if (s->flags & SIG_FLAG_BYPASS) {
792  json_array_append_new(js_flags, json_string("bypass"));
793  }
794  if (s->flags & SIG_FLAG_PREFILTER) {
795  json_array_append_new(js_flags, json_string("prefilter"));
796  }
797  if (s->flags & SIG_FLAG_PDONLY) {
798  json_array_append_new(js_flags, json_string("proto_detect_only"));
799  }
800  if (s->flags & SIG_FLAG_SRC_IS_TARGET) {
801  json_array_append_new(js_flags, json_string("src_is_target"));
802  }
803  if (s->flags & SIG_FLAG_DEST_IS_TARGET) {
804  json_array_append_new(js_flags, json_string("dst_is_target"));
805  }
806  json_object_set_new(ctx.js, "flags", js_flags);
807  }
808 
809  json_t *js_pkt_array = json_array();
811  for ( ; pkt != NULL; pkt = pkt->next) {
812  const char *name = DetectBufferTypeGetNameById(de_ctx, pkt->sm_list);
813  if (name == NULL) {
814  switch (pkt->sm_list) {
816  name = "payload";
817  break;
819  name = "packet";
820  break;
821  default:
822  name = "unknown";
823  break;
824  }
825  }
826  json_t *js_engine = json_object();
827  if (js_engine != NULL) {
828  json_object_set_new(js_engine, "name", json_string(name));
829 
830  json_object_set_new(js_engine, "is_mpm", json_boolean(pkt->mpm));
831 
832  DumpMatches(&ctx, js_engine, pkt->smd);
833 
834  json_array_append_new(js_pkt_array, js_engine);
835  }
836  }
837  json_object_set_new(ctx.js, "pkt_engines", js_pkt_array);
838 
840  bool has_stream = false;
841  bool has_client_body_mpm = false;
842  bool has_file_data_mpm = false;
843 
844  json_t *js_array = json_array();
846  for ( ; app != NULL; app = app->next) {
847  const char *name = DetectBufferTypeGetNameById(de_ctx, app->sm_list);
848  if (name == NULL) {
849  switch (app->sm_list) {
851  name = "stream";
852  break;
853  default:
854  name = "unknown";
855  break;
856  }
857  }
858 
859  if (app->sm_list == DETECT_SM_LIST_PMATCH && !app->mpm) {
860  has_stream = true;
861  } else if (app->mpm && strcmp(name, "http_client_body") == 0) {
862  has_client_body_mpm = true;
863  } else if (app->mpm && strcmp(name, "file_data") == 0) {
864  has_file_data_mpm = true;
865  }
866 
867  json_t *js_engine = json_object();
868  if (js_engine != NULL) {
869  json_object_set_new(js_engine, "name", json_string(name));
870 
871  const char *direction = app->dir == 0 ? "toserver" : "toclient";
872  json_object_set_new(js_engine, "direction", json_string(direction));
873  json_object_set_new(js_engine, "is_mpm", json_boolean(app->mpm));
874  json_object_set_new(js_engine, "app_proto", json_string(AppProtoToString(app->alproto)));
875  json_object_set_new(js_engine, "progress", json_integer(app->progress));
876 
877  DumpMatches(&ctx, js_engine, app->smd);
878 
879  json_array_append_new(js_array, js_engine);
880  }
881  }
882  json_object_set_new(ctx.js, "engines", js_array);
883 
884  if (has_stream && has_client_body_mpm)
885  AnalyzerNote(&ctx, (char *)"mpm in http_client_body combined with stream match leads to stream buffering");
886  if (has_stream && has_file_data_mpm)
887  AnalyzerNote(&ctx, (char *)"mpm in file_data combined with stream match leads to stream buffering");
888  }
889 
890  json_t *js_lists = json_object();
891  for (int i = 0; i < DETECT_SM_LIST_MAX; i++) {
892  if (s->sm_arrays[i] != NULL) {
893  json_t *js_list = json_object();
894  if (js_list != NULL) {
895  DumpMatches(&ctx, js_list, s->sm_arrays[i]);
896  json_object_set_new(js_lists, DetectSigmatchListEnumToString(i), js_list);
897  }
898  }
899  }
900  json_object_set_new(ctx.js, "lists", js_lists);
901 
902  if (ctx.js_warnings) {
903  json_object_set_new(ctx.js, "warnings", ctx.js_warnings);
904  }
905  if (ctx.js_notes) {
906  json_object_set_new(ctx.js, "notes", ctx.js_notes);
907  }
908 
909  const char *filename = "rules.json";
910  const char *log_dir = ConfigGetLogDirectory();
911  char json_path[PATH_MAX] = "";
912  snprintf(json_path, sizeof(json_path), "%s/%s", log_dir, filename);
913 
914  MemBuffer *mbuf = MemBufferCreateNew(4096);
915  if (mbuf != NULL) {
916  OutputJSONMemBufferWrapper wrapper = {
917  .buffer = &mbuf,
918  .expand_by = 4096,
919  };
920 
921  int r = json_dump_callback(ctx.js, OutputJSONMemBufferCallback, &wrapper,
922  JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
924  if (r != 0) {
925  SCLogWarning(SC_ERR_SOCKET, "unable to serialize JSON object");
926  } else {
927  MemBufferWriteString(mbuf, "\n");
928  SCMutexLock(&g_rules_analyzer_write_m);
929  FILE *fp = fopen(json_path, "a");
930  if (fp != NULL) {
931  MemBufferPrintToFPAsString(mbuf, fp);
932  fclose(fp);
933  }
934  SCMutexUnlock(&g_rules_analyzer_write_m);
935  }
936 
937  MemBufferFree(mbuf);
938  }
939  json_object_clear(ctx.js);
940  json_decref(ctx.js);
941  SCReturn;
942 }
943 
944 static void EngineAnalysisItemsReset(void)
945 {
946  for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {
947  analyzer_items[i].item_seen = false;
948  }
949 }
950 
951 static void EngineAnalysisItemsInit(void)
952 {
953  static bool analyzer_init = false;
954 
955  if (analyzer_init) {
956  EngineAnalysisItemsReset();
957  return;
958  }
959 
960  memset(analyzer_item_map, -1, sizeof(analyzer_item_map));
961 
962  for (size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {
963  DetectEngineAnalyzerItems *analyzer_item = &analyzer_items[i];
964 
965  analyzer_item->item_id = DetectBufferTypeGetByName(analyzer_item->item_name);
966  analyzer_item->item_seen = false;
967 
968  if (analyzer_item->export_item_seen) {
969  for (size_t k = 0; k < ARRAY_SIZE(exposed_item_seen_list); k++) {
970  if (0 == strcmp(exposed_item_seen_list[k].bufname, analyzer_item->item_name))
971  exposed_item_seen_list[k].item_seen_ptr = &analyzer_item->item_seen;
972  }
973 
974  }
975  analyzer_item_map[analyzer_item->item_id] = (int16_t) i;
976  }
977 
978  analyzer_init = true;
979 }
980 
981 /**
982  * \brief Prints analysis of loaded rules.
983  *
984  * Warns if potential rule issues are detected. For example,
985  * warns if a rule uses a construct that may perform poorly,
986  * e.g. pcre without content or with http_method content only;
987  * warns if a rule uses a construct that may not be consistent with intent,
988  * e.g. client side ports only, http and content without any http_* modifiers, etc.
989  *
990  * \param s Pointer to the signature.
991  */
993  const Signature *s, const char *line)
994 {
995  uint32_t rule_bidirectional = 0;
996  uint32_t rule_pcre = 0;
997  uint32_t rule_pcre_http = 0;
998  uint32_t rule_content = 0;
999  uint32_t rule_flow = 0;
1000  uint32_t rule_flags = 0;
1001  uint32_t rule_flow_toserver = 0;
1002  uint32_t rule_flow_toclient = 0;
1003  uint32_t rule_flow_nostream = 0;
1004  uint32_t rule_ipv4_only = 0;
1005  uint32_t rule_ipv6_only = 0;
1006  uint32_t rule_flowbits = 0;
1007  uint32_t rule_flowint = 0;
1008  uint32_t rule_content_http = 0;
1009  uint32_t rule_content_offset_depth = 0;
1010  int32_t list_id = 0;
1011  uint32_t rule_warning = 0;
1012  uint32_t stream_buf = 0;
1013  uint32_t packet_buf = 0;
1014  uint32_t warn_pcre_no_content = 0;
1015  uint32_t warn_pcre_http_content = 0;
1016  uint32_t warn_pcre_http = 0;
1017  uint32_t warn_content_http_content = 0;
1018  uint32_t warn_content_http = 0;
1019  uint32_t warn_tcp_no_flow = 0;
1020  uint32_t warn_client_ports = 0;
1021  uint32_t warn_direction = 0;
1022  uint32_t warn_method_toclient = 0;
1023  uint32_t warn_method_serverbody = 0;
1024  uint32_t warn_pcre_method = 0;
1025  uint32_t warn_encoding_norm_http_buf = 0;
1026  uint32_t warn_offset_depth_pkt_stream = 0;
1027  uint32_t warn_offset_depth_alproto = 0;
1028  uint32_t warn_non_alproto_fp_for_alproto_sig = 0;
1029  uint32_t warn_no_direction = 0;
1030  uint32_t warn_both_direction = 0;
1031 
1032  EngineAnalysisItemsInit();
1033 
1034  bool *http_method_item_seen_ptr = exposed_item_seen_list[0].item_seen_ptr;
1035  bool *http_server_body_item_seen_ptr = exposed_item_seen_list[1].item_seen_ptr;
1036 
1038  rule_bidirectional = 1;
1039  }
1040 
1041  if (s->flags & SIG_FLAG_REQUIRE_PACKET) {
1042  packet_buf += 1;
1043  }
1044  if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
1045  stream_buf += 1;
1046  }
1047 
1048  if (s->proto.flags & DETECT_PROTO_IPV4) {
1049  rule_ipv4_only += 1;
1050  }
1051  if (s->proto.flags & DETECT_PROTO_IPV6) {
1052  rule_ipv6_only += 1;
1053  }
1054 
1055  for (list_id = 0; list_id < (int)s->init_data->smlists_array_size; list_id++) {
1056  SigMatch *sm = NULL;
1057  for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) {
1058  int16_t item_slot = analyzer_item_map[list_id];
1059  if (sm->type == DETECT_PCRE) {
1060  if (item_slot == -1) {
1061  rule_pcre++;
1062  continue;
1063  }
1064 
1065  rule_pcre_http++;
1066  analyzer_items[item_slot].item_seen = true;
1067  } else if (sm->type == DETECT_CONTENT) {
1068  if (item_slot == -1) {
1069  rule_content++;
1070  if (list_id == DETECT_SM_LIST_PMATCH) {
1073  rule_content_offset_depth++;
1074  }
1075  }
1076  continue;
1077  }
1078 
1079  rule_content_http++;
1080  analyzer_items[item_slot].item_seen = true;
1081 
1082  if (analyzer_items[item_slot].check_encoding_match) {
1084  if (cd != NULL && PerCentEncodingMatch(cd->content, cd->content_len) > 0) {
1085  warn_encoding_norm_http_buf += 1;
1086  }
1087  }
1088  }
1089  else if (sm->type == DETECT_FLOW) {
1090  rule_flow += 1;
1091  if ((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_TOCLIENT)) {
1092  rule_flow_toserver = 1;
1093  }
1094  else if ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_TOSERVER)) {
1095  rule_flow_toclient = 1;
1096  }
1097  DetectFlowData *fd = (DetectFlowData *)sm->ctx;
1098  if (fd != NULL) {
1099  if (fd->flags & DETECT_FLOW_FLAG_NOSTREAM)
1100  rule_flow_nostream = 1;
1101  }
1102  }
1103  else if (sm->type == DETECT_FLOWBITS) {
1104  if (list_id == DETECT_SM_LIST_MATCH) {
1105  rule_flowbits += 1;
1106  }
1107  }
1108  else if (sm->type == DETECT_FLOWINT) {
1109  if (list_id == DETECT_SM_LIST_MATCH) {
1110  rule_flowint += 1;
1111  }
1112  }
1113  else if (sm->type == DETECT_FLAGS) {
1114  DetectFlagsData *fd = (DetectFlagsData *)sm->ctx;
1115  if (fd != NULL) {
1116  rule_flags = 1;
1117  }
1118  }
1119  } /* for (sm = s->sm_lists[list_id]; sm != NULL; sm = sm->next) */
1120 
1121  } /* for ( ; list_id < DETECT_SM_LIST_MAX; list_id++) */
1122 
1123 
1124  if (rule_pcre > 0 && rule_content == 0 && rule_content_http == 0) {
1125  rule_warning += 1;
1126  warn_pcre_no_content = 1;
1127  }
1128 
1129  if (rule_content_http > 0 && rule_pcre > 0 && rule_pcre_http == 0) {
1130  rule_warning += 1;
1131  warn_pcre_http_content = 1;
1132  }
1133  else if (s->alproto == ALPROTO_HTTP && rule_pcre > 0 && rule_pcre_http == 0) {
1134  rule_warning += 1;
1135  warn_pcre_http = 1;
1136  }
1137 
1138  if (rule_content > 0 && rule_content_http > 0) {
1139  rule_warning += 1;
1140  warn_content_http_content = 1;
1141  }
1142  if (s->alproto == ALPROTO_HTTP && rule_content > 0 && rule_content_http == 0) {
1143  rule_warning += 1;
1144  warn_content_http = 1;
1145  }
1146  if (rule_content == 1) {
1147  //todo: warning if content is weak, separate warning for pcre + weak content
1148  }
1149  if (rule_flow == 0 && rule_flags == 0
1150  && !(s->proto.flags & DETECT_PROTO_ANY) && DetectProtoContainsProto(&s->proto, IPPROTO_TCP)
1151  && (rule_content || rule_content_http || rule_pcre || rule_pcre_http || rule_flowbits)) {
1152  rule_warning += 1;
1153  warn_tcp_no_flow = 1;
1154  }
1155  if (rule_flow && !rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)
1156  && !((s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))) {
1157  if (((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))
1158  || ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_DP_ANY) && (s->flags & SIG_FLAG_SP_ANY))) {
1159  rule_warning += 1;
1160  warn_client_ports = 1;
1161  }
1162  }
1163  if (rule_flow && rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)) {
1164  rule_warning += 1;
1165  warn_direction = 1;
1166  }
1167 
1168  if (*http_method_item_seen_ptr) {
1169  if (rule_flow && rule_flow_toclient) {
1170  rule_warning += 1;
1171  warn_method_toclient = 1;
1172  }
1173  if (*http_server_body_item_seen_ptr) {
1174  rule_warning += 1;
1175  warn_method_serverbody = 1;
1176  }
1177  if (rule_content == 0 && rule_content_http == 0 && (rule_pcre > 0 || rule_pcre_http > 0)) {
1178  rule_warning += 1;
1179  warn_pcre_method = 1;
1180  }
1181  }
1182  if (rule_content_offset_depth > 0 && stream_buf && packet_buf) {
1183  rule_warning += 1;
1184  warn_offset_depth_pkt_stream = 1;
1185  }
1186  if (rule_content_offset_depth > 0 && !stream_buf && packet_buf && s->alproto != ALPROTO_UNKNOWN) {
1187  rule_warning += 1;
1188  warn_offset_depth_alproto = 1;
1189  }
1190  if (s->init_data->mpm_sm != NULL && s->alproto == ALPROTO_HTTP &&
1192  rule_warning += 1;
1193  warn_non_alproto_fp_for_alproto_sig = 1;
1194  }
1195 
1196  if ((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) {
1197  warn_no_direction += 1;
1198  rule_warning += 1;
1199  }
1200 
1201  /* No warning about direction for ICMP protos */
1202  if (!(DetectProtoContainsProto(&s->proto, IPPROTO_ICMPV6) && DetectProtoContainsProto(&s->proto, IPPROTO_ICMP))) {
1204  warn_both_direction += 1;
1205  rule_warning += 1;
1206  }
1207  }
1208 
1209  if (!rule_warnings_only || (rule_warnings_only && rule_warning > 0)) {
1210  fprintf(rule_engine_analysis_FD, "== Sid: %u ==\n", s->id);
1211  fprintf(rule_engine_analysis_FD, "%s\n", line);
1212 
1213  if (s->flags & SIG_FLAG_IPONLY) fprintf(rule_engine_analysis_FD, " Rule is ip only.\n");
1214  if (s->flags & SIG_FLAG_PDONLY) fprintf(rule_engine_analysis_FD, " Rule is PD only.\n");
1215  if (rule_ipv6_only) fprintf(rule_engine_analysis_FD, " Rule is IPv6 only.\n");
1216  if (rule_ipv4_only) fprintf(rule_engine_analysis_FD, " Rule is IPv4 only.\n");
1217  if (packet_buf) fprintf(rule_engine_analysis_FD, " Rule matches on packets.\n");
1218  if (!rule_flow_nostream && stream_buf && (rule_flow || rule_flowbits || rule_content || rule_pcre)) {
1219  fprintf(rule_engine_analysis_FD, " Rule matches on reassembled stream.\n");
1220  }
1221  for(size_t i = 0; i < ARRAY_SIZE(analyzer_items); i++) {
1222  DetectEngineAnalyzerItems *ai = &analyzer_items[i];
1223  if (ai->item_seen) {
1224  fprintf(rule_engine_analysis_FD, " Rule matches on %s buffer.\n", ai->display_name);
1225  }
1226  }
1227  if (s->alproto != ALPROTO_UNKNOWN) {
1228  fprintf(rule_engine_analysis_FD, " App layer protocol is %s.\n", AppProtoToString(s->alproto));
1229  }
1230  if (rule_content || rule_content_http || rule_pcre || rule_pcre_http) {
1231  fprintf(rule_engine_analysis_FD, " Rule contains %d content options, %d http content options, %d pcre options, and %d pcre options with http modifiers.\n", rule_content, rule_content_http, rule_pcre, rule_pcre_http);
1232  }
1233 
1234  /* print fast pattern info */
1235  if (s->init_data->prefilter_sm) {
1236  fprintf(rule_engine_analysis_FD, " Prefilter on: %s.\n",
1238  } else {
1239  EngineAnalysisRulesPrintFP(de_ctx, s);
1240  }
1241 
1242  /* this is where the warnings start */
1243  if (warn_pcre_no_content /*rule_pcre > 0 && rule_content == 0 && rule_content_http == 0*/) {
1244  fprintf(rule_engine_analysis_FD, " Warning: Rule uses pcre without a content option present.\n"
1245  " -Consider adding a content to improve performance of this rule.\n");
1246  }
1247  if (warn_pcre_http_content /*rule_content_http > 0 && rule_pcre > 0 && rule_pcre_http == 0*/) {
1248  fprintf(rule_engine_analysis_FD, " Warning: Rule uses content options with http_* and pcre options without http modifiers.\n"
1249  " -Consider adding http pcre modifier.\n");
1250  }
1251  else if (warn_pcre_http /*s->alproto == ALPROTO_HTTP && rule_pcre > 0 && rule_pcre_http == 0*/) {
1252  fprintf(rule_engine_analysis_FD, " Warning: Rule app layer protocol is http, but pcre options do not have http modifiers.\n"
1253  " -Consider adding http pcre modifiers.\n");
1254  }
1255  if (warn_content_http_content /*rule_content > 0 && rule_content_http > 0*/) {
1256  fprintf(rule_engine_analysis_FD, " Warning: Rule contains content with http_* and content without http_*.\n"
1257  " -Consider adding http content modifiers.\n");
1258  }
1259  if (warn_content_http /*s->alproto == ALPROTO_HTTP && rule_content > 0 && rule_content_http == 0*/) {
1260  fprintf(rule_engine_analysis_FD, " Warning: Rule app layer protocol is http, but content options do not have http_* modifiers.\n"
1261  " -Consider adding http content modifiers.\n");
1262  }
1263  if (rule_content == 1) {
1264  //todo: warning if content is weak, separate warning for pcre + weak content
1265  }
1266  if (warn_encoding_norm_http_buf) {
1267  fprintf(rule_engine_analysis_FD, " Warning: Rule may contain percent encoded content for a normalized http buffer match.\n");
1268  }
1269  if (warn_tcp_no_flow /*rule_flow == 0 && rule_flags == 0
1270  && !(s->proto.flags & DETECT_PROTO_ANY) && DetectProtoContainsProto(&s->proto, IPPROTO_TCP)*/) {
1271  fprintf(rule_engine_analysis_FD, " Warning: TCP rule without a flow or flags option.\n"
1272  " -Consider adding flow or flags to improve performance of this rule.\n");
1273  }
1274  if (warn_client_ports /*rule_flow && !rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)
1275  && !((s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY)))
1276  if (((s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_SP_ANY) && (s->flags & SIG_FLAG_DP_ANY))
1277  || ((s->flags & SIG_FLAG_TOCLIENT) && !(s->flags & SIG_FLAG_DP_ANY) && (s->flags & SIG_FLAG_SP_ANY))*/) {
1278  fprintf(rule_engine_analysis_FD, " Warning: Rule contains ports or port variables only on the client side.\n"
1279  " -Flow direction possibly inconsistent with rule.\n");
1280  }
1281  if (warn_direction /*rule_flow && rule_bidirectional && (rule_flow_toserver || rule_flow_toclient)*/) {
1282  fprintf(rule_engine_analysis_FD, " Warning: Rule is bidirectional and has a flow option with a specific direction.\n");
1283  }
1284  if (warn_method_toclient /*http_method_buf && rule_flow && rule_flow_toclient*/) {
1285  fprintf(rule_engine_analysis_FD, " Warning: Rule uses content or pcre for http_method with flow:to_client or from_server\n");
1286  }
1287  if (warn_method_serverbody /*http_method_buf && http_server_body_buf*/) {
1288  fprintf(rule_engine_analysis_FD, " Warning: Rule uses content or pcre for http_method with content or pcre for http_server_body.\n");
1289  }
1290  if (warn_pcre_method /*http_method_buf && rule_content == 0 && rule_content_http == 0
1291  && (rule_pcre > 0 || rule_pcre_http > 0)*/) {
1292  fprintf(rule_engine_analysis_FD, " Warning: Rule uses pcre with only a http_method content; possible performance issue.\n");
1293  }
1294  if (warn_offset_depth_pkt_stream) {
1295  fprintf(rule_engine_analysis_FD, " Warning: Rule has depth"
1296  "/offset with raw content keywords. Please note the "
1297  "offset/depth will be checked against both packet "
1298  "payloads and stream. If you meant to have the offset/"
1299  "depth checked against just the payload, you can update "
1300  "the signature as \"alert tcp-pkt...\"\n");
1301  }
1302  if (warn_offset_depth_alproto) {
1303  fprintf(rule_engine_analysis_FD, " Warning: Rule has "
1304  "offset/depth set along with a match on a specific "
1305  "app layer protocol - %d. This can lead to FNs if we "
1306  "have a offset/depth content match on a packet payload "
1307  "before we can detect the app layer protocol for the "
1308  "flow.\n", s->alproto);
1309  }
1310  if (warn_non_alproto_fp_for_alproto_sig) {
1311  fprintf(rule_engine_analysis_FD, " Warning: Rule app layer "
1312  "protocol is http, but the fast_pattern is set on the raw "
1313  "stream. Consider adding fast_pattern over a http "
1314  "buffer for increased performance.");
1315  }
1316  if (warn_no_direction) {
1317  fprintf(rule_engine_analysis_FD, " Warning: Rule has no direction indicator.\n");
1318  }
1319  if (warn_both_direction) {
1320  fprintf(rule_engine_analysis_FD, " Warning: Rule is inspecting both the request and the response.\n");
1321  }
1322  if (rule_warning == 0) {
1323  fprintf(rule_engine_analysis_FD, " No warnings for this rule.\n");
1324  }
1325  fprintf(rule_engine_analysis_FD, "\n");
1326  }
1327  return;
1328 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:234
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
void CleanupFPAnalyzer(void)
#define SCMutex
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1448
DetectProto proto
Definition: detect.h:539
SignatureInitData * init_data
Definition: detect.h:591
#define DETECT_PROTO_IPV6
#define SIG_FLAG_PDONLY
Definition: detect.h:247
DetectEngineAnalyzerItems analyzer_items[]
int PerCentEncodingMatch(uint8_t *content, uint8_t content_len)
Checks for % encoding in content.
struct RuleAnalyzer RuleAnalyzer
#define SIG_FLAG_PREFILTER
Definition: detect.h:243
#define MemBufferWriteString(dst,...)
Write a string buffer to the Membuffer dst.
Definition: util-buffer.h:162
SigMatch * prefilter_sm
Definition: detect.h:495
uint32_t flags
Definition: detect.h:523
char * msg
Definition: detect.h:580
uint32_t id
Definition: detect.h:555
#define DETECT_PROTO_ANY
#define DETECT_CONTENT_FAST_PATTERN
#define unlikely(expr)
Definition: util-optimize.h:35
uint8_t is_last
Definition: detect.h:329
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
SCMutex g_rules_analyzer_write_m
#define SIG_FLAG_REQUIRE_STREAM
Definition: detect.h:224
#define DETECT_PERCENT_ENCODING_REGEX
Data needed for Match()
Definition: detect.h:327
uint16_t flags
Definition: detect-flow.h:38
DetectEngineTransforms transforms
Definition: detect.h:434
#define DETECT_CONTENT_DISTANCE
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:232
#define SIG_FLAG_REQUIRE_PACKET
Definition: detect.h:223
#define SIG_FLAG_TLSSTORE
Definition: detect.h:239
const char * name
Definition: detect.h:1200
#define SIG_FLAG_DEST_IS_TARGET
Definition: detect.h:251
Signature container.
Definition: detect.h:522
#define SIG_MASK_REQUIRE_ENGINE_EVENT
Definition: detect.h:275
#define DETECT_CONTENT_DEPTH
#define SCMutexLock(mut)
#define SIG_FLAG_APPLAYER
Definition: detect.h:218
struct DetectEngineAnalyzerItems DetectEngineAnalyzerItems
enum @34 __attribute__
DNP3 application header.
int16_t analyzer_item_map[256]
#define SIG_FLAG_SRC_IS_TARGET
Definition: detect.h:249
struct SigMatch_ * next
Definition: detect.h:322
main detection engine ctx
Definition: detect.h:761
#define json_boolean(val)
struct DetectEnginePktInspectionEngine * next
Definition: detect.h:464
#define SIG_MASK_REQUIRE_FLAGS_INITDEINIT
Definition: detect.h:270
#define SIG_FLAG_DST_ANY
Definition: detect.h:212
#define SIG_FLAG_MPM_NEG
Definition: detect.h:226
#define SIG_FLAG_INIT_BIDIREC
Definition: detect.h:259
int DetectBufferTypeGetByName(const char *name)
#define str(s)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define SCMutexUnlock(mut)
#define SIG_FLAG_TOCLIENT
Definition: detect.h:237
struct tm * SCLocalTime(time_t timep, struct tm *result)
Definition: util-time.c:240
#define SIG_FLAG_SP_ANY
Definition: detect.h:213
#define ARRAY_SIZE(arr)
#define SIG_FLAG_IPONLY
Definition: detect.h:219
#define SCMUTEX_INITIALIZER
#define SIG_MASK_REQUIRE_FLOW
Definition: detect.h:269
void CleanupRuleAnalyzer(void)
const char * AppProtoToString(AppProto alproto)
Maps the ALPROTO_*, to its string equivalent.
struct FpPatternStats_ FpPatternStats
struct DetectEngineAppInspectionEngine_ * next
Definition: detect.h:421
#define SIG_FLAG_FLUSH
Definition: detect.h:228
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void EngineAnalysisRules(const DetectEngineCtx *de_ctx, const Signature *s, const char *line)
Prints analysis of loaded rules.
#define MemBufferPrintToFPAsString(mem_buffer, fp)
Write a buffer to the file pointer as a printable char string.
Definition: util-buffer.h:93
#define SIG_MASK_REQUIRE_FLAGS_UNUSUAL
Definition: detect.h:271
#define SIG_FLAG_TOSERVER
Definition: detect.h:236
#define DETECT_CONTENT_STARTS_WITH
#define SCEnter(...)
Definition: util-debug.h:337
#define MAX_ENCODED_CHARS
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:575
void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
int SignatureHasPacketContent(const Signature *s)
check if a signature has patterns that are to be inspected against a packets payload (as opposed to t...
json_t * SCJsonString(const char *val)
Definition: output-json.c:107
AppProto alproto
Definition: detect.h:526
void PrintRawUriFp(FILE *fp, uint8_t *buf, uint32_t buflen)
Definition: util-print.c:93
int PerCentEncodingSetup()
Compiles regex for rule analysis.
const char * DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
#define CHECK(pat)
uint32_t gid
Definition: detect.h:556
uint8_t type
Definition: detect.h:319
#define DETECT_CONTENT_FAST_PATTERN_ONLY
#define SIG_FLAG_INIT_STATE_MATCH
Definition: detect.h:262
#define DETECT_PROTO_IPV4
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
#define DETECT_CONTENT_ENDS_WITH
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
struct SigMatch_ ** smlists
Definition: detect.h:516
void EngineAnalysisRulesFailure(char *line, char *file, int lineno)
int OutputJSONMemBufferCallback(const char *str, size_t size, void *data)
Definition: output-json.c:796
#define SIG_MASK_REQUIRE_PAYLOAD
Definition: detect.h:268
uint32_t smlists_array_size
Definition: detect.h:514
#define DETECT_CONTENT_MPM
Definition: conf.h:32
void EngineAnalysisFP(const DetectEngineCtx *de_ctx, const Signature *s, char *line)
SigMatchCtx * ctx
Definition: detect.h:321
const char * ConfigGetLogDirectory()
Definition: util-conf.c:36
#define SCMalloc(a)
Definition: util-mem.h:222
DetectBufferType ** buffer_type_map
Definition: detect.h:917
#define JSON_ESCAPE_SLASH
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
uint8_t type
Definition: detect.h:328
#define DETECT_CONTENT_NEGATED
char * sig_str
Definition: detect.h:589
#define SCFree(a)
Definition: util-mem.h:322
#define SIG_FLAG_NOALERT
Definition: detect.h:216
DetectEnginePktInspectionEngine * pkt_inspect
Definition: detect.h:571
#define SIG_FLAG_DSIZE
Definition: detect.h:217
uint32_t init_flags
Definition: detect.h:486
DetectEngineAppInspectionEngine * app_inspect
Definition: detect.h:570
uint32_t rev
Definition: detect.h:557
const char * DetectBufferTypeGetDescriptionById(const DetectEngineCtx *de_ctx, const int id)
#define SIG_FLAG_BYPASS
Definition: detect.h:241
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define DETECT_CONTENT_WITHIN
int SignatureHasStreamContent(const Signature *s)
check if a signature has patterns that are to be inspected against the stream payload (as opposed to ...
int SetupFPAnalyzer(void)
Sets up the fast pattern analyzer according to the config.
#define SIG_FLAG_DP_ANY
Definition: detect.h:214
#define DETECT_CONTENT_FAST_PATTERN_CHOP
#define DETECT_CONTENT_NOCASE
struct ExposedItemSeen exposed_item_seen_list[]
#define SCReturn
Definition: util-debug.h:339
uint8_t len
SignatureMask mask
Definition: detect.h:531
int SigMatchListSMBelongsTo(const Signature *s, const SigMatch *key_sm)
Definition: detect-parse.c:619
#define DETECT_FLOW_FLAG_NOSTREAM
Definition: detect-flow.h:33
#define SIG_FLAG_SRC_ANY
Definition: detect.h:211
#define SIG_MASK_REQUIRE_DCERPC
Definition: detect.h:273
const char * DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id)
#define SIG_MASK_REQUIRE_NO_PAYLOAD
Definition: detect.h:272
SigMatch * mpm_sm
Definition: detect.h:493
#define DETECT_CONTENT_OFFSET
int DetectProtoContainsProto(const DetectProto *dp, int proto)
see if a DetectProto contains a certain proto
a single match condition for a signature
Definition: detect.h:318
int SetupRuleAnalyzer(void)
Sets up the rule analyzer according to the config.
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
SigMatchCtx * ctx
Definition: detect.h:330