suricata
detect.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2017 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Basic detection engine
24  */
25 
26 #include "suricata-common.h"
27 #include "suricata.h"
28 #include "conf.h"
29 
30 #include "decode.h"
31 #include "flow.h"
32 #include "stream-tcp.h"
33 #include "app-layer.h"
34 #include "app-layer-parser.h"
35 
36 #include "detect.h"
37 #include "detect-engine.h"
38 #include "detect-engine-profile.h"
39 
40 #include "detect-engine-alert.h"
41 #include "detect-engine-siggroup.h"
42 #include "detect-engine-address.h"
43 #include "detect-engine-proto.h"
44 #include "detect-engine-port.h"
45 #include "detect-engine-mpm.h"
46 #include "detect-engine-iponly.h"
49 #include "detect-engine-state.h"
50 #include "detect-engine-analyzer.h"
51 
52 #include "detect-engine-payload.h"
53 #include "detect-engine-event.h"
54 
55 #include "detect-filestore.h"
56 #include "detect-flowvar.h"
57 #include "detect-replace.h"
58 
59 #include "util-validate.h"
60 #include "util-detect.h"
61 
62 typedef struct DetectRunScratchpad {
64  const uint8_t flow_flags; /* flow/state flags: STREAM_* */
65  const bool app_decoder_events;
66  const SigGroupHead *sgh;
69 
70 /* prototypes */
71 static DetectRunScratchpad DetectRunSetup(const DetectEngineCtx *de_ctx,
72  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow);
73 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
74  DetectEngineThreadCtx *det_ctx, Flow * const pflow, Packet * const p);
75 static inline void DetectRunGetRuleGroup(const DetectEngineCtx *de_ctx,
76  Packet * const p, Flow * const pflow, DetectRunScratchpad *scratch);
77 static inline void DetectRunPrefilterPkt(ThreadVars *tv,
78  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p,
79  DetectRunScratchpad *scratch);
80 static inline void DetectRulePacketRules(ThreadVars * const tv,
81  DetectEngineCtx * const de_ctx, DetectEngineThreadCtx * const det_ctx,
82  Packet * const p, Flow * const pflow, const DetectRunScratchpad *scratch);
83 static void DetectRunTx(ThreadVars *tv, DetectEngineCtx *de_ctx,
84  DetectEngineThreadCtx *det_ctx, Packet *p,
85  Flow *f, DetectRunScratchpad *scratch);
86 static inline void DetectRunPostRules(ThreadVars *tv, DetectEngineCtx *de_ctx,
87  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow,
88  DetectRunScratchpad *scratch);
89 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
90  Packet *p, Flow * const pflow);
91 
92 /** \internal
93  */
94 static void DetectRun(ThreadVars *th_v,
95  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
96  Packet *p)
97 {
98  SCEnter();
99  SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt);
100 
101  /* bail early if packet should not be inspected */
102  if (p->flags & PKT_NOPACKET_INSPECTION) {
103  /* nothing to do */
104  SCReturn;
105  }
106 
107  /* Load the Packet's flow early, even though it might not be needed.
108  * Mark as a constant pointer, although the flow itself can change. */
109  Flow * const pflow = p->flow;
110 
111  DetectRunScratchpad scratch = DetectRunSetup(de_ctx, det_ctx, p, pflow);
112 
113  /* run the IPonly engine */
114  DetectRunInspectIPOnly(th_v, de_ctx, det_ctx, pflow, p);
115 
116  /* get our rule group */
117  DetectRunGetRuleGroup(de_ctx, p, pflow, &scratch);
118  /* if we didn't get a sig group head, we
119  * have nothing to do.... */
120  if (scratch.sgh == NULL) {
121  SCLogDebug("no sgh for this packet, nothing to match against");
122  goto end;
123  }
124 
125  /* run the prefilters for packets */
126  DetectRunPrefilterPkt(th_v, de_ctx, det_ctx, p, &scratch);
127 
129  /* inspect the rules against the packet */
130  DetectRulePacketRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
132 
133  /* run tx/state inspection */
134  if (pflow && pflow->alstate) {
136  DetectRunTx(th_v, de_ctx, det_ctx, p, pflow, &scratch);
138  }
139 
140 end:
141  DetectRunPostRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
142 
143  DetectRunCleanup(det_ctx, p, pflow);
144  SCReturn;
145 }
146 
147 static void DetectRunPostMatch(ThreadVars *tv,
148  DetectEngineThreadCtx *det_ctx, Packet *p,
149  const Signature *s)
150 {
151  /* run the packet match functions */
153  if (smd != NULL) {
155 
156  SCLogDebug("running match functions, sm %p", smd);
157 
158  while (1) {
160  (void)sigmatch_table[smd->type].Match(tv, det_ctx, p, s, smd->ctx);
161  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
162  if (smd->is_last)
163  break;
164  smd++;
165  }
166  }
167 
168  DetectReplaceExecute(p, det_ctx);
169 
170  if (s->flags & SIG_FLAG_FILESTORE)
171  DetectFilestorePostMatch(tv, det_ctx, p, s);
172 
173  return;
174 }
175 
176 /**
177  * \brief Get the SigGroupHead for a packet.
178  *
179  * \param de_ctx detection engine context
180  * \param p packet
181  *
182  * \retval sgh the SigGroupHead or NULL if non applies to the packet
183  */
185  const Packet *p)
186 {
187  SCEnter();
188 
189  int f;
190  SigGroupHead *sgh = NULL;
191 
192  /* if the packet proto is 0 (not set), we're inspecting it against
193  * the decoder events sgh we have. */
194  if (p->proto == 0 && p->events.cnt > 0) {
195  SCReturnPtr(de_ctx->decoder_event_sgh, "SigGroupHead");
196  } else if (p->proto == 0) {
197  if (!(PKT_IS_IPV4(p) || PKT_IS_IPV6(p))) {
198  /* not IP, so nothing to do */
199  SCReturnPtr(NULL, "SigGroupHead");
200  }
201  }
202 
203  /* select the flow_gh */
204  if (p->flowflags & FLOW_PKT_TOCLIENT)
205  f = 0;
206  else
207  f = 1;
208 
209  int proto = IP_GET_IPPROTO(p);
210  if (proto == IPPROTO_TCP) {
211  DetectPort *list = de_ctx->flow_gh[f].tcp;
212  SCLogDebug("tcp toserver %p, tcp toclient %p: going to use %p",
213  de_ctx->flow_gh[1].tcp, de_ctx->flow_gh[0].tcp, de_ctx->flow_gh[f].tcp);
214  uint16_t port = f ? p->dp : p->sp;
215  SCLogDebug("tcp port %u -> %u:%u", port, p->sp, p->dp);
216  DetectPort *sghport = DetectPortLookupGroup(list, port);
217  if (sghport != NULL)
218  sgh = sghport->sh;
219  SCLogDebug("TCP list %p, port %u, direction %s, sghport %p, sgh %p",
220  list, port, f ? "toserver" : "toclient", sghport, sgh);
221  } else if (proto == IPPROTO_UDP) {
222  DetectPort *list = de_ctx->flow_gh[f].udp;
223  uint16_t port = f ? p->dp : p->sp;
224  DetectPort *sghport = DetectPortLookupGroup(list, port);
225  if (sghport != NULL)
226  sgh = sghport->sh;
227  SCLogDebug("UDP list %p, port %u, direction %s, sghport %p, sgh %p",
228  list, port, f ? "toserver" : "toclient", sghport, sgh);
229  } else {
230  sgh = de_ctx->flow_gh[f].sgh[proto];
231  }
232 
233  SCReturnPtr(sgh, "SigGroupHead");
234 }
235 
236 static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
237  DetectEngineThreadCtx *det_ctx)
238 {
239  SigIntId mpm, nonmpm;
240  det_ctx->match_array_cnt = 0;
241  SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array;
242  SigIntId *nonmpm_ptr = det_ctx->non_pf_id_array;
243  uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt;
244  uint32_t n_cnt = det_ctx->non_pf_id_cnt;
245  SigIntId *final_ptr;
246  uint32_t final_cnt;
247  SigIntId id;
248  SigIntId previous_id = (SigIntId)-1;
249  Signature **sig_array = de_ctx->sig_array;
250  Signature **match_array = det_ctx->match_array;
251  Signature *s;
252 
253  SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt);
254 
255  /* Load first values. */
256  if (likely(m_cnt)) {
257  mpm = *mpm_ptr;
258  } else {
259  /* mpm list is empty */
260  final_ptr = nonmpm_ptr;
261  final_cnt = n_cnt;
262  goto final;
263  }
264  if (likely(n_cnt)) {
265  nonmpm = *nonmpm_ptr;
266  } else {
267  /* non-mpm list is empty. */
268  final_ptr = mpm_ptr;
269  final_cnt = m_cnt;
270  goto final;
271  }
272  while (1) {
273  if (mpm < nonmpm) {
274  /* Take from mpm list */
275  id = mpm;
276 
277  s = sig_array[id];
278  /* As the mpm list can contain duplicates, check for that here. */
279  if (likely(id != previous_id)) {
280  *match_array++ = s;
281  previous_id = id;
282  }
283  if (unlikely(--m_cnt == 0)) {
284  /* mpm list is now empty */
285  final_ptr = nonmpm_ptr;
286  final_cnt = n_cnt;
287  goto final;
288  }
289  mpm_ptr++;
290  mpm = *mpm_ptr;
291  } else if (mpm > nonmpm) {
292  id = nonmpm;
293 
294  s = sig_array[id];
295  /* As the mpm list can contain duplicates, check for that here. */
296  if (likely(id != previous_id)) {
297  *match_array++ = s;
298  previous_id = id;
299  }
300  if (unlikely(--n_cnt == 0)) {
301  final_ptr = mpm_ptr;
302  final_cnt = m_cnt;
303  goto final;
304  }
305  nonmpm_ptr++;
306  nonmpm = *nonmpm_ptr;
307 
308  } else { /* implied mpm == nonmpm */
309  /* special case: if on both lists, it's a negated mpm pattern */
310 
311  /* mpm list may have dups, so skip past them here */
312  while (--m_cnt != 0) {
313  mpm_ptr++;
314  mpm = *mpm_ptr;
315  if (mpm != nonmpm)
316  break;
317  }
318  /* if mpm is done, update nonmpm_ptrs and jump to final */
319  if (unlikely(m_cnt == 0)) {
320  n_cnt--;
321 
322  /* mpm list is now empty */
323  final_ptr = ++nonmpm_ptr;
324  final_cnt = n_cnt;
325  goto final;
326  }
327  /* otherwise, if nonmpm is done jump to final for mpm
328  * mpm ptrs alrady updated */
329  if (unlikely(--n_cnt == 0)) {
330  final_ptr = mpm_ptr;
331  final_cnt = m_cnt;
332  goto final;
333  }
334 
335  /* not at end of the lists, update nonmpm. Mpm already
336  * updated in while loop above. */
337  nonmpm_ptr++;
338  nonmpm = *nonmpm_ptr;
339  }
340  }
341 
342  final: /* Only one list remaining. Just walk that list. */
343 
344  while (final_cnt-- > 0) {
345  id = *final_ptr++;
346  s = sig_array[id];
347 
348  /* As the mpm list can contain duplicates, check for that here. */
349  if (likely(id != previous_id)) {
350  *match_array++ = s;
351  previous_id = id;
352  }
353  }
354 
355  det_ctx->match_array_cnt = match_array - det_ctx->match_array;
356 
357  BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_pf_id_cnt) < det_ctx->match_array_cnt);
358 }
359 
360 static inline void
361 DetectPrefilterBuildNonPrefilterList(DetectEngineThreadCtx *det_ctx, SignatureMask mask, uint8_t alproto)
362 {
363  uint32_t x = 0;
364  for (x = 0; x < det_ctx->non_pf_store_cnt; x++) {
365  /* only if the mask matches this rule can possibly match,
366  * so build the non_mpm array only for match candidates */
367  const SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask;
368  const uint8_t rule_alproto = det_ctx->non_pf_store_ptr[x].alproto;
369  if ((rule_mask & mask) == rule_mask && (rule_alproto == 0 || rule_alproto == alproto)) {
370  det_ctx->non_pf_id_array[det_ctx->non_pf_id_cnt++] = det_ctx->non_pf_store_ptr[x].id;
371  }
372  }
373 }
374 
375 /** \internal
376  * \brief select non-mpm list
377  * Based on the packet properties, select the non-mpm list to use
378  * \todo move non_pf_store* into scratchpad */
379 static inline void
380 DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx, DetectRunScratchpad *scratch)
381 {
382  if ((p->proto == IPPROTO_TCP) && (p->tcph != NULL) && (p->tcph->th_flags & TH_SYN)) {
383  det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_syn_store_array;
384  det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_syn_store_cnt;
385  } else {
386  det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_other_store_array;
387  det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_other_store_cnt;
388  }
389  SCLogDebug("sgh non_pf ptr %p cnt %u (syn %p/%u, other %p/%u)",
390  det_ctx->non_pf_store_ptr, det_ctx->non_pf_store_cnt,
391  scratch->sgh->non_pf_syn_store_array, scratch->sgh->non_pf_syn_store_cnt,
393 }
394 
395 /** \internal
396  * \brief update flow's file tracking flags based on the detection engine
397  */
398 static inline void
399 DetectPostInspectFileFlagsUpdate(Flow *pflow, const SigGroupHead *sgh, uint8_t direction)
400 {
401  /* see if this sgh requires us to consider file storing */
402  if (!FileForceFilestore() && (sgh == NULL ||
403  sgh->filestore_cnt == 0))
404  {
405  FileDisableStoring(pflow, direction);
406  }
407 #ifdef HAVE_MAGIC
408  /* see if this sgh requires us to consider file magic */
409  if (!FileForceMagic() && (sgh == NULL ||
410  !(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)))
411  {
412  SCLogDebug("disabling magic for flow");
413  FileDisableMagic(pflow, direction);
414  }
415 #endif
416  /* see if this sgh requires us to consider file md5 */
417  if (!FileForceMd5() && (sgh == NULL ||
419  {
420  SCLogDebug("disabling md5 for flow");
421  FileDisableMd5(pflow, direction);
422  }
423 
424  /* see if this sgh requires us to consider file sha1 */
425  if (!FileForceSha1() && (sgh == NULL ||
427  {
428  SCLogDebug("disabling sha1 for flow");
429  FileDisableSha1(pflow, direction);
430  }
431 
432  /* see if this sgh requires us to consider file sha256 */
433  if (!FileForceSha256() && (sgh == NULL ||
435  {
436  SCLogDebug("disabling sha256 for flow");
437  FileDisableSha256(pflow, direction);
438  }
439 
440  /* see if this sgh requires us to consider filesize */
441  if (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE))
442  {
443  SCLogDebug("disabling filesize for flow");
444  FileDisableFilesize(pflow, direction);
445  }
446 }
447 
448 static inline void
449 DetectRunPostGetFirstRuleGroup(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
450 {
451  if ((p->flowflags & FLOW_PKT_TOSERVER) && !(pflow->flags & FLOW_SGH_TOSERVER)) {
452  /* first time we see this toserver sgh, store it */
453  pflow->sgh_toserver = sgh;
454  pflow->flags |= FLOW_SGH_TOSERVER;
455 
456  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
457  if (pflow->protoctx != NULL) {
458  TcpSession *ssn = pflow->protoctx;
459  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.client");
461  }
462  }
463 
464  DetectPostInspectFileFlagsUpdate(pflow,
465  pflow->sgh_toserver, STREAM_TOSERVER);
466 
467  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && !(pflow->flags & FLOW_SGH_TOCLIENT)) {
468  pflow->sgh_toclient = sgh;
469  pflow->flags |= FLOW_SGH_TOCLIENT;
470 
471  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
472  if (pflow->protoctx != NULL) {
473  TcpSession *ssn = pflow->protoctx;
474  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.server");
476  }
477  }
478 
479  DetectPostInspectFileFlagsUpdate(pflow,
480  pflow->sgh_toclient, STREAM_TOCLIENT);
481  }
482 }
483 
484 static inline void DetectRunGetRuleGroup(
485  const DetectEngineCtx *de_ctx,
486  Packet * const p, Flow * const pflow,
487  DetectRunScratchpad *scratch)
488 {
489  const SigGroupHead *sgh = NULL;
490 
491  if (pflow) {
492  bool use_flow_sgh = false;
493  /* Get the stored sgh from the flow (if any). Make sure we're not using
494  * the sgh for icmp error packets part of the same stream. */
495  if (IP_GET_IPPROTO(p) == pflow->proto) { /* filter out icmp */
497  if ((p->flowflags & FLOW_PKT_TOSERVER) && (pflow->flags & FLOW_SGH_TOSERVER)) {
498  sgh = pflow->sgh_toserver;
499  SCLogDebug("sgh = pflow->sgh_toserver; => %p", sgh);
500  use_flow_sgh = true;
501  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && (pflow->flags & FLOW_SGH_TOCLIENT)) {
502  sgh = pflow->sgh_toclient;
503  SCLogDebug("sgh = pflow->sgh_toclient; => %p", sgh);
504  use_flow_sgh = true;
505  }
507  }
508 
509  if (!(use_flow_sgh)) {
511  sgh = SigMatchSignaturesGetSgh(de_ctx, p);
513 
514  /* HACK: prevent the wrong sgh (or NULL) from being stored in the
515  * flow's sgh pointers */
517  ; /* no-op */
518  } else {
519  /* store the found sgh (or NULL) in the flow to save us
520  * from looking it up again for the next packet.
521  * Also run other tasks */
522  DetectRunPostGetFirstRuleGroup(p, pflow, sgh);
523  }
524  }
525  } else { /* p->flags & PKT_HAS_FLOW */
526  /* no flow */
527 
529  sgh = SigMatchSignaturesGetSgh(de_ctx, p);
531  }
532 
533  scratch->sgh = sgh;
534 }
535 
536 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
537  DetectEngineThreadCtx *det_ctx,
538  Flow * const pflow, Packet * const p)
539 {
540  if (pflow) {
541  /* set the iponly stuff */
542  if (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)
544  if (pflow->flags & FLOW_TOSERVER_IPONLY_SET)
546 
549  {
550  SCLogDebug("testing against \"ip-only\" signatures");
551 
553  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, &det_ctx->io_ctx, p);
555 
556  /* save in the flow that we scanned this direction... */
557  FlowSetIPOnlyFlag(pflow, p->flowflags & FLOW_PKT_TOSERVER ? 1 : 0);
558 
559  } else if (((p->flowflags & FLOW_PKT_TOSERVER) &&
560  (pflow->flags & FLOW_TOSERVER_IPONLY_SET)) ||
561  ((p->flowflags & FLOW_PKT_TOCLIENT) &&
562  (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)))
563  {
564  /* If we have a drop from IP only module,
565  * we will drop the rest of the flow packets
566  * This will apply only to inline/IPS */
567  if (pflow->flags & FLOW_ACTION_DROP) {
568  PACKET_DROP(p);
569  }
570  }
571  } else { /* p->flags & PKT_HAS_FLOW */
572  /* no flow */
573 
574  /* Even without flow we should match the packet src/dst */
576  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx,
577  &det_ctx->io_ctx, p);
579  }
580 }
581 
582 /* returns 0 if no match, 1 if match */
583 static inline int DetectRunInspectRuleHeader(
584  const Packet *p,
585  const Flow *f,
586  const Signature *s,
587  const uint32_t sflags,
588  const uint8_t s_proto_flags)
589 {
590  /* check if this signature has a requirement for flowvars of some type
591  * and if so, if we actually have any in the flow. If not, the sig
592  * can't match and we skip it. */
593  if ((p->flags & PKT_HAS_FLOW) && (sflags & SIG_FLAG_REQUIRE_FLOWVAR)) {
594  DEBUG_VALIDATE_BUG_ON(f == NULL);
595 
596  int m = f->flowvar ? 1 : 0;
597 
598  /* no flowvars? skip this sig */
599  if (m == 0) {
600  SCLogDebug("skipping sig as the flow has no flowvars and sig "
601  "has SIG_FLAG_REQUIRE_FLOWVAR flag set.");
602  return 0;
603  }
604  }
605 
606  if ((s_proto_flags & DETECT_PROTO_IPV4) && !PKT_IS_IPV4(p)) {
607  SCLogDebug("ip version didn't match");
608  return 0;
609  }
610  if ((s_proto_flags & DETECT_PROTO_IPV6) && !PKT_IS_IPV6(p)) {
611  SCLogDebug("ip version didn't match");
612  return 0;
613  }
614 
615  if (DetectProtoContainsProto(&s->proto, IP_GET_IPPROTO(p)) == 0) {
616  SCLogDebug("proto didn't match");
617  return 0;
618  }
619 
620  /* check the source & dst port in the sig */
621  if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP || p->proto == IPPROTO_SCTP) {
622  if (!(sflags & SIG_FLAG_DP_ANY)) {
623  if (p->flags & PKT_IS_FRAGMENT)
624  return 0;
625  DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp);
626  if (dport == NULL) {
627  SCLogDebug("dport didn't match.");
628  return 0;
629  }
630  }
631  if (!(sflags & SIG_FLAG_SP_ANY)) {
632  if (p->flags & PKT_IS_FRAGMENT)
633  return 0;
634  DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp);
635  if (sport == NULL) {
636  SCLogDebug("sport didn't match.");
637  return 0;
638  }
639  }
640  } else if ((sflags & (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) != (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) {
641  SCLogDebug("port-less protocol and sig needs ports");
642  return 0;
643  }
644 
645  /* check the destination address */
646  if (!(sflags & SIG_FLAG_DST_ANY)) {
647  if (PKT_IS_IPV4(p)) {
649  return 0;
650  } else if (PKT_IS_IPV6(p)) {
652  return 0;
653  }
654  }
655  /* check the source address */
656  if (!(sflags & SIG_FLAG_SRC_ANY)) {
657  if (PKT_IS_IPV4(p)) {
659  return 0;
660  } else if (PKT_IS_IPV6(p)) {
662  return 0;
663  }
664  }
665 
666  return 1;
667 }
668 
669 /* returns 0 if no match, 1 if match */
670 static inline int DetectRunInspectRulePacketMatches(
671  ThreadVars *tv,
672  DetectEngineThreadCtx *det_ctx,
673  Packet *p,
674  const Flow *f,
675  const Signature *s)
676 {
677  /* run the packet match functions */
678  if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) {
681 
682  SCLogDebug("running match functions, sm %p", smd);
683  if (smd != NULL) {
684  while (1) {
686  if (sigmatch_table[smd->type].Match(tv, det_ctx, p, s, smd->ctx) <= 0) {
687  KEYWORD_PROFILING_END(det_ctx, smd->type, 0);
688  SCLogDebug("no match");
689  return 0;
690  }
691  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
692  if (smd->is_last) {
693  SCLogDebug("match and is_last");
694  break;
695  }
696  smd++;
697  }
698  }
699  }
700  return 1;
701 }
702 
703 /** \internal
704  * \brief run packet/stream prefilter engines
705  */
706 static inline void DetectRunPrefilterPkt(
707  ThreadVars *tv,
708  DetectEngineCtx *de_ctx,
709  DetectEngineThreadCtx *det_ctx,
710  Packet *p,
711  DetectRunScratchpad *scratch
712 )
713 {
714  DetectPrefilterSetNonPrefilterList(p, det_ctx, scratch);
715 
716  /* create our prefilter mask */
717  PacketCreateMask(p, &scratch->pkt_mask, scratch->alproto, scratch->app_decoder_events);
718 
719  /* build and prefilter non_pf list against the mask of the packet */
721  det_ctx->non_pf_id_cnt = 0;
722  if (likely(det_ctx->non_pf_store_cnt > 0)) {
723  DetectPrefilterBuildNonPrefilterList(det_ctx, scratch->pkt_mask, scratch->alproto);
724  }
726 
727  /* run the prefilter engines */
728  Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags);
729  /* create match list if we have non-pf and/or pf */
730  if (det_ctx->non_pf_store_cnt || det_ctx->pmq.rule_id_array_cnt) {
732  DetectPrefilterMergeSort(de_ctx, det_ctx);
734  }
735 
736 #ifdef PROFILING
737  if (tv) {
738  StatsAddUI64(tv, det_ctx->counter_mpm_list,
739  (uint64_t)det_ctx->pmq.rule_id_array_cnt);
740  StatsAddUI64(tv, det_ctx->counter_nonmpm_list,
741  (uint64_t)det_ctx->non_pf_store_cnt);
742  /* non mpm sigs after mask prefilter */
743  StatsAddUI64(tv, det_ctx->counter_fnonmpm_list,
744  (uint64_t)det_ctx->non_pf_id_cnt);
745  }
746 #endif
747 }
748 
749 static inline void DetectRulePacketRules(
750  ThreadVars * const tv,
751  DetectEngineCtx * const de_ctx,
752  DetectEngineThreadCtx * const det_ctx,
753  Packet * const p,
754  Flow * const pflow,
755  const DetectRunScratchpad *scratch
756 )
757 {
758  const Signature *s = NULL;
759  const Signature *next_s = NULL;
760 
761  /* inspect the sigs against the packet */
762  /* Prefetch the next signature. */
763  SigIntId match_cnt = det_ctx->match_array_cnt;
764 #ifdef PROFILING
765  if (tv) {
766  StatsAddUI64(tv, det_ctx->counter_match_list,
767  (uint64_t)match_cnt);
768  }
769 #endif
770  Signature **match_array = det_ctx->match_array;
771 
772  SGH_PROFILING_RECORD(det_ctx, scratch->sgh);
773 #ifdef PROFILING
774 #ifdef HAVE_LIBJANSSON
775  if (match_cnt >= de_ctx->profile_match_logging_threshold)
776  RulesDumpMatchArray(det_ctx, scratch->sgh, p);
777 #endif
778 #endif
779 
780  uint32_t sflags, next_sflags = 0;
781  if (match_cnt) {
782  next_s = *match_array++;
783  next_sflags = next_s->flags;
784  }
785  while (match_cnt--) {
787  uint8_t alert_flags = 0;
788  bool state_alert = false;
789 #ifdef PROFILING
790  bool smatch = false; /* signature match */
791 #endif
792  s = next_s;
793  sflags = next_sflags;
794  if (match_cnt) {
795  next_s = *match_array++;
796  next_sflags = next_s->flags;
797  }
798  const uint8_t s_proto_flags = s->proto.flags;
799 
800  SCLogDebug("inspecting signature id %"PRIu32"", s->id);
801 
802  if (s->app_inspect != NULL) {
803  goto next; // handle sig in DetectRunTx
804  }
805 
806  /* don't run mask check for stateful rules.
807  * There we depend on prefilter */
808  if ((s->mask & scratch->pkt_mask) != s->mask) {
809  SCLogDebug("mask mismatch %x & %x != %x", s->mask, scratch->pkt_mask, s->mask);
810  goto next;
811  }
812 
813  if (unlikely(sflags & SIG_FLAG_DSIZE)) {
814  if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) {
815  SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u",
816  p->payload_len, s->dsize_low, s->dsize_high);
817  goto next;
818  }
819  }
820 
821  /* if the sig has alproto and the session as well they should match */
822  if (likely(sflags & SIG_FLAG_APPLAYER)) {
823  if (s->alproto != ALPROTO_UNKNOWN && s->alproto != scratch->alproto) {
824  if (s->alproto == ALPROTO_DCERPC) {
825  if (scratch->alproto != ALPROTO_SMB && scratch->alproto != ALPROTO_SMB2) {
826  SCLogDebug("DCERPC sig, alproto not SMB or SMB2");
827  goto next;
828  }
829  } else {
830  SCLogDebug("alproto mismatch");
831  goto next;
832  }
833  }
834  }
835 
836  if (DetectRunInspectRuleHeader(p, pflow, s, sflags, s_proto_flags) == 0) {
837  goto next;
838  }
839 
840  /* Check the payload keywords. If we are a MPM sig and we've made
841  * to here, we've had at least one of the patterns match */
842  if (s->sm_arrays[DETECT_SM_LIST_PMATCH] != NULL) {
844  /* if we have stream msgs, inspect against those first,
845  * but not for a "dsize" signature */
846  if (sflags & SIG_FLAG_REQUIRE_STREAM) {
847  int pmatch = 0;
848  if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
849  pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, pflow, p);
850  if (pmatch) {
852  /* Tell the engine that this reassembled stream can drop the
853  * rest of the pkts with no further inspection */
854  if (s->action & ACTION_DROP)
855  alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
856 
857  alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
858  }
859  }
860  /* no match? then inspect packet payload */
861  if (pmatch == 0) {
862  SCLogDebug("no match in stream, fall back to packet payload");
863 
864  /* skip if we don't have to inspect the packet and segment was
865  * added to stream */
866  if (!(sflags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
867  goto next;
868  }
869 
870  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, pflow, p) != 1) {
871  goto next;
872  }
873  }
874  } else {
875  if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, pflow, p) != 1) {
876  goto next;
877  }
878  }
879  }
880 
881  if (DetectRunInspectRulePacketMatches(tv, det_ctx, p, pflow, s) == 0)
882  goto next;
883 
884 #ifdef PROFILING
885  smatch = true;
886 #endif
887  DetectRunPostMatch(tv, det_ctx, p, s);
888 
889  if (!(sflags & SIG_FLAG_NOALERT)) {
890  /* stateful sigs call PacketAlertAppend from DeStateDetectStartDetection */
891  if (!state_alert)
892  PacketAlertAppend(det_ctx, s, p, 0, alert_flags);
893  } else {
894  /* apply actions even if not alerting */
895  DetectSignatureApplyActions(p, s, alert_flags);
896  }
897 next:
898  DetectVarProcessList(det_ctx, pflow, p);
899  DetectReplaceFree(det_ctx);
900  RULE_PROFILING_END(det_ctx, s, smatch, p);
901 
902  det_ctx->flags = 0;
903  continue;
904  }
905 }
906 
907 static DetectRunScratchpad DetectRunSetup(
908  const DetectEngineCtx *de_ctx,
909  DetectEngineThreadCtx *det_ctx,
910  Packet * const p, Flow * const pflow)
911 {
912  AppProto alproto = ALPROTO_UNKNOWN;
913  uint8_t flow_flags = 0; /* flow/state flags */
914  bool app_decoder_events = false;
915 
917 
918 #ifdef UNITTESTS
919  p->alerts.cnt = 0;
920 #endif
921  det_ctx->ticker++;
922  det_ctx->filestore_cnt = 0;
923  det_ctx->base64_decoded_len = 0;
924  det_ctx->raw_stream_progress = 0;
925 
926 #ifdef DEBUG
927  if (p->flags & PKT_STREAM_ADD) {
928  det_ctx->pkt_stream_add_cnt++;
929  }
930 #endif
931 
932  /* grab the protocol state we will detect on */
933  if (p->flags & PKT_HAS_FLOW) {
934  DEBUG_VALIDATE_BUG_ON(pflow == NULL);
935 
936  if (p->flowflags & FLOW_PKT_TOSERVER) {
937  flow_flags = STREAM_TOSERVER;
938  SCLogDebug("flag STREAM_TOSERVER set");
939  } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
940  flow_flags = STREAM_TOCLIENT;
941  SCLogDebug("flag STREAM_TOCLIENT set");
942  }
943  SCLogDebug("p->flowflags 0x%02x", p->flowflags);
944 
945  if (p->flags & PKT_STREAM_EOF) {
946  flow_flags |= STREAM_EOF;
947  SCLogDebug("STREAM_EOF set");
948  }
949 
950  /* store tenant_id in the flow so that we can use it
951  * for creating pseudo packets */
952  if (p->tenant_id > 0 && pflow->tenant_id == 0) {
953  pflow->tenant_id = p->tenant_id;
954  }
955 
956  /* live ruleswap check for flow updates */
957  if (pflow->de_ctx_version == 0) {
958  /* first time this flow is inspected, set id */
959  pflow->de_ctx_version = de_ctx->version;
960  } else if (pflow->de_ctx_version != de_ctx->version) {
961  /* first time we inspect flow with this de_ctx, reset */
962  pflow->flags &= ~FLOW_SGH_TOSERVER;
963  pflow->flags &= ~FLOW_SGH_TOCLIENT;
964  pflow->sgh_toserver = NULL;
965  pflow->sgh_toclient = NULL;
966 
967  pflow->de_ctx_version = de_ctx->version;
968  GenericVarFree(pflow->flowvar);
969  pflow->flowvar = NULL;
970 
972  }
973 
974  /* Retrieve the app layer state and protocol and the tcp reassembled
975  * stream chunks. */
976  if ((p->proto == IPPROTO_TCP && (p->flags & PKT_STREAM_EST)) ||
977  (p->proto == IPPROTO_UDP) ||
978  (p->proto == IPPROTO_SCTP && (p->flowflags & FLOW_PKT_ESTABLISHED)))
979  {
980  /* update flow flags with knowledge on disruptions */
981  flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
982  alproto = FlowGetAppProtocol(pflow);
983  if (p->proto == IPPROTO_TCP && pflow->protoctx &&
986  flow_flags |= STREAM_FLUSH;
987  }
988  SCLogDebug("alproto %u", alproto);
989  } else {
990  SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
991  }
992 
993  app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
994  }
995 
998  return pad;
999 }
1000 
1001 static inline void DetectRunPostRules(
1002  ThreadVars *tv,
1003  DetectEngineCtx *de_ctx,
1004  DetectEngineThreadCtx *det_ctx,
1005  Packet * const p,
1006  Flow * const pflow,
1007  DetectRunScratchpad *scratch)
1008 {
1009  /* see if we need to increment the inspect_id and reset the de_state */
1010  if (pflow && pflow->alstate && AppLayerParserProtocolSupportsTxs(p->proto, scratch->alproto)) {
1012  DeStateUpdateInspectTransactionId(pflow, scratch->flow_flags, (scratch->sgh == NULL));
1014  }
1015 
1016  /* so now let's iterate the alerts and remove the ones after a pass rule
1017  * matched (if any). This is done inside PacketAlertFinalize() */
1018  /* PR: installed "tag" keywords are handled after the threshold inspection */
1019 
1021  PacketAlertFinalize(de_ctx, det_ctx, p);
1022  if (p->alerts.cnt > 0) {
1023  StatsAddUI64(tv, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt);
1024  }
1026 }
1027 
1028 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
1029  Packet *p, Flow * const pflow)
1030 {
1032  /* cleanup pkt specific part of the patternmatcher */
1033  PacketPatternCleanup(det_ctx);
1034 
1035  if (pflow != NULL) {
1036  /* update inspected tracker for raw reassembly */
1037  if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL &&
1038  (p->flags & PKT_STREAM_EST))
1039  {
1041  det_ctx->raw_stream_progress);
1042  }
1043  }
1045  SCReturn;
1046 }
1047 
1049 {
1051  det_ctx->tx_candidates = SCCalloc(size, sizeof(RuleMatchCandidateTx));
1052  if (det_ctx->tx_candidates == NULL) {
1053  FatalError(SC_ERR_MEM_ALLOC, "failed to allocate %"PRIu64" bytes",
1054  (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
1055  }
1056  det_ctx->tx_candidates_size = size;
1057  SCLogDebug("array initialized to %u elements (%"PRIu64" bytes)",
1058  size, (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
1059 }
1060 
1062 {
1063  SCFree(det_ctx->tx_candidates);
1064  det_ctx->tx_candidates_size = 0;
1065 }
1066 
1067 /* if size >= cur_space */
1068 static inline bool RuleMatchCandidateTxArrayHasSpace(const DetectEngineThreadCtx *det_ctx,
1069  const uint32_t need)
1070 {
1071  if (det_ctx->tx_candidates_size >= need)
1072  return 1;
1073  return 0;
1074 }
1075 
1076 /* realloc */
1077 static int RuleMatchCandidateTxArrayExpand(DetectEngineThreadCtx *det_ctx, const uint32_t needed)
1078 {
1079  const uint32_t old_size = det_ctx->tx_candidates_size;
1080  uint32_t new_size = needed;
1081  void *ptmp = SCRealloc(det_ctx->tx_candidates, (new_size * sizeof(RuleMatchCandidateTx)));
1082  if (ptmp == NULL) {
1083  FatalError(SC_ERR_MEM_ALLOC, "failed to expand to %"PRIu64" bytes",
1084  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx)));
1085  // TODO can this be handled more gracefully?
1086  }
1087  det_ctx->tx_candidates = ptmp;
1088  det_ctx->tx_candidates_size = new_size;
1089  SCLogDebug("array expanded from %u to %u elements (%"PRIu64" bytes -> %"PRIu64" bytes)",
1090  old_size, new_size, (uint64_t)(old_size * sizeof(RuleMatchCandidateTx)),
1091  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx))); (void)old_size;
1092  return 1;
1093 }
1094 
1095 
1096 /* TODO maybe let one with flags win if equal? */
1097 static int
1098 DetectRunTxSortHelper(const void *a, const void *b)
1099 {
1100  const RuleMatchCandidateTx *s0 = a;
1101  const RuleMatchCandidateTx *s1 = b;
1102  if (s1->id == s0->id)
1103  return 0;
1104  else
1105  return s0->id > s1->id ? -1 : 1;
1106 }
1107 
1108 #if 0
1109 #define TRACE_SID_TXS(sid,txs,...) \
1110  do { \
1111  char _trace_buf[2048]; \
1112  snprintf(_trace_buf, sizeof(_trace_buf), __VA_ARGS__); \
1113  SCLogNotice("%p/%"PRIu64"/%u: %s", txs->tx_ptr, txs->tx_id, sid, _trace_buf); \
1114  } while(0)
1115 #else
1116 #define TRACE_SID_TXS(sid,txs,...)
1117 #endif
1118 
1119 /** \internal
1120  * \brief inspect a rule against a transaction
1121  *
1122  * Inspect a rule. New detection or continued stateful
1123  * detection.
1124  *
1125  * \param stored_flags pointer to stored flags or NULL.
1126  * If stored_flags is set it means we're continueing
1127  * inspection from an earlier run.
1128  *
1129  * \retval bool true sig matched, false didn't match
1130  */
1131 static bool DetectRunTxInspectRule(ThreadVars *tv,
1132  DetectEngineCtx *de_ctx,
1133  DetectEngineThreadCtx *det_ctx,
1134  Packet *p,
1135  Flow *f,
1136  const uint8_t in_flow_flags, // direction, EOF, etc
1137  void *alstate,
1138  DetectTransaction *tx,
1139  const Signature *s,
1140  uint32_t *stored_flags,
1141  RuleMatchCandidateTx *can,
1142  DetectRunScratchpad *scratch)
1143 {
1144  uint8_t flow_flags = in_flow_flags;
1145  const int direction = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1146  uint32_t inspect_flags = stored_flags ? *stored_flags : 0;
1147  int total_matches = 0;
1148  int file_no_match = 0;
1149  bool retval = false;
1150  bool mpm_before_progress = false; // is mpm engine before progress?
1151  bool mpm_in_progress = false; // is mpm engine in a buffer we will revisit?
1152 
1153  /* see if we want to pass on the FLUSH flag */
1154  if ((s->flags & SIG_FLAG_FLUSH) == 0)
1155  flow_flags &=~ STREAM_FLUSH;
1156 
1157  TRACE_SID_TXS(s->id, tx, "starting %s", direction ? "toclient" : "toserver");
1158 
1159  /* for a new inspection we inspect pkt header and packet matches */
1160  if (likely(stored_flags == NULL)) {
1161  TRACE_SID_TXS(s->id, tx, "first inspect, run packet matches");
1162  if (DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags) == 0) {
1163  TRACE_SID_TXS(s->id, tx, "DetectRunInspectRuleHeader() no match");
1164  return false;
1165  }
1166  if (DetectRunInspectRulePacketMatches(tv, det_ctx, p, f, s) == 0) {
1167  TRACE_SID_TXS(s->id, tx, "DetectRunInspectRulePacketMatches no match");
1168  return false;
1169  }
1170  /* stream mpm and negated mpm sigs can end up here with wrong proto */
1171  if (!(f->alproto == s->alproto || s->alproto == ALPROTO_UNKNOWN)) {
1172  TRACE_SID_TXS(s->id, tx, "alproto mismatch");
1173  return false;
1174  }
1175  }
1176 
1177  const DetectEngineAppInspectionEngine *engine = s->app_inspect;
1178  while (engine != NULL) { // TODO could be do {} while as s->app_inspect cannot be null
1179  TRACE_SID_TXS(s->id, tx, "engine %p inspect_flags %x", engine, inspect_flags);
1180  if (!(inspect_flags & BIT_U32(engine->id)) &&
1181  direction == engine->dir)
1182  {
1183  const bool skip_engine = (engine->alproto != 0 && engine->alproto != f->alproto);
1184  /* special case: file_data on 'alert tcp' will have engines
1185  * in the list that are not for us. */
1186  if (unlikely(skip_engine)) {
1187  engine = engine->next;
1188  continue;
1189  }
1190 
1191  /* engines are sorted per progress, except that the one with
1192  * mpm/prefilter enabled is first */
1193  if (tx->tx_progress < engine->progress) {
1194  SCLogDebug("tx progress %d < engine progress %d",
1195  tx->tx_progress, engine->progress);
1196  break;
1197  }
1198  if (engine->mpm) {
1199  if (tx->tx_progress > engine->progress) {
1200  mpm_before_progress = true;
1201  } else if (tx->tx_progress == engine->progress) {
1202  mpm_in_progress = true;
1203  }
1204  }
1205 
1206  /* run callback: but bypass stream callback if we can */
1207  int match;
1208  if (unlikely(engine->stream && can->stream_stored)) {
1209  match = can->stream_result;
1210  TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match);
1211  } else {
1212  KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list);
1213  if (engine->Callback) {
1214  match = engine->Callback(tv, de_ctx, det_ctx,
1215  s, engine->smd, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
1216  } else {
1217  BUG_ON(engine->v2.Callback == NULL);
1218  match = engine->v2.Callback(de_ctx, det_ctx, engine,
1219  s, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
1220  }
1221  TRACE_SID_TXS(s->id, tx, "engine %p match %d", engine, match);
1222  if (engine->stream) {
1223  can->stream_stored = true;
1224  can->stream_result = match;
1225  TRACE_SID_TXS(s->id, tx, "stream ran, store result %d for next tx (if any)", match);
1226  }
1227  }
1228  if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) {
1229  inspect_flags |= BIT_U32(engine->id);
1230  engine = engine->next;
1231  total_matches++;
1232  continue;
1233  } else if (match == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES) {
1234  /* if the file engine matched, but indicated more
1235  * files are still in progress, we don't set inspect
1236  * flags as these would end inspection for this tx */
1237  engine = engine->next;
1238  total_matches++;
1239  continue;
1240  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
1241  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1242  inspect_flags |= BIT_U32(engine->id);
1243  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) {
1244  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1245  inspect_flags |= BIT_U32(engine->id);
1246  file_no_match = 1;
1247  }
1248  /* implied DETECT_ENGINE_INSPECT_SIG_NO_MATCH */
1249  if (engine->mpm && mpm_before_progress) {
1250  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1251  inspect_flags |= BIT_U32(engine->id);
1252  }
1253  break;
1254  }
1255  engine = engine->next;
1256  }
1257  TRACE_SID_TXS(s->id, tx, "inspect_flags %x, total_matches %u, engine %p",
1258  inspect_flags, total_matches, engine);
1259 
1260  if (engine == NULL && total_matches) {
1261  inspect_flags |= DE_STATE_FLAG_FULL_INSPECT;
1262  TRACE_SID_TXS(s->id, tx, "MATCH");
1263  retval = true;
1264  }
1265 
1266  if (stored_flags) {
1267  *stored_flags = inspect_flags;
1268  TRACE_SID_TXS(s->id, tx, "continue inspect flags %08x", inspect_flags);
1269  } else {
1270  // store... or? If tx is done we might not want to come back to this tx
1271 
1272  // also... if mpmid tracking is enabled, we won't do a sig again for this tx...
1273  TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags);
1274  if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) {
1275  if (file_no_match) {
1276  /* if we have a mismatch on a file sig, we need to keep state.
1277  * We may get another file on the same tx (for http and smtp
1278  * at least), so for a new file we need to re-eval the sig.
1279  * Thoughts / TODO:
1280  * - not for some protos that have 1 file per tx (e.g. nfs)
1281  * - maybe we only need this for file sigs that mix with
1282  * other matches? E.g. 'POST + filename', is different than
1283  * just 'filename'.
1284  */
1285  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1286  inspect_flags, flow_flags, file_no_match);
1287  }
1288  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) {
1289  TRACE_SID_TXS(s->id, tx, "no need to store match sig, "
1290  "mpm won't trigger for it anymore");
1291 
1292  if (inspect_flags & DE_STATE_FLAG_FILE_INSPECT) {
1293  TRACE_SID_TXS(s->id, tx, "except that for new files, "
1294  "we may have to revisit anyway");
1295  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1296  inspect_flags, flow_flags, file_no_match);
1297  }
1298  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) {
1299  TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, "
1300  "mpm will revisit it");
1301  } else {
1302  TRACE_SID_TXS(s->id, tx, "storing state: flags %08x", inspect_flags);
1303  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1304  inspect_flags, flow_flags, file_no_match);
1305  }
1306  }
1307 
1308  return retval;
1309 }
1310 
1311 /** \internal
1312  * \brief get a DetectTransaction object
1313  * \retval struct filled with relevant info or all nulls/0s
1314  */
1315 static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto,
1316  void *alstate, const uint64_t tx_id, void *tx_ptr, const int tx_end_state,
1317  const uint8_t flow_flags)
1318 {
1319  const uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx_ptr, flow_flags);
1320  if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) {
1321  SCLogDebug("%"PRIu64" tx already fully inspected for %s. Flags %016"PRIx64,
1322  tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1323  detect_flags);
1324  DetectTransaction no_tx = { NULL, 0, NULL, 0, 0, 0, 0, 0, };
1325  return no_tx;
1326  }
1327 
1328  const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags);
1329  const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1330  DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(ipproto, alproto, tx_ptr);
1331  DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL;
1332  uint64_t prefilter_flags = detect_flags & APP_LAYER_TX_PREFILTER_MASK;
1333 
1334  DetectTransaction tx = {
1335  .tx_ptr = tx_ptr,
1336  .tx_id = tx_id,
1337  .de_state = tx_dir_state,
1338  .detect_flags = detect_flags,
1339  .prefilter_flags = prefilter_flags,
1340  .prefilter_flags_orig = prefilter_flags,
1341  .tx_progress = tx_progress,
1342  .tx_end_state = tx_end_state,
1343  };
1344  return tx;
1345 }
1346 
1347 static void DetectRunTx(ThreadVars *tv,
1348  DetectEngineCtx *de_ctx,
1349  DetectEngineThreadCtx *det_ctx,
1350  Packet *p,
1351  Flow *f,
1352  DetectRunScratchpad *scratch)
1353 {
1354  const uint8_t flow_flags = scratch->flow_flags;
1355  const SigGroupHead * const sgh = scratch->sgh;
1356  void * const alstate = f->alstate;
1357  const uint8_t ipproto = f->proto;
1358  const AppProto alproto = f->alproto;
1359 
1360  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
1361  uint64_t tx_id_min = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags);
1362  const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags);
1363 
1364  AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
1365  AppLayerGetTxIterState state;
1366  memset(&state, 0, sizeof(state));
1367 
1368  while (1) {
1369  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id_min, total_txs, &state);
1370  if (ires.tx_ptr == NULL)
1371  break;
1372 
1373  DetectTransaction tx = GetDetectTx(ipproto, alproto,
1374  alstate, ires.tx_id, ires.tx_ptr, tx_end_state, flow_flags);
1375  if (tx.tx_ptr == NULL) {
1376  SCLogDebug("%p/%"PRIu64" no transaction to inspect",
1377  tx.tx_ptr, tx_id_min);
1378 
1379  tx_id_min++; // next (if any) run look for +1
1380  goto next;
1381  }
1382  tx_id_min = tx.tx_id + 1; // next look for cur + 1
1383 
1384  uint32_t array_idx = 0;
1385  uint32_t total_rules = det_ctx->match_array_cnt;
1386  total_rules += (tx.de_state ? tx.de_state->cnt : 0);
1387 
1388  /* run prefilter engines and merge results into a candidates array */
1389  if (sgh->tx_engines) {
1391  DetectRunPrefilterTx(det_ctx, sgh, p, ipproto, flow_flags, alproto,
1392  alstate, &tx);
1394  SCLogDebug("%p/%"PRIu64" rules added from prefilter: %u candidates",
1395  tx.tx_ptr, tx.tx_id, det_ctx->pmq.rule_id_array_cnt);
1396 
1397  total_rules += det_ctx->pmq.rule_id_array_cnt;
1398  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1399  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1400  }
1401 
1402  for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) {
1403  const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]];
1404  const SigIntId id = s->num;
1405  det_ctx->tx_candidates[array_idx].s = s;
1406  det_ctx->tx_candidates[array_idx].id = id;
1407  det_ctx->tx_candidates[array_idx].flags = NULL;
1408  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1409  array_idx++;
1410  }
1411  } else {
1412  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1413  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1414  }
1415  }
1416 
1417  /* merge 'state' rules from the regular prefilter */
1418  uint32_t x = array_idx;
1419  for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) {
1420  const Signature *s = det_ctx->match_array[i];
1421  if (s->app_inspect != NULL) {
1422  const SigIntId id = s->num;
1423  det_ctx->tx_candidates[array_idx].s = s;
1424  det_ctx->tx_candidates[array_idx].id = id;
1425  det_ctx->tx_candidates[array_idx].flags = NULL;
1426  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1427  array_idx++;
1428 
1429  SCLogDebug("%p/%"PRIu64" rule %u (%u) added from 'match' list",
1430  tx.tx_ptr, tx.tx_id, s->id, id);
1431  }
1432  }
1433  SCLogDebug("%p/%"PRIu64" rules added from 'match' list: %u",
1434  tx.tx_ptr, tx.tx_id, array_idx - x); (void)x;
1435 
1436  /* merge stored state into results */
1437  if (tx.de_state != NULL) {
1438  const uint32_t old = array_idx;
1439 
1440  /* if tx.de_state->flags has 'new file' set and sig below has
1441  * 'file inspected' flag, reset the file part of the state */
1442  const bool have_new_file = (tx.de_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_NEW);
1443  if (have_new_file) {
1444  SCLogDebug("%p/%"PRIu64" destate: need to consider new file",
1445  tx.tx_ptr, tx.tx_id);
1447  }
1448 
1449  SigIntId state_cnt = 0;
1450  DeStateStore *tx_store = tx.de_state->head;
1451  for (; tx_store != NULL; tx_store = tx_store->next) {
1452  SCLogDebug("tx_store %p", tx_store);
1453 
1454  SigIntId store_cnt = 0;
1455  for (store_cnt = 0;
1456  store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < tx.de_state->cnt;
1457  store_cnt++, state_cnt++)
1458  {
1459  DeStateStoreItem *item = &tx_store->store[store_cnt];
1460  SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags);
1461  if (have_new_file && (item->flags & DE_STATE_FLAG_FILE_INSPECT)) {
1462  /* remove part of the state. File inspect engine will now
1463  * be able to run again */
1465  SCLogDebug("rule id %u, post file reset inspect_flags %u", item->sid, item->flags);
1466  }
1467  det_ctx->tx_candidates[array_idx].s = de_ctx->sig_array[item->sid];
1468  det_ctx->tx_candidates[array_idx].id = item->sid;
1469  det_ctx->tx_candidates[array_idx].flags = &item->flags;
1470  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1471  array_idx++;
1472  }
1473  }
1474  if (old && old != array_idx) {
1475  qsort(det_ctx->tx_candidates, array_idx, sizeof(RuleMatchCandidateTx),
1476  DetectRunTxSortHelper);
1477 
1478  SCLogDebug("%p/%"PRIu64" rules added from 'continue' list: %u",
1479  tx.tx_ptr, tx.tx_id, array_idx - old);
1480  }
1481  }
1482 
1483  det_ctx->tx_id = tx.tx_id;
1484  det_ctx->tx_id_set = 1;
1485  det_ctx->p = p;
1486 
1487  /* run rules: inspect the match candidates */
1488  for (uint32_t i = 0; i < array_idx; i++) {
1489  RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i];
1490  const Signature *s = det_ctx->tx_candidates[i].s;
1491  uint32_t *inspect_flags = det_ctx->tx_candidates[i].flags;
1492 
1493  /* deduplicate: rules_array is sorted, but not deduplicated:
1494  * both mpm and stored state could give us the same sid.
1495  * As they are back to back in that case we can check for it
1496  * here. We select the stored state one. */
1497  if ((i + 1) < array_idx) {
1498  if (det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i+1].s) {
1499  if (det_ctx->tx_candidates[i].flags != NULL) {
1500  i++;
1501  SCLogDebug("%p/%"PRIu64" inspecting SKIP NEXT: sid %u (%u), flags %08x",
1502  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1503  } else if (det_ctx->tx_candidates[i+1].flags != NULL) {
1504  SCLogDebug("%p/%"PRIu64" inspecting SKIP CURRENT: sid %u (%u), flags %08x",
1505  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1506  continue;
1507  } else {
1508  // if it's all the same, inspect the current one and skip next.
1509  i++;
1510  SCLogDebug("%p/%"PRIu64" inspecting SKIP NEXT: sid %u (%u), flags %08x",
1511  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1512  }
1513  }
1514  }
1515 
1516  SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x",
1517  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1518 
1519  if (inspect_flags) {
1521  SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x ALREADY COMPLETE",
1522  tx.tx_ptr, tx.tx_id, s->id, s->num, *inspect_flags);
1523  continue;
1524  }
1525  }
1526 
1527  if (inspect_flags) {
1528  /* continue previous inspection */
1529  SCLogDebug("%p/%"PRIu64" Continueing sid %u", tx.tx_ptr, tx.tx_id, s->id);
1530  } else {
1531  /* start new inspection */
1532  SCLogDebug("%p/%"PRIu64" Start sid %u", tx.tx_ptr, tx.tx_id, s->id);
1533  }
1534 
1535  /* call individual rule inspection */
1537  const int r = DetectRunTxInspectRule(tv, de_ctx, det_ctx, p, f, flow_flags,
1538  alstate, &tx, s, inspect_flags, can, scratch);
1539  if (r == 1) {
1540  /* match */
1541  DetectRunPostMatch(tv, det_ctx, p, s);
1542 
1543  uint8_t alert_flags = (PACKET_ALERT_FLAG_STATE_MATCH|PACKET_ALERT_FLAG_TX);
1544  if (s->action & ACTION_DROP)
1545  alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
1546 
1547  SCLogDebug("%p/%"PRIu64" sig %u (%u) matched", tx.tx_ptr, tx.tx_id, s->id, s->num);
1548  if (!(s->flags & SIG_FLAG_NOALERT)) {
1549  PacketAlertAppend(det_ctx, s, p, tx.tx_id, alert_flags);
1550  } else {
1551  DetectSignatureApplyActions(p, s, alert_flags);
1552  }
1553  }
1554  DetectVarProcessList(det_ctx, p->flow, p);
1555  RULE_PROFILING_END(det_ctx, s, r, p);
1556  }
1557 
1558  det_ctx->tx_id = 0;
1559  det_ctx->tx_id_set = 0;
1560  det_ctx->p = NULL;
1561 
1562  /* see if we have any updated state to store in the tx */
1563 
1564  uint64_t new_detect_flags = 0;
1565  /* this side of the tx is done */
1566  if (tx.tx_progress >= tx.tx_end_state) {
1567  new_detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
1568  SCLogDebug("%p/%"PRIu64" tx is done for direction %s. Flag %016"PRIx64,
1569  tx.tx_ptr, tx.tx_id,
1570  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1571  new_detect_flags);
1572  }
1573  if (tx.prefilter_flags != tx.prefilter_flags_orig) {
1574  new_detect_flags |= tx.prefilter_flags;
1575  SCLogDebug("%p/%"PRIu64" updated prefilter flags %016"PRIx64" "
1576  "(was: %016"PRIx64") for direction %s. Flag %016"PRIx64,
1578  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1579  new_detect_flags);
1580  }
1581  if (new_detect_flags != 0 &&
1582  (new_detect_flags | tx.detect_flags) != tx.detect_flags)
1583  {
1584  new_detect_flags |= tx.detect_flags;
1585  SCLogDebug("%p/%"PRIu64" Storing new flags %016"PRIx64" (was %016"PRIx64")",
1586  tx.tx_ptr, tx.tx_id, new_detect_flags, tx.detect_flags);
1587  AppLayerParserSetTxDetectFlags(ipproto, alproto, tx.tx_ptr,
1588  flow_flags, new_detect_flags);
1589  }
1590 next:
1591  InspectionBufferClean(det_ctx);
1592 
1593  if (!ires.has_next)
1594  break;
1595  }
1596 }
1597 
1598 /** \brief Apply action(s) and Set 'drop' sig info,
1599  * if applicable */
1601  const Signature *s, const uint8_t alert_flags)
1602 {
1604 
1605  if (s->action & ACTION_DROP) {
1606  if (p->alerts.drop.action == 0) {
1607  p->alerts.drop.num = s->num;
1608  p->alerts.drop.action = s->action;
1609  p->alerts.drop.s = (Signature *)s;
1610  }
1611  } else if (s->action & ACTION_PASS) {
1612  /* if an stream/app-layer match we enforce the pass for the flow */
1613  if ((p->flow != NULL) &&
1615  {
1616  FlowSetNoPacketInspectionFlag(p->flow);
1617  }
1618 
1619  }
1620 }
1621 
1622 static DetectEngineThreadCtx *GetTenantById(HashTable *h, uint32_t id)
1623 {
1624  /* technically we need to pass a DetectEngineThreadCtx struct with the
1625  * tentant_id member. But as that member is the first in the struct, we
1626  * can use the id directly. */
1627  return HashTableLookup(h, &id, 0);
1628 }
1629 
1630 static void DetectFlow(ThreadVars *tv,
1631  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1632  Packet *p)
1633 {
1634  if (p->flags & PKT_NOPACKET_INSPECTION) {
1635  /* hack: if we are in pass the entire flow mode, we need to still
1636  * update the inspect_id forward. So test for the condition here,
1637  * and call the update code if necessary. */
1638  const int pass = ((p->flow->flags & FLOW_NOPACKET_INSPECTION));
1639  const AppProto alproto = FlowGetAppProtocol(p->flow);
1640  if (pass && AppLayerParserProtocolSupportsTxs(p->proto, alproto)) {
1641  uint8_t flags;
1642  if (p->flowflags & FLOW_PKT_TOSERVER) {
1643  flags = STREAM_TOSERVER;
1644  } else {
1645  flags = STREAM_TOCLIENT;
1646  }
1647  flags = FlowGetDisruptionFlags(p->flow, flags);
1648  DeStateUpdateInspectTransactionId(p->flow, flags, true);
1649  }
1650  SCLogDebug("p->pcap %"PRIu64": no detection on packet, "
1651  "PKT_NOPACKET_INSPECTION is set", p->pcap_cnt);
1652  return;
1653  }
1654 
1655  /* see if the packet matches one or more of the sigs */
1656  (void)DetectRun(tv, de_ctx, det_ctx, p);
1657 }
1658 
1659 
1660 static void DetectNoFlow(ThreadVars *tv,
1661  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1662  Packet *p)
1663 {
1664  /* No need to perform any detection on this packet, if the the given flag is set.*/
1665  if ((p->flags & PKT_NOPACKET_INSPECTION) ||
1667  {
1668  return;
1669  }
1670 
1671  /* see if the packet matches one or more of the sigs */
1672  DetectRun(tv, de_ctx, det_ctx, p);
1673  return;
1674 }
1675 
1676 /** \brief Detection engine thread wrapper.
1677  * \param tv thread vars
1678  * \param p packet to inspect
1679  * \param data thread specific data
1680  * \param pq packet queue
1681  * \retval TM_ECODE_FAILED error
1682  * \retval TM_ECODE_OK ok
1683  */
1684 TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
1685 {
1687 
1688  DetectEngineCtx *de_ctx = NULL;
1689  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
1690  if (det_ctx == NULL) {
1691  printf("ERROR: Detect has no thread ctx\n");
1692  goto error;
1693  }
1694 
1695  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
1696  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
1697  SCLogDebug("Detect Engine using new det_ctx - %p",
1698  det_ctx);
1699  }
1700 
1701  /* if in MT mode _and_ we have tenants registered, use
1702  * MT logic. */
1703  if (det_ctx->mt_det_ctxs_cnt > 0 && det_ctx->TenantGetId != NULL)
1704  {
1705  uint32_t tenant_id = p->tenant_id;
1706  if (tenant_id == 0)
1707  tenant_id = det_ctx->TenantGetId(det_ctx, p);
1708  if (tenant_id > 0 && tenant_id < det_ctx->mt_det_ctxs_cnt) {
1709  p->tenant_id = tenant_id;
1710  det_ctx = GetTenantById(det_ctx->mt_det_ctxs_hash, tenant_id);
1711  if (det_ctx == NULL)
1712  return TM_ECODE_OK;
1713  de_ctx = det_ctx->de_ctx;
1714  if (de_ctx == NULL)
1715  return TM_ECODE_OK;
1716 
1717  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
1718  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
1719  SCLogDebug("MT de_ctx %p det_ctx %p (tenant %u)", de_ctx, det_ctx, tenant_id);
1720  }
1721  } else {
1722  /* use default if no tenants are registered for this packet */
1723  de_ctx = det_ctx->de_ctx;
1724  }
1725  } else {
1726  de_ctx = det_ctx->de_ctx;
1727  }
1728 
1729  if (p->flow) {
1730  DetectFlow(tv, de_ctx, det_ctx, p);
1731  } else {
1732  DetectNoFlow(tv, de_ctx, det_ctx, p);
1733  }
1734  return TM_ECODE_OK;
1735 error:
1736  return TM_ECODE_FAILED;
1737 }
1738 
1739 /** \brief disable file features we don't need
1740  * Called if we have no detection engine.
1741  */
1743 {
1744  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOSERVER);
1745  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOCLIENT);
1746 }
1747 
1748 #ifdef UNITTESTS
1749 /**
1750  * \brief wrapper for old tests
1751  */
1753  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1754  Packet *p)
1755 {
1756  DetectRun(th_v, de_ctx, det_ctx, p);
1757 }
1758 #endif
1759 
1760 /*
1761  * TESTS
1762  */
1763 
1764 #ifdef UNITTESTS
1765 #include "tests/detect.c"
1766 #endif
1767 
void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size)
Definition: detect.c:1048
#define SIG_FLAG_FILESTORE
Definition: detect.h:241
uint16_t filestore_cnt
Definition: detect.h:1311
#define PACKET_ALERT_FLAG_STREAM_MATCH
Definition: decode.h:283
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1403
SignatureNonPrefilterStore * non_pf_syn_store_array
Definition: detect.h:1307
DetectProto proto
Definition: detect.h:509
uint16_t flags
#define DETECT_PROTO_IPV6
#define SCLogDebug(...)
Definition: util-debug.h:335
Signature ** sig_array
Definition: detect.h:735
uint32_t tx_candidates_size
Definition: detect.h:1050
struct Flow_ * flow
Definition: decode.h:444
#define KEYWORD_PROFILING_END(ctx, type, m)
#define SignatureMask
Definition: detect.h:284
void PacketPatternCleanup(DetectEngineThreadCtx *det_ctx)
cleans up the mpm instance after a match
struct HtpBodyChunk_ * next
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:285
const struct SigGroupHead_ * sgh_toclient
Definition: flow.h:440
const Signature * s
Definition: detect.h:956
#define KEYWORD_PROFILING_SET_LIST(ctx, list)
uint16_t dsize_high
Definition: detect.h:499
int FileForceSha256(void)
Definition: util-file.c:145
#define BUG_ON(x)
uint32_t non_pf_store_cnt
Definition: detect.h:1053
#define PACKET_TEST_ACTION(p, a)
Definition: decode.h:870
uint32_t flags
Definition: detect.h:493
uint8_t proto
Definition: flow.h:346
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
uint32_t id
Definition: detect.h:525
#define ICMPV4_DEST_UNREACH_IS_VALID(p)
struct SigGroupHead_ * sh
Definition: detect.h:209
#define DE_STATE_FLAG_SIG_CANT_MATCH
int DetectEngineInspectStreamPayload(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p)
Do the content inspection & validation for a signature on the raw stream.
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
#define DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH
Definition: detect.h:286
uint32_t non_pf_other_store_cnt
Definition: detect.h:1303
uint16_t counter_fnonmpm_list
Definition: detect.h:1003
#define unlikely(expr)
Definition: util-optimize.h:35
#define SIG_GROUP_HEAD_HAVEFILEMD5
Definition: detect.h:1192
uint8_t is_last
Definition: detect.h:335
Port sp
Definition: decode.h:414
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void FlowSetIPOnlyFlag(Flow *f, int direction)
Set the IPOnly scanned flag for &#39;direction&#39;.
Definition: flow.c:186
#define FLOW_TOCLIENT_IPONLY_SET
Definition: flow.h:55
void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx)
Definition: detect.c:1061
void FileDisableFilesize(Flow *f, uint8_t direction)
disable file size tracking for this flow
Definition: util-file.c:1153
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
#define PKT_IS_FRAGMENT
Definition: decode.h:1116
uint32_t tenant_id
Definition: flow.h:372
DetectEngineStateDirection dir_state[2]
Port dp
Definition: decode.h:422
void FileDisableSha1(Flow *f, uint8_t direction)
disable file sha1 calc for this flow
Definition: util-file.c:1075
void FileDisableMd5(Flow *f, uint8_t direction)
disable file md5 calc for this flow
Definition: util-file.c:1036
uint8_t action
Definition: decode.h:270
#define FLOW_ACTION_DROP
Definition: flow.h:63
#define SIG_GROUP_HEAD_HAVEFILESHA256
Definition: detect.h:1195
RuleMatchCandidateTx * tx_candidates
Definition: detect.h:1049
#define SIG_FLAG_REQUIRE_STREAM
Definition: detect.h:231
SigIntId * non_pf_id_array
Definition: detect.h:973
uint16_t pad
Address dst
Definition: decode.h:412
DetectEngineStateDirection * de_state
#define FLOW_PKT_ESTABLISHED
Definition: flow.h:195
#define PKT_IS_IPV6(p)
Definition: decode.h:251
void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, const Packet *p)
Data needed for Match()
Definition: detect.h:333
Container for matching data for a signature group.
Definition: detect.h:1295
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
uint64_t pcap_cnt
Definition: decode.h:566
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:239
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
#define SIG_FLAG_REQUIRE_PACKET
Definition: detect.h:230
#define FLOW_TOSERVER_IPONLY_SET
Definition: flow.h:53
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:992
uint32_t non_pf_syn_store_cnt
Definition: detect.h:1304
uint16_t AppProto
TCPHdr * tcph
Definition: decode.h:525
#define PACKET_ALERT_FLAG_DROP_FLOW
Definition: decode.h:279
Signature container.
Definition: detect.h:492
struct DetectEngineAppInspectionEngine_::@100 v2
#define DEBUG_VALIDATE_PACKET(p)
uint8_t action
Definition: detect.h:505
uint16_t addr_src_match6_cnt
Definition: detect.h:518
void IPOnlyMatchPacket(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const DetectEngineIPOnlyCtx *io_ctx, DetectEngineIPOnlyThreadCtx *io_tctx, Packet *p)
Match a packet against the IP Only detection engine contexts.
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get &#39;disruption&#39; flags: GAP/DEPTH/PASS
Definition: flow.c:1009
#define SIG_FLAG_APPLAYER
Definition: detect.h:225
void DetectRunStoreStateTx(const SigGroupHead *sgh, Flow *f, void *tx, uint64_t tx_id, const Signature *s, uint32_t inspect_flags, uint8_t flow_flags, const uint16_t file_no_match)
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:818
#define PKT_IS_IPV4(p)
Definition: decode.h:250
void * protoctx
Definition: flow.h:398
SigIntId num
Definition: decode.h:269
struct DeStateStore_ * next
DetectMatchAddressIPv6 * addr_dst_match6
Definition: detect.h:522
SigIntId num
Definition: detect.h:502
uint64_t AppLayerParserGetTransactionInspectId(AppLayerParserState *pstate, uint8_t direction)
int FileForceMd5(void)
Definition: util-file.c:135
main detection engine ctx
Definition: detect.h:720
uint16_t addr_src_match4_cnt
Definition: detect.h:516
PrefilterEngine * tx_engines
Definition: detect.h:1317
const uint8_t flow_flags
Definition: detect.c:64
const uint64_t prefilter_flags_orig
int DetectAddressMatchIPv6(const DetectMatchAddressIPv6 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
const struct Signature_ * s
Definition: decode.h:272
DetectPort * udp
Definition: detect.h:659
#define SIG_FLAG_DST_ANY
Definition: detect.h:219
void * alstate
Definition: flow.h:436
#define APP_LAYER_TX_INSPECTED_FLAG
DetectMatchAddressIPv4 * addr_dst_match4
Definition: detect.h:519
int DetectEngineInspectPacketPayload(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p)
Do the content inspection & validation for a signature.
#define KEYWORD_PROFILING_START
const bool app_decoder_events
Definition: detect.c:65
#define FLOW_NOPACKET_INSPECTION
Definition: flow.h:58
#define SCCalloc(nm, a)
Definition: util-mem.h:205
DetectMatchAddressIPv6 * addr_src_match6
Definition: detect.h:523
DetectPort * dp
Definition: detect.h:531
#define SIG_FLAG_SP_ANY
Definition: detect.h:220
DetectPort * tcp
Definition: detect.h:658
void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t ipproto, const uint8_t flow_flags, const AppProto alproto, void *alstate, DetectTransaction *tx)
run prefilter engines on a transaction
uint8_t proto
Definition: decode.h:429
struct SigGroupHead_ * sgh[256]
Definition: detect.h:660
#define DETECT_ENGINE_INSPECT_SIG_MATCH
DetectPort * DetectPortLookupGroup(DetectPort *dp, uint16_t port)
Function that find the group matching address in a group head.
uint16_t filestore_cnt
Definition: detect.h:996
Data structures and function prototypes for keeping state for the detection engine.
void FileDisableMagic(Flow *f, uint8_t direction)
disable file magic lookups for this flow
Definition: util-file.c:1005
struct DetectEngineAppInspectionEngine_ * next
Definition: detect.h:426
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1131
#define SIG_FLAG_FLUSH
Definition: detect.h:235
uint16_t counter_alerts
Definition: detect.h:999
uint32_t(* TenantGetId)(const void *, const Packet *p)
Definition: detect.h:983
void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t flags)
DetectMatchAddressIPv4 * addr_src_match4
Definition: detect.h:520
#define STREAM_EOF
Definition: stream.h:30
#define APP_LAYER_TX_PREFILTER_MASK
void DeStateUpdateInspectTransactionId(Flow *f, const uint8_t flags, const bool tag_txs_as_inspected)
update flow&#39;s inspection id&#39;s
void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
wrapper for old tests
Definition: detect.c:1752
#define SCEnter(...)
Definition: util-debug.h:337
PrefilterRuleStore pmq
Definition: detect.h:1061
uint32_t profile_match_logging_threshold
Definition: detect.h:845
uint8_t flowflags
Definition: decode.h:438
uint16_t counter_match_list
Definition: detect.h:1004
PacketEngineEvents events
Definition: decode.h:570
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:544
uint16_t addr_dst_match6_cnt
Definition: detect.h:517
#define FLOW_SGH_TOCLIENT
Definition: flow.h:68
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PACKET_PROFILING_DETECT_END(p, id)
#define FLOW_PKT_TOSERVER
Definition: flow.h:193
HashTable * mt_det_ctxs_hash
Definition: detect.h:978
SigIntId match_array_cnt
Definition: detect.h:1047
AppProto alproto
Definition: detect.h:496
#define SIG_GROUP_HEAD_HAVEFILESHA1
Definition: detect.h:1194
void DetectSignatureApplyActions(Packet *p, const Signature *s, const uint8_t alert_flags)
Apply action(s) and Set &#39;drop&#39; sig info, if applicable.
Definition: detect.c:1600
SigIntId * rule_id_array
#define DETECT_ENGINE_STATE_FLAG_FILE_NEW
uint32_t flags
Definition: detect.h:1296
void FileDisableStoring(Flow *f, uint8_t direction)
disable file storage for a flow
Definition: util-file.c:973
uint8_t proto
uint32_t stream_reset
Definition: detect.h:953
const AppProto alproto
Definition: detect.c:63
InspectEngineFuncPtr Callback
Definition: detect.h:415
#define RULE_PROFILING_START(p)
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:766
GenericVar * flowvar
Definition: flow.h:446
#define PACKET_UPDATE_ACTION(p, a)
Definition: decode.h:875
const uint64_t detect_flags
#define SIG_GROUP_HEAD_HAVERAWSTREAM
Definition: detect.h:1188
#define DETECT_PROTO_IPV4
void DetectEngineStateResetTxs(Flow *f)
Reset de state for active tx&#39; To be used on detect engine reload.
#define SCRealloc(x, a)
Definition: util-mem.h:190
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
const SigGroupHead * SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx, const Packet *p)
Get the SigGroupHead for a packet.
Definition: detect.c:184
uint32_t version
Definition: detect.h:814
int FileForceFilestore(void)
Definition: util-file.c:111
int FileForceMagic(void)
Definition: util-file.c:130
SignatureMask pkt_mask
Definition: detect.c:67
void DisableDetectFlowFileFlags(Flow *f)
disable file features we don&#39;t need Called if we have no detection engine.
Definition: detect.c:1742
DeStateStoreItem store[DE_STATE_CHUNK_SIZE]
#define PACKET_DROP(p)
Definition: decode.h:860
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:208
void * HashTableLookup(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:193
DetectEngineCtx * de_ctx
Definition: detect.h:1086
const SigGroupHead * sgh
Definition: detect.c:66
uint16_t counter_nonmpm_list
Definition: detect.h:1002
uint64_t raw_stream_progress
Definition: detect.h:987
struct DetectRunScratchpad DetectRunScratchpad
uint8_t type
Definition: detect.h:334
#define TH_SYN
Definition: decode-tcp.h:36
#define SCFree(a)
Definition: util-mem.h:236
#define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES
#define PKT_STREAM_EOF
Definition: decode.h:1100
#define SIG_FLAG_NOALERT
Definition: detect.h:223
DetectPort * sp
Definition: detect.h:531
uint16_t addr_dst_match4_cnt
Definition: detect.h:515
uint16_t tx_id
#define FLOW_PKT_TOSERVER_IPONLY_SET
Definition: flow.h:196
#define SIG_FLAG_DSIZE
Definition: detect.h:224
void PacketAlertFinalize(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
Check the threshold of the sigs that match, set actions, break on pass action This function iterate t...
#define STREAM_FLUSH
Definition: stream.h:36
PacketAlert drop
Definition: decode.h:296
void GenericVarFree(GenericVar *gv)
Definition: util-var.c:47
#define PKT_NOPACKET_INSPECTION
Definition: decode.h:1094
DetectEngineAppInspectionEngine * app_inspect
Definition: detect.h:540
int(* Match)(ThreadVars *, DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1129
const struct SigGroupHead_ * sgh_toserver
Definition: flow.h:443
void InspectionBufferClean(DetectEngineThreadCtx *det_ctx)
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE
#define PKT_IS_ICMPV4(p)
Definition: decode.h:254
#define FatalError(x,...)
Definition: util-debug.h:539
uint32_t rule_id_array_cnt
#define PKT_STREAM_ADD
Definition: decode.h:1098
#define STREAM_TOSERVER
Definition: stream.h:31
#define IP_GET_IPPROTO(p)
Definition: decode.h:262
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
SCMutex m
Definition: flow-hash.h:105
#define PACKET_PROFILING_DETECT_START(p, id)
TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
Detection engine thread wrapper.
Definition: detect.c:1684
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:193
PacketAlerts alerts
Definition: decode.h:560
#define TRACE_SID_TXS(sid, txs,...)
Definition: detect.c:1116
int DetectAddressMatchIPv4(const DetectMatchAddressIPv4 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
Signature ** match_array
Definition: detect.h:1042
SignatureMask mask
Definition: detect.h:940
int FileForceSha1(void)
Definition: util-file.c:140
#define PKT_HAS_FLOW
Definition: decode.h:1101
SignatureNonPrefilterStore * non_pf_store_ptr
Definition: detect.h:1052
int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, const Signature *s, Packet *p, uint64_t tx_id, uint8_t flags)
append a signature match to a packet
#define PACKET_ALERT_FLAG_STATE_MATCH
Definition: decode.h:281
#define BIT_U32(n)
uint32_t de_ctx_version
Definition: flow.h:421
uint32_t * flags
Definition: detect.h:947
#define SIG_FLAG_DP_ANY
Definition: detect.h:221
DetectEngineIPOnlyThreadCtx io_ctx
Definition: detect.h:1068
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:754
#define ACTION_PASS
uint8_t stream_result
Definition: detect.h:951
uint16_t cnt
Definition: decode.h:292
SignatureNonPrefilterStore * non_pf_other_store_array
Definition: detect.h:1305
#define SCReturn
Definition: util-debug.h:339
uint32_t tenant_id
Definition: decode.h:599
Per thread variable structure.
Definition: threadvars.h:57
void FileDisableSha256(Flow *f, uint8_t direction)
disable file sha256 calc for this flow
Definition: util-file.c:1114
#define FLOW_PKT_TOCLIENT_IPONLY_SET
Definition: flow.h:197
SignatureMask mask
Definition: detect.h:501
#define FLOW_PKT_TOCLIENT
Definition: flow.h:194
AppProto alproto
application level protocol
Definition: flow.h:407
int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s)
post-match function for filestore
uint16_t counter_mpm_list
Definition: detect.h:1001
#define ACTION_DROP
uint32_t flags
Definition: decode.h:442
DetectEngineState * AppLayerParserGetTxDetectState(uint8_t ipproto, AppProto alproto, void *tx)
void AppLayerParserSetTxDetectFlags(uint8_t ipproto, AppProto alproto, void *tx, uint8_t dir, uint64_t flags)
uint16_t payload_len
Definition: decode.h:546
void StatsAddUI64(ThreadVars *tv, uint16_t id, uint64_t x)
Adds a value of type uint64_t to the local counter.
Definition: counters.c:142
#define likely(expr)
Definition: util-optimize.h:32
uint32_t non_pf_id_cnt
Definition: detect.h:974
#define DE_STATE_FLAG_FULL_INSPECT
#define SIG_FLAG_SRC_ANY
Definition: detect.h:218
uint16_t dsize_low
Definition: detect.h:498
bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
#define FLOW_SGH_TOSERVER
Definition: flow.h:66
#define DE_STATE_FLAG_FILE_INSPECT
Flow data structure.
Definition: flow.h:327
uint32_t mt_det_ctxs_cnt
Definition: detect.h:976
int DetectProtoContainsProto(const DetectProto *dp, int proto)
see if a DetectProto contains a certain proto
#define SigIntId
uint32_t flags
Definition: flow.h:377
int AppLayerParserProtocolSupportsTxs(uint8_t ipproto, AppProto alproto)
#define RULE_PROFILING_END(ctx, r, m, p)
#define PKT_STREAM_EST
Definition: decode.h:1099
Port structure for detection engine.
Definition: detect.h:198
#define SIG_GROUP_HEAD_HAVEFILESIZE
Definition: detect.h:1193
AppLayerParserState * alparser
Definition: flow.h:435
Address src
Definition: decode.h:411
SigMatchCtx * ctx
Definition: detect.h:336
uint64_t AppLayerParserGetTxDetectFlags(uint8_t ipproto, AppProto alproto, void *tx, uint8_t dir)
#define DE_STATE_CHUNK_SIZE
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SGH_PROFILING_RECORD(det_ctx, sgh)
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype