suricata
detect.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2017 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Basic detection engine
24  */
25 
26 #include "suricata-common.h"
27 #include "suricata.h"
28 #include "conf.h"
29 
30 #include "decode.h"
31 #include "flow.h"
32 #include "stream-tcp.h"
33 #include "app-layer.h"
34 #include "app-layer-parser.h"
35 
36 #include "detect.h"
37 #include "detect-engine.h"
38 #include "detect-engine-profile.h"
39 
40 #include "detect-engine-alert.h"
41 #include "detect-engine-siggroup.h"
42 #include "detect-engine-address.h"
43 #include "detect-engine-proto.h"
44 #include "detect-engine-port.h"
45 #include "detect-engine-mpm.h"
46 #include "detect-engine-iponly.h"
49 #include "detect-engine-state.h"
50 #include "detect-engine-analyzer.h"
51 
52 #include "detect-engine-payload.h"
53 #include "detect-engine-event.h"
54 
55 #include "detect-filestore.h"
56 #include "detect-flowvar.h"
57 #include "detect-replace.h"
58 
59 #include "util-validate.h"
60 #include "util-detect.h"
61 
62 typedef struct DetectRunScratchpad {
64  const uint8_t flow_flags; /* flow/state flags: STREAM_* */
65  const bool app_decoder_events;
66  const SigGroupHead *sgh;
69 
70 /* prototypes */
71 static DetectRunScratchpad DetectRunSetup(const DetectEngineCtx *de_ctx,
72  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow);
73 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
74  DetectEngineThreadCtx *det_ctx, Flow * const pflow, Packet * const p);
75 static inline void DetectRunGetRuleGroup(const DetectEngineCtx *de_ctx,
76  Packet * const p, Flow * const pflow, DetectRunScratchpad *scratch);
77 static inline void DetectRunPrefilterPkt(ThreadVars *tv,
78  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p,
79  DetectRunScratchpad *scratch);
80 static inline void DetectRulePacketRules(ThreadVars * const tv,
81  DetectEngineCtx * const de_ctx, DetectEngineThreadCtx * const det_ctx,
82  Packet * const p, Flow * const pflow, const DetectRunScratchpad *scratch);
83 static void DetectRunTx(ThreadVars *tv, DetectEngineCtx *de_ctx,
84  DetectEngineThreadCtx *det_ctx, Packet *p,
85  Flow *f, DetectRunScratchpad *scratch);
86 static inline void DetectRunPostRules(ThreadVars *tv, DetectEngineCtx *de_ctx,
87  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow,
88  DetectRunScratchpad *scratch);
89 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
90  Packet *p, Flow * const pflow);
91 
92 /** \internal
93  */
94 static void DetectRun(ThreadVars *th_v,
95  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
96  Packet *p)
97 {
98  SCEnter();
99  SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt);
100 
101  /* bail early if packet should not be inspected */
102  if (p->flags & PKT_NOPACKET_INSPECTION) {
103  /* nothing to do */
104  SCReturn;
105  }
106 
107  /* Load the Packet's flow early, even though it might not be needed.
108  * Mark as a constant pointer, although the flow itself can change. */
109  Flow * const pflow = p->flow;
110 
111  DetectRunScratchpad scratch = DetectRunSetup(de_ctx, det_ctx, p, pflow);
112 
113  /* run the IPonly engine */
114  DetectRunInspectIPOnly(th_v, de_ctx, det_ctx, pflow, p);
115 
116  /* get our rule group */
117  DetectRunGetRuleGroup(de_ctx, p, pflow, &scratch);
118  /* if we didn't get a sig group head, we
119  * have nothing to do.... */
120  if (scratch.sgh == NULL) {
121  SCLogDebug("no sgh for this packet, nothing to match against");
122  goto end;
123  }
124 
125  /* run the prefilters for packets */
126  DetectRunPrefilterPkt(th_v, de_ctx, det_ctx, p, &scratch);
127 
129  /* inspect the rules against the packet */
130  DetectRulePacketRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
132 
133  /* run tx/state inspection. Don't call for ICMP error msgs. */
134  if (pflow && pflow->alstate && likely(pflow->proto == p->proto)) {
136  DetectRunTx(th_v, de_ctx, det_ctx, p, pflow, &scratch);
138  }
139 
140 end:
141  DetectRunPostRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
142 
143  DetectRunCleanup(det_ctx, p, pflow);
144  SCReturn;
145 }
146 
147 static void DetectRunPostMatch(ThreadVars *tv,
148  DetectEngineThreadCtx *det_ctx, Packet *p,
149  const Signature *s)
150 {
151  /* run the packet match functions */
153  if (smd != NULL) {
155 
156  SCLogDebug("running match functions, sm %p", smd);
157 
158  while (1) {
160  (void)sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx);
161  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
162  if (smd->is_last)
163  break;
164  smd++;
165  }
166  }
167 }
168 
169 /**
170  * \brief Get the SigGroupHead for a packet.
171  *
172  * \param de_ctx detection engine context
173  * \param p packet
174  *
175  * \retval sgh the SigGroupHead or NULL if non applies to the packet
176  */
178  const Packet *p)
179 {
180  SCEnter();
181 
182  int f;
183  SigGroupHead *sgh = NULL;
184 
185  /* if the packet proto is 0 (not set), we're inspecting it against
186  * the decoder events sgh we have. */
187  if (p->proto == 0 && p->events.cnt > 0) {
188  SCReturnPtr(de_ctx->decoder_event_sgh, "SigGroupHead");
189  } else if (p->proto == 0) {
190  if (!(PKT_IS_IPV4(p) || PKT_IS_IPV6(p))) {
191  /* not IP, so nothing to do */
192  SCReturnPtr(NULL, "SigGroupHead");
193  }
194  }
195 
196  /* select the flow_gh */
197  if (p->flowflags & FLOW_PKT_TOCLIENT)
198  f = 0;
199  else
200  f = 1;
201 
202  int proto = IP_GET_IPPROTO(p);
203  if (proto == IPPROTO_TCP) {
204  DetectPort *list = de_ctx->flow_gh[f].tcp;
205  SCLogDebug("tcp toserver %p, tcp toclient %p: going to use %p",
206  de_ctx->flow_gh[1].tcp, de_ctx->flow_gh[0].tcp, de_ctx->flow_gh[f].tcp);
207  uint16_t port = f ? p->dp : p->sp;
208  SCLogDebug("tcp port %u -> %u:%u", port, p->sp, p->dp);
209  DetectPort *sghport = DetectPortLookupGroup(list, port);
210  if (sghport != NULL)
211  sgh = sghport->sh;
212  SCLogDebug("TCP list %p, port %u, direction %s, sghport %p, sgh %p",
213  list, port, f ? "toserver" : "toclient", sghport, sgh);
214  } else if (proto == IPPROTO_UDP) {
215  DetectPort *list = de_ctx->flow_gh[f].udp;
216  uint16_t port = f ? p->dp : p->sp;
217  DetectPort *sghport = DetectPortLookupGroup(list, port);
218  if (sghport != NULL)
219  sgh = sghport->sh;
220  SCLogDebug("UDP list %p, port %u, direction %s, sghport %p, sgh %p",
221  list, port, f ? "toserver" : "toclient", sghport, sgh);
222  } else {
223  sgh = de_ctx->flow_gh[f].sgh[proto];
224  }
225 
226  SCReturnPtr(sgh, "SigGroupHead");
227 }
228 
229 static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
230  DetectEngineThreadCtx *det_ctx)
231 {
232  SigIntId mpm, nonmpm;
233  SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array;
234  SigIntId *nonmpm_ptr = det_ctx->non_pf_id_array;
235  uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt;
236  uint32_t n_cnt = det_ctx->non_pf_id_cnt;
237  SigIntId *final_ptr;
238  uint32_t final_cnt;
239  SigIntId id;
240  SigIntId previous_id = (SigIntId)-1;
241  Signature **sig_array = de_ctx->sig_array;
242  Signature **match_array = det_ctx->match_array;
243  Signature *s;
244 
245  SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt);
246 
247  /* Load first values. */
248  if (likely(m_cnt)) {
249  mpm = *mpm_ptr;
250  } else {
251  /* mpm list is empty */
252  final_ptr = nonmpm_ptr;
253  final_cnt = n_cnt;
254  goto final;
255  }
256  if (likely(n_cnt)) {
257  nonmpm = *nonmpm_ptr;
258  } else {
259  /* non-mpm list is empty. */
260  final_ptr = mpm_ptr;
261  final_cnt = m_cnt;
262  goto final;
263  }
264  while (1) {
265  if (mpm < nonmpm) {
266  /* Take from mpm list */
267  id = mpm;
268 
269  s = sig_array[id];
270  /* As the mpm list can contain duplicates, check for that here. */
271  if (likely(id != previous_id)) {
272  *match_array++ = s;
273  previous_id = id;
274  }
275  if (unlikely(--m_cnt == 0)) {
276  /* mpm list is now empty */
277  final_ptr = nonmpm_ptr;
278  final_cnt = n_cnt;
279  goto final;
280  }
281  mpm_ptr++;
282  mpm = *mpm_ptr;
283  } else if (mpm > nonmpm) {
284  id = nonmpm;
285 
286  s = sig_array[id];
287  /* As the mpm list can contain duplicates, check for that here. */
288  if (likely(id != previous_id)) {
289  *match_array++ = s;
290  previous_id = id;
291  }
292  if (unlikely(--n_cnt == 0)) {
293  final_ptr = mpm_ptr;
294  final_cnt = m_cnt;
295  goto final;
296  }
297  nonmpm_ptr++;
298  nonmpm = *nonmpm_ptr;
299 
300  } else { /* implied mpm == nonmpm */
301  /* special case: if on both lists, it's a negated mpm pattern */
302 
303  /* mpm list may have dups, so skip past them here */
304  while (--m_cnt != 0) {
305  mpm_ptr++;
306  mpm = *mpm_ptr;
307  if (mpm != nonmpm)
308  break;
309  }
310  /* if mpm is done, update nonmpm_ptrs and jump to final */
311  if (unlikely(m_cnt == 0)) {
312  n_cnt--;
313 
314  /* mpm list is now empty */
315  final_ptr = ++nonmpm_ptr;
316  final_cnt = n_cnt;
317  goto final;
318  }
319  /* otherwise, if nonmpm is done jump to final for mpm
320  * mpm ptrs alrady updated */
321  if (unlikely(--n_cnt == 0)) {
322  final_ptr = mpm_ptr;
323  final_cnt = m_cnt;
324  goto final;
325  }
326 
327  /* not at end of the lists, update nonmpm. Mpm already
328  * updated in while loop above. */
329  nonmpm_ptr++;
330  nonmpm = *nonmpm_ptr;
331  }
332  }
333 
334  final: /* Only one list remaining. Just walk that list. */
335 
336  while (final_cnt-- > 0) {
337  id = *final_ptr++;
338  s = sig_array[id];
339 
340  /* As the mpm list can contain duplicates, check for that here. */
341  if (likely(id != previous_id)) {
342  *match_array++ = s;
343  previous_id = id;
344  }
345  }
346 
347  det_ctx->match_array_cnt = match_array - det_ctx->match_array;
348 
349  DEBUG_VALIDATE_BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_pf_id_cnt) < det_ctx->match_array_cnt);
350 }
351 
352 static inline void
353 DetectPrefilterBuildNonPrefilterList(DetectEngineThreadCtx *det_ctx, SignatureMask mask, uint8_t alproto)
354 {
355  uint32_t x = 0;
356  for (x = 0; x < det_ctx->non_pf_store_cnt; x++) {
357  /* only if the mask matches this rule can possibly match,
358  * so build the non_mpm array only for match candidates */
359  const SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask;
360  const uint8_t rule_alproto = det_ctx->non_pf_store_ptr[x].alproto;
361  if ((rule_mask & mask) == rule_mask && (rule_alproto == 0 || rule_alproto == alproto)) {
362  det_ctx->non_pf_id_array[det_ctx->non_pf_id_cnt++] = det_ctx->non_pf_store_ptr[x].id;
363  }
364  }
365 }
366 
367 /** \internal
368  * \brief select non-mpm list
369  * Based on the packet properties, select the non-mpm list to use
370  * \todo move non_pf_store* into scratchpad */
371 static inline void
372 DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx, DetectRunScratchpad *scratch)
373 {
374  if ((p->proto == IPPROTO_TCP) && (p->tcph != NULL) && (p->tcph->th_flags & TH_SYN)) {
375  det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_syn_store_array;
376  det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_syn_store_cnt;
377  } else {
378  det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_other_store_array;
379  det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_other_store_cnt;
380  }
381  SCLogDebug("sgh non_pf ptr %p cnt %u (syn %p/%u, other %p/%u)",
382  det_ctx->non_pf_store_ptr, det_ctx->non_pf_store_cnt,
383  scratch->sgh->non_pf_syn_store_array, scratch->sgh->non_pf_syn_store_cnt,
385 }
386 
387 /** \internal
388  * \brief update flow's file tracking flags based on the detection engine
389  */
390 static inline void
391 DetectPostInspectFileFlagsUpdate(Flow *pflow, const SigGroupHead *sgh, uint8_t direction)
392 {
393  /* see if this sgh requires us to consider file storing */
394  if (!FileForceFilestore() && (sgh == NULL ||
395  sgh->filestore_cnt == 0))
396  {
397  FileDisableStoring(pflow, direction);
398  }
399 #ifdef HAVE_MAGIC
400  /* see if this sgh requires us to consider file magic */
401  if (!FileForceMagic() && (sgh == NULL ||
402  !(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)))
403  {
404  SCLogDebug("disabling magic for flow");
405  FileDisableMagic(pflow, direction);
406  }
407 #endif
408  /* see if this sgh requires us to consider file md5 */
409  if (!FileForceMd5() && (sgh == NULL ||
411  {
412  SCLogDebug("disabling md5 for flow");
413  FileDisableMd5(pflow, direction);
414  }
415 
416  /* see if this sgh requires us to consider file sha1 */
417  if (!FileForceSha1() && (sgh == NULL ||
419  {
420  SCLogDebug("disabling sha1 for flow");
421  FileDisableSha1(pflow, direction);
422  }
423 
424  /* see if this sgh requires us to consider file sha256 */
425  if (!FileForceSha256() && (sgh == NULL ||
427  {
428  SCLogDebug("disabling sha256 for flow");
429  FileDisableSha256(pflow, direction);
430  }
431 
432  /* see if this sgh requires us to consider filesize */
433  if (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE))
434  {
435  SCLogDebug("disabling filesize for flow");
436  FileDisableFilesize(pflow, direction);
437  }
438 }
439 
440 static inline void
441 DetectRunPostGetFirstRuleGroup(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
442 {
443  if ((p->flowflags & FLOW_PKT_TOSERVER) && !(pflow->flags & FLOW_SGH_TOSERVER)) {
444  /* first time we see this toserver sgh, store it */
445  pflow->sgh_toserver = sgh;
446  pflow->flags |= FLOW_SGH_TOSERVER;
447 
448  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
449  if (pflow->protoctx != NULL) {
450  TcpSession *ssn = pflow->protoctx;
451  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.client");
453  }
454  }
455 
456  DetectPostInspectFileFlagsUpdate(pflow,
457  pflow->sgh_toserver, STREAM_TOSERVER);
458 
459  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && !(pflow->flags & FLOW_SGH_TOCLIENT)) {
460  pflow->sgh_toclient = sgh;
461  pflow->flags |= FLOW_SGH_TOCLIENT;
462 
463  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
464  if (pflow->protoctx != NULL) {
465  TcpSession *ssn = pflow->protoctx;
466  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.server");
468  }
469  }
470 
471  DetectPostInspectFileFlagsUpdate(pflow,
472  pflow->sgh_toclient, STREAM_TOCLIENT);
473  }
474 }
475 
476 static inline void DetectRunGetRuleGroup(
477  const DetectEngineCtx *de_ctx,
478  Packet * const p, Flow * const pflow,
479  DetectRunScratchpad *scratch)
480 {
481  const SigGroupHead *sgh = NULL;
482 
483  if (pflow) {
484  bool use_flow_sgh = false;
485  /* Get the stored sgh from the flow (if any). Make sure we're not using
486  * the sgh for icmp error packets part of the same stream. */
487  if (IP_GET_IPPROTO(p) == pflow->proto) { /* filter out icmp */
489  if ((p->flowflags & FLOW_PKT_TOSERVER) && (pflow->flags & FLOW_SGH_TOSERVER)) {
490  sgh = pflow->sgh_toserver;
491  SCLogDebug("sgh = pflow->sgh_toserver; => %p", sgh);
492  use_flow_sgh = true;
493  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && (pflow->flags & FLOW_SGH_TOCLIENT)) {
494  sgh = pflow->sgh_toclient;
495  SCLogDebug("sgh = pflow->sgh_toclient; => %p", sgh);
496  use_flow_sgh = true;
497  }
499  }
500 
501  if (!(use_flow_sgh)) {
503  sgh = SigMatchSignaturesGetSgh(de_ctx, p);
505 
506  /* HACK: prevent the wrong sgh (or NULL) from being stored in the
507  * flow's sgh pointers */
509  ; /* no-op */
510  } else {
511  /* store the found sgh (or NULL) in the flow to save us
512  * from looking it up again for the next packet.
513  * Also run other tasks */
514  DetectRunPostGetFirstRuleGroup(p, pflow, sgh);
515  }
516  }
517  } else { /* p->flags & PKT_HAS_FLOW */
518  /* no flow */
519 
521  sgh = SigMatchSignaturesGetSgh(de_ctx, p);
523  }
524 
525  scratch->sgh = sgh;
526 }
527 
528 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
529  DetectEngineThreadCtx *det_ctx,
530  Flow * const pflow, Packet * const p)
531 {
532  if (pflow) {
533  /* set the iponly stuff */
534  if (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)
536  if (pflow->flags & FLOW_TOSERVER_IPONLY_SET)
538 
541  {
542  SCLogDebug("testing against \"ip-only\" signatures");
543 
545  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, &det_ctx->io_ctx, p);
547 
548  /* save in the flow that we scanned this direction... */
549  FlowSetIPOnlyFlag(pflow, p->flowflags & FLOW_PKT_TOSERVER ? 1 : 0);
550 
551  } else if (((p->flowflags & FLOW_PKT_TOSERVER) &&
552  (pflow->flags & FLOW_TOSERVER_IPONLY_SET)) ||
553  ((p->flowflags & FLOW_PKT_TOCLIENT) &&
554  (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)))
555  {
556  /* If we have a drop from IP only module,
557  * we will drop the rest of the flow packets
558  * This will apply only to inline/IPS */
559  if (pflow->flags & FLOW_ACTION_DROP) {
560  PACKET_DROP(p);
561  }
562  }
563  } else { /* p->flags & PKT_HAS_FLOW */
564  /* no flow */
565 
566  /* Even without flow we should match the packet src/dst */
568  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx,
569  &det_ctx->io_ctx, p);
571  }
572 }
573 
574 /* returns 0 if no match, 1 if match */
575 static inline int DetectRunInspectRuleHeader(
576  const Packet *p,
577  const Flow *f,
578  const Signature *s,
579  const uint32_t sflags,
580  const uint8_t s_proto_flags)
581 {
582  /* check if this signature has a requirement for flowvars of some type
583  * and if so, if we actually have any in the flow. If not, the sig
584  * can't match and we skip it. */
585  if ((p->flags & PKT_HAS_FLOW) && (sflags & SIG_FLAG_REQUIRE_FLOWVAR)) {
586  DEBUG_VALIDATE_BUG_ON(f == NULL);
587 
588  int m = f->flowvar ? 1 : 0;
589 
590  /* no flowvars? skip this sig */
591  if (m == 0) {
592  SCLogDebug("skipping sig as the flow has no flowvars and sig "
593  "has SIG_FLAG_REQUIRE_FLOWVAR flag set.");
594  return 0;
595  }
596  }
597 
598  if ((s_proto_flags & DETECT_PROTO_IPV4) && !PKT_IS_IPV4(p)) {
599  SCLogDebug("ip version didn't match");
600  return 0;
601  }
602  if ((s_proto_flags & DETECT_PROTO_IPV6) && !PKT_IS_IPV6(p)) {
603  SCLogDebug("ip version didn't match");
604  return 0;
605  }
606 
607  if (DetectProtoContainsProto(&s->proto, IP_GET_IPPROTO(p)) == 0) {
608  SCLogDebug("proto didn't match");
609  return 0;
610  }
611 
612  /* check the source & dst port in the sig */
613  if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP || p->proto == IPPROTO_SCTP) {
614  if (!(sflags & SIG_FLAG_DP_ANY)) {
615  if (p->flags & PKT_IS_FRAGMENT)
616  return 0;
617  DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp);
618  if (dport == NULL) {
619  SCLogDebug("dport didn't match.");
620  return 0;
621  }
622  }
623  if (!(sflags & SIG_FLAG_SP_ANY)) {
624  if (p->flags & PKT_IS_FRAGMENT)
625  return 0;
626  DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp);
627  if (sport == NULL) {
628  SCLogDebug("sport didn't match.");
629  return 0;
630  }
631  }
632  } else if ((sflags & (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) != (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) {
633  SCLogDebug("port-less protocol and sig needs ports");
634  return 0;
635  }
636 
637  /* check the destination address */
638  if (!(sflags & SIG_FLAG_DST_ANY)) {
639  if (PKT_IS_IPV4(p)) {
641  return 0;
642  } else if (PKT_IS_IPV6(p)) {
644  return 0;
645  }
646  }
647  /* check the source address */
648  if (!(sflags & SIG_FLAG_SRC_ANY)) {
649  if (PKT_IS_IPV4(p)) {
651  return 0;
652  } else if (PKT_IS_IPV6(p)) {
654  return 0;
655  }
656  }
657 
658  return 1;
659 }
660 
661 /** \internal
662  * \brief run packet/stream prefilter engines
663  */
664 static inline void DetectRunPrefilterPkt(
665  ThreadVars *tv,
666  DetectEngineCtx *de_ctx,
667  DetectEngineThreadCtx *det_ctx,
668  Packet *p,
669  DetectRunScratchpad *scratch
670 )
671 {
672  DetectPrefilterSetNonPrefilterList(p, det_ctx, scratch);
673 
674  /* create our prefilter mask */
675  PacketCreateMask(p, &scratch->pkt_mask, scratch->alproto, scratch->app_decoder_events);
676 
677  /* build and prefilter non_pf list against the mask of the packet */
679  det_ctx->non_pf_id_cnt = 0;
680  if (likely(det_ctx->non_pf_store_cnt > 0)) {
681  DetectPrefilterBuildNonPrefilterList(det_ctx, scratch->pkt_mask, scratch->alproto);
682  }
684 
685  /* run the prefilter engines */
686  Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags);
687  /* create match list if we have non-pf and/or pf */
688  if (det_ctx->non_pf_store_cnt || det_ctx->pmq.rule_id_array_cnt) {
690  DetectPrefilterMergeSort(de_ctx, det_ctx);
692  }
693 
694 #ifdef PROFILING
695  if (tv) {
696  StatsAddUI64(tv, det_ctx->counter_mpm_list,
697  (uint64_t)det_ctx->pmq.rule_id_array_cnt);
698  StatsAddUI64(tv, det_ctx->counter_nonmpm_list,
699  (uint64_t)det_ctx->non_pf_store_cnt);
700  /* non mpm sigs after mask prefilter */
701  StatsAddUI64(tv, det_ctx->counter_fnonmpm_list,
702  (uint64_t)det_ctx->non_pf_id_cnt);
703  }
704 #endif
705 }
706 
707 static inline void DetectRulePacketRules(
708  ThreadVars * const tv,
709  DetectEngineCtx * const de_ctx,
710  DetectEngineThreadCtx * const det_ctx,
711  Packet * const p,
712  Flow * const pflow,
713  const DetectRunScratchpad *scratch
714 )
715 {
716  const Signature *s = NULL;
717  const Signature *next_s = NULL;
718 
719  /* inspect the sigs against the packet */
720  /* Prefetch the next signature. */
721  SigIntId match_cnt = det_ctx->match_array_cnt;
722 #ifdef PROFILING
723  if (tv) {
724  StatsAddUI64(tv, det_ctx->counter_match_list,
725  (uint64_t)match_cnt);
726  }
727 #endif
728  Signature **match_array = det_ctx->match_array;
729 
730  SGH_PROFILING_RECORD(det_ctx, scratch->sgh);
731 #ifdef PROFILING
732  if (match_cnt >= de_ctx->profile_match_logging_threshold)
733  RulesDumpMatchArray(det_ctx, scratch->sgh, p);
734 #endif
735 
736  uint32_t sflags, next_sflags = 0;
737  if (match_cnt) {
738  next_s = *match_array++;
739  next_sflags = next_s->flags;
740  }
741  while (match_cnt--) {
743  uint8_t alert_flags = 0;
744  bool state_alert = false;
745 #ifdef PROFILING
746  bool smatch = false; /* signature match */
747 #endif
748  s = next_s;
749  sflags = next_sflags;
750  if (match_cnt) {
751  next_s = *match_array++;
752  next_sflags = next_s->flags;
753  }
754  const uint8_t s_proto_flags = s->proto.flags;
755 
756  SCLogDebug("inspecting signature id %"PRIu32"", s->id);
757 
758  if (s->app_inspect != NULL) {
759  goto next; // handle sig in DetectRunTx
760  }
761 
762  /* don't run mask check for stateful rules.
763  * There we depend on prefilter */
764  if ((s->mask & scratch->pkt_mask) != s->mask) {
765  SCLogDebug("mask mismatch %x & %x != %x", s->mask, scratch->pkt_mask, s->mask);
766  goto next;
767  }
768 
769  if (unlikely(sflags & SIG_FLAG_DSIZE)) {
770  if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) {
771  SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u",
772  p->payload_len, s->dsize_low, s->dsize_high);
773  goto next;
774  }
775  }
776 
777  /* if the sig has alproto and the session as well they should match */
778  if (likely(sflags & SIG_FLAG_APPLAYER)) {
779  if (s->alproto != ALPROTO_UNKNOWN && s->alproto != scratch->alproto) {
780  if (s->alproto == ALPROTO_DCERPC) {
781  if (scratch->alproto != ALPROTO_SMB) {
782  SCLogDebug("DCERPC sig, alproto not SMB");
783  goto next;
784  }
785  } else {
786  SCLogDebug("alproto mismatch");
787  goto next;
788  }
789  }
790  }
791 
792  if (DetectRunInspectRuleHeader(p, pflow, s, sflags, s_proto_flags) == 0) {
793  goto next;
794  }
795 
796  if (DetectEnginePktInspectionRun(tv, det_ctx, s, pflow, p, &alert_flags) == false) {
797  goto next;
798  }
799 
800 #ifdef PROFILING
801  smatch = true;
802 #endif
803  DetectRunPostMatch(tv, det_ctx, p, s);
804 
805  if (!(sflags & SIG_FLAG_NOALERT)) {
806  /* stateful sigs call PacketAlertAppend from DeStateDetectStartDetection */
807  if (!state_alert)
808  PacketAlertAppend(det_ctx, s, p, 0, alert_flags);
809  } else {
810  /* apply actions even if not alerting */
811  DetectSignatureApplyActions(p, s, alert_flags);
812  }
813 next:
814  DetectVarProcessList(det_ctx, pflow, p);
815  DetectReplaceFree(det_ctx);
816  RULE_PROFILING_END(det_ctx, s, smatch, p);
817 
818  det_ctx->flags = 0;
819  continue;
820  }
821 }
822 
823 static DetectRunScratchpad DetectRunSetup(
824  const DetectEngineCtx *de_ctx,
825  DetectEngineThreadCtx *det_ctx,
826  Packet * const p, Flow * const pflow)
827 {
828  AppProto alproto = ALPROTO_UNKNOWN;
829  uint8_t flow_flags = 0; /* flow/state flags */
830  bool app_decoder_events = false;
831 
833 
834 #ifdef UNITTESTS
835  p->alerts.cnt = 0;
836 #endif
837  det_ctx->ticker++;
838  det_ctx->filestore_cnt = 0;
839  det_ctx->base64_decoded_len = 0;
840  det_ctx->raw_stream_progress = 0;
841  det_ctx->match_array_cnt = 0;
842 
843 #ifdef DEBUG
844  if (p->flags & PKT_STREAM_ADD) {
845  det_ctx->pkt_stream_add_cnt++;
846  }
847 #endif
848 
849  /* grab the protocol state we will detect on */
850  if (p->flags & PKT_HAS_FLOW) {
851  DEBUG_VALIDATE_BUG_ON(pflow == NULL);
852 
853  if (p->flowflags & FLOW_PKT_TOSERVER) {
854  flow_flags = STREAM_TOSERVER;
855  SCLogDebug("flag STREAM_TOSERVER set");
856  } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
857  flow_flags = STREAM_TOCLIENT;
858  SCLogDebug("flag STREAM_TOCLIENT set");
859  }
860  SCLogDebug("p->flowflags 0x%02x", p->flowflags);
861 
862  if (p->flags & PKT_STREAM_EOF) {
863  flow_flags |= STREAM_EOF;
864  SCLogDebug("STREAM_EOF set");
865  }
866 
867  /* store tenant_id in the flow so that we can use it
868  * for creating pseudo packets */
869  if (p->tenant_id > 0 && pflow->tenant_id == 0) {
870  pflow->tenant_id = p->tenant_id;
871  }
872 
873  /* live ruleswap check for flow updates */
874  if (pflow->de_ctx_version == 0) {
875  /* first time this flow is inspected, set id */
876  pflow->de_ctx_version = de_ctx->version;
877  } else if (pflow->de_ctx_version != de_ctx->version) {
878  /* first time we inspect flow with this de_ctx, reset */
879  pflow->flags &= ~FLOW_SGH_TOSERVER;
880  pflow->flags &= ~FLOW_SGH_TOCLIENT;
881  pflow->sgh_toserver = NULL;
882  pflow->sgh_toclient = NULL;
883 
884  pflow->de_ctx_version = de_ctx->version;
885  GenericVarFree(pflow->flowvar);
886  pflow->flowvar = NULL;
887 
889  }
890 
891  /* Retrieve the app layer state and protocol and the tcp reassembled
892  * stream chunks. */
893  if ((p->proto == IPPROTO_TCP && (p->flags & PKT_STREAM_EST)) ||
894  (p->proto == IPPROTO_UDP) ||
895  (p->proto == IPPROTO_SCTP && (p->flowflags & FLOW_PKT_ESTABLISHED)))
896  {
897  /* update flow flags with knowledge on disruptions */
898  flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
899  alproto = FlowGetAppProtocol(pflow);
900  if (p->proto == IPPROTO_TCP && pflow->protoctx &&
903  flow_flags |= STREAM_FLUSH;
904  }
905  SCLogDebug("alproto %u", alproto);
906  } else {
907  SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
908  }
909 
910  app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
911  }
912 
915  return pad;
916 }
917 
918 static inline void DetectRunPostRules(
919  ThreadVars *tv,
920  DetectEngineCtx *de_ctx,
921  DetectEngineThreadCtx *det_ctx,
922  Packet * const p,
923  Flow * const pflow,
924  DetectRunScratchpad *scratch)
925 {
926  /* see if we need to increment the inspect_id and reset the de_state */
927  if (pflow && pflow->alstate) {
929  DeStateUpdateInspectTransactionId(pflow, scratch->flow_flags, (scratch->sgh == NULL));
931  }
932 
933  /* so now let's iterate the alerts and remove the ones after a pass rule
934  * matched (if any). This is done inside PacketAlertFinalize() */
935  /* PR: installed "tag" keywords are handled after the threshold inspection */
936 
938  PacketAlertFinalize(de_ctx, det_ctx, p);
939  if (p->alerts.cnt > 0) {
940  StatsAddUI64(tv, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt);
941  }
943 }
944 
945 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
946  Packet *p, Flow * const pflow)
947 {
949  /* cleanup pkt specific part of the patternmatcher */
950  PacketPatternCleanup(det_ctx);
951 
952  if (pflow != NULL) {
953  /* update inspected tracker for raw reassembly */
954  if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL &&
955  (p->flags & PKT_STREAM_EST))
956  {
958  det_ctx->raw_stream_progress);
959  }
960  }
962  SCReturn;
963 }
964 
966 {
968  det_ctx->tx_candidates = SCCalloc(size, sizeof(RuleMatchCandidateTx));
969  if (det_ctx->tx_candidates == NULL) {
970  FatalError(SC_ERR_MEM_ALLOC, "failed to allocate %"PRIu64" bytes",
971  (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
972  }
973  det_ctx->tx_candidates_size = size;
974  SCLogDebug("array initialized to %u elements (%"PRIu64" bytes)",
975  size, (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
976 }
977 
979 {
980  SCFree(det_ctx->tx_candidates);
981  det_ctx->tx_candidates_size = 0;
982 }
983 
984 /* if size >= cur_space */
985 static inline bool RuleMatchCandidateTxArrayHasSpace(const DetectEngineThreadCtx *det_ctx,
986  const uint32_t need)
987 {
988  if (det_ctx->tx_candidates_size >= need)
989  return 1;
990  return 0;
991 }
992 
993 /* realloc */
994 static int RuleMatchCandidateTxArrayExpand(DetectEngineThreadCtx *det_ctx, const uint32_t needed)
995 {
996  const uint32_t old_size = det_ctx->tx_candidates_size;
997  uint32_t new_size = needed;
998  void *ptmp = SCRealloc(det_ctx->tx_candidates, (new_size * sizeof(RuleMatchCandidateTx)));
999  if (ptmp == NULL) {
1000  FatalError(SC_ERR_MEM_ALLOC, "failed to expand to %"PRIu64" bytes",
1001  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx)));
1002  // TODO can this be handled more gracefully?
1003  }
1004  det_ctx->tx_candidates = ptmp;
1005  det_ctx->tx_candidates_size = new_size;
1006  SCLogDebug("array expanded from %u to %u elements (%"PRIu64" bytes -> %"PRIu64" bytes)",
1007  old_size, new_size, (uint64_t)(old_size * sizeof(RuleMatchCandidateTx)),
1008  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx))); (void)old_size;
1009  return 1;
1010 }
1011 
1012 
1013 /* TODO maybe let one with flags win if equal? */
1014 static int
1015 DetectRunTxSortHelper(const void *a, const void *b)
1016 {
1017  const RuleMatchCandidateTx *s0 = a;
1018  const RuleMatchCandidateTx *s1 = b;
1019  if (s1->id == s0->id)
1020  return 0;
1021  else
1022  return s0->id > s1->id ? -1 : 1;
1023 }
1024 
1025 #if 0
1026 #define TRACE_SID_TXS(sid,txs,...) \
1027  do { \
1028  char _trace_buf[2048]; \
1029  snprintf(_trace_buf, sizeof(_trace_buf), __VA_ARGS__); \
1030  SCLogNotice("%p/%"PRIu64"/%u: %s", txs->tx_ptr, txs->tx_id, sid, _trace_buf); \
1031  } while(0)
1032 #else
1033 #define TRACE_SID_TXS(sid,txs,...)
1034 #endif
1035 
1036 /** \internal
1037  * \brief inspect a rule against a transaction
1038  *
1039  * Inspect a rule. New detection or continued stateful
1040  * detection.
1041  *
1042  * \param stored_flags pointer to stored flags or NULL.
1043  * If stored_flags is set it means we're continueing
1044  * inspection from an earlier run.
1045  *
1046  * \retval bool true sig matched, false didn't match
1047  */
1048 static bool DetectRunTxInspectRule(ThreadVars *tv,
1049  DetectEngineCtx *de_ctx,
1050  DetectEngineThreadCtx *det_ctx,
1051  Packet *p,
1052  Flow *f,
1053  const uint8_t in_flow_flags, // direction, EOF, etc
1054  void *alstate,
1055  DetectTransaction *tx,
1056  const Signature *s,
1057  uint32_t *stored_flags,
1058  RuleMatchCandidateTx *can,
1059  DetectRunScratchpad *scratch)
1060 {
1061  uint8_t flow_flags = in_flow_flags;
1062  const int direction = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1063  uint32_t inspect_flags = stored_flags ? *stored_flags : 0;
1064  int total_matches = 0;
1065  int file_no_match = 0;
1066  bool retval = false;
1067  bool mpm_before_progress = false; // is mpm engine before progress?
1068  bool mpm_in_progress = false; // is mpm engine in a buffer we will revisit?
1069 
1070  /* see if we want to pass on the FLUSH flag */
1071  if ((s->flags & SIG_FLAG_FLUSH) == 0)
1072  flow_flags &=~ STREAM_FLUSH;
1073 
1074  TRACE_SID_TXS(s->id, tx, "starting %s", direction ? "toclient" : "toserver");
1075  TRACE_SID_TXS(s->id, tx, "FLUSH? %s", (flow_flags & STREAM_FLUSH)?"true":"false");
1076 
1077  /* for a new inspection we inspect pkt header and packet matches */
1078  if (likely(stored_flags == NULL)) {
1079  TRACE_SID_TXS(s->id, tx, "first inspect, run packet matches");
1080  if (DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags) == 0) {
1081  TRACE_SID_TXS(s->id, tx, "DetectRunInspectRuleHeader() no match");
1082  return false;
1083  }
1084  if (DetectEnginePktInspectionRun(tv, det_ctx, s, f, p, NULL) == false) {
1085  TRACE_SID_TXS(s->id, tx, "DetectEnginePktInspectionRun no match");
1086  return false;
1087  }
1088  /* stream mpm and negated mpm sigs can end up here with wrong proto */
1089  if (!(f->alproto == s->alproto || s->alproto == ALPROTO_UNKNOWN)) {
1090  TRACE_SID_TXS(s->id, tx, "alproto mismatch");
1091  return false;
1092  }
1093  }
1094 
1095  const DetectEngineAppInspectionEngine *engine = s->app_inspect;
1096  while (engine != NULL) { // TODO could be do {} while as s->app_inspect cannot be null
1097  TRACE_SID_TXS(s->id, tx, "engine %p inspect_flags %x", engine, inspect_flags);
1098  if (!(inspect_flags & BIT_U32(engine->id)) &&
1099  direction == engine->dir)
1100  {
1101  const bool skip_engine = (engine->alproto != 0 && engine->alproto != f->alproto);
1102  /* special case: file_data on 'alert tcp' will have engines
1103  * in the list that are not for us. */
1104  if (unlikely(skip_engine)) {
1105  engine = engine->next;
1106  continue;
1107  }
1108 
1109  /* engines are sorted per progress, except that the one with
1110  * mpm/prefilter enabled is first */
1111  if (tx->tx_progress < engine->progress) {
1112  SCLogDebug("tx progress %d < engine progress %d",
1113  tx->tx_progress, engine->progress);
1114  break;
1115  }
1116  if (engine->mpm) {
1117  if (tx->tx_progress > engine->progress) {
1118  mpm_before_progress = true;
1119  } else if (tx->tx_progress == engine->progress) {
1120  mpm_in_progress = true;
1121  }
1122  }
1123 
1124  /* run callback: but bypass stream callback if we can */
1125  int match;
1126  if (unlikely(engine->stream && can->stream_stored)) {
1127  match = can->stream_result;
1128  TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match);
1129  } else {
1130  KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list);
1131  if (engine->Callback) {
1132  match = engine->Callback(tv, de_ctx, det_ctx,
1133  s, engine->smd, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
1134  } else {
1135  BUG_ON(engine->v2.Callback == NULL);
1136  match = engine->v2.Callback(de_ctx, det_ctx, engine,
1137  s, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
1138  }
1139  TRACE_SID_TXS(s->id, tx, "engine %p match %d", engine, match);
1140  if (engine->stream) {
1141  can->stream_stored = true;
1142  can->stream_result = match;
1143  TRACE_SID_TXS(s->id, tx, "stream ran, store result %d for next tx (if any)", match);
1144  }
1145  }
1146  if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) {
1147  inspect_flags |= BIT_U32(engine->id);
1148  engine = engine->next;
1149  total_matches++;
1150  continue;
1151  } else if (match == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES) {
1152  /* if the file engine matched, but indicated more
1153  * files are still in progress, we don't set inspect
1154  * flags as these would end inspection for this tx */
1155  engine = engine->next;
1156  total_matches++;
1157  continue;
1158  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
1159  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1160  inspect_flags |= BIT_U32(engine->id);
1161  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES) {
1162  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1163  inspect_flags |= BIT_U32(engine->id);
1164  file_no_match = 1;
1165  }
1166  /* implied DETECT_ENGINE_INSPECT_SIG_NO_MATCH */
1167  if (engine->mpm && mpm_before_progress) {
1168  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1169  inspect_flags |= BIT_U32(engine->id);
1170  }
1171  break;
1172  }
1173  engine = engine->next;
1174  }
1175  TRACE_SID_TXS(s->id, tx, "inspect_flags %x, total_matches %u, engine %p",
1176  inspect_flags, total_matches, engine);
1177 
1178  if (engine == NULL && total_matches) {
1179  inspect_flags |= DE_STATE_FLAG_FULL_INSPECT;
1180  TRACE_SID_TXS(s->id, tx, "MATCH");
1181  retval = true;
1182  }
1183 
1184  if (stored_flags) {
1185  *stored_flags = inspect_flags;
1186  TRACE_SID_TXS(s->id, tx, "continue inspect flags %08x", inspect_flags);
1187  } else {
1188  // store... or? If tx is done we might not want to come back to this tx
1189 
1190  // also... if mpmid tracking is enabled, we won't do a sig again for this tx...
1191  TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags);
1192  if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) {
1193  if (file_no_match) {
1194  /* if we have a mismatch on a file sig, we need to keep state.
1195  * We may get another file on the same tx (for http and smtp
1196  * at least), so for a new file we need to re-eval the sig.
1197  * Thoughts / TODO:
1198  * - not for some protos that have 1 file per tx (e.g. nfs)
1199  * - maybe we only need this for file sigs that mix with
1200  * other matches? E.g. 'POST + filename', is different than
1201  * just 'filename'.
1202  */
1203  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1204  inspect_flags, flow_flags, file_no_match);
1205  }
1206  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) {
1207  TRACE_SID_TXS(s->id, tx, "no need to store match sig, "
1208  "mpm won't trigger for it anymore");
1209 
1210  if (inspect_flags & DE_STATE_FLAG_FILE_INSPECT) {
1211  TRACE_SID_TXS(s->id, tx, "except that for new files, "
1212  "we may have to revisit anyway");
1213  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1214  inspect_flags, flow_flags, file_no_match);
1215  }
1216  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) {
1217  TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, "
1218  "mpm will revisit it");
1219  } else {
1220  TRACE_SID_TXS(s->id, tx, "storing state: flags %08x", inspect_flags);
1221  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1222  inspect_flags, flow_flags, file_no_match);
1223  }
1224  }
1225 
1226  return retval;
1227 }
1228 
1229 /** \internal
1230  * \brief get a DetectTransaction object
1231  * \retval struct filled with relevant info or all nulls/0s
1232  */
1233 static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto,
1234  void *alstate, const uint64_t tx_id, void *tx_ptr, const int tx_end_state,
1235  const uint8_t flow_flags)
1236 {
1237  const uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx_ptr, flow_flags);
1238  if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) {
1239  SCLogDebug("%"PRIu64" tx already fully inspected for %s. Flags %016"PRIx64,
1240  tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1241  detect_flags);
1242  DetectTransaction no_tx = { NULL, 0, NULL, 0, 0, 0, 0, 0, };
1243  return no_tx;
1244  }
1245 
1246  const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags);
1247  const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1248  DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(ipproto, alproto, tx_ptr);
1249  DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL;
1250  uint64_t prefilter_flags = detect_flags & APP_LAYER_TX_PREFILTER_MASK;
1251 
1252  DetectTransaction tx = {
1253  .tx_ptr = tx_ptr,
1254  .tx_id = tx_id,
1255  .de_state = tx_dir_state,
1256  .detect_flags = detect_flags,
1257  .prefilter_flags = prefilter_flags,
1258  .prefilter_flags_orig = prefilter_flags,
1259  .tx_progress = tx_progress,
1260  .tx_end_state = tx_end_state,
1261  };
1262  return tx;
1263 }
1264 
1265 static void DetectRunTx(ThreadVars *tv,
1266  DetectEngineCtx *de_ctx,
1267  DetectEngineThreadCtx *det_ctx,
1268  Packet *p,
1269  Flow *f,
1270  DetectRunScratchpad *scratch)
1271 {
1272  const uint8_t flow_flags = scratch->flow_flags;
1273  const SigGroupHead * const sgh = scratch->sgh;
1274  void * const alstate = f->alstate;
1275  const uint8_t ipproto = f->proto;
1276  const AppProto alproto = f->alproto;
1277 
1278  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
1279  uint64_t tx_id_min = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags);
1280  const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags);
1281 
1282  AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
1283  AppLayerGetTxIterState state;
1284  memset(&state, 0, sizeof(state));
1285 
1286  while (1) {
1287  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id_min, total_txs, &state);
1288  if (ires.tx_ptr == NULL)
1289  break;
1290 
1291  DetectTransaction tx = GetDetectTx(ipproto, alproto,
1292  alstate, ires.tx_id, ires.tx_ptr, tx_end_state, flow_flags);
1293  if (tx.tx_ptr == NULL) {
1294  SCLogDebug("%p/%"PRIu64" no transaction to inspect",
1295  tx.tx_ptr, tx_id_min);
1296 
1297  tx_id_min++; // next (if any) run look for +1
1298  goto next;
1299  }
1300  tx_id_min = tx.tx_id + 1; // next look for cur + 1
1301 
1302  uint32_t array_idx = 0;
1303  uint32_t total_rules = det_ctx->match_array_cnt;
1304  total_rules += (tx.de_state ? tx.de_state->cnt : 0);
1305 
1306  /* run prefilter engines and merge results into a candidates array */
1307  if (sgh->tx_engines) {
1309  DetectRunPrefilterTx(det_ctx, sgh, p, ipproto, flow_flags, alproto,
1310  alstate, &tx);
1312  SCLogDebug("%p/%"PRIu64" rules added from prefilter: %u candidates",
1313  tx.tx_ptr, tx.tx_id, det_ctx->pmq.rule_id_array_cnt);
1314 
1315  total_rules += det_ctx->pmq.rule_id_array_cnt;
1316  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1317  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1318  }
1319 
1320  for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) {
1321  const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]];
1322  const SigIntId id = s->num;
1323  det_ctx->tx_candidates[array_idx].s = s;
1324  det_ctx->tx_candidates[array_idx].id = id;
1325  det_ctx->tx_candidates[array_idx].flags = NULL;
1326  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1327  array_idx++;
1328  }
1329  } else {
1330  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1331  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1332  }
1333  }
1334 
1335  /* merge 'state' rules from the regular prefilter */
1336  uint32_t x = array_idx;
1337  for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) {
1338  const Signature *s = det_ctx->match_array[i];
1339  if (s->app_inspect != NULL) {
1340  const SigIntId id = s->num;
1341  det_ctx->tx_candidates[array_idx].s = s;
1342  det_ctx->tx_candidates[array_idx].id = id;
1343  det_ctx->tx_candidates[array_idx].flags = NULL;
1344  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1345  array_idx++;
1346 
1347  SCLogDebug("%p/%"PRIu64" rule %u (%u) added from 'match' list",
1348  tx.tx_ptr, tx.tx_id, s->id, id);
1349  }
1350  }
1351  SCLogDebug("%p/%"PRIu64" rules added from 'match' list: %u",
1352  tx.tx_ptr, tx.tx_id, array_idx - x); (void)x;
1353 
1354  /* merge stored state into results */
1355  if (tx.de_state != NULL) {
1356  const uint32_t old = array_idx;
1357 
1358  /* if tx.de_state->flags has 'new file' set and sig below has
1359  * 'file inspected' flag, reset the file part of the state */
1360  const bool have_new_file = (tx.de_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_NEW);
1361  if (have_new_file) {
1362  SCLogDebug("%p/%"PRIu64" destate: need to consider new file",
1363  tx.tx_ptr, tx.tx_id);
1365  }
1366 
1367  SigIntId state_cnt = 0;
1368  DeStateStore *tx_store = tx.de_state->head;
1369  for (; tx_store != NULL; tx_store = tx_store->next) {
1370  SCLogDebug("tx_store %p", tx_store);
1371 
1372  SigIntId store_cnt = 0;
1373  for (store_cnt = 0;
1374  store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < tx.de_state->cnt;
1375  store_cnt++, state_cnt++)
1376  {
1377  DeStateStoreItem *item = &tx_store->store[store_cnt];
1378  SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags);
1379  if (have_new_file && (item->flags & DE_STATE_FLAG_FILE_INSPECT)) {
1380  /* remove part of the state. File inspect engine will now
1381  * be able to run again */
1383  SCLogDebug("rule id %u, post file reset inspect_flags %u", item->sid, item->flags);
1384  }
1385  det_ctx->tx_candidates[array_idx].s = de_ctx->sig_array[item->sid];
1386  det_ctx->tx_candidates[array_idx].id = item->sid;
1387  det_ctx->tx_candidates[array_idx].flags = &item->flags;
1388  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1389  array_idx++;
1390  }
1391  }
1392  if (old && old != array_idx) {
1393  qsort(det_ctx->tx_candidates, array_idx, sizeof(RuleMatchCandidateTx),
1394  DetectRunTxSortHelper);
1395 
1396  SCLogDebug("%p/%"PRIu64" rules added from 'continue' list: %u",
1397  tx.tx_ptr, tx.tx_id, array_idx - old);
1398  }
1399  }
1400 
1401  det_ctx->tx_id = tx.tx_id;
1402  det_ctx->tx_id_set = 1;
1403  det_ctx->p = p;
1404 
1405  /* run rules: inspect the match candidates */
1406  for (uint32_t i = 0; i < array_idx; i++) {
1407  RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i];
1408  const Signature *s = det_ctx->tx_candidates[i].s;
1409  uint32_t *inspect_flags = det_ctx->tx_candidates[i].flags;
1410 
1411  /* deduplicate: rules_array is sorted, but not deduplicated:
1412  * both mpm and stored state could give us the same sid.
1413  * As they are back to back in that case we can check for it
1414  * here. We select the stored state one. */
1415  if ((i + 1) < array_idx) {
1416  if (det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i+1].s) {
1417  if (det_ctx->tx_candidates[i].flags != NULL) {
1418  i++;
1419  SCLogDebug("%p/%"PRIu64" inspecting SKIP NEXT: sid %u (%u), flags %08x",
1420  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1421  } else if (det_ctx->tx_candidates[i+1].flags != NULL) {
1422  SCLogDebug("%p/%"PRIu64" inspecting SKIP CURRENT: sid %u (%u), flags %08x",
1423  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1424  continue;
1425  } else {
1426  // if it's all the same, inspect the current one and skip next.
1427  i++;
1428  SCLogDebug("%p/%"PRIu64" inspecting SKIP NEXT: sid %u (%u), flags %08x",
1429  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1430  }
1431  }
1432  }
1433 
1434  SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x",
1435  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1436 
1437  if (inspect_flags) {
1439  SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x ALREADY COMPLETE",
1440  tx.tx_ptr, tx.tx_id, s->id, s->num, *inspect_flags);
1441  continue;
1442  }
1443  }
1444 
1445  if (inspect_flags) {
1446  /* continue previous inspection */
1447  SCLogDebug("%p/%"PRIu64" Continueing sid %u", tx.tx_ptr, tx.tx_id, s->id);
1448  } else {
1449  /* start new inspection */
1450  SCLogDebug("%p/%"PRIu64" Start sid %u", tx.tx_ptr, tx.tx_id, s->id);
1451  }
1452 
1453  /* call individual rule inspection */
1455  const int r = DetectRunTxInspectRule(tv, de_ctx, det_ctx, p, f, flow_flags,
1456  alstate, &tx, s, inspect_flags, can, scratch);
1457  if (r == 1) {
1458  /* match */
1459  DetectRunPostMatch(tv, det_ctx, p, s);
1460 
1461  uint8_t alert_flags = (PACKET_ALERT_FLAG_STATE_MATCH|PACKET_ALERT_FLAG_TX);
1462  if (s->action & ACTION_DROP)
1463  alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
1464 
1465  SCLogDebug("%p/%"PRIu64" sig %u (%u) matched", tx.tx_ptr, tx.tx_id, s->id, s->num);
1466  if (!(s->flags & SIG_FLAG_NOALERT)) {
1467  PacketAlertAppend(det_ctx, s, p, tx.tx_id, alert_flags);
1468  } else {
1469  DetectSignatureApplyActions(p, s, alert_flags);
1470  }
1471  }
1472  DetectVarProcessList(det_ctx, p->flow, p);
1473  RULE_PROFILING_END(det_ctx, s, r, p);
1474  }
1475 
1476  det_ctx->tx_id = 0;
1477  det_ctx->tx_id_set = 0;
1478  det_ctx->p = NULL;
1479 
1480  /* see if we have any updated state to store in the tx */
1481 
1482  uint64_t new_detect_flags = 0;
1483  /* this side of the tx is done */
1484  if (tx.tx_progress >= tx.tx_end_state) {
1485  new_detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
1486  SCLogDebug("%p/%"PRIu64" tx is done for direction %s. Flag %016"PRIx64,
1487  tx.tx_ptr, tx.tx_id,
1488  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1489  new_detect_flags);
1490  }
1491  if (tx.prefilter_flags != tx.prefilter_flags_orig) {
1492  new_detect_flags |= tx.prefilter_flags;
1493  SCLogDebug("%p/%"PRIu64" updated prefilter flags %016"PRIx64" "
1494  "(was: %016"PRIx64") for direction %s. Flag %016"PRIx64,
1496  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1497  new_detect_flags);
1498  }
1499  if (new_detect_flags != 0 &&
1500  (new_detect_flags | tx.detect_flags) != tx.detect_flags)
1501  {
1502  new_detect_flags |= tx.detect_flags;
1503  SCLogDebug("%p/%"PRIu64" Storing new flags %016"PRIx64" (was %016"PRIx64")",
1504  tx.tx_ptr, tx.tx_id, new_detect_flags, tx.detect_flags);
1505  AppLayerParserSetTxDetectFlags(ipproto, alproto, tx.tx_ptr,
1506  flow_flags, new_detect_flags);
1507  }
1508 next:
1509  InspectionBufferClean(det_ctx);
1510 
1511  if (!ires.has_next)
1512  break;
1513  }
1514 }
1515 
1516 /** \brief Apply action(s) and Set 'drop' sig info,
1517  * if applicable */
1519  const Signature *s, const uint8_t alert_flags)
1520 {
1522 
1523  if (s->action & ACTION_DROP) {
1524  if (p->alerts.drop.action == 0) {
1525  p->alerts.drop.num = s->num;
1526  p->alerts.drop.action = s->action;
1527  p->alerts.drop.s = (Signature *)s;
1528  }
1529  } else if (s->action & ACTION_PASS) {
1530  /* if an stream/app-layer match we enforce the pass for the flow */
1531  if ((p->flow != NULL) &&
1533  {
1534  FlowSetNoPacketInspectionFlag(p->flow);
1535  }
1536 
1537  }
1538 }
1539 
1540 static DetectEngineThreadCtx *GetTenantById(HashTable *h, uint32_t id)
1541 {
1542  /* technically we need to pass a DetectEngineThreadCtx struct with the
1543  * tentant_id member. But as that member is the first in the struct, we
1544  * can use the id directly. */
1545  return HashTableLookup(h, &id, 0);
1546 }
1547 
1548 static void DetectFlow(ThreadVars *tv,
1549  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1550  Packet *p)
1551 {
1552  if (p->flags & PKT_NOPACKET_INSPECTION) {
1553  /* hack: if we are in pass the entire flow mode, we need to still
1554  * update the inspect_id forward. So test for the condition here,
1555  * and call the update code if necessary. */
1556  const int pass = ((p->flow->flags & FLOW_NOPACKET_INSPECTION));
1557  if (pass) {
1558  uint8_t flags;
1559  if (p->flowflags & FLOW_PKT_TOSERVER) {
1560  flags = STREAM_TOSERVER;
1561  } else {
1562  flags = STREAM_TOCLIENT;
1563  }
1564  flags = FlowGetDisruptionFlags(p->flow, flags);
1565  DeStateUpdateInspectTransactionId(p->flow, flags, true);
1566  }
1567  SCLogDebug("p->pcap %"PRIu64": no detection on packet, "
1568  "PKT_NOPACKET_INSPECTION is set", p->pcap_cnt);
1569  return;
1570  }
1571 
1572  /* see if the packet matches one or more of the sigs */
1573  (void)DetectRun(tv, de_ctx, det_ctx, p);
1574 }
1575 
1576 
1577 static void DetectNoFlow(ThreadVars *tv,
1578  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1579  Packet *p)
1580 {
1581  /* No need to perform any detection on this packet, if the the given flag is set.*/
1582  if ((p->flags & PKT_NOPACKET_INSPECTION) ||
1584  {
1585  return;
1586  }
1587 
1588  /* see if the packet matches one or more of the sigs */
1589  DetectRun(tv, de_ctx, det_ctx, p);
1590  return;
1591 }
1592 
1593 /** \brief Detection engine thread wrapper.
1594  * \param tv thread vars
1595  * \param p packet to inspect
1596  * \param data thread specific data
1597  * \param pq packet queue
1598  * \retval TM_ECODE_FAILED error
1599  * \retval TM_ECODE_OK ok
1600  */
1601 TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
1602 {
1604 
1605  DetectEngineCtx *de_ctx = NULL;
1606  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
1607  if (det_ctx == NULL) {
1608  printf("ERROR: Detect has no thread ctx\n");
1609  goto error;
1610  }
1611 
1612  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
1613  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
1614  SCLogDebug("Detect Engine using new det_ctx - %p",
1615  det_ctx);
1616  }
1617 
1618  /* if in MT mode _and_ we have tenants registered, use
1619  * MT logic. */
1620  if (det_ctx->mt_det_ctxs_cnt > 0 && det_ctx->TenantGetId != NULL)
1621  {
1622  uint32_t tenant_id = p->tenant_id;
1623  if (tenant_id == 0)
1624  tenant_id = det_ctx->TenantGetId(det_ctx, p);
1625  if (tenant_id > 0 && tenant_id < det_ctx->mt_det_ctxs_cnt) {
1626  p->tenant_id = tenant_id;
1627  det_ctx = GetTenantById(det_ctx->mt_det_ctxs_hash, tenant_id);
1628  if (det_ctx == NULL)
1629  return TM_ECODE_OK;
1630  de_ctx = det_ctx->de_ctx;
1631  if (de_ctx == NULL)
1632  return TM_ECODE_OK;
1633 
1634  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
1635  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
1636  SCLogDebug("MT de_ctx %p det_ctx %p (tenant %u)", de_ctx, det_ctx, tenant_id);
1637  }
1638  } else {
1639  /* use default if no tenants are registered for this packet */
1640  de_ctx = det_ctx->de_ctx;
1641  }
1642  } else {
1643  de_ctx = det_ctx->de_ctx;
1644  }
1645 
1646  if (p->flow) {
1647  DetectFlow(tv, de_ctx, det_ctx, p);
1648  } else {
1649  DetectNoFlow(tv, de_ctx, det_ctx, p);
1650  }
1651  return TM_ECODE_OK;
1652 error:
1653  return TM_ECODE_FAILED;
1654 }
1655 
1656 /** \brief disable file features we don't need
1657  * Called if we have no detection engine.
1658  */
1660 {
1661  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOSERVER);
1662  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOCLIENT);
1663 }
1664 
1665 #ifdef UNITTESTS
1666 /**
1667  * \brief wrapper for old tests
1668  */
1670  DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1671  Packet *p)
1672 {
1673  DetectRun(th_v, de_ctx, det_ctx, p);
1674 }
1675 #endif
1676 
1677 /*
1678  * TESTS
1679  */
1680 
1681 #ifdef UNITTESTS
1682 #include "tests/detect.c"
1683 #endif
1684 
void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size)
Definition: detect.c:965
uint16_t filestore_cnt
Definition: detect.h:1352
#define PACKET_ALERT_FLAG_STREAM_MATCH
Definition: decode.h:284
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1448
SignatureNonPrefilterStore * non_pf_syn_store_array
Definition: detect.h:1348
DetectProto proto
Definition: detect.h:539
uint16_t flags
#define DETECT_PROTO_IPV6
#define SCLogDebug(...)
Definition: util-debug.h:335
Signature ** sig_array
Definition: detect.h:776
uint32_t tx_candidates_size
Definition: detect.h:1091
struct Flow_ * flow
Definition: decode.h:445
#define KEYWORD_PROFILING_END(ctx, type, m)
#define SignatureMask
Definition: detect.h:278
void PacketPatternCleanup(DetectEngineThreadCtx *det_ctx)
cleans up the mpm instance after a match
struct HtpBodyChunk_ * next
void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, const Packet *p)
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:286
struct DetectEngineAppInspectionEngine_::@93 v2
const struct SigGroupHead_ * sgh_toclient
Definition: flow.h:442
const Signature * s
Definition: detect.h:997
#define KEYWORD_PROFILING_SET_LIST(ctx, list)
uint16_t dsize_high
Definition: detect.h:529
int FileForceSha256(void)
Definition: util-file.c:145
#define BUG_ON(x)
uint32_t non_pf_store_cnt
Definition: detect.h:1094
#define PACKET_TEST_ACTION(p, a)
Definition: decode.h:857
uint32_t flags
Definition: detect.h:523
uint8_t proto
Definition: flow.h:344
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
uint32_t id
Definition: detect.h:555
#define ICMPV4_DEST_UNREACH_IS_VALID(p)
struct SigGroupHead_ * sh
Definition: detect.h:202
#define DE_STATE_FLAG_SIG_CANT_MATCH
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
uint32_t non_pf_other_store_cnt
Definition: detect.h:1344
uint16_t counter_fnonmpm_list
Definition: detect.h:1044
#define unlikely(expr)
Definition: util-optimize.h:35
#define SIG_GROUP_HEAD_HAVEFILEMD5
Definition: detect.h:1232
uint8_t is_last
Definition: detect.h:329
Port sp
Definition: decode.h:415
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void FlowSetIPOnlyFlag(Flow *f, int direction)
Set the IPOnly scanned flag for &#39;direction&#39;.
Definition: flow.c:186
#define FLOW_TOCLIENT_IPONLY_SET
Definition: flow.h:56
void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx)
Definition: detect.c:978
void FileDisableFilesize(Flow *f, uint8_t direction)
disable file size tracking for this flow
Definition: util-file.c:1185
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
#define PKT_IS_FRAGMENT
Definition: decode.h:1108
uint32_t tenant_id
Definition: flow.h:374
DetectEngineStateDirection dir_state[2]
Port dp
Definition: decode.h:423
void FileDisableSha1(Flow *f, uint8_t direction)
disable file sha1 calc for this flow
Definition: util-file.c:1107
void FileDisableMd5(Flow *f, uint8_t direction)
disable file md5 calc for this flow
Definition: util-file.c:1068
uint8_t action
Definition: decode.h:271
#define FLOW_ACTION_DROP
Definition: flow.h:64
#define SIG_GROUP_HEAD_HAVEFILESHA256
Definition: detect.h:1235
RuleMatchCandidateTx * tx_candidates
Definition: detect.h:1090
SigIntId * non_pf_id_array
Definition: detect.h:1014
uint16_t pad
Address dst
Definition: decode.h:413
DetectEngineStateDirection * de_state
#define FLOW_PKT_ESTABLISHED
Definition: flow.h:203
#define PKT_IS_IPV6(p)
Definition: decode.h:252
Data needed for Match()
Definition: detect.h:327
Container for matching data for a signature group.
Definition: detect.h:1336
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
uint64_t pcap_cnt
Definition: decode.h:561
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:232
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
#define FLOW_TOSERVER_IPONLY_SET
Definition: flow.h:54
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1063
uint32_t non_pf_syn_store_cnt
Definition: detect.h:1345
uint16_t AppProto
TCPHdr * tcph
Definition: decode.h:522
#define PACKET_ALERT_FLAG_DROP_FLOW
Definition: decode.h:280
Signature container.
Definition: detect.h:522
#define DEBUG_VALIDATE_PACKET(p)
bool DetectEnginePktInspectionRun(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p, uint8_t *alert_flags)
uint8_t action
Definition: detect.h:535
uint16_t addr_src_match6_cnt
Definition: detect.h:548
void IPOnlyMatchPacket(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const DetectEngineIPOnlyCtx *io_ctx, DetectEngineIPOnlyThreadCtx *io_tctx, Packet *p)
Match a packet against the IP Only detection engine contexts.
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get &#39;disruption&#39; flags: GAP/DEPTH/PASS
Definition: flow.c:1080
#define SIG_FLAG_APPLAYER
Definition: detect.h:218
void DetectRunStoreStateTx(const SigGroupHead *sgh, Flow *f, void *tx, uint64_t tx_id, const Signature *s, uint32_t inspect_flags, uint8_t flow_flags, const uint16_t file_no_match)
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:859
#define PKT_IS_IPV4(p)
Definition: decode.h:251
void * protoctx
Definition: flow.h:400
SigIntId num
Definition: decode.h:270
struct DeStateStore_ * next
DetectMatchAddressIPv6 * addr_dst_match6
Definition: detect.h:552
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES
SigIntId num
Definition: detect.h:532
uint64_t AppLayerParserGetTransactionInspectId(AppLayerParserState *pstate, uint8_t direction)
int FileForceMd5(void)
Definition: util-file.c:135
main detection engine ctx
Definition: detect.h:761
uint16_t addr_src_match4_cnt
Definition: detect.h:546
PrefilterEngine * tx_engines
Definition: detect.h:1358
const uint8_t flow_flags
Definition: detect.c:64
const uint64_t prefilter_flags_orig
int DetectAddressMatchIPv6(const DetectMatchAddressIPv6 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
const struct Signature_ * s
Definition: decode.h:273
DetectPort * udp
Definition: detect.h:699
#define SIG_FLAG_DST_ANY
Definition: detect.h:212
void * alstate
Definition: flow.h:438
#define APP_LAYER_TX_INSPECTED_FLAG
DetectMatchAddressIPv4 * addr_dst_match4
Definition: detect.h:549
#define KEYWORD_PROFILING_START
const bool app_decoder_events
Definition: detect.c:65
#define FLOW_NOPACKET_INSPECTION
Definition: flow.h:59
#define SCCalloc(nm, a)
Definition: util-mem.h:253
DetectMatchAddressIPv6 * addr_src_match6
Definition: detect.h:553
DetectPort * dp
Definition: detect.h:561
#define SIG_FLAG_SP_ANY
Definition: detect.h:213
DetectPort * tcp
Definition: detect.h:698
void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t ipproto, const uint8_t flow_flags, const AppProto alproto, void *alstate, DetectTransaction *tx)
run prefilter engines on a transaction
uint8_t proto
Definition: decode.h:430
struct SigGroupHead_ * sgh[256]
Definition: detect.h:700
#define DETECT_ENGINE_INSPECT_SIG_MATCH
DetectPort * DetectPortLookupGroup(DetectPort *dp, uint16_t port)
Function that find the group matching address in a group head.
uint16_t filestore_cnt
Definition: detect.h:1037
Data structures and function prototypes for keeping state for the detection engine.
void FileDisableMagic(Flow *f, uint8_t direction)
disable file magic lookups for this flow
Definition: util-file.c:1037
struct DetectEngineAppInspectionEngine_ * next
Definition: detect.h:421
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1123
#define SIG_FLAG_FLUSH
Definition: detect.h:228
uint16_t counter_alerts
Definition: detect.h:1040
uint32_t(* TenantGetId)(const void *, const Packet *p)
Definition: detect.h:1024
void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t flags)
DetectMatchAddressIPv4 * addr_src_match4
Definition: detect.h:550
#define STREAM_EOF
Definition: stream.h:30
#define APP_LAYER_TX_PREFILTER_MASK
void DeStateUpdateInspectTransactionId(Flow *f, const uint8_t flags, const bool tag_txs_as_inspected)
update flow&#39;s inspection id&#39;s
void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
wrapper for old tests
Definition: detect.c:1669
#define SCEnter(...)
Definition: util-debug.h:337
PrefilterRuleStore pmq
Definition: detect.h:1102
uint32_t profile_match_logging_threshold
Definition: detect.h:887
uint8_t flowflags
Definition: decode.h:439
uint16_t counter_match_list
Definition: detect.h:1045
PacketEngineEvents events
Definition: decode.h:565
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:575
uint16_t addr_dst_match6_cnt
Definition: detect.h:547
#define FLOW_SGH_TOCLIENT
Definition: flow.h:69
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PACKET_PROFILING_DETECT_END(p, id)
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
HashTable * mt_det_ctxs_hash
Definition: detect.h:1019
SigIntId match_array_cnt
Definition: detect.h:1088
AppProto alproto
Definition: detect.h:526
#define SIG_GROUP_HEAD_HAVEFILESHA1
Definition: detect.h:1234
void DetectSignatureApplyActions(Packet *p, const Signature *s, const uint8_t alert_flags)
Apply action(s) and Set &#39;drop&#39; sig info, if applicable.
Definition: detect.c:1518
SigIntId * rule_id_array
#define DETECT_ENGINE_STATE_FLAG_FILE_NEW
int(* Match)(DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1170
uint32_t flags
Definition: detect.h:1337
void FileDisableStoring(Flow *f, uint8_t direction)
disable file storage for a flow
Definition: util-file.c:1005
uint8_t proto
uint32_t stream_reset
Definition: detect.h:994
const AppProto alproto
Definition: detect.c:63
InspectEngineFuncPtr Callback
Definition: detect.h:410
#define RULE_PROFILING_START(p)
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:807
GenericVar * flowvar
Definition: flow.h:448
#define PACKET_UPDATE_ACTION(p, a)
Definition: decode.h:862
const uint64_t detect_flags
#define SIG_GROUP_HEAD_HAVERAWSTREAM
Definition: detect.h:1228
#define DETECT_PROTO_IPV4
void DetectEngineStateResetTxs(Flow *f)
Reset de state for active tx&#39; To be used on detect engine reload.
#define SCRealloc(x, a)
Definition: util-mem.h:238
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
const SigGroupHead * SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx, const Packet *p)
Get the SigGroupHead for a packet.
Definition: detect.c:177
uint32_t version
Definition: detect.h:855
int FileForceFilestore(void)
Definition: util-file.c:111
int FileForceMagic(void)
Definition: util-file.c:130
SignatureMask pkt_mask
Definition: detect.c:67
void DisableDetectFlowFileFlags(Flow *f)
disable file features we don&#39;t need Called if we have no detection engine.
Definition: detect.c:1659
DeStateStoreItem store[DE_STATE_CHUNK_SIZE]
#define PACKET_DROP(p)
Definition: decode.h:847
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:207
void * HashTableLookup(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:193
DetectEngineCtx * de_ctx
Definition: detect.h:1127
const SigGroupHead * sgh
Definition: detect.c:66
uint16_t counter_nonmpm_list
Definition: detect.h:1043
uint64_t raw_stream_progress
Definition: detect.h:1028
struct DetectRunScratchpad DetectRunScratchpad
uint8_t type
Definition: detect.h:328
#define TH_SYN
Definition: decode-tcp.h:36
#define SCFree(a)
Definition: util-mem.h:322
#define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES
#define PKT_STREAM_EOF
Definition: decode.h:1092
#define SIG_FLAG_NOALERT
Definition: detect.h:216
DetectPort * sp
Definition: detect.h:561
uint16_t addr_dst_match4_cnt
Definition: detect.h:545
uint16_t tx_id
#define FLOW_PKT_TOSERVER_IPONLY_SET
Definition: flow.h:204
#define SIG_FLAG_DSIZE
Definition: detect.h:217
void PacketAlertFinalize(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
Check the threshold of the sigs that match, set actions, break on pass action This function iterate t...
#define STREAM_FLUSH
Definition: stream.h:36
PacketAlert drop
Definition: decode.h:297
void GenericVarFree(GenericVar *gv)
Definition: util-var.c:47
#define PKT_NOPACKET_INSPECTION
Definition: decode.h:1086
DetectEngineAppInspectionEngine * app_inspect
Definition: detect.h:570
const struct SigGroupHead_ * sgh_toserver
Definition: flow.h:445
void InspectionBufferClean(DetectEngineThreadCtx *det_ctx)
#define PKT_IS_ICMPV4(p)
Definition: decode.h:255
#define FatalError(x,...)
Definition: util-debug.h:539
uint32_t rule_id_array_cnt
#define PKT_STREAM_ADD
Definition: decode.h:1090
#define STREAM_TOSERVER
Definition: stream.h:31
#define IP_GET_IPPROTO(p)
Definition: decode.h:263
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
SCMutex m
Definition: flow-hash.h:105
#define PACKET_PROFILING_DETECT_START(p, id)
TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
Detection engine thread wrapper.
Definition: detect.c:1601
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
PacketAlerts alerts
Definition: decode.h:555
#define TRACE_SID_TXS(sid, txs,...)
Definition: detect.c:1033
int DetectAddressMatchIPv4(const DetectMatchAddressIPv4 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
Signature ** match_array
Definition: detect.h:1083
SignatureMask mask
Definition: detect.h:981
int FileForceSha1(void)
Definition: util-file.c:140
#define PKT_HAS_FLOW
Definition: decode.h:1093
SignatureNonPrefilterStore * non_pf_store_ptr
Definition: detect.h:1093
int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, const Signature *s, Packet *p, uint64_t tx_id, uint8_t flags)
append a signature match to a packet
#define PACKET_ALERT_FLAG_STATE_MATCH
Definition: decode.h:282
#define BIT_U32(n)
uint32_t de_ctx_version
Definition: flow.h:423
uint32_t * flags
Definition: detect.h:988
#define SIG_FLAG_DP_ANY
Definition: detect.h:214
DetectEngineIPOnlyThreadCtx io_ctx
Definition: detect.h:1109
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:795
#define ACTION_PASS
uint8_t stream_result
Definition: detect.h:992
uint16_t cnt
Definition: decode.h:293
SignatureNonPrefilterStore * non_pf_other_store_array
Definition: detect.h:1346
#define SCReturn
Definition: util-debug.h:339
uint32_t tenant_id
Definition: decode.h:594
Per thread variable structure.
Definition: threadvars.h:57
void FileDisableSha256(Flow *f, uint8_t direction)
disable file sha256 calc for this flow
Definition: util-file.c:1146
#define FLOW_PKT_TOCLIENT_IPONLY_SET
Definition: flow.h:205
SignatureMask mask
Definition: detect.h:531
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
uint16_t counter_mpm_list
Definition: detect.h:1042
#define ACTION_DROP
uint32_t flags
Definition: decode.h:443
DetectEngineState * AppLayerParserGetTxDetectState(uint8_t ipproto, AppProto alproto, void *tx)
void AppLayerParserSetTxDetectFlags(uint8_t ipproto, AppProto alproto, void *tx, uint8_t dir, uint64_t flags)
uint16_t payload_len
Definition: decode.h:541
void StatsAddUI64(ThreadVars *tv, uint16_t id, uint64_t x)
Adds a value of type uint64_t to the local counter.
Definition: counters.c:147
#define likely(expr)
Definition: util-optimize.h:32
uint32_t non_pf_id_cnt
Definition: detect.h:1015
#define DE_STATE_FLAG_FULL_INSPECT
#define SIG_FLAG_SRC_ANY
Definition: detect.h:211
uint16_t dsize_low
Definition: detect.h:528
bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
#define FLOW_SGH_TOSERVER
Definition: flow.h:67
#define DE_STATE_FLAG_FILE_INSPECT
Flow data structure.
Definition: flow.h:325
uint32_t mt_det_ctxs_cnt
Definition: detect.h:1017
int DetectProtoContainsProto(const DetectProto *dp, int proto)
see if a DetectProto contains a certain proto
#define SigIntId
uint32_t flags
Definition: flow.h:379
#define RULE_PROFILING_END(ctx, r, m, p)
#define PKT_STREAM_EST
Definition: decode.h:1091
Port structure for detection engine.
Definition: detect.h:191
#define SIG_GROUP_HEAD_HAVEFILESIZE
Definition: detect.h:1233
AppLayerParserState * alparser
Definition: flow.h:437
Address src
Definition: decode.h:412
SigMatchCtx * ctx
Definition: detect.h:330
uint64_t AppLayerParserGetTxDetectFlags(uint8_t ipproto, AppProto alproto, void *tx, uint8_t dir)
#define DE_STATE_CHUNK_SIZE
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SGH_PROFILING_RECORD(det_ctx, sgh)
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype