suricata
detect.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2025 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Basic detection engine
24  */
25 
26 #include "suricata-common.h"
27 #include "suricata.h"
28 
29 #include "decode.h"
30 #include "packet.h"
31 #include "flow.h"
32 #include "stream-tcp.h"
33 #include "app-layer.h"
34 #include "app-layer-parser.h"
35 #include "app-layer-frames.h"
36 
37 #include "detect.h"
38 #include "detect-dsize.h"
39 #include "detect-engine.h"
40 #include "detect-engine-build.h"
41 #include "detect-engine-frame.h"
42 #include "detect-engine-profile.h"
43 
44 #include "detect-engine-alert.h"
45 #include "detect-engine-siggroup.h"
46 #include "detect-engine-address.h"
47 #include "detect-engine-proto.h"
48 #include "detect-engine-port.h"
49 #include "detect-engine-mpm.h"
50 #include "detect-engine-iponly.h"
53 #include "detect-engine-state.h"
54 #include "detect-engine-analyzer.h"
55 
56 #include "detect-engine-payload.h"
57 #include "detect-engine-event.h"
58 
59 #include "detect-filestore.h"
60 #include "detect-flowvar.h"
61 #include "detect-replace.h"
62 
63 #include "util-validate.h"
64 #include "util-detect.h"
65 #include "util-profiling.h"
66 
67 #include "action-globals.h"
68 
69 typedef struct DetectRunScratchpad {
71  const uint8_t flow_flags; /* flow/state flags: STREAM_* */
72  const bool app_decoder_events;
73  const SigGroupHead *sgh;
75 
76 /* prototypes */
77 static DetectRunScratchpad DetectRunSetup(const DetectEngineCtx *de_ctx,
78  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow);
79 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
80  DetectEngineThreadCtx *det_ctx, Flow * const pflow, Packet * const p);
81 static inline void DetectRunGetRuleGroup(const DetectEngineCtx *de_ctx,
82  Packet * const p, Flow * const pflow, DetectRunScratchpad *scratch);
83 static inline void DetectRunPrefilterPkt(ThreadVars *tv,
85  DetectRunScratchpad *scratch);
86 static inline uint8_t DetectRulePacketRules(ThreadVars *const tv, DetectEngineCtx *const de_ctx,
87  DetectEngineThreadCtx *const det_ctx, Packet *const p, Flow *const pflow,
88  const DetectRunScratchpad *scratch);
89 static void DetectRunTx(ThreadVars *tv, DetectEngineCtx *de_ctx,
90  DetectEngineThreadCtx *det_ctx, Packet *p,
91  Flow *f, DetectRunScratchpad *scratch);
92 static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
93  Packet *p, Flow *f, DetectRunScratchpad *scratch);
94 static inline void DetectRunPostRules(ThreadVars *tv, DetectEngineCtx *de_ctx,
95  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow,
96  DetectRunScratchpad *scratch);
97 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
98  Packet *p, Flow * const pflow);
99 static inline void DetectRunAppendDefaultAccept(DetectEngineThreadCtx *det_ctx, Packet *p);
100 
101 /** \internal
102  */
103 static void DetectRun(ThreadVars *th_v,
105  Packet *p)
106 {
107  SCEnter();
108  SCLogDebug("p->pcap_cnt %" PRIu64 " direction %s pkt_src %s", p->pcap_cnt,
109  p->flow ? (FlowGetPacketDirection(p->flow, p) == TOSERVER ? "toserver" : "toclient")
110  : "noflow",
111  PktSrcToString(p->pkt_src));
112 
113  /* Load the Packet's flow early, even though it might not be needed.
114  * Mark as a constant pointer, although the flow itself can change. */
115  Flow * const pflow = p->flow;
116 
117  DetectRunScratchpad scratch = DetectRunSetup(de_ctx, det_ctx, p, pflow);
118 
119  /* run the IPonly engine */
120  DetectRunInspectIPOnly(th_v, de_ctx, det_ctx, pflow, p);
121 
122  /* get our rule group */
123  DetectRunGetRuleGroup(de_ctx, p, pflow, &scratch);
124  /* if we didn't get a sig group head, we
125  * have nothing to do.... */
126  if (scratch.sgh == NULL) {
127  SCLogDebug("no sgh for this packet, nothing to match against");
128  goto end;
129  }
130 
131  /* run the prefilters for packets */
132  DetectRunPrefilterPkt(th_v, de_ctx, det_ctx, p, &scratch);
133 
135  /* inspect the rules against the packet */
136  const uint8_t pkt_policy = DetectRulePacketRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
138 
139  /* Only FW rules will already have set the action, IDS rules go through PacketAlertFinalize
140  *
141  * If rules told us to drop or accept:packet/accept:flow, we skip app_filter and app_td.
142  *
143  * accept:hook won't have set the pkt_policy, so we simply continue.
144  *
145  * TODO what about app state progression, cleanup and such? */
146  if (pkt_policy & (ACTION_DROP | ACTION_ACCEPT)) {
147  goto end;
148  }
149 
150  /* run tx/state inspection. Don't call for ICMP error msgs. */
151  if (pflow && pflow->alstate && likely(pflow->proto == p->proto)) {
152  if (p->proto == IPPROTO_TCP) {
153  if ((p->flags & PKT_STREAM_EST) == 0) {
154  SCLogDebug("packet %" PRIu64 ": skip tcp non-established", p->pcap_cnt);
155  DetectRunAppendDefaultAccept(det_ctx, p);
156  goto end;
157  }
158  const TcpSession *ssn = p->flow->protoctx;
159  bool setting_nopayload = p->flow->alparser &&
163  // we may be right after disabling app-layer (ssh)
164  if (ssn &&
165  ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0 || setting_nopayload)) {
166  // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX);
167  DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch);
168  // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_TX);
169  }
170  // no update to transactions
171  if (!PKT_IS_PSEUDOPKT(p) && p->app_update_direction == 0 &&
172  ((PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATED) == 0) ||
173  (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATED) == 0))) {
174  SCLogDebug("packet %" PRIu64 ": no app-layer update", p->pcap_cnt);
175  DetectRunAppendDefaultAccept(det_ctx, p);
176  goto end;
177  }
178  } else if (p->proto == IPPROTO_UDP) {
179  DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch);
180  }
181 
183  DetectRunTx(th_v, de_ctx, det_ctx, p, pflow, &scratch);
185  /* see if we need to increment the inspect_id and reset the de_state */
188  pflow, pflow->alparser, pflow->alstate, scratch.flow_flags, (scratch.sgh == NULL));
190  } else {
191  SCLogDebug("packet %" PRIu64 ": no flow / app-layer", p->pcap_cnt);
192  DetectRunAppendDefaultAccept(det_ctx, p);
193  }
194 
195 end:
196  DetectRunPostRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
197 
198  DetectRunCleanup(det_ctx, p, pflow);
199  SCReturn;
200 }
201 
202 static void DetectRunPostMatch(ThreadVars *tv,
203  DetectEngineThreadCtx *det_ctx, Packet *p,
204  const Signature *s)
205 {
206  /* run the packet match functions */
208  if (smd != NULL) {
210 
211  SCLogDebug("running match functions, sm %p", smd);
212 
213  while (1) {
215  (void)sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx);
216  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
217  if (smd->is_last)
218  break;
219  smd++;
220  }
221  }
222 }
223 
224 /**
225  * \brief Get the SigGroupHead for a packet.
226  *
227  * \param de_ctx detection engine context
228  * \param p packet
229  *
230  * \retval sgh the SigGroupHead or NULL if non applies to the packet
231  */
233  const Packet *p)
234 {
235  SCEnter();
236  SigGroupHead *sgh = NULL;
237 
238  /* if the packet proto is 0 (not set), we're inspecting it against
239  * the decoder events sgh we have. */
240  if (p->proto == 0 && p->events.cnt > 0) {
241  SCReturnPtr(de_ctx->decoder_event_sgh, "SigGroupHead");
242  } else if (p->proto == 0) {
243  if (!(PacketIsIPv4(p) || PacketIsIPv6(p))) {
244  /* not IP, so nothing to do */
245  SCReturnPtr(NULL, "SigGroupHead");
246  }
247  }
248 
249  /* select the flow_gh */
250  const int dir = (p->flowflags & FLOW_PKT_TOCLIENT) == 0;
251 
252  int proto = PacketGetIPProto(p);
253  if (proto == IPPROTO_TCP) {
254  DetectPort *list = de_ctx->flow_gh[dir].tcp;
255  SCLogDebug("tcp toserver %p, tcp toclient %p: going to use %p", de_ctx->flow_gh[1].tcp,
256  de_ctx->flow_gh[0].tcp, de_ctx->flow_gh[dir].tcp);
257  const uint16_t port = dir ? p->dp : p->sp;
258  SCLogDebug("tcp port %u -> %u:%u", port, p->sp, p->dp);
259  DetectPort *sghport = DetectPortLookupGroup(list, port);
260  if (sghport != NULL)
261  sgh = sghport->sh;
262  SCLogDebug("TCP list %p, port %u, direction %s, sghport %p, sgh %p", list, port,
263  dir ? "toserver" : "toclient", sghport, sgh);
264  } else if (proto == IPPROTO_UDP) {
265  DetectPort *list = de_ctx->flow_gh[dir].udp;
266  uint16_t port = dir ? p->dp : p->sp;
267  DetectPort *sghport = DetectPortLookupGroup(list, port);
268  if (sghport != NULL)
269  sgh = sghport->sh;
270  SCLogDebug("UDP list %p, port %u, direction %s, sghport %p, sgh %p", list, port,
271  dir ? "toserver" : "toclient", sghport, sgh);
272  } else {
273  sgh = de_ctx->flow_gh[dir].sgh[proto];
274  }
275 
276  SCReturnPtr(sgh, "SigGroupHead");
277 }
278 
279 static inline void DetectPrefilterCopyDeDup(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
280 {
281  SigIntId *pf_ptr = det_ctx->pmq.rule_id_array;
282  uint32_t final_cnt = det_ctx->pmq.rule_id_array_cnt;
283  Signature **sig_array = de_ctx->sig_array;
284  Signature **match_array = det_ctx->match_array;
285  SigIntId previous_id = (SigIntId)-1;
286  while (final_cnt-- > 0) {
287  SigIntId id = *pf_ptr++;
288  Signature *s = sig_array[id];
289 
290  /* As the prefilter list can contain duplicates, check for that here. */
291  if (likely(id != previous_id)) {
292  *match_array++ = s;
293  previous_id = id;
294  }
295  }
296 
297  det_ctx->match_array_cnt = (uint32_t)(match_array - det_ctx->match_array);
299  PMQ_RESET(&det_ctx->pmq);
300 }
301 
302 /** \internal
303  * \brief update flow's file tracking flags based on the detection engine
304  * A set of flags is prepared that is sent to the File API. The
305  File API may reject one or more based on the global force settings.
306  */
307 static inline void
308 DetectPostInspectFileFlagsUpdate(Flow *f, const SigGroupHead *sgh, uint8_t direction)
309 {
310  uint16_t flow_file_flags = FLOWFILE_INIT;
311 
312  if (sgh == NULL) {
313  SCLogDebug("requesting disabling all file features for flow");
314  flow_file_flags = FLOWFILE_NONE;
315  } else {
316  if (sgh->filestore_cnt == 0) {
317  SCLogDebug("requesting disabling filestore for flow");
318  flow_file_flags |= (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC);
319  }
320 #ifdef HAVE_MAGIC
321  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)) {
322  SCLogDebug("requesting disabling magic for flow");
323  flow_file_flags |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC);
324  }
325 #endif
326  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5)) {
327  SCLogDebug("requesting disabling md5 for flow");
328  flow_file_flags |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC);
329  }
330  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA1)) {
331  SCLogDebug("requesting disabling sha1 for flow");
332  flow_file_flags |= (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC);
333  }
334  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA256)) {
335  SCLogDebug("requesting disabling sha256 for flow");
336  flow_file_flags |= (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC);
337  }
338  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE)) {
339  SCLogDebug("requesting disabling filesize for flow");
340  flow_file_flags |= (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC);
341  }
342  }
343  if (flow_file_flags != 0) {
344  FileUpdateFlowFileFlags(f, flow_file_flags, direction);
345  }
346 }
347 
348 static inline void
349 DetectRunPostGetFirstRuleGroup(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
350 {
351  if ((p->flowflags & FLOW_PKT_TOSERVER) && !(pflow->flags & FLOW_SGH_TOSERVER)) {
352  /* first time we see this toserver sgh, store it */
353  pflow->sgh_toserver = sgh;
354  pflow->flags |= FLOW_SGH_TOSERVER;
355 
356  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
357  if (pflow->protoctx != NULL) {
358  TcpSession *ssn = pflow->protoctx;
359  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.client");
361  }
362  }
363 
364  DetectPostInspectFileFlagsUpdate(pflow,
365  pflow->sgh_toserver, STREAM_TOSERVER);
366 
367  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && !(pflow->flags & FLOW_SGH_TOCLIENT)) {
368  pflow->sgh_toclient = sgh;
369  pflow->flags |= FLOW_SGH_TOCLIENT;
370 
371  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
372  if (pflow->protoctx != NULL) {
373  TcpSession *ssn = pflow->protoctx;
374  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.server");
376  }
377  }
378 
379  DetectPostInspectFileFlagsUpdate(pflow,
380  pflow->sgh_toclient, STREAM_TOCLIENT);
381  }
382 }
383 
384 static inline void DetectRunGetRuleGroup(
385  const DetectEngineCtx *de_ctx,
386  Packet * const p, Flow * const pflow,
387  DetectRunScratchpad *scratch)
388 {
389  const SigGroupHead *sgh = NULL;
390 
391  if (pflow) {
392  bool use_flow_sgh = false;
393  /* Get the stored sgh from the flow (if any). Make sure we're not using
394  * the sgh for icmp error packets part of the same stream. */
395  if (PacketGetIPProto(p) == pflow->proto) { /* filter out icmp */
397  if ((p->flowflags & FLOW_PKT_TOSERVER) && (pflow->flags & FLOW_SGH_TOSERVER)) {
398  sgh = pflow->sgh_toserver;
399  SCLogDebug("sgh = pflow->sgh_toserver; => %p", sgh);
400  use_flow_sgh = true;
401  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && (pflow->flags & FLOW_SGH_TOCLIENT)) {
402  sgh = pflow->sgh_toclient;
403  SCLogDebug("sgh = pflow->sgh_toclient; => %p", sgh);
404  use_flow_sgh = true;
405  }
407  }
408 
409  if (!(use_flow_sgh)) {
413 
414  /* HACK: prevent the wrong sgh (or NULL) from being stored in the
415  * flow's sgh pointers */
416  if (PacketIsICMPv4(p) && ICMPV4_DEST_UNREACH_IS_VALID(p)) {
417  ; /* no-op */
418  } else {
419  /* store the found sgh (or NULL) in the flow to save us
420  * from looking it up again for the next packet.
421  * Also run other tasks */
422  DetectRunPostGetFirstRuleGroup(p, pflow, sgh);
423  }
424  }
425  } else { /* p->flags & PKT_HAS_FLOW */
426  /* no flow */
427 
431  }
432 
433  scratch->sgh = sgh;
434 }
435 
436 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
437  DetectEngineThreadCtx *det_ctx,
438  Flow * const pflow, Packet * const p)
439 {
440  if (pflow) {
442  SCLogDebug("testing against \"ip-only\" signatures");
443 
445  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, p);
447  }
448  } else { /* p->flags & PKT_HAS_FLOW */
449  /* no flow */
450 
451  /* Even without flow we should match the packet src/dst */
453  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, p);
455  }
456 }
457 
458 /** \internal
459  * \brief inspect the rule header: protocol, ports, etc
460  * \retval bool false if no match, true if match */
461 static inline bool DetectRunInspectRuleHeader(const Packet *p, const Flow *f, const Signature *s,
462  const uint32_t sflags, const uint8_t s_proto_flags)
463 {
464  /* check if this signature has a requirement for flowvars of some type
465  * and if so, if we actually have any in the flow. If not, the sig
466  * can't match and we skip it. */
467  if ((p->flags & PKT_HAS_FLOW) && (sflags & SIG_FLAG_REQUIRE_FLOWVAR)) {
468  DEBUG_VALIDATE_BUG_ON(f == NULL);
469 
470  /* no flowvars? skip this sig */
471  const bool fv = f->flowvar != NULL;
472  if (!fv) {
473  SCLogDebug("skipping sig as the flow has no flowvars and sig "
474  "has SIG_FLAG_REQUIRE_FLOWVAR flag set.");
475  return false;
476  }
477  }
478 
479  if ((s_proto_flags & DETECT_PROTO_IPV4) && !PacketIsIPv4(p)) {
480  SCLogDebug("ip version didn't match");
481  return false;
482  }
483  if ((s_proto_flags & DETECT_PROTO_IPV6) && !PacketIsIPv6(p)) {
484  SCLogDebug("ip version didn't match");
485  return false;
486  }
487 
488  if (DetectProtoContainsProto(&s->proto, PacketGetIPProto(p)) == 0) {
489  SCLogDebug("proto didn't match");
490  return false;
491  }
492 
493  /* check the source & dst port in the sig */
494  if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP || p->proto == IPPROTO_SCTP) {
495  if (!(sflags & SIG_FLAG_DP_ANY)) {
496  if (p->flags & PKT_IS_FRAGMENT)
497  return false;
498  const DetectPort *dport = DetectPortLookupGroup(s->dp, p->dp);
499  if (dport == NULL) {
500  SCLogDebug("dport didn't match.");
501  return false;
502  }
503  }
504  if (!(sflags & SIG_FLAG_SP_ANY)) {
505  if (p->flags & PKT_IS_FRAGMENT)
506  return false;
507  const DetectPort *sport = DetectPortLookupGroup(s->sp, p->sp);
508  if (sport == NULL) {
509  SCLogDebug("sport didn't match.");
510  return false;
511  }
512  }
513  } else if ((sflags & (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) != (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) {
514  SCLogDebug("port-less protocol and sig needs ports");
515  return false;
516  }
517 
518  /* check the destination address */
519  if (!(sflags & SIG_FLAG_DST_ANY)) {
520  if (PacketIsIPv4(p)) {
522  return false;
523  } else if (PacketIsIPv6(p)) {
525  return false;
526  }
527  }
528  /* check the source address */
529  if (!(sflags & SIG_FLAG_SRC_ANY)) {
530  if (PacketIsIPv4(p)) {
532  return false;
533  } else if (PacketIsIPv6(p)) {
535  return false;
536  }
537  }
538 
539  return true;
540 }
541 
542 /** \internal
543  * \brief run packet/stream prefilter engines
544  */
545 static inline void DetectRunPrefilterPkt(
546  ThreadVars *tv,
548  DetectEngineThreadCtx *det_ctx,
549  Packet *p,
550  DetectRunScratchpad *scratch
551 )
552 {
553  /* create our prefilter mask */
554  PacketCreateMask(p, &p->sig_mask, scratch->alproto, scratch->app_decoder_events);
555  /* run the prefilter engines */
556  Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, p->sig_mask);
557  /* create match list if we have non-pf and/or pf */
558  if (det_ctx->pmq.rule_id_array_cnt) {
559 #ifdef PROFILING
560  if (tv) {
561  StatsAddUI64(tv, det_ctx->counter_mpm_list, (uint64_t)det_ctx->pmq.rule_id_array_cnt);
562  }
563 #endif
565  DetectPrefilterCopyDeDup(de_ctx, det_ctx);
567  }
568 }
569 
570 /** \internal
571  * \brief check if the tx whose id is given is the only one
572  * live transaction for the flow in the given direction
573  *
574  * \param f flow
575  * \param txid transaction id
576  * \param dir direction
577  *
578  * \retval bool true if we are sure this tx is the only one live in said direction
579  */
580 static bool IsOnlyTxInDirection(Flow *f, uint64_t txid, uint8_t dir)
581 {
582  uint64_t tx_cnt = AppLayerParserGetTxCnt(f, f->alstate);
583  if (tx_cnt == txid + 1) {
584  // only live tx
585  return true;
586  }
587  if (tx_cnt == txid + 2) {
588  // 2 live txs, one after us
589  void *tx = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, txid + 1);
590  if (tx) {
592  // test if the other tx is unidirectional in the other way
593  if ((dir == STREAM_TOSERVER && (txd->flags & APP_LAYER_TX_SKIP_INSPECT_TS)) ||
594  (dir == STREAM_TOCLIENT && (txd->flags & APP_LAYER_TX_SKIP_INSPECT_TC))) {
595  return true;
596  }
597  }
598  }
599  return false;
600 }
601 
602 static int SortHelper(const void *a, const void *b)
603 {
604  const Signature *sa = *(const Signature **)a;
605  const Signature *sb = *(const Signature **)b;
606  if (sa->num == sb->num)
607  return 0;
608  return sa->num > sb->num ? 1 : -1;
609 }
610 
611 static inline uint8_t DetectRulePacketRules(ThreadVars *const tv, DetectEngineCtx *const de_ctx,
612  DetectEngineThreadCtx *const det_ctx, Packet *const p, Flow *const pflow,
613  const DetectRunScratchpad *scratch)
614 {
615  uint8_t action = 0;
616  bool fw_verdict = false;
617  const bool have_fw_rules = (de_ctx->flags & DE_HAS_FIREWALL) != 0;
618  const Signature *next_s = NULL;
619 
620  /* inspect the sigs against the packet */
621  /* Prefetch the next signature. */
622  SigIntId match_cnt = det_ctx->match_array_cnt;
623 #ifdef PROFILING
624  if (tv) {
626  (uint64_t)match_cnt);
627  }
628 #endif
629  Signature **match_array = det_ctx->match_array;
630 
631  SGH_PROFILING_RECORD(det_ctx, scratch->sgh);
632 #ifdef PROFILING
633  if (match_cnt >= de_ctx->profile_match_logging_threshold)
634  RulesDumpMatchArray(det_ctx, scratch->sgh, p);
635 #endif
636 
637  bool skip_fw = false;
638  uint32_t sflags, next_sflags = 0;
639  if (match_cnt) {
640  next_s = *match_array++;
641  next_sflags = next_s->flags;
642  }
643  while (match_cnt--) {
645  bool break_out_of_packet_filter = false;
646  uint8_t alert_flags = 0;
647 #ifdef PROFILE_RULES
648  bool smatch = false; /* signature match */
649 #endif
650  const Signature *s = next_s;
651  sflags = next_sflags;
652  if (match_cnt) {
653  next_s = *match_array++;
654  next_sflags = next_s->flags;
655  }
656  const uint8_t s_proto_flags = s->proto.flags;
657 
658  SCLogDebug("packet %" PRIu64 ": inspecting signature id %" PRIu32 "", p->pcap_cnt, s->id);
659 
660  /* if we accept:hook'd the `packet_filter` hook, we skip the rest of the firewall rules. */
661  if (s->flags & SIG_FLAG_FIREWALL) {
662  if (skip_fw) {
663  SCLogDebug("skipping firewall rule %u", s->id);
664  goto next;
665  }
666  } else if (have_fw_rules) {
667  /* fw mode, we skip anything after the fw rules if:
668  * - flow pass is set
669  * - packet pass (e.g. exception policy) */
670  if (p->flags & PKT_NOPACKET_INSPECTION ||
671  (pflow != NULL && pflow->flags & (FLOW_ACTION_PASS))) {
672  SCLogDebug("skipping firewall rule %u", s->id);
673  break_out_of_packet_filter = true;
674  goto next;
675  }
676  }
677 
678  if (s->app_inspect != NULL) {
679  goto next; // handle sig in DetectRunTx
680  }
681  if (s->frame_inspect != NULL) {
682  goto next; // handle sig in DetectRunFrame
683  }
684 
685  /* skip pkt sigs for flow end packets */
686  if ((p->flags & PKT_PSEUDO_STREAM_END) != 0 && s->type == SIG_TYPE_PKT)
687  goto next;
688 
689  /* don't run mask check for stateful rules.
690  * There we depend on prefilter */
691  if ((s->mask & p->sig_mask) != s->mask) {
692  SCLogDebug("mask mismatch %x & %x != %x", s->mask, p->sig_mask, s->mask);
693  goto next;
694  }
695 
696  if (SigDsizePrefilter(p, s, sflags))
697  goto next;
698 
699  /* if the sig has alproto and the session as well they should match */
700  if (likely(sflags & SIG_FLAG_APPLAYER)) {
701  if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, scratch->alproto)) {
702  SCLogDebug("alproto mismatch");
703  goto next;
704  }
705  }
706 
707  if (!DetectRunInspectRuleHeader(p, pflow, s, sflags, s_proto_flags)) {
708  goto next;
709  }
710 
711  if (!DetectEnginePktInspectionRun(tv, det_ctx, s, pflow, p, &alert_flags)) {
712  goto next;
713  }
714 
715 #ifdef PROFILE_RULES
716  smatch = true;
717 #endif
718  DetectRunPostMatch(tv, det_ctx, p, s);
719 
720  uint64_t txid = PACKET_ALERT_NOTX;
721  if (pflow && pflow->alstate) {
722  uint8_t dir = (p->flowflags & FLOW_PKT_TOCLIENT) ? STREAM_TOCLIENT : STREAM_TOSERVER;
724  if ((s->alproto != ALPROTO_UNKNOWN && pflow->proto == IPPROTO_UDP) ||
725  (de_ctx->guess_applayer && IsOnlyTxInDirection(pflow, txid, dir))) {
726  // if there is a UDP specific app-layer signature,
727  // or only one live transaction
728  // try to use the good tx for the packet direction
729  void *tx_ptr =
730  AppLayerParserGetTx(pflow->proto, pflow->alproto, pflow->alstate, txid);
731  AppLayerTxData *txd =
732  tx_ptr ? AppLayerParserGetTxData(pflow->proto, pflow->alproto, tx_ptr)
733  : NULL;
734  if (txd && txd->guessed_applayer_logged < de_ctx->guess_applayer_log_limit) {
735  alert_flags |= PACKET_ALERT_FLAG_TX;
736  if (pflow->proto != IPPROTO_UDP) {
737  alert_flags |= PACKET_ALERT_FLAG_TX_GUESSED;
738  }
739  txd->guessed_applayer_logged++;
740  }
741  }
742  }
743  AlertQueueAppend(det_ctx, s, p, txid, alert_flags);
744 
745  if (det_ctx->post_rule_work_queue.len > 0) {
746  /* run post match prefilter engines on work queue */
747  PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, pflow);
748 
749  if (det_ctx->pmq.rule_id_array_cnt > 0) {
750  /* undo "prefetch" */
751  if (next_s)
752  match_array--;
753  /* create temporary rule pointer array starting
754  * at where we are in the current match array */
755  const Signature *replace[de_ctx->sig_array_len]; // TODO heap?
756  SCLogDebug("sig_array_len %u det_ctx->pmq.rule_id_array_cnt %u",
758  const Signature **r = replace;
759  for (uint32_t x = 0; x < match_cnt; x++) {
760  *r++ = match_array[x];
761  SCLogDebug("appended %u", match_array[x]->id);
762  }
763  /* append the prefilter results, then sort it */
764  for (uint32_t x = 0; x < det_ctx->pmq.rule_id_array_cnt; x++) {
765  SCLogDebug("adding iid %u", det_ctx->pmq.rule_id_array[x]);
766  Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[x]];
767  SCLogDebug("adding id %u", ts->id);
768  if (ts->app_inspect == NULL) {
769  *r++ = ts;
770  match_cnt++;
771  }
772  }
773  if (match_cnt > 1) {
774  qsort(replace, match_cnt, sizeof(Signature *), SortHelper);
775  }
776  /* rewrite match_array to include the new additions, and deduplicate
777  * while at it. */
778  Signature **m = match_array;
779  Signature *last_sig = NULL;
780  uint32_t skipped = 0;
781  for (uint32_t x = 0; x < match_cnt; x++) {
782  /* de-duplicate */
783  if (last_sig == *m) {
784  skipped++;
785  continue;
786  }
787  last_sig = *m;
788  *m++ = (Signature *)replace[x];
789  }
790  match_cnt -= skipped;
791  /* prefetch next */
792  next_s = *match_array++;
793  next_sflags = next_s->flags;
794  SCLogDebug("%u rules added", det_ctx->pmq.rule_id_array_cnt);
795  det_ctx->post_rule_work_queue.len = 0;
796  PMQ_RESET(&det_ctx->pmq);
797  }
798  }
799 
800  /* firewall logic in the packet:filter table:
801  * 1. firewall rules preceed the packet:td rules in the list
802  * 2. if no rule issues an accept, we drop
803  * 3. drop is immediate
804  * 4. accept:
805  * - hook: skip rest of fw rules, inspect packet:td rules
806  * - packet: immediate accept, no packet:td or app:* inspect
807  * - flow: as packet, but applied to all future packets in the
808  * flow as well
809  */
810  if (s->flags & SIG_FLAG_FIREWALL) {
811  if (s->action & (ACTION_ACCEPT)) {
812  fw_verdict = true;
813 
814  enum ActionScope as = s->action_scope;
815  if (as == ACTION_SCOPE_HOOK) {
816  /* accept:hook: jump to first TD. Implemented as:
817  * skip until the first TD rule.
818  * Don't update action as we're just continuing to the next hook. */
819  skip_fw = true;
820 
821  } else if (as == ACTION_SCOPE_PACKET) {
822  /* accept:packet: break loop, return accept */
823  action |= s->action;
824  break_out_of_packet_filter = true;
825 
826  } else if (as == ACTION_SCOPE_FLOW) {
827  /* accept:flow: break loop, return accept */
828  action |= s->action;
829  break_out_of_packet_filter = true;
830 
831  /* set immediately, as we're in hook "packet_filter" */
832  if (pflow) {
833  pflow->flags |= FLOW_ACTION_ACCEPT;
834  }
835  }
836  } else if (s->action & ACTION_DROP) {
837  /* apply a drop immediately here */
838  fw_verdict = true;
839  action |= s->action;
840  break_out_of_packet_filter = true;
841  }
842  }
843 next:
844  DetectVarProcessList(det_ctx, pflow, p);
845  DetectReplaceFree(det_ctx);
846  RULE_PROFILING_END(det_ctx, s, smatch, p);
847 
848  /* fw accept:packet or accept:flow means we're done here */
849  if (break_out_of_packet_filter)
850  break;
851 
852  continue;
853  }
854 
855  /* if no rule told us to accept, and no rule explicitly dropped, we invoke the default drop
856  * policy
857  */
858  if (have_fw_rules) {
859  if (!fw_verdict) {
862  action |= ACTION_DROP;
863  } else {
864  /* apply fw action */
865  p->action |= action;
866  }
867  }
868  return action;
869 }
870 
871 static DetectRunScratchpad DetectRunSetup(
872  const DetectEngineCtx *de_ctx,
873  DetectEngineThreadCtx *det_ctx,
874  Packet * const p, Flow * const pflow)
875 {
876  AppProto alproto = ALPROTO_UNKNOWN;
877  uint8_t flow_flags = 0; /* flow/state flags */
878  bool app_decoder_events = false;
879 
881 
882 #ifdef UNITTESTS
883  p->alerts.cnt = 0;
884  p->alerts.discarded = 0;
885  p->alerts.suppressed = 0;
886 #endif
887  det_ctx->filestore_cnt = 0;
888  det_ctx->base64_decoded_len = 0;
889  det_ctx->raw_stream_progress = 0;
890  det_ctx->match_array_cnt = 0;
891 
892  det_ctx->alert_queue_size = 0;
893  p->alerts.drop.action = 0;
894 
895 #ifdef DEBUG
896  if (p->flags & PKT_STREAM_ADD) {
897  det_ctx->pkt_stream_add_cnt++;
898  }
899 #endif
900 
901  /* grab the protocol state we will detect on */
902  if (p->flags & PKT_HAS_FLOW) {
903  DEBUG_VALIDATE_BUG_ON(pflow == NULL);
904 
905  if (p->flowflags & FLOW_PKT_TOSERVER) {
906  flow_flags = STREAM_TOSERVER;
907  SCLogDebug("flag STREAM_TOSERVER set");
908  } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
909  flow_flags = STREAM_TOCLIENT;
910  SCLogDebug("flag STREAM_TOCLIENT set");
911  }
912  SCLogDebug("p->flowflags 0x%02x", p->flowflags);
913 
914  if (p->flags & PKT_STREAM_EOF) {
915  flow_flags |= STREAM_EOF;
916  SCLogDebug("STREAM_EOF set");
917  }
918 
919  /* store tenant_id in the flow so that we can use it
920  * for creating pseudo packets */
921  if (p->tenant_id > 0 && pflow->tenant_id == 0) {
922  pflow->tenant_id = p->tenant_id;
923  }
924 
925  /* live ruleswap check for flow updates */
926  if (pflow->de_ctx_version == 0) {
927  /* first time this flow is inspected, set id */
928  pflow->de_ctx_version = de_ctx->version;
929  } else if (pflow->de_ctx_version != de_ctx->version) {
930  /* first time we inspect flow with this de_ctx, reset */
931  pflow->flags &= ~FLOW_SGH_TOSERVER;
932  pflow->flags &= ~FLOW_SGH_TOCLIENT;
933  pflow->sgh_toserver = NULL;
934  pflow->sgh_toclient = NULL;
935 
936  pflow->de_ctx_version = de_ctx->version;
937  GenericVarFree(pflow->flowvar);
938  pflow->flowvar = NULL;
939 
941  }
942 
943  /* Retrieve the app layer state and protocol and the tcp reassembled
944  * stream chunks. */
945  if ((p->proto == IPPROTO_TCP && (p->flags & PKT_STREAM_EST)) ||
946  (p->proto == IPPROTO_UDP) ||
948  {
949  /* update flow flags with knowledge on disruptions */
950  flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
951  alproto = FlowGetAppProtocol(pflow);
952  if (p->proto == IPPROTO_TCP && pflow->protoctx &&
955  }
956  SCLogDebug("alproto %u", alproto);
957  } else {
958  SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
959  }
960 
961  app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
962  }
963 
964  DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL };
966  return pad;
967 }
968 
969 static inline void DetectRunPostRules(
970  ThreadVars *tv,
972  DetectEngineThreadCtx *det_ctx,
973  Packet * const p,
974  Flow * const pflow,
975  DetectRunScratchpad *scratch)
976 {
977  /* so now let's iterate the alerts and remove the ones after a pass rule
978  * matched (if any). This is done inside PacketAlertFinalize() */
979  /* PR: installed "tag" keywords are handled after the threshold inspection */
980 
982  PacketAlertFinalize(de_ctx, det_ctx, p);
983  if (p->alerts.cnt > 0) {
984  StatsAddUI64(tv, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt);
985  }
986  if (p->alerts.discarded > 0) {
987  StatsAddUI64(tv, det_ctx->counter_alerts_overflow, (uint64_t)p->alerts.discarded);
988  }
989  if (p->alerts.suppressed > 0) {
990  StatsAddUI64(tv, det_ctx->counter_alerts_suppressed, (uint64_t)p->alerts.suppressed);
991  }
993 
994  /* firewall: "fail" closed if we don't have an ACCEPT. This can happen
995  * if there was no rule group. */
996  // TODO review packet src types here
997  if (de_ctx->flags & DE_HAS_FIREWALL && !(p->action & ACTION_ACCEPT) &&
998  p->pkt_src == PKT_SRC_WIRE) {
999  SCLogDebug("packet %" PRIu64 ": droppit as no ACCEPT set %02x (pkt %s)", p->pcap_cnt,
1000  p->action, PktSrcToString(p->pkt_src));
1002  }
1003 }
1004 
1005 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
1006  Packet *p, Flow * const pflow)
1007 {
1009  InspectionBufferClean(det_ctx);
1010 
1011  if (pflow != NULL) {
1012  /* update inspected tracker for raw reassembly */
1013  if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL &&
1016  det_ctx->raw_stream_progress);
1017  }
1018  }
1020  SCReturn;
1021 }
1022 
1024 {
1026  det_ctx->tx_candidates = SCCalloc(size, sizeof(RuleMatchCandidateTx));
1027  if (det_ctx->tx_candidates == NULL) {
1028  FatalError("failed to allocate %" PRIu64 " bytes",
1029  (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
1030  }
1031  det_ctx->tx_candidates_size = size;
1032  SCLogDebug("array initialized to %u elements (%"PRIu64" bytes)",
1033  size, (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
1034 }
1035 
1037 {
1038  SCFree(det_ctx->tx_candidates);
1039  det_ctx->tx_candidates_size = 0;
1040 }
1041 
1042 /* if size >= cur_space */
1043 static inline bool RuleMatchCandidateTxArrayHasSpace(const DetectEngineThreadCtx *det_ctx,
1044  const uint32_t need)
1045 {
1046  if (det_ctx->tx_candidates_size >= need)
1047  return 1;
1048  return 0;
1049 }
1050 
1051 /* realloc */
1052 static int RuleMatchCandidateTxArrayExpand(DetectEngineThreadCtx *det_ctx, const uint32_t needed)
1053 {
1054  const uint32_t old_size = det_ctx->tx_candidates_size;
1055  uint32_t new_size = needed;
1056  void *ptmp = SCRealloc(det_ctx->tx_candidates, (new_size * sizeof(RuleMatchCandidateTx)));
1057  if (ptmp == NULL) {
1058  FatalError("failed to expand to %" PRIu64 " bytes",
1059  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx)));
1060  // TODO can this be handled more gracefully?
1061  }
1062  det_ctx->tx_candidates = ptmp;
1063  det_ctx->tx_candidates_size = new_size;
1064  SCLogDebug("array expanded from %u to %u elements (%"PRIu64" bytes -> %"PRIu64" bytes)",
1065  old_size, new_size, (uint64_t)(old_size * sizeof(RuleMatchCandidateTx)),
1066  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx))); (void)old_size;
1067  return 1;
1068 }
1069 
1070 /** \internal
1071  * \brief sort helper for sorting match candidates by id: ascending
1072  *
1073  * The id field is set from Signature::num, so we sort the candidates to match the signature
1074  * sort order (ascending), where candidates that have flags go first.
1075  */
1076 static int
1077 DetectRunTxSortHelper(const void *a, const void *b)
1078 {
1079  const RuleMatchCandidateTx *s0 = a;
1080  const RuleMatchCandidateTx *s1 = b;
1081  if (s1->id == s0->id) {
1082  if (s1->flags && !s0->flags)
1083  return 1;
1084  else if (!s1->flags && s0->flags)
1085  return -1;
1086  return 0;
1087  } else
1088  return s0->id > s1->id ? 1 : -1;
1089 }
1090 
1091 #if 0
1092 #define TRACE_SID_TXS(sid,txs,...) \
1093  do { \
1094  char _trace_buf[2048]; \
1095  snprintf(_trace_buf, sizeof(_trace_buf), __VA_ARGS__); \
1096  SCLogNotice("%p/%"PRIu64"/%u: %s", txs->tx_ptr, txs->tx_id, sid, _trace_buf); \
1097  } while(0)
1098 #else
1099 #define TRACE_SID_TXS(sid,txs,...)
1100 #endif
1101 
1102 // Get inner transaction for engine
1103 void *DetectGetInnerTx(void *tx_ptr, AppProto alproto, AppProto engine_alproto, uint8_t flow_flags)
1104 {
1105  if (unlikely(alproto == ALPROTO_DOH2)) {
1106  if (engine_alproto == ALPROTO_DNS) {
1107  // need to get the dns tx pointer
1108  tx_ptr = SCDoH2GetDnsTx(tx_ptr, flow_flags);
1109  } else if (engine_alproto != ALPROTO_HTTP2 && engine_alproto != ALPROTO_UNKNOWN) {
1110  // incompatible engine->alproto with flow alproto
1111  tx_ptr = NULL;
1112  }
1113  } else if (engine_alproto != alproto && engine_alproto != ALPROTO_UNKNOWN) {
1114  // incompatible engine->alproto with flow alproto
1115  tx_ptr = NULL;
1116  }
1117  return tx_ptr;
1118 }
1119 
1120 /** \internal
1121  * \brief inspect a rule against a transaction
1122  *
1123  * Inspect a rule. New detection or continued stateful
1124  * detection.
1125  *
1126  * \param stored_flags pointer to stored flags or NULL.
1127  * If stored_flags is set it means we're continuing
1128  * inspection from an earlier run.
1129  *
1130  * \retval bool true sig matched, false didn't match
1131  */
1132 static bool DetectRunTxInspectRule(ThreadVars *tv,
1134  DetectEngineThreadCtx *det_ctx,
1135  Packet *p,
1136  Flow *f,
1137  const uint8_t in_flow_flags, // direction, EOF, etc
1138  void *alstate,
1139  DetectTransaction *tx,
1140  const Signature *s,
1141  uint32_t *stored_flags,
1142  RuleMatchCandidateTx *can,
1143  DetectRunScratchpad *scratch)
1144 {
1145  const uint8_t flow_flags = in_flow_flags;
1146  const int direction = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1147  uint32_t inspect_flags = stored_flags ? *stored_flags : 0;
1148  int total_matches = 0;
1149  uint16_t file_no_match = 0;
1150  bool retval = false;
1151  bool mpm_before_progress = false; // is mpm engine before progress?
1152  bool mpm_in_progress = false; // is mpm engine in a buffer we will revisit?
1153 
1154  TRACE_SID_TXS(s->id, tx, "starting %s", direction ? "toclient" : "toserver");
1155 
1156  /* for a new inspection we inspect pkt header and packet matches */
1157  if (likely(stored_flags == NULL)) {
1158  TRACE_SID_TXS(s->id, tx, "first inspect, run packet matches");
1159  if (!DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags)) {
1160  TRACE_SID_TXS(s->id, tx, "DetectRunInspectRuleHeader() no match");
1161  return false;
1162  }
1163  if (!DetectEnginePktInspectionRun(tv, det_ctx, s, f, p, NULL)) {
1164  TRACE_SID_TXS(s->id, tx, "DetectEnginePktInspectionRun no match");
1165  return false;
1166  }
1167  /* stream mpm and negated mpm sigs can end up here with wrong proto */
1168  if (!(AppProtoEquals(s->alproto, f->alproto) || s->alproto == ALPROTO_UNKNOWN)) {
1169  TRACE_SID_TXS(s->id, tx, "alproto mismatch");
1170  return false;
1171  }
1172  }
1173 
1174  const DetectEngineAppInspectionEngine *engine = s->app_inspect;
1175  do {
1176  TRACE_SID_TXS(s->id, tx, "engine %p inspect_flags %x", engine, inspect_flags);
1177  // also if it is not the same direction, but
1178  // this is a transactional signature, and we are toclient
1179  if (!(inspect_flags & BIT_U32(engine->id)) &&
1180  (direction == engine->dir || ((s->flags & SIG_FLAG_TXBOTHDIR) && direction == 1))) {
1181 
1182  void *tx_ptr = DetectGetInnerTx(tx->tx_ptr, f->alproto, engine->alproto, flow_flags);
1183  if (tx_ptr == NULL) {
1184  if (engine->alproto != ALPROTO_UNKNOWN) {
1185  /* special case: file_data on 'alert tcp' will have engines
1186  * in the list that are not for us. */
1187  engine = engine->next;
1188  continue;
1189  } else {
1190  tx_ptr = tx->tx_ptr;
1191  }
1192  }
1193 
1194  /* engines are sorted per progress, except that the one with
1195  * mpm/prefilter enabled is first */
1196  if (tx->tx_progress < engine->progress) {
1197  SCLogDebug("tx progress %d < engine progress %d",
1198  tx->tx_progress, engine->progress);
1199  break;
1200  }
1201  if (engine->mpm) {
1202  if (tx->tx_progress > engine->progress) {
1203  TRACE_SID_TXS(s->id, tx,
1204  "engine->mpm: t->tx_progress %u > engine->progress %u, so set "
1205  "mpm_before_progress",
1206  tx->tx_progress, engine->progress);
1207  mpm_before_progress = true;
1208  } else if (tx->tx_progress == engine->progress) {
1209  TRACE_SID_TXS(s->id, tx,
1210  "engine->mpm: t->tx_progress %u == engine->progress %u, so set "
1211  "mpm_in_progress",
1212  tx->tx_progress, engine->progress);
1213  mpm_in_progress = true;
1214  }
1215  }
1216 
1217  uint8_t engine_flags = flow_flags;
1218  if (direction != engine->dir) {
1219  engine_flags = flow_flags ^ (STREAM_TOCLIENT | STREAM_TOSERVER);
1220  }
1221  /* run callback: but bypass stream callback if we can */
1222  uint8_t match;
1223  if (unlikely(engine->stream && can->stream_stored)) {
1224  match = can->stream_result;
1225  TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match);
1226  } else if (engine->v2.Callback == NULL) {
1227  /* TODO is this the cleanest way to support a non-app sig on a app hook? */
1228 
1229  if (tx->tx_progress > engine->progress) {
1230  mpm_before_progress = true; // TODO needs a new name now
1231  }
1232 
1233  /* we don't have to store a "hook" match, also don't want to keep any state to make
1234  * sure the hook gets invoked again until tx progress progresses. */
1235  if (tx->tx_progress <= engine->progress)
1237 
1238  /* if progress > engine progress, track state to avoid additional matches */
1240  } else {
1241  KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list);
1242  DEBUG_VALIDATE_BUG_ON(engine->v2.Callback == NULL);
1243  match = engine->v2.Callback(
1244  de_ctx, det_ctx, engine, s, f, engine_flags, alstate, tx_ptr, tx->tx_id);
1245  TRACE_SID_TXS(s->id, tx, "engine %p match %d", engine, match);
1246  if (engine->stream) {
1247  can->stream_stored = true;
1248  can->stream_result = match;
1249  TRACE_SID_TXS(s->id, tx, "stream ran, store result %d for next tx (if any)", match);
1250  }
1251  }
1252  if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) {
1253  inspect_flags |= BIT_U32(engine->id);
1254  engine = engine->next;
1255  total_matches++;
1256  continue;
1257  } else if (match == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES) {
1258  /* if the file engine matched, but indicated more
1259  * files are still in progress, we don't set inspect
1260  * flags as these would end inspection for this tx */
1261  engine = engine->next;
1262  total_matches++;
1263  continue;
1264  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
1265  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1266  inspect_flags |= BIT_U32(engine->id);
1267  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES) {
1268  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1269  inspect_flags |= BIT_U32(engine->id);
1270  file_no_match = 1;
1271  }
1272  /* implied DETECT_ENGINE_INSPECT_SIG_NO_MATCH */
1273  if (engine->mpm && mpm_before_progress) {
1274  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1275  inspect_flags |= BIT_U32(engine->id);
1276  }
1277  break;
1278  } else if (!(inspect_flags & BIT_U32(engine->id)) && s->flags & SIG_FLAG_TXBOTHDIR &&
1279  direction != engine->dir) {
1280  // for transactional rules, the engines on the opposite direction
1281  // are ordered by progress on the different side
1282  // so we have a two mixed-up lists, and we skip the elements
1283  if (direction == 0 && engine->next == NULL) {
1284  // do not match yet on request only
1285  break;
1286  }
1287  engine = engine->next;
1288  continue;
1289  }
1290 
1291  engine = engine->next;
1292  } while (engine != NULL);
1293  TRACE_SID_TXS(s->id, tx, "inspect_flags %x, total_matches %u, engine %p",
1294  inspect_flags, total_matches, engine);
1295 
1296  if (engine == NULL && total_matches) {
1297  inspect_flags |= DE_STATE_FLAG_FULL_INSPECT;
1298  TRACE_SID_TXS(s->id, tx, "MATCH");
1299  retval = true;
1300  }
1301 
1302  if (stored_flags) {
1303  *stored_flags = inspect_flags;
1304  TRACE_SID_TXS(s->id, tx, "continue inspect flags %08x", inspect_flags);
1305  } else {
1306  // store... or? If tx is done we might not want to come back to this tx
1307 
1308  // also... if mpmid tracking is enabled, we won't do a sig again for this tx...
1309  TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags);
1310  if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) {
1311  if (file_no_match) {
1312  /* if we have a mismatch on a file sig, we need to keep state.
1313  * We may get another file on the same tx (for http and smtp
1314  * at least), so for a new file we need to re-eval the sig.
1315  * Thoughts / TODO:
1316  * - not for some protos that have 1 file per tx (e.g. nfs)
1317  * - maybe we only need this for file sigs that mix with
1318  * other matches? E.g. 'POST + filename', is different than
1319  * just 'filename'.
1320  */
1321  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1322  inspect_flags, flow_flags, file_no_match);
1323  }
1324  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) {
1325  TRACE_SID_TXS(s->id, tx, "no need to store match sig, "
1326  "mpm won't trigger for it anymore");
1327 
1328  if (inspect_flags & DE_STATE_FLAG_FILE_INSPECT) {
1329  TRACE_SID_TXS(s->id, tx, "except that for new files, "
1330  "we may have to revisit anyway");
1331  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1332  inspect_flags, flow_flags, file_no_match);
1333  }
1334  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) {
1335  TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, "
1336  "mpm will revisit it");
1337  } else if (inspect_flags != 0 || file_no_match != 0) {
1338  TRACE_SID_TXS(s->id, tx, "storing state: flags %08x", inspect_flags);
1339  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1340  inspect_flags, flow_flags, file_no_match);
1341  }
1342  }
1343 
1344  return retval;
1345 }
1346 
1347 #define NO_TX \
1348  { \
1349  NULL, 0, NULL, NULL, 0, 0, 0, 0, \
1350  }
1351 
1352 /** \internal
1353  * \brief get a DetectTransaction object
1354  * \retval struct filled with relevant info or all nulls/0s
1355  */
1356 static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto,
1357  const uint64_t tx_id, void *tx_ptr, const int tx_end_state, const uint8_t flow_flags)
1358 {
1359  AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx_ptr);
1360  const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags);
1361  bool updated = (flow_flags & STREAM_TOSERVER) ? txd->updated_ts : txd->updated_tc;
1362  if (!updated && tx_progress < tx_end_state && ((flow_flags & STREAM_EOF) == 0)) {
1363  DetectTransaction no_tx = NO_TX;
1364  return no_tx;
1365  }
1366  const uint8_t inspected_flag =
1367  (flow_flags & STREAM_TOSERVER) ? APP_LAYER_TX_INSPECTED_TS : APP_LAYER_TX_INSPECTED_TC;
1368  if (unlikely(txd->flags & inspected_flag)) {
1369  SCLogDebug("%" PRIu64 " tx already fully inspected for %s. Flags %02x", tx_id,
1370  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", txd->flags);
1371  DetectTransaction no_tx = NO_TX;
1372  return no_tx;
1373  }
1374  const uint8_t skip_flag = (flow_flags & STREAM_TOSERVER) ? APP_LAYER_TX_SKIP_INSPECT_TS
1376  if (unlikely(txd->flags & skip_flag)) {
1377  SCLogDebug("%" PRIu64 " tx should not be inspected in direction %s. Flags %02x", tx_id,
1378  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", txd->flags);
1379  DetectTransaction no_tx = NO_TX;
1380  return no_tx;
1381  }
1382 
1383  const uint8_t detect_progress =
1384  (flow_flags & STREAM_TOSERVER) ? txd->detect_progress_ts : txd->detect_progress_tc;
1385 
1386  const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1387  DetectEngineState *tx_de_state = txd->de_state;
1388  DetectEngineStateDirection *tx_dir_state =
1389  tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL;
1390  DetectTransaction tx = {
1391  .tx_ptr = tx_ptr,
1392  .tx_id = tx_id,
1393  .tx_data_ptr = (struct AppLayerTxData *)txd,
1394  .de_state = tx_dir_state,
1395  .detect_progress = detect_progress,
1396  .detect_progress_orig = detect_progress,
1397  .tx_progress = tx_progress,
1398  .tx_end_state = tx_end_state,
1399  };
1400  return tx;
1401 }
1402 
1403 static inline void StoreDetectProgress(
1404  DetectTransaction *tx, const uint8_t flow_flags, const uint8_t progress)
1405 {
1407  if (flow_flags & STREAM_TOSERVER) {
1408  txd->detect_progress_ts = progress;
1409  } else {
1410  txd->detect_progress_tc = progress;
1411  }
1412 }
1413 
1414 // Merge 'state' rules from the regular prefilter
1415 // updates array_idx on the way
1416 static inline void RuleMatchCandidateMergeStateRules(
1417  DetectEngineThreadCtx *det_ctx, uint32_t *array_idx)
1418 {
1419  // Now, we will merge 2 sorted lists :
1420  // the one in det_ctx->tx_candidates
1421  // and the one in det_ctx->match_array
1422  // For match_array, we take only the relevant elements where s->app_inspect != NULL
1423 
1424  // Basically, we iterate at the same time over the 2 lists
1425  // comparing and taking an element from either.
1426 
1427  // Trick is to do so in place in det_ctx->tx_candidates,
1428  // so as to minimize the number of moves in det_ctx->tx_candidates.
1429  // For this, the algorithm traverses the lists in reverse order.
1430  // Otherwise, if the first element of match_array was to be put before
1431  // all tx_candidates, we would need to shift all tx_candidates
1432 
1433  // Retain the number of elements sorted in tx_candidates before merge
1434  uint32_t j = *array_idx;
1435  // First loop only counting the number of elements to add
1436  for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) {
1437  const Signature *s = det_ctx->match_array[i];
1438  if (s->app_inspect != NULL) {
1439  (*array_idx)++;
1440  }
1441  }
1442  // Future number of elements in tx_candidates after merge
1443  uint32_t k = *array_idx;
1444 
1445  if (k == j) {
1446  // no new element from match_array to merge in tx_candidates
1447  return;
1448  }
1449 
1450  // variable i is for all elements of match_array (even not relevant ones)
1451  // variable j is for elements of tx_candidates before merge
1452  // variable k is for elements of tx_candidates after merge
1453  for (uint32_t i = det_ctx->match_array_cnt; i > 0;) {
1454  const Signature *s = det_ctx->match_array[i - 1];
1455  if (s->app_inspect == NULL) {
1456  // no relevant element, get the next one from match_array
1457  i--;
1458  continue;
1459  }
1460  // we have one element from match_array to merge in tx_candidates
1461  k--;
1462  if (j > 0) {
1463  // j > 0 means there is still at least one element in tx_candidates to merge
1464  const RuleMatchCandidateTx *s0 = &det_ctx->tx_candidates[j - 1];
1465  if (s->num <= s0->id) {
1466  // get next element from previous tx_candidates
1467  j--;
1468  // take the element from tx_candidates before merge
1469  det_ctx->tx_candidates[k].s = det_ctx->tx_candidates[j].s;
1470  det_ctx->tx_candidates[k].id = det_ctx->tx_candidates[j].id;
1471  det_ctx->tx_candidates[k].flags = det_ctx->tx_candidates[j].flags;
1472  det_ctx->tx_candidates[k].stream_reset = det_ctx->tx_candidates[j].stream_reset;
1473  continue;
1474  }
1475  } // otherwise
1476  // get next element from match_array
1477  i--;
1478  // take the element from match_array
1479  det_ctx->tx_candidates[k].s = s;
1480  det_ctx->tx_candidates[k].id = s->num;
1481  det_ctx->tx_candidates[k].flags = NULL;
1482  det_ctx->tx_candidates[k].stream_reset = 0;
1483  }
1484  // Even if k > 0 or j > 0, the loop is over. (Note that j == k now)
1485  // The remaining elements in tx_candidates up to k were already sorted
1486  // and come before any other element later in the list
1487 }
1488 
1489 /**
1490  * \internal
1491  * \brief Check and update firewall rules state.
1492  *
1493  * \param skip_fw_hook bool to indicate firewall rules skips
1494  * For state `skip_before_progress` should be skipped.
1495  *
1496  * \param skip_before_progress progress value to skip rules before.
1497  * Only used if `skip_fw_hook` is set.
1498  *
1499  * \param last_for_progress[out] set to true if this is the last rule for a progress value
1500  *
1501  * \param fw_next_progress_missing[out] set to true if the next fw rule does not target the next
1502  * progress value, or there is no fw rule for that value.
1503  *
1504  * \retval 0 no action needed
1505  * \retval 1 rest of rules shouldn't inspected
1506  * \retval -1 skip this rule
1507  */
1508 static int DetectRunTxCheckFirewallPolicy(DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f,
1509  DetectTransaction *tx, const Signature *s, const uint32_t can_idx, const uint32_t can_size,
1510  bool *skip_fw_hook, const uint8_t skip_before_progress, bool *last_for_progress,
1511  bool *fw_next_progress_missing)
1512 {
1513  if (s->flags & SIG_FLAG_FIREWALL) {
1514  /* check if the next sig is on the same progress hook. If not, we need to apply our
1515  * default policy in case the current sig doesn't apply one. If the next sig has a
1516  * progress beyond our progress + 1, it means the next progress has no rules and needs
1517  * the default policy applied. But only after we evaluate the current rule first, as
1518  * that may override it.
1519  * TODO should we do this after dedup below? */
1520 
1521  if (can_idx + 1 < can_size) {
1522  const Signature *next_s = det_ctx->tx_candidates[can_idx + 1].s;
1523  SCLogDebug(
1524  "peek: peeking at sid %u / progress %u", next_s->id, next_s->app_progress_hook);
1525  if (next_s->flags & SIG_FLAG_FIREWALL) {
1526  if (s->app_progress_hook != next_s->app_progress_hook) {
1527  SCLogDebug("peek: next sid progress %u != current progress %u, so current "
1528  "is last for progress",
1529  next_s->app_progress_hook, s->app_progress_hook);
1530  *last_for_progress = true;
1531 
1532  if (next_s->app_progress_hook - s->app_progress_hook > 1) {
1533  SCLogDebug("peek: missing progress, so we'll drop that unless we get a "
1534  "sweeping accept first");
1535  *fw_next_progress_missing = true;
1536  }
1537  }
1538  } else {
1539  SCLogDebug("peek: next sid not a fw rule, so current is last for progress");
1540  *last_for_progress = true;
1541  }
1542  } else {
1543  SCLogDebug("peek: no peek beyond last rule");
1544  if (s->app_progress_hook < tx->tx_progress) {
1545  SCLogDebug("peek: there are no rules to allow the state after this rule");
1546  *fw_next_progress_missing = true;
1547  }
1548  }
1549 
1550  if ((*skip_fw_hook) == true) {
1551  if (s->app_progress_hook <= skip_before_progress) {
1552  return -1;
1553  }
1554  *skip_fw_hook = false;
1555  }
1556  } else {
1557  /* fw mode, we skip anything after the fw rules if:
1558  * - flow pass is set
1559  * - packet pass (e.g. exception policy) */
1560  if (p->flags & PKT_NOPACKET_INSPECTION || (f->flags & (FLOW_ACTION_PASS))) {
1561  SCLogDebug("skipping firewall rule %u", s->id);
1562  return 1;
1563  }
1564  }
1565  return 0;
1566 }
1567 
1568 // TODO move into det_ctx?
1570 static inline void DetectRunAppendDefaultAccept(DetectEngineThreadCtx *det_ctx, Packet *p)
1571 {
1572  if (det_ctx->de_ctx->flags & DE_HAS_FIREWALL) {
1573  memset(&default_accept, 0, sizeof(default_accept));
1576  default_accept.num = UINT32_MAX;
1580  }
1581 }
1582 
1583 /** \internal
1584  * \brief see if the accept rule needs to apply to the packet
1585  */
1586 static inline bool ApplyAcceptToPacket(
1587  const uint64_t total_txs, const DetectTransaction *tx, const Signature *s)
1588 {
1589  if ((s->flags & SIG_FLAG_FIREWALL) == 0) {
1590  return false;
1591  }
1592  if ((s->action & ACTION_ACCEPT) == 0) {
1593  return false;
1594  }
1595 
1596  /* for accept:tx we need:
1597  * - packet will only be accepted if this is set on the last tx
1598  */
1599  if (s->action_scope == ACTION_SCOPE_TX) {
1600  if (total_txs == tx->tx_id + 1) {
1601  return true;
1602  }
1603  }
1604  /* for accept:hook we need a bit more checking:
1605  * - packet will only be accepted if this is set on the last tx
1606  * - the hook accepted should be the last progress available. */
1607  if (s->action_scope == ACTION_SCOPE_HOOK) {
1608  if ((total_txs == tx->tx_id + 1) && /* last tx */
1609  (s->app_progress_hook == tx->tx_progress)) {
1610  return true;
1611  }
1612  }
1613  return false;
1614 }
1615 
1616 /** \internal
1617  * \retval bool true: break_out_of_app_filter, false: don't break out */
1618 static bool ApplyAccept(Packet *p, const uint8_t flow_flags, const Signature *s,
1619  DetectTransaction *tx, const int tx_end_state, const bool fw_next_progress_missing,
1620  bool *tx_fw_verdict, bool *skip_fw_hook, uint8_t *skip_before_progress)
1621 {
1622  *tx_fw_verdict = true;
1623 
1624  const enum ActionScope as = s->action_scope;
1625  /* accept:hook: jump to first rule of next state.
1626  * Implemented as skip until the first rule of next state. */
1627  if (as == ACTION_SCOPE_HOOK) {
1628  *skip_fw_hook = true;
1629  *skip_before_progress = s->app_progress_hook;
1630 
1631  /* if there is no fw rule for the next progress value,
1632  * we invoke the default drop policy. */
1633  if (fw_next_progress_missing) {
1634  SCLogDebug("%" PRIu64 ": %s default drop for progress", p->pcap_cnt,
1635  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient");
1637  p->flow->flags |= FLOW_ACTION_DROP;
1638  return true;
1639  }
1640  return false;
1641  } else if (as == ACTION_SCOPE_TX) {
1642  tx->tx_data_ptr->flags |= APP_LAYER_TX_ACCEPT;
1643  *skip_fw_hook = true;
1644  *skip_before_progress = (uint8_t)tx_end_state + 1; // skip all hooks
1645  SCLogDebug(
1646  "accept:tx applied, skip_fw_hook, skip_before_progress %u", *skip_before_progress);
1647  return false;
1648  } else if (as == ACTION_SCOPE_PACKET) {
1649  return true;
1650  } else if (as == ACTION_SCOPE_FLOW) {
1651  return true;
1652  }
1653  return false;
1654 }
1655 
1656 static void DetectRunTx(ThreadVars *tv,
1658  DetectEngineThreadCtx *det_ctx,
1659  Packet *p,
1660  Flow *f,
1661  DetectRunScratchpad *scratch)
1662 {
1663  const uint8_t flow_flags = scratch->flow_flags;
1664  const SigGroupHead * const sgh = scratch->sgh;
1665  void * const alstate = f->alstate;
1666  const uint8_t ipproto = f->proto;
1667  const AppProto alproto = f->alproto;
1668 
1669  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
1670  uint64_t tx_id_min = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags);
1671  const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags);
1672 
1673  AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
1674  AppLayerGetTxIterState state = { 0 };
1675 
1676  uint32_t fw_verdicted = 0;
1677  uint32_t tx_inspected = 0;
1678  const bool have_fw_rules = (de_ctx->flags & DE_HAS_FIREWALL) != 0;
1679 
1680  SCLogDebug("packet %" PRIu64, p->pcap_cnt);
1681 
1682  while (1) {
1683  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id_min, total_txs, &state);
1684  if (ires.tx_ptr == NULL)
1685  break;
1686 
1687  DetectTransaction tx =
1688  GetDetectTx(ipproto, alproto, ires.tx_id, ires.tx_ptr, tx_end_state, flow_flags);
1689  if (tx.tx_ptr == NULL) {
1690  SCLogDebug("%p/%"PRIu64" no transaction to inspect",
1691  tx.tx_ptr, tx_id_min);
1692 
1693  tx_id_min++; // next (if any) run look for +1
1694  goto next;
1695  }
1696  tx_id_min = tx.tx_id + 1; // next look for cur + 1
1697  tx_inspected++;
1698 
1699  SCLogDebug("%p/%" PRIu64 " txd flags %02x", tx.tx_ptr, tx_id_min, tx.tx_data_ptr->flags);
1700 
1701  bool do_sort = false; // do we need to sort the tx candidate list?
1702  uint32_t array_idx = 0;
1703  uint32_t total_rules = det_ctx->match_array_cnt;
1704  total_rules += (tx.de_state ? tx.de_state->cnt : 0);
1705 
1706  /* run prefilter engines and merge results into a candidates array */
1707  if (sgh->tx_engines) {
1709  DetectRunPrefilterTx(det_ctx, sgh, p, ipproto, flow_flags, alproto,
1710  alstate, &tx);
1712  SCLogDebug("%p/%"PRIu64" rules added from prefilter: %u candidates",
1713  tx.tx_ptr, tx.tx_id, det_ctx->pmq.rule_id_array_cnt);
1714 
1715  total_rules += det_ctx->pmq.rule_id_array_cnt;
1716  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1717  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1718  }
1719 
1720  for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) {
1721  const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]];
1722  const SigIntId id = s->num;
1723  det_ctx->tx_candidates[array_idx].s = s;
1724  det_ctx->tx_candidates[array_idx].id = id;
1725  det_ctx->tx_candidates[array_idx].flags = NULL;
1726  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1727  array_idx++;
1728  }
1729  PMQ_RESET(&det_ctx->pmq);
1730  } else {
1731  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1732  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1733  }
1734  }
1735 
1736  /* merge 'state' rules from the regular prefilter */
1737 #ifdef PROFILING
1738  uint32_t x = array_idx;
1739 #endif
1740  RuleMatchCandidateMergeStateRules(det_ctx, &array_idx);
1741 
1742  /* merge stored state into results */
1743  if (tx.de_state != NULL) {
1744  const uint32_t old = array_idx;
1745 
1746  /* if tx.de_state->flags has 'new file' set and sig below has
1747  * 'file inspected' flag, reset the file part of the state */
1748  const bool have_new_file = (tx.de_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_NEW);
1749  if (have_new_file) {
1750  SCLogDebug("%p/%"PRIu64" destate: need to consider new file",
1751  tx.tx_ptr, tx.tx_id);
1753  }
1754 
1755  SigIntId state_cnt = 0;
1756  DeStateStore *tx_store = tx.de_state->head;
1757  for (; tx_store != NULL; tx_store = tx_store->next) {
1758  SCLogDebug("tx_store %p", tx_store);
1759 
1760  SigIntId store_cnt = 0;
1761  for (store_cnt = 0;
1762  store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < tx.de_state->cnt;
1763  store_cnt++, state_cnt++)
1764  {
1765  DeStateStoreItem *item = &tx_store->store[store_cnt];
1766  SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags);
1767  if (have_new_file && (item->flags & DE_STATE_FLAG_FILE_INSPECT)) {
1768  /* remove part of the state. File inspect engine will now
1769  * be able to run again */
1771  SCLogDebug("rule id %u, post file reset inspect_flags %u", item->sid, item->flags);
1772  }
1773  det_ctx->tx_candidates[array_idx].s = de_ctx->sig_array[item->sid];
1774  det_ctx->tx_candidates[array_idx].id = item->sid;
1775  det_ctx->tx_candidates[array_idx].flags = &item->flags;
1776  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1777  array_idx++;
1778  }
1779  }
1780  do_sort |= (old && old != array_idx); // sort if continue list adds sids
1781  SCLogDebug("%p/%" PRIu64 " rules added from 'continue' list: %u", tx.tx_ptr, tx.tx_id,
1782  array_idx - old);
1783  }
1784  if (do_sort) {
1785  qsort(det_ctx->tx_candidates, array_idx, sizeof(RuleMatchCandidateTx),
1786  DetectRunTxSortHelper);
1787  }
1788 
1789 #ifdef PROFILING
1790  if (array_idx >= de_ctx->profile_match_logging_threshold)
1791  RulesDumpTxMatchArray(det_ctx, scratch->sgh, p, tx.tx_id, array_idx, x);
1792 #endif
1793  det_ctx->tx_id = tx.tx_id;
1794  det_ctx->tx_id_set = true;
1795  det_ctx->p = p;
1796 
1797 #ifdef DEBUG
1798  for (uint32_t i = 0; i < array_idx; i++) {
1799  RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i];
1800  const Signature *s = det_ctx->tx_candidates[i].s;
1801  SCLogDebug("%u: sid %u flags %p", i, s->id, can->flags);
1802  }
1803 #endif
1804  bool skip_fw_hook = false;
1805  uint8_t skip_before_progress = 0;
1806  bool fw_next_progress_missing = false;
1807 
1808  /* if there are no rules / rule candidates, make sure we don't
1809  * invoke the default drop */
1810  if (have_fw_rules && array_idx == 0 && (tx.tx_data_ptr->flags & APP_LAYER_TX_ACCEPT)) {
1811  fw_verdicted++;
1812 
1813  /* current tx is the last we have, append a blank accept:packet */
1814  if (total_txs == tx.tx_id + 1) {
1815  DetectRunAppendDefaultAccept(det_ctx, p);
1816  return;
1817  }
1818  goto next;
1819  }
1820 
1821  bool tx_fw_verdict = false;
1822  /* run rules: inspect the match candidates */
1823  for (uint32_t i = 0; i < array_idx; i++) {
1824  RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i];
1825  const Signature *s = det_ctx->tx_candidates[i].s;
1826  uint32_t *inspect_flags = det_ctx->tx_candidates[i].flags;
1827  bool break_out_of_app_filter = false;
1828 
1829  SCLogDebug("%" PRIu64 ": sid:%u: %s tx %u/%u/%u sig %u", p->pcap_cnt, s->id,
1830  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", tx.tx_progress,
1832 
1833  /* deduplicate: rules_array is sorted, but not deduplicated:
1834  * both mpm and stored state could give us the same sid.
1835  * As they are back to back in that case we can check for it
1836  * here. We select the stored state one as that comes first
1837  * in the array. */
1838  while ((i + 1) < array_idx &&
1839  det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i + 1].s) {
1840  SCLogDebug("%p/%" PRIu64 " inspecting SKIP NEXT: sid %u (%u), flags %08x",
1841  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1842  i++;
1843  }
1844 
1845  /* skip fw rules if we're in accept:tx mode */
1846  if (have_fw_rules && (tx.tx_data_ptr->flags & APP_LAYER_TX_ACCEPT)) {
1847  /* append a blank accept:packet action for the APP_LAYER_TX_ACCEPT,
1848  * if this is the last tx */
1849  if (!tx_fw_verdict) {
1850  const bool accept_tx_applies_to_packet = total_txs == tx.tx_id + 1;
1851  if (accept_tx_applies_to_packet) {
1852  SCLogDebug("accept:(tx|hook): should be applied to the packet");
1853  DetectRunAppendDefaultAccept(det_ctx, p);
1854  }
1855  }
1856  tx_fw_verdict = true;
1857 
1858  if (s->flags & SIG_FLAG_FIREWALL) {
1859  SCLogDebug("APP_LAYER_TX_ACCEPT, so skip rule");
1860  continue;
1861  }
1862 
1863  /* threat detect rules will be inspected */
1864  }
1865 
1866  SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x",
1867  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1868 
1869  if (inspect_flags) {
1870  if (*inspect_flags & DE_STATE_FLAG_FULL_INSPECT) {
1871  SCLogDebug("%p/%" PRIu64
1872  " inspecting: sid %u (%u), flags %08x DE_STATE_FLAG_FULL_INSPECT",
1873  tx.tx_ptr, tx.tx_id, s->id, s->num, *inspect_flags);
1874 
1875  /* if we're still in the same progress state as an earlier full
1876  * match, we need to apply the same accept */
1877  if (have_fw_rules && (s->flags & SIG_FLAG_FIREWALL) &&
1878  (s->action & ACTION_ACCEPT) && s->app_progress_hook == tx.tx_progress) {
1879  const bool fw_accept_to_packet = ApplyAcceptToPacket(total_txs, &tx, s);
1880  break_out_of_app_filter = ApplyAccept(p, flow_flags, s, &tx, tx_end_state,
1881  fw_next_progress_missing, &tx_fw_verdict, &skip_fw_hook,
1882  &skip_before_progress);
1883  if (fw_accept_to_packet)
1884  DetectRunAppendDefaultAccept(det_ctx, p);
1885  if (break_out_of_app_filter)
1886  break;
1887  }
1888  continue;
1889  }
1890  if (*inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) {
1891  SCLogDebug("%p/%" PRIu64
1892  " inspecting: sid %u (%u), flags %08x DE_STATE_FLAG_SIG_CANT_MATCH",
1893  tx.tx_ptr, tx.tx_id, s->id, s->num, *inspect_flags);
1894  continue;
1895  }
1896  }
1897 
1898  if (inspect_flags) {
1899  /* continue previous inspection */
1900  SCLogDebug("%p/%" PRIu64 " Continuing sid %u", tx.tx_ptr, tx.tx_id, s->id);
1901  } else {
1902  /* start new inspection */
1903  SCLogDebug("%p/%"PRIu64" Start sid %u", tx.tx_ptr, tx.tx_id, s->id);
1904  }
1905 
1906  bool last_for_progress = false;
1907  if (have_fw_rules) {
1908  int fw_r = DetectRunTxCheckFirewallPolicy(det_ctx, p, f, &tx, s, i, array_idx,
1909  &skip_fw_hook, skip_before_progress, &last_for_progress,
1910  &fw_next_progress_missing);
1911  if (fw_r == -1)
1912  continue;
1913  if (fw_r == 1)
1914  break;
1915  }
1916 
1917  /* call individual rule inspection */
1919  const int r = DetectRunTxInspectRule(tv, de_ctx, det_ctx, p, f, flow_flags,
1920  alstate, &tx, s, inspect_flags, can, scratch);
1921  if (r == 1) {
1922  /* match */
1923  DetectRunPostMatch(tv, det_ctx, p, s);
1924 
1925  /* see if we need to apply tx/hook accept to the packet. This can be needed when
1926  * we've completed the inspection so far for an incomplete tx, and an accept:tx or
1927  * accept:hook is the last match.*/
1928  const bool fw_accept_to_packet = ApplyAcceptToPacket(total_txs, &tx, s);
1929 
1930  uint8_t alert_flags = (PACKET_ALERT_FLAG_STATE_MATCH | PACKET_ALERT_FLAG_TX);
1931  if (fw_accept_to_packet) {
1932  SCLogDebug("accept:(tx|hook): should be applied to the packet");
1934  }
1935 
1936  SCLogDebug("%p/%"PRIu64" sig %u (%u) matched", tx.tx_ptr, tx.tx_id, s->id, s->num);
1937  AlertQueueAppend(det_ctx, s, p, tx.tx_id, alert_flags);
1938 
1939  if ((s->flags & SIG_FLAG_FIREWALL) && (s->action & ACTION_ACCEPT)) {
1940  break_out_of_app_filter = ApplyAccept(p, flow_flags, s, &tx, tx_end_state,
1941  fw_next_progress_missing, &tx_fw_verdict, &skip_fw_hook,
1942  &skip_before_progress);
1943  }
1944  } else if (last_for_progress) {
1945  SCLogDebug("sid %u: not a match: %s rule, last_for_progress %s", s->id,
1946  (s->flags & SIG_FLAG_FIREWALL) ? "firewall" : "regular",
1947  BOOL2STR(last_for_progress));
1948  if (s->flags & SIG_FLAG_FIREWALL) {
1949  SCLogDebug("%" PRIu64 ": %s default drop for progress", p->pcap_cnt,
1950  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient");
1951  /* if this rule was the last for our progress state, and it didn't match,
1952  * we have to invoke the default drop policy. */
1954  p->flow->flags |= FLOW_ACTION_DROP;
1955  break_out_of_app_filter = true;
1956  tx_fw_verdict = true;
1957  }
1958  }
1959  DetectVarProcessList(det_ctx, p->flow, p);
1960  RULE_PROFILING_END(det_ctx, s, r, p);
1961 
1962  if (det_ctx->post_rule_work_queue.len > 0) {
1963  SCLogDebug("%p/%" PRIu64 " post_rule_work_queue len %u", tx.tx_ptr, tx.tx_id,
1964  det_ctx->post_rule_work_queue.len);
1965  /* run post match prefilter engines on work queue */
1966  PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, f);
1967 
1968  uint32_t prev_array_idx = array_idx;
1969  for (uint32_t j = 0; j < det_ctx->pmq.rule_id_array_cnt; j++) {
1970  const Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[j]];
1971  if (ts->app_inspect != NULL) {
1972  const SigIntId id = ts->num;
1973  det_ctx->tx_candidates[array_idx].s = ts;
1974  det_ctx->tx_candidates[array_idx].id = id;
1975  det_ctx->tx_candidates[array_idx].flags = NULL;
1976  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1977  array_idx++;
1978 
1979  SCLogDebug("%p/%" PRIu64 " rule %u (%u) added from 'post match' prefilter",
1980  tx.tx_ptr, tx.tx_id, ts->id, id);
1981  }
1982  }
1983  SCLogDebug("%p/%" PRIu64 " rules added from 'post match' prefilter: %u", tx.tx_ptr,
1984  tx.tx_id, array_idx - prev_array_idx);
1985  if (prev_array_idx != array_idx) {
1986  /* sort, but only part of array we're still going to process */
1987  qsort(det_ctx->tx_candidates + i, array_idx - i, sizeof(RuleMatchCandidateTx),
1988  DetectRunTxSortHelper);
1989  }
1990  det_ctx->post_rule_work_queue.len = 0;
1991  PMQ_RESET(&det_ctx->pmq);
1992  }
1993 
1994  if (break_out_of_app_filter)
1995  break;
1996  }
1997  if (tx_fw_verdict)
1998  fw_verdicted++;
1999 
2000  det_ctx->tx_id = 0;
2001  det_ctx->tx_id_set = false;
2002  det_ctx->p = NULL;
2003 
2004  /* see if we have any updated state to store in the tx */
2005 
2006  /* this side of the tx is done */
2007  if (tx.tx_progress >= tx.tx_end_state) {
2008  SCLogDebug("%" PRIu64 ": %s tx done", p->pcap_cnt,
2009  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient");
2010  const uint8_t inspected_flag = (flow_flags & STREAM_TOSERVER)
2013  tx.tx_data_ptr->flags |= inspected_flag;
2014  SCLogDebug("%p/%" PRIu64 " tx is done for direction %s. Progress %02x", tx.tx_ptr,
2015  tx.tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
2016  tx.detect_progress);
2017  }
2018 
2019  if (tx.detect_progress != tx.detect_progress_orig) {
2020  SCLogDebug("%" PRIu64 ": %s tx state change %u -> %u", p->pcap_cnt,
2021  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", tx.detect_progress_orig,
2022  tx.detect_progress);
2023  SCLogDebug("%p/%" PRIu64 " Storing new progress %02x (was %02x)", tx.tx_ptr, tx.tx_id,
2025 
2026  StoreDetectProgress(&tx, flow_flags, tx.detect_progress);
2027  }
2028 
2029  InspectionBufferClean(det_ctx);
2030 
2031  next:
2032  if (!ires.has_next)
2033  break;
2034  }
2035 
2036  /* apply default policy if there were txs to inspect, we have fw rules and non of the rules
2037  * applied a policy. */
2038  SCLogDebug("packet %" PRIu64 ": tx_inspected %u fw_verdicted %u", p->pcap_cnt, tx_inspected,
2039  fw_verdicted);
2040  if (tx_inspected && have_fw_rules && tx_inspected != fw_verdicted) {
2041  SCLogDebug("%" PRIu64 ": %s default drop", p->pcap_cnt,
2042  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient");
2044  p->flow->flags |= FLOW_ACTION_DROP;
2045  return;
2046  }
2047  /* if all tables have been bypassed, we accept:packet */
2048  if (tx_inspected == 0 && fw_verdicted == 0 && have_fw_rules) {
2049  DetectRunAppendDefaultAccept(det_ctx, p);
2050  }
2051 }
2052 
2053 static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
2054  Packet *p, Flow *f, DetectRunScratchpad *scratch)
2055 {
2056  const SigGroupHead *const sgh = scratch->sgh;
2057  const AppProto alproto = f->alproto;
2058 
2059  /* for TCP, limit inspection to pseudo packets or real packet that did
2060  * an app-layer update. */
2061  if (p->proto == IPPROTO_TCP && !PKT_IS_PSEUDOPKT(p) &&
2062  ((PKT_IS_TOSERVER(p) && (f->flags & FLOW_TS_APP_UPDATED) == 0) ||
2063  (PKT_IS_TOCLIENT(p) && (f->flags & FLOW_TC_APP_UPDATED) == 0))) {
2064  SCLogDebug("pcap_cnt %" PRIu64 ": %s: skip frame inspection for TCP w/o APP UPDATE",
2065  p->pcap_cnt, PKT_IS_TOSERVER(p) ? "toserver" : "toclient");
2066  return;
2067  }
2068  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
2069  if (frames_container == NULL) {
2070  return;
2071  }
2072  Frames *frames;
2073  if (PKT_IS_TOSERVER(p)) {
2074  frames = &frames_container->toserver;
2075  } else {
2076  frames = &frames_container->toclient;
2077  }
2078 
2079  for (uint32_t idx = 0; idx < frames->cnt; idx++) {
2080  SCLogDebug("frame %u", idx);
2081  Frame *frame = FrameGetByIndex(frames, idx);
2082  if (frame == NULL) {
2083  continue;
2084  }
2085 
2086  det_ctx->frame_inspect_progress = 0;
2087  uint32_t array_idx = 0;
2088  uint32_t total_rules = det_ctx->match_array_cnt;
2089 
2090  /* run prefilter engines and merge results into a candidates array */
2091  if (sgh->frame_engines) {
2092  // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_TX);
2093  DetectRunPrefilterFrame(det_ctx, sgh, p, frames, frame, alproto);
2094  // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_TX);
2095  SCLogDebug("%p/%" PRIi64 " rules added from prefilter: %u candidates", frame, frame->id,
2096  det_ctx->pmq.rule_id_array_cnt);
2097 
2098  total_rules += det_ctx->pmq.rule_id_array_cnt;
2099 
2100  if (!(RuleMatchCandidateTxArrayHasSpace(
2101  det_ctx, total_rules))) { // TODO is it safe to overload?
2102  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
2103  }
2104 
2105  for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) {
2106  const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]];
2107  const SigIntId id = s->num;
2108  det_ctx->tx_candidates[array_idx].s = s;
2109  det_ctx->tx_candidates[array_idx].id = id;
2110  det_ctx->tx_candidates[array_idx].flags = NULL;
2111  det_ctx->tx_candidates[array_idx].stream_reset = 0;
2112  array_idx++;
2113  }
2114  PMQ_RESET(&det_ctx->pmq);
2115  }
2116  /* merge 'state' rules from the regular prefilter */
2117  uint32_t x = array_idx;
2118  for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) {
2119  const Signature *s = det_ctx->match_array[i];
2120  if (s->frame_inspect != NULL) {
2121  const SigIntId id = s->num;
2122  det_ctx->tx_candidates[array_idx].s = s;
2123  det_ctx->tx_candidates[array_idx].id = id;
2124  det_ctx->tx_candidates[array_idx].flags = NULL;
2125  det_ctx->tx_candidates[array_idx].stream_reset = 0;
2126  array_idx++;
2127 
2128  SCLogDebug("%p/%" PRIi64 " rule %u (%u) added from 'match' list", frame, frame->id,
2129  s->id, id);
2130  }
2131  }
2132  SCLogDebug("%p/%" PRIi64 " rules added from 'match' list: %u", frame, frame->id,
2133  array_idx - x);
2134  (void)x;
2135 
2136  /* run rules: inspect the match candidates */
2137  for (uint32_t i = 0; i < array_idx; i++) {
2138  const Signature *s = det_ctx->tx_candidates[i].s;
2139 
2140  /* deduplicate: rules_array is sorted, but not deduplicated.
2141  * As they are back to back in that case we can check for it
2142  * here. We select the stored state one as that comes first
2143  * in the array. */
2144  while ((i + 1) < array_idx &&
2145  det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i + 1].s) {
2146  i++;
2147  }
2148  SCLogDebug("%p/%" PRIi64 " inspecting: sid %u (%u)", frame, frame->id, s->id, s->num);
2149 
2150  /* start new inspection */
2151  SCLogDebug("%p/%" PRIi64 " Start sid %u", frame, frame->id, s->id);
2152 
2153  /* call individual rule inspection */
2155  bool r = DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags);
2156  if (r) {
2157  r = DetectRunFrameInspectRule(tv, det_ctx, s, f, p, frames, frame);
2158  if (r) {
2159  /* match */
2160  DetectRunPostMatch(tv, det_ctx, p, s);
2161 
2162  uint8_t alert_flags = (PACKET_ALERT_FLAG_STATE_MATCH | PACKET_ALERT_FLAG_FRAME);
2163  det_ctx->frame_id = frame->id;
2164  SCLogDebug(
2165  "%p/%" PRIi64 " sig %u (%u) matched", frame, frame->id, s->id, s->num);
2166  if (frame->flags & FRAME_FLAG_TX_ID_SET) {
2167  alert_flags |= PACKET_ALERT_FLAG_TX;
2168  }
2169  AlertQueueAppend(det_ctx, s, p, frame->tx_id, alert_flags);
2170  }
2171  }
2172  DetectVarProcessList(det_ctx, p->flow, p);
2173  RULE_PROFILING_END(det_ctx, s, r, p);
2174  }
2175 
2176  /* update Frame::inspect_progress here instead of in the code above. The reason is that a
2177  * frame might be used more than once in buffers with transforms. */
2178  if (frame->inspect_progress < det_ctx->frame_inspect_progress) {
2179  frame->inspect_progress = det_ctx->frame_inspect_progress;
2180  SCLogDebug("frame->inspect_progress: %" PRIu64 " -> updated", frame->inspect_progress);
2181  } else {
2182  SCLogDebug(
2183  "frame->inspect_progress: %" PRIu64 " -> not updated", frame->inspect_progress);
2184  }
2185 
2186  SCLogDebug("%p/%" PRIi64 " rules inspected, running cleanup", frame, frame->id);
2187  InspectionBufferClean(det_ctx);
2188  }
2189 }
2190 
2191 static DetectEngineThreadCtx *GetTenantById(HashTable *h, uint32_t id)
2192 {
2193  /* technically we need to pass a DetectEngineThreadCtx struct with the
2194  * tenant_id member. But as that member is the first in the struct, we
2195  * can use the id directly. */
2196  return HashTableLookup(h, &id, 0);
2197 }
2198 
2199 static void DetectFlow(ThreadVars *tv,
2201  Packet *p)
2202 {
2203  Flow *const f = p->flow;
2204 
2205  /* we check the flow drop here, and not the packet drop. This is
2206  * to allow stream engine "invalid" drop packets to still be
2207  * evaluated by the stream event rules. */
2208  if (f->flags & FLOW_ACTION_DROP) {
2210  SCReturn;
2211  }
2212 
2213  /* in firewall mode, we still need to run the fw rulesets even for exception policy pass */
2214  bool skip = false;
2215  if (de_ctx->flags & DE_HAS_FIREWALL) {
2216  skip = (f->flags & (FLOW_ACTION_ACCEPT));
2217 
2218  } else {
2219  skip = (p->flags & PKT_NOPACKET_INSPECTION || f->flags & (FLOW_ACTION_PASS));
2220  }
2221  if (skip) {
2222  /* enfore prior accept:flow */
2223  if (f->flags & FLOW_ACTION_ACCEPT) {
2224  p->action |= ACTION_ACCEPT;
2225  }
2226  /* hack: if we are in pass the entire flow mode, we need to still
2227  * update the inspect_id forward. So test for the condition here,
2228  * and call the update code if necessary. */
2229  const int pass = (f->flags & (FLOW_ACTION_PASS | FLOW_ACTION_ACCEPT));
2230  if (pass) {
2231  uint8_t flags = STREAM_FLAGS_FOR_PACKET(p);
2233  if (f->alstate) {
2235  }
2236  }
2237  SCLogDebug("p->pcap %"PRIu64": no detection on packet, "
2238  "PKT_NOPACKET_INSPECTION is set", p->pcap_cnt);
2239  return;
2240  }
2241 
2242  /* see if the packet matches one or more of the sigs */
2243  DetectRun(tv, de_ctx, det_ctx, p);
2244 }
2245 
2246 
2247 static void DetectNoFlow(ThreadVars *tv,
2249  Packet *p)
2250 {
2251  /* No need to perform any detection on this packet, if the given flag is set.*/
2253  return;
2254  }
2255 
2256  /* see if the packet matches one or more of the sigs */
2257  DetectRun(tv, de_ctx, det_ctx, p);
2258 }
2259 
2260 /** \brief Detection engine thread wrapper.
2261  * \param tv thread vars
2262  * \param p packet to inspect
2263  * \param data thread specific data
2264  * \param pq packet queue
2265  * \retval TM_ECODE_FAILED error
2266  * \retval TM_ECODE_OK ok
2267  */
2268 TmEcode Detect(ThreadVars *tv, Packet *p, void *data)
2269 {
2271 
2272  DetectEngineCtx *de_ctx = NULL;
2273  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
2274  if (det_ctx == NULL) {
2275  printf("ERROR: Detect has no thread ctx\n");
2276  goto error;
2277  }
2278 
2279  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
2280  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
2281  SCLogDebug("Detect Engine using new det_ctx - %p",
2282  det_ctx);
2283  }
2284 
2285  /* if in MT mode _and_ we have tenants registered, use
2286  * MT logic. */
2287  if (det_ctx->mt_det_ctxs_cnt > 0 && det_ctx->TenantGetId != NULL)
2288  {
2289  uint32_t tenant_id = p->tenant_id;
2290  if (tenant_id == 0)
2291  tenant_id = det_ctx->TenantGetId(det_ctx, p);
2292  if (tenant_id > 0 && tenant_id < det_ctx->mt_det_ctxs_cnt) {
2293  p->tenant_id = tenant_id;
2294  det_ctx = GetTenantById(det_ctx->mt_det_ctxs_hash, tenant_id);
2295  if (det_ctx == NULL)
2296  return TM_ECODE_OK;
2297  de_ctx = det_ctx->de_ctx;
2298  if (de_ctx == NULL)
2299  return TM_ECODE_OK;
2300 
2301  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
2302  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
2303  SCLogDebug("MT de_ctx %p det_ctx %p (tenant %u)", de_ctx, det_ctx, tenant_id);
2304  }
2305  } else {
2306  /* use default if no tenants are registered for this packet */
2307  de_ctx = det_ctx->de_ctx;
2308  }
2309  } else {
2310  de_ctx = det_ctx->de_ctx;
2311  }
2312 
2313  if (p->flow) {
2314  DetectFlow(tv, de_ctx, det_ctx, p);
2315  } else {
2316  DetectNoFlow(tv, de_ctx, det_ctx, p);
2317  }
2318 
2319 #ifdef PROFILE_RULES
2320  /* aggregate statistics */
2321  struct timeval ts;
2322  gettimeofday(&ts, NULL);
2323  if (ts.tv_sec != det_ctx->rule_perf_last_sync) {
2324  SCProfilingRuleThreatAggregate(det_ctx);
2325  det_ctx->rule_perf_last_sync = ts.tv_sec;
2326  }
2327 #endif
2328 
2329  return TM_ECODE_OK;
2330 error:
2331  return TM_ECODE_FAILED;
2332 }
2333 
2334 /** \brief disable file features we don't need
2335  * Called if we have no detection engine.
2336  */
2338 {
2339  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOSERVER);
2340  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOCLIENT);
2341 }
2342 
2343 #ifdef UNITTESTS
2344 /**
2345  * \brief wrapper for old tests
2346  */
2349 {
2350  if (p->flow) {
2351  DetectFlow(tv, de_ctx, det_ctx, p);
2352  } else {
2353  DetectNoFlow(tv, de_ctx, det_ctx, p);
2354  }
2355 }
2356 #endif
2357 
2358 /*
2359  * TESTS
2360  */
2361 
2362 #ifdef UNITTESTS
2363 #include "tests/detect.c"
2364 #endif
PKT_IS_TOCLIENT
#define PKT_IS_TOCLIENT(p)
Definition: decode.h:239
default_accept
thread_local Signature default_accept
Definition: detect.c:1569
DetectEngineAppInspectionEngine_::stream
bool stream
Definition: detect.h:442
PacketCheckAction
bool PacketCheckAction(const Packet *p, const uint8_t a)
Definition: packet.c:49
FLOWFILE_NO_MD5_TS
#define FLOWFILE_NO_MD5_TS
Definition: flow.h:137
RulesDumpTxMatchArray
void RulesDumpTxMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, const Packet *p, const uint64_t tx_id, const uint32_t rule_cnt, const uint32_t pkt_prefilter_cnt)
Definition: detect-engine-profile.c:34
SIG_GROUP_HEAD_HAVEFILEMD5
#define SIG_GROUP_HEAD_HAVEFILEMD5
Definition: detect.h:1461
PACKET_ALERT_FLAG_TX_GUESSED
#define PACKET_ALERT_FLAG_TX_GUESSED
Definition: decode.h:272
FLOWFILE_NO_MD5_TC
#define FLOWFILE_NO_MD5_TC
Definition: flow.h:138
SigGroupHead_::tx_engines
PrefilterEngine * tx_engines
Definition: detect.h:1610
DetectEngineAppInspectionEngine_
Definition: detect.h:437
Packet_::proto
uint8_t proto
Definition: decode.h:514
DetectTransaction_::tx_data_ptr
struct AppLayerTxData * tx_data_ptr
Definition: detect-engine-prefilter.h:34
DetectEngineAppInspectionEngine_::mpm
bool mpm
Definition: detect.h:441
DE_STATE_CHUNK_SIZE
#define DE_STATE_CHUNK_SIZE
Definition: detect-engine-state.h:52
RuleMatchCandidateTx::stream_stored
bool stream_stored
Definition: detect.h:1192
FLOWFILE_NO_SIZE_TS
#define FLOWFILE_NO_SIZE_TS
Definition: flow.h:149
Frame::inspect_progress
uint64_t inspect_progress
Definition: app-layer-frames.h:55
DetectEngineThreadCtx_::alert_queue_size
uint16_t alert_queue_size
Definition: detect.h:1304
PROF_DETECT_GETSGH
@ PROF_DETECT_GETSGH
Definition: suricata-common.h:458
ts
uint64_t ts
Definition: source-erf-file.c:55
FLOWFILE_NO_SIZE_TC
#define FLOWFILE_NO_SIZE_TC
Definition: flow.h:150
detect-engine.h
DE_HAS_FIREWALL
#define DE_HAS_FIREWALL
Definition: detect.h:330
detect-engine-proto.h
DetectEngineThreadCtx_::match_array_cnt
SigIntId match_array_cnt
Definition: detect.h:1315
Frame::tx_id
uint64_t tx_id
Definition: app-layer-frames.h:54
DetectEngineStateDirection_::flags
uint8_t flags
Definition: detect-engine-state.h:88
detect-dsize.h
Signature_::addr_src_match6
DetectMatchAddressIPv6 * addr_src_match6
Definition: detect.h:713
sigmatch_table
SigTableElmt * sigmatch_table
Definition: detect-parse.c:155
PKT_HAS_FLOW
#define PKT_HAS_FLOW
Definition: decode.h:1250
DetectEngineCtx_::decoder_event_sgh
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:1016
FlowGetPacketDirection
int FlowGetPacketDirection(const Flow *f, const Packet *p)
determine the direction of the packet compared to the flow
Definition: flow.c:278
DetectEngineCtx_::flow_gh
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:958
DE_STATE_FLAG_SIG_CANT_MATCH
#define DE_STATE_FLAG_SIG_CANT_MATCH
Definition: detect-engine-state.h:56
DetectEngineThreadCtx_::counter_match_list
uint16_t counter_match_list
Definition: detect.h:1273
ALPROTO_DNS
@ ALPROTO_DNS
Definition: app-layer-protos.h:47
DetectEngineAppInspectionEngine_::next
struct DetectEngineAppInspectionEngine_ * next
Definition: detect.h:461
DETECT_PROTO_IPV6
#define DETECT_PROTO_IPV6
Definition: detect-engine-proto.h:32
detect-engine-siggroup.h
SigGroupHead_::flags
uint16_t flags
Definition: detect.h:1599
APP_LAYER_TX_SKIP_INSPECT_TC
#define APP_LAYER_TX_SKIP_INSPECT_TC
Definition: app-layer-parser.h:54
PostRuleMatchWorkQueue::len
uint32_t len
Definition: detect.h:1216
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1305
DetectEngineState_
Definition: detect-engine-state.h:92
Signature_::num
SigIntId num
Definition: detect.h:682
stream-tcp.h
PrefilterRuleStore_::rule_id_array_cnt
uint32_t rule_id_array_cnt
Definition: util-prefilter.h:40
detect-engine-event.h
SigGroupHead_
Container for matching data for a signature group.
Definition: detect.h:1598
SIG_GROUP_HEAD_HAVEFILESIZE
#define SIG_GROUP_HEAD_HAVEFILESIZE
Definition: detect.h:1462
PACKET_ALERT_FLAG_STATE_MATCH
#define PACKET_ALERT_FLAG_STATE_MATCH
Definition: decode.h:262
DetectEngineCtx_::guess_applayer
bool guess_applayer
Definition: detect.h:978
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
detect.c
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
Signature_::app_progress_hook
uint8_t app_progress_hook
Definition: detect.h:707
KEYWORD_PROFILING_SET_LIST
#define KEYWORD_PROFILING_SET_LIST(ctx, list)
Definition: util-profiling.h:46
DetectAddressMatchIPv4
int DetectAddressMatchIPv4(const DetectMatchAddressIPv4 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
Definition: detect-engine-address.c:1589
DetectEngineAppInspectionEngine_::Callback
InspectEngineFuncPtr Callback
Definition: detect.h:454
Signature_::alproto
AppProto alproto
Definition: detect.h:675
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
FLOW_SGH_TOCLIENT
#define FLOW_SGH_TOCLIENT
Definition: flow.h:75
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:617
AppLayerParserSetTransactionInspectId
void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *pstate, void *alstate, const uint8_t flags, bool tag_txs_as_inspected)
Definition: app-layer-parser.c:753
DetectEngineStateDirection_::cnt
SigIntId cnt
Definition: detect-engine-state.h:86
AppLayerGetTxIterator
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
Definition: app-layer-parser.c:706
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
SigMatchData_::is_last
bool is_last
Definition: detect.h:367
Flow_::proto
uint8_t proto
Definition: flow.h:378
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:85
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:281
AppLayerParserGetStateProgress
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
Definition: app-layer-parser.c:1086
DetectEngineThreadCtx_::tx_id
uint64_t tx_id
Definition: detect.h:1295
SigMatchData_::ctx
SigMatchCtx * ctx
Definition: detect.h:368
action-globals.h
FLOWFILE_NO_MAGIC_TS
#define FLOWFILE_NO_MAGIC_TS
Definition: flow.h:130
FramesContainer::toserver
Frames toserver
Definition: app-layer-frames.h:74
Packet_::flags
uint32_t flags
Definition: decode.h:535
Packet_::action
uint8_t action
Definition: decode.h:600
ICMPV4_DEST_UNREACH_IS_VALID
#define ICMPV4_DEST_UNREACH_IS_VALID(p)
Definition: decode-icmpv4.h:253
Frame
Definition: app-layer-frames.h:45
Flow_
Flow data structure.
Definition: flow.h:356
DetectTransaction_::tx_end_state
const int tx_end_state
Definition: detect-engine-prefilter.h:44
PROF_DETECT_ALERT
@ PROF_DETECT_ALERT
Definition: suricata-common.h:469
DetectEngineThreadCtx_::pmq
PrefilterRuleStore pmq
Definition: detect.h:1324
SIG_GROUP_HEAD_HAVEFILESHA1
#define SIG_GROUP_HEAD_HAVEFILESHA1
Definition: detect.h:1463
FLOW_TC_APP_UPDATED
#define FLOW_TC_APP_UPDATED
Definition: flow.h:120
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:931
AppLayerParserGetStateProgressCompletionStatus
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
Definition: app-layer-parser.c:1115
DetectRunScratchpad
struct DetectRunScratchpad DetectRunScratchpad
PROF_DETECT_PF_TX
@ PROF_DETECT_PF_TX
Definition: suricata-common.h:464
Frames::cnt
uint16_t cnt
Definition: app-layer-frames.h:61
Frame::id
int64_t id
Definition: app-layer-frames.h:53
DetectEngineThreadCtx_::p
Packet * p
Definition: detect.h:1299
DetectEngineAppInspectionEngine_::v2
struct DetectEngineAppInspectionEngine_::@79 v2
DetectEngineState_::dir_state
DetectEngineStateDirection dir_state[2]
Definition: detect-engine-state.h:93
FrameGetByIndex
Frame * FrameGetByIndex(Frames *frames, const uint32_t idx)
Definition: app-layer-frames.c:144
RuleMatchCandidateTx::id
SigIntId id
Definition: detect.h:1188
PROF_DETECT_CLEANUP
@ PROF_DETECT_CLEANUP
Definition: suricata-common.h:471
SIG_FLAG_DST_ANY
#define SIG_FLAG_DST_ANY
Definition: detect.h:241
ACTION_SCOPE_FLOW
@ ACTION_SCOPE_FLOW
Definition: action-globals.h:45
NO_TX
#define NO_TX
Definition: detect.c:1347
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:233
HashTable_
Definition: util-hash.h:35
SIG_FLAG_TXBOTHDIR
#define SIG_FLAG_TXBOTHDIR
Definition: detect.h:249
Packet_::sig_mask
SignatureMask sig_mask
Definition: decode.h:529
Frames
Definition: app-layer-frames.h:60
FramesContainer
Definition: app-layer-frames.h:73
PacketAlerts_::drop
PacketAlert drop
Definition: decode.h:287
DetectRunScratchpad
Definition: detect.c:69
SIG_GROUP_HEAD_HAVERAWSTREAM
#define SIG_GROUP_HEAD_HAVERAWSTREAM
Definition: detect.h:1457
FLOW_ACTION_DROP
#define FLOW_ACTION_DROP
Definition: flow.h:70
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
FLOWFILE_NONE
#define FLOWFILE_NONE
Definition: flow.h:168
detect-engine-frame.h
SigMatchSignatures
void SigMatchSignatures(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
wrapper for old tests
Definition: detect.c:2347
Signature_::sm_arrays
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:733
proto
uint8_t proto
Definition: decode-template.h:0
m
SCMutex m
Definition: flow-hash.h:6
DetectPort_::sh
struct SigGroupHead_ * sh
Definition: detect.h:230
PKT_NOPAYLOAD_INSPECTION
#define PKT_NOPAYLOAD_INSPECTION
Definition: decode.h:1239
PACKET_PROFILING_DETECT_END
#define PACKET_PROFILING_DETECT_END(p, id)
Definition: util-profiling.h:221
Detect
TmEcode Detect(ThreadVars *tv, Packet *p, void *data)
Detection engine thread wrapper.
Definition: detect.c:2268
detect-engine-payload.h
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:523
SIG_FLAG_SRC_ANY
#define SIG_FLAG_SRC_ANY
Definition: detect.h:240
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:82
DetectRunFrameInspectRule
bool DetectRunFrameInspectRule(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p, const Frames *frames, const Frame *frame)
Definition: detect-engine-frame.c:228
Flow_::protoctx
void * protoctx
Definition: flow.h:441
SigMatchData_
Data needed for Match()
Definition: detect.h:365
DeStateStoreItem_::sid
SigIntId sid
Definition: detect-engine-state.h:74
RuleMatchCandidateTx::s
const Signature * s
Definition: detect.h:1198
KEYWORD_PROFILING_START
#define KEYWORD_PROFILING_START
Definition: util-profiling.h:50
SigMatchData_::type
uint16_t type
Definition: detect.h:366
DetectEngineCtx_::version
uint32_t version
Definition: detect.h:1012
DisableDetectFlowFileFlags
void DisableDetectFlowFileFlags(Flow *f)
disable file features we don't need Called if we have no detection engine.
Definition: detect.c:2337
AppLayerParserGetTransactionInspectId
uint64_t AppLayerParserGetTransactionInspectId(AppLayerParserState *pstate, uint8_t direction)
Definition: app-layer-parser.c:731
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:611
DetectTransaction_::tx_progress
const int tx_progress
Definition: detect-engine-prefilter.h:43
detect-engine-prefilter.h
DetectRunPrefilterFrame
void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const Frames *frames, const Frame *frame, const AppProto alproto)
Definition: detect-engine-frame.c:73
Packet_::events
PacketEngineEvents events
Definition: decode.h:621
Signature_::frame_inspect
DetectEngineFrameInspectionEngine * frame_inspect
Definition: detect.h:729
PROF_DETECT_PF_SORT2
@ PROF_DETECT_PF_SORT2
Definition: suricata-common.h:467
pad
uint16_t pad
Definition: source-erf-file.c:61
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:81
PacketCreateMask
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
Definition: detect-engine-build.c:419
SIG_FLAG_APPLAYER
#define SIG_FLAG_APPLAYER
Definition: detect.h:248
KEYWORD_PROFILING_END
#define KEYWORD_PROFILING_END(ctx, type, m)
Definition: util-profiling.h:64
detect-flowvar.h
DetectEngineThreadCtx_::mt_det_ctxs_hash
HashTable * mt_det_ctxs_hash
Definition: detect.h:1235
PacketAlert_::action
uint8_t action
Definition: decode.h:245
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:294
DetectEnginePktInspectionRun
bool DetectEnginePktInspectionRun(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p, uint8_t *alert_flags)
Definition: detect-engine.c:1918
SIG_FLAG_FIREWALL
#define SIG_FLAG_FIREWALL
Definition: detect.h:245
Flow_::sgh_toserver
const struct SigGroupHead_ * sgh_toserver
Definition: flow.h:486
PROF_DETECT_TX_UPDATE
@ PROF_DETECT_TX_UPDATE
Definition: suricata-common.h:470
DetectEngineAppInspectionEngine_::id
uint8_t id
Definition: detect.h:440
DetectEngineThreadCtx_::counter_alerts_suppressed
uint16_t counter_alerts_suppressed
Definition: detect.h:1268
DetectRunStoreStateTx
void DetectRunStoreStateTx(const SigGroupHead *sgh, Flow *f, void *tx, uint64_t tx_id, const Signature *s, uint32_t inspect_flags, uint8_t flow_flags, const uint16_t file_no_match)
Definition: detect-engine-state.c:214
DetectEngineAppInspectionEngine_::sm_list
uint16_t sm_list
Definition: detect.h:445
FLOWFILE_INIT
#define FLOWFILE_INIT
Definition: flow.h:127
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:478
FLOW_ACTION_PASS
#define FLOW_ACTION_PASS
Definition: flow.h:117
DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES
Definition: detect-engine-state.h:43
FLOWFILE_NO_SHA1_TC
#define FLOWFILE_NO_SHA1_TC
Definition: flow.h:142
DETECT_SM_LIST_POSTMATCH
@ DETECT_SM_LIST_POSTMATCH
Definition: detect.h:126
FramesContainer::toclient
Frames toclient
Definition: app-layer-frames.h:75
DetectEngineThreadCtx_::tx_candidates
RuleMatchCandidateTx * tx_candidates
Definition: detect.h:1317
SIG_TYPE_PKT
@ SIG_TYPE_PKT
Definition: detect.h:71
PKT_DROP_REASON_DEFAULT_APP_POLICY
@ PKT_DROP_REASON_DEFAULT_APP_POLICY
Definition: decode.h:392
Signature_::addr_src_match4
DetectMatchAddressIPv4 * addr_src_match4
Definition: detect.h:710
decode.h
DetectEngineThreadCtx_::counter_alerts_overflow
uint16_t counter_alerts_overflow
Definition: detect.h:1266
PKT_SRC_WIRE
@ PKT_SRC_WIRE
Definition: decode.h:52
TOSERVER
#define TOSERVER
Definition: flow.h:45
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:18
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:238
Prefilter
void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t flags, const SignatureMask mask)
Definition: detect-engine-prefilter.c:216
DetectTransaction_::detect_progress_orig
const uint8_t detect_progress_orig
Definition: detect-engine-prefilter.h:41
DetectEngineThreadCtx_
Definition: detect.h:1223
DEBUG_VALIDATE_PACKET
#define DEBUG_VALIDATE_PACKET(p)
Definition: util-validate.h:101
DeStateStoreItem_::flags
uint32_t flags
Definition: detect-engine-state.h:73
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1245
DetectTransaction_::detect_progress
uint8_t detect_progress
Definition: detect-engine-prefilter.h:39
FLOWFILE_NO_STORE_TS
#define FLOWFILE_NO_STORE_TS
Definition: flow.h:134
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:417
DetectEngineThreadCtx_::tx_candidates_size
uint32_t tx_candidates_size
Definition: detect.h:1318
FRAME_FLAG_TX_ID_SET
#define FRAME_FLAG_TX_ID_SET
Definition: app-layer-frames.h:38
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1252
PKT_IS_FRAGMENT
#define PKT_IS_FRAGMENT
Definition: decode.h:1274
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
STREAM_FLAGS_FOR_PACKET
#define STREAM_FLAGS_FOR_PACKET(p)
Definition: stream.h:30
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
PMQ_RESET
#define PMQ_RESET(pmq)
Definition: util-prefilter.h:46
detect-engine-mpm.h
DetectEngineLookupFlow_::sgh
struct SigGroupHead_ * sgh[256]
Definition: detect.h:865
HashTableLookup
void * HashTableLookup(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:183
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
DeStateStore_::next
struct DeStateStore_ * next
Definition: detect-engine-state.h:79
Packet_::sp
Port sp
Definition: decode.h:499
FLOWFILE_NO_SHA256_TS
#define FLOWFILE_NO_SHA256_TS
Definition: flow.h:145
DETECT_ENGINE_INSPECT_SIG_MATCH
#define DETECT_ENGINE_INSPECT_SIG_MATCH
Definition: detect-engine-state.h:38
PKT_DETECT_HAS_STREAMDATA
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1289
FLOW_PKT_TOCLIENT_FIRST
#define FLOW_PKT_TOCLIENT_FIRST
Definition: flow.h:237
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1512
detect-engine-port.h
PktSrcToString
const char * PktSrcToString(enum PktSrcEnum pkt_src)
Definition: decode.c:826
DetectPort_
Port structure for detection engine.
Definition: detect.h:219
PacketAlerts_::discarded
uint16_t discarded
Definition: decode.h:282
app-layer-parser.h
Signature_::app_inspect
DetectEngineAppInspectionEngine * app_inspect
Definition: detect.h:727
util-detect.h
detect-engine-profile.h
Flow_::sgh_toclient
const struct SigGroupHead_ * sgh_toclient
Definition: flow.h:483
DetectEngineThreadCtx_::base64_decoded_len
int base64_decoded_len
Definition: detect.h:1302
SIG_FLAG_REQUIRE_FLOWVAR
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:266
RuleMatchCandidateTx::stream_result
uint8_t stream_result
Definition: detect.h:1193
Signature_::action
uint8_t action
Definition: detect.h:685
util-profiling.h
DetectTransaction_::tx_id
const uint64_t tx_id
Definition: detect-engine-prefilter.h:33
DetectEngineLookupFlow_::udp
DetectPort * udp
Definition: detect.h:864
DetectEngineThreadCtx_::raw_stream_progress
uint64_t raw_stream_progress
Definition: detect.h:1244
FileUpdateFlowFileFlags
void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction)
set a flow's file flags
Definition: util-file.c:1120
SCReturn
#define SCReturn
Definition: util-debug.h:273
Signature_::flags
uint32_t flags
Definition: detect.h:671
AlertQueueAppend
void AlertQueueAppend(DetectEngineThreadCtx *det_ctx, const Signature *s, Packet *p, uint64_t tx_id, uint8_t alert_flags)
Append signature to local packet alert queue for later preprocessing.
Definition: detect-engine-alert.c:298
RuleMatchCandidateTxArrayFree
void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx)
Definition: detect.c:1036
AppLayerGetTxIterState
Definition: app-layer-parser.h:121
Packet_
Definition: decode.h:492
detect-engine-build.h
AppLayerFramesGetContainer
FramesContainer * AppLayerFramesGetContainer(Flow *f)
Definition: app-layer-parser.c:182
ACTION_SCOPE_TX
@ ACTION_SCOPE_TX
Definition: action-globals.h:47
DetectEngineThreadCtx_::frame_id
int64_t frame_id
Definition: detect.h:1296
detect-engine-alert.h
DetectEngineThreadCtx_::filestore_cnt
uint16_t filestore_cnt
Definition: detect.h:1261
PROF_DETECT_IPONLY
@ PROF_DETECT_IPONLY
Definition: suricata-common.h:459
RULE_PROFILING_END
#define RULE_PROFILING_END(a, b, c, p)
Definition: util-profiling.h:422
TmEcode
TmEcode
Definition: tm-threads-common.h:80
RULE_PROFILING_START
#define RULE_PROFILING_START(p)
Definition: util-profiling.h:421
ALPROTO_DOH2
@ ALPROTO_DOH2
Definition: app-layer-protos.h:66
PKT_STREAM_EOF
#define PKT_STREAM_EOF
Definition: decode.h:1249
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:287
DetectTransaction_::de_state
DetectEngineStateDirection * de_state
Definition: detect-engine-prefilter.h:35
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
detect-filestore.h
SigTableElmt_::Match
int(* Match)(DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1396
detect-replace.h
RulesDumpMatchArray
void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, const Packet *p)
Definition: detect-engine-profile.c:78
ALPROTO_HTTP2
@ ALPROTO_HTTP2
Definition: app-layer-protos.h:69
Signature_::addr_dst_match6_cnt
uint16_t addr_dst_match6_cnt
Definition: detect.h:697
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:234
SigGroupHead_::frame_engines
PrefilterEngine * frame_engines
Definition: detect.h:1611
PACKET_ALERT_NOTX
#define PACKET_ALERT_NOTX
Definition: detect.h:54
Signature_::sp
DetectPort * sp
Definition: detect.h:721
APP_LAYER_PARSER_NO_INSPECTION
#define APP_LAYER_PARSER_NO_INSPECTION
Definition: app-layer-parser.h:35
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1108
DetectRunScratchpad::sgh
const SigGroupHead * sgh
Definition: detect.c:73
PACKET_ALERT_FLAG_APPLY_ACTION_TO_PACKET
#define PACKET_ALERT_FLAG_APPLY_ACTION_TO_PACKET
Definition: decode.h:274
DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
Definition: detect-engine-state.h:39
DETECT_ENGINE_STATE_FLAG_FILE_NEW
#define DETECT_ENGINE_STATE_FLAG_FILE_NEW
Definition: detect-engine-state.h:70
Flow_::flowvar
GenericVar * flowvar
Definition: flow.h:489
PKT_DROP_REASON_DEFAULT_PACKET_POLICY
@ PKT_DROP_REASON_DEFAULT_PACKET_POLICY
Definition: decode.h:391
RuleMatchCandidateTx::flags
uint32_t * flags
Definition: detect.h:1189
PACKET_ALERT_FLAG_FRAME
#define PACKET_ALERT_FLAG_FRAME
Definition: decode.h:270
Frame::flags
uint8_t flags
Definition: app-layer-frames.h:47
DetectEngineAppInspectionEngine_::alproto
AppProto alproto
Definition: detect.h:438
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
detect-engine-analyzer.h
FLOWFILE_NO_SHA256_TC
#define FLOWFILE_NO_SHA256_TC
Definition: flow.h:146
DetectEngineStateDirection_::head
DeStateStore * head
Definition: detect-engine-state.h:83
DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES
#define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES
Definition: detect-engine-state.h:49
DetectProto_::flags
uint8_t flags
Definition: detect-engine-proto.h:37
AppLayerTxData
struct AppLayerTxData AppLayerTxData
Definition: detect.h:1494
DETECT_PROTO_IPV4
#define DETECT_PROTO_IPV4
Definition: detect-engine-proto.h:31
app-layer-frames.h
DetectAddressMatchIPv6
int DetectAddressMatchIPv6(const DetectMatchAddressIPv6 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
Definition: detect-engine-address.c:1622
Packet_::tenant_id
uint32_t tenant_id
Definition: decode.h:656
Packet_::flow
struct Flow_ * flow
Definition: decode.h:537
DetectEngineStateResetTxs
void DetectEngineStateResetTxs(Flow *f)
Reset de state for active tx' To be used on detect engine reload.
Definition: detect-engine-state.c:260
FLOWFILE_NO_MAGIC_TC
#define FLOWFILE_NO_MAGIC_TC
Definition: flow.h:131
flags
uint8_t flags
Definition: decode-gre.h:0
Signature_::proto
DetectProto proto
Definition: detect.h:689
suricata-common.h
SIG_FLAG_SP_ANY
#define SIG_FLAG_SP_ANY
Definition: detect.h:242
ActionScope
ActionScope
Definition: action-globals.h:42
ACTION_SCOPE_HOOK
@ ACTION_SCOPE_HOOK
Definition: action-globals.h:46
Signature_::action_scope
uint8_t action_scope
Definition: detect.h:692
packet.h
DeStateStore_
Definition: detect-engine-state.h:77
ACTION_DROP
#define ACTION_DROP
Definition: action-globals.h:30
DetectTransaction_
Definition: detect-engine-prefilter.h:31
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:526
PROF_DETECT_SETUP
@ PROF_DETECT_SETUP
Definition: suricata-common.h:457
DetectEngineStateDirection_
Definition: detect-engine-state.h:82
DetectEngineThreadCtx_::frame_inspect_progress
uint64_t frame_inspect_progress
Definition: detect.h:1297
DetectEngineCtx_::profile_match_logging_threshold
uint32_t profile_match_logging_threshold
Definition: detect.h:1048
FLOW_TS_APP_UPDATED
#define FLOW_TS_APP_UPDATED
Definition: flow.h:119
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1182
FatalError
#define FatalError(...)
Definition: util-debug.h:502
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:297
FLOWFILE_NO_STORE_TC
#define FLOWFILE_NO_STORE_TC
Definition: flow.h:135
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
DetectEngineAppInspectionEngine_::progress
int16_t progress
Definition: detect.h:447
FLOW_ACTION_ACCEPT
#define FLOW_ACTION_ACCEPT
Definition: flow.h:62
util-validate.h
DetectTransaction_::tx_ptr
void * tx_ptr
Definition: detect-engine-prefilter.h:32
Signature_::addr_src_match6_cnt
uint16_t addr_src_match6_cnt
Definition: detect.h:698
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1558
StatsAddUI64
void StatsAddUI64(ThreadVars *tv, uint16_t id, uint64_t x)
Adds a value of type uint64_t to the local counter.
Definition: counters.c:146
FLOWFILE_NO_SHA1_TS
#define FLOWFILE_NO_SHA1_TS
Definition: flow.h:141
PROF_DETECT_RULES
@ PROF_DETECT_RULES
Definition: suricata-common.h:460
APP_LAYER_TX_INSPECTED_TS
#define APP_LAYER_TX_INSPECTED_TS
Definition: app-layer-parser.h:56
DetectRunScratchpad::app_decoder_events
const bool app_decoder_events
Definition: detect.c:72
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:296
Signature_::dp
DetectPort * dp
Definition: detect.h:721
PacketAlerts_::suppressed
uint16_t suppressed
Definition: decode.h:283
PACKET_PROFILING_DETECT_START
#define PACKET_PROFILING_DETECT_START(p, id)
Definition: util-profiling.h:214
FLOW_SGH_TOSERVER
#define FLOW_SGH_TOSERVER
Definition: flow.h:73
APP_LAYER_TX_ACCEPT
#define APP_LAYER_TX_ACCEPT
Definition: app-layer-parser.h:59
DetectEngineThreadCtx_::counter_alerts
uint16_t counter_alerts
Definition: detect.h:1264
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Packet_::pkt_src
uint8_t pkt_src
Definition: decode.h:602
SGH_PROFILING_RECORD
#define SGH_PROFILING_RECORD(det_ctx, sgh)
Definition: util-profiling.h:252
PacketAlertFinalize
void PacketAlertFinalize(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
Check the threshold of the sigs that match, set actions, break on pass action This function iterate t...
Definition: detect-engine-alert.c:507
Signature_::addr_dst_match6
DetectMatchAddressIPv6 * addr_dst_match6
Definition: detect.h:712
Flow_::alstate
void * alstate
Definition: flow.h:479
Signature_::id
uint32_t id
Definition: detect.h:715
DeStateStore_::store
DeStateStoreItem store[DE_STATE_CHUNK_SIZE]
Definition: detect-engine-state.h:78
DetectEngineCtx_::guess_applayer_log_limit
uint8_t guess_applayer_log_limit
Definition: detect.h:975
Flow_::flags
uint32_t flags
Definition: flow.h:421
RuleMatchCandidateTx::stream_reset
uint32_t stream_reset
Definition: detect.h:1195
ACTION_SCOPE_PACKET
@ ACTION_SCOPE_PACKET
Definition: action-globals.h:44
detect-engine-iponly.h
Signature_
Signature container.
Definition: detect.h:670
InspectionBufferClean
void InspectionBufferClean(DetectEngineThreadCtx *det_ctx)
Definition: detect-engine.c:1401
PACKET_ALERT_FLAG_TX
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:266
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
TRACE_SID_TXS
#define TRACE_SID_TXS(sid, txs,...)
Definition: detect.c:1099
FLOW_PKT_ESTABLISHED
#define FLOW_PKT_ESTABLISHED
Definition: flow.h:235
DetectEngineThreadCtx_::tx_id_set
bool tx_id_set
Definition: detect.h:1293
PacketDrop
void PacketDrop(Packet *p, const uint8_t action, enum PacketDropReason r)
issue drop action
Definition: packet.c:33
ACTION_ACCEPT
#define ACTION_ACCEPT
Definition: action-globals.h:36
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
DetectRunScratchpad::flow_flags
const uint8_t flow_flags
Definition: detect.c:71
DetectEngineThreadCtx_::de_ctx
DetectEngineCtx * de_ctx
Definition: detect.h:1339
suricata.h
IPOnlyMatchPacket
void IPOnlyMatchPacket(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const DetectEngineIPOnlyCtx *io_ctx, Packet *p)
Match a packet against the IP Only detection engine contexts.
Definition: detect-engine-iponly.c:998
DetectEngineCtx_::sig_array
Signature ** sig_array
Definition: detect.h:949
Packet_::dst
Address dst
Definition: decode.h:497
PROF_DETECT_TX
@ PROF_DETECT_TX
Definition: suricata-common.h:461
DetectEngineAppInspectionEngine_::dir
uint8_t dir
Definition: detect.h:439
PKT_NOPACKET_INSPECTION
#define PKT_NOPACKET_INSPECTION
Definition: decode.h:1234
PrefilterPostRuleMatch
void PrefilterPostRuleMatch(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, Flow *f)
invoke post-rule match "prefilter" engines
Definition: detect-engine-prefilter.c:193
DetectGetInnerTx
void * DetectGetInnerTx(void *tx_ptr, AppProto alproto, AppProto engine_alproto, uint8_t flow_flags)
Definition: detect.c:1103
DetectRunScratchpad::alproto
const AppProto alproto
Definition: detect.c:70
DE_STATE_FLAG_FULL_INSPECT
#define DE_STATE_FLAG_FULL_INSPECT
Definition: detect-engine-state.h:55
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
DetectEngineThreadCtx_::mt_det_ctxs_cnt
uint32_t mt_det_ctxs_cnt
Definition: detect.h:1233
AppLayerGetTxIteratorFunc
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype
Definition: app-layer-parser.h:130
likely
#define likely(expr)
Definition: util-optimize.h:32
IPPROTO_SCTP
#define IPPROTO_SCTP
Definition: decode.h:1215
DE_STATE_FLAG_FILE_INSPECT
#define DE_STATE_FLAG_FILE_INSPECT
Definition: detect-engine-state.h:60
id
uint32_t id
Definition: detect-flowbits.c:933
DetectEngineCtx_::flags
uint8_t flags
Definition: detect.h:933
DetectEngineCtx_::io_ctx
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:969
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
Signature_::addr_dst_match4
DetectMatchAddressIPv4 * addr_dst_match4
Definition: detect.h:709
FlowGetDisruptionFlags
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get 'disruption' flags: GAP/DEPTH/PASS
Definition: flow.c:1140
TcpSession_
Definition: stream-tcp-private.h:283
flow.h
Signature_::addr_src_match4_cnt
uint16_t addr_src_match4_cnt
Definition: detect.h:696
SigIntId
#define SigIntId
Definition: suricata-common.h:332
AppLayerParserStateIssetFlag
uint16_t AppLayerParserStateIssetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1828
DeStateStoreItem_
Definition: detect-engine-state.h:72
GenericVarFree
void GenericVarFree(GenericVar *gv)
Definition: util-var.c:48
Signature_::addr_dst_match4_cnt
uint16_t addr_dst_match4_cnt
Definition: detect.h:695
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
Packet_::dp
Port dp
Definition: decode.h:507
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
DetectEngineCtx_::sig_array_len
uint32_t sig_array_len
Definition: detect.h:950
SIG_GROUP_HEAD_HAVEFILESHA256
#define SIG_GROUP_HEAD_HAVEFILESHA256
Definition: detect.h:1464
Signature_::type
enum SignatureType type
Definition: detect.h:673
SigGroupHead_::filestore_cnt
uint16_t filestore_cnt
Definition: detect.h:1604
AppLayerParserGetTxCnt
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
Definition: app-layer-parser.c:1101
AppLayerParserHasDecoderEvents
bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
Definition: app-layer-parser.c:1512
FLOW_PKT_TOSERVER_FIRST
#define FLOW_PKT_TOSERVER_FIRST
Definition: flow.h:236
DetectEngineLookupFlow_::tcp
DetectPort * tcp
Definition: detect.h:863
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
detect-engine-address.h
Flow_::tenant_id
uint32_t tenant_id
Definition: flow.h:416
Packet_::src
Address src
Definition: decode.h:496
DetectEngineThreadCtx_::counter_mpm_list
uint16_t counter_mpm_list
Definition: detect.h:1270
SigMatchSignaturesGetSgh
const SigGroupHead * SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx, const Packet *p)
Get the SigGroupHead for a packet.
Definition: detect.c:232
DetectRunPrefilterTx
void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t ipproto, const uint8_t flow_flags, const AppProto alproto, void *alstate, DetectTransaction *tx)
run prefilter engines on a transaction
Definition: detect-engine-prefilter.c:95
RuleMatchCandidateTxArrayInit
void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size)
Definition: detect.c:1023
APP_LAYER_TX_INSPECTED_TC
#define APP_LAYER_TX_INSPECTED_TC
Definition: app-layer-parser.h:57
SIG_FLAG_DP_ANY
#define SIG_FLAG_DP_ANY
Definition: detect.h:243
DetectPortLookupGroup
DetectPort * DetectPortLookupGroup(DetectPort *dp, uint16_t port)
Function that find the group matching port in a group head.
Definition: detect-engine-port.c:613
DetectEngineThreadCtx_::post_rule_work_queue
PostRuleMatchWorkQueue post_rule_work_queue
Definition: detect.h:1322
PKT_STREAM_EST
#define PKT_STREAM_EST
Definition: decode.h:1247
detect-engine-threshold.h
APP_LAYER_TX_SKIP_INSPECT_TS
#define APP_LAYER_TX_SKIP_INSPECT_TS
Definition: app-layer-parser.h:53
Signature_::mask
SignatureMask mask
Definition: detect.h:681
app-layer.h
RuleMatchCandidateTx
Definition: detect.h:1187
DetectEngineThreadCtx_::TenantGetId
uint32_t(* TenantGetId)(const void *, const Packet *p)
Definition: detect.h:1240
PrefilterRuleStore_::rule_id_array
SigIntId * rule_id_array
Definition: util-prefilter.h:38
PacketEngineEvents_::cnt
uint8_t cnt
Definition: decode.h:300
Flow_::de_ctx_version
uint32_t de_ctx_version
Definition: flow.h:464
DetectProtoContainsProto
int DetectProtoContainsProto(const DetectProto *dp, int proto)
see if a DetectProto contains a certain proto
Definition: detect-engine-proto.c:135
DetectEngineThreadCtx_::match_array
Signature ** match_array
Definition: detect.h:1310