suricata
detect.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Basic detection engine
24  */
25 
26 #include "suricata-common.h"
27 #include "suricata.h"
28 #include "conf.h"
29 
30 #include "decode.h"
31 #include "packet.h"
32 #include "flow.h"
33 #include "stream-tcp.h"
34 #include "app-layer.h"
35 #include "app-layer-parser.h"
36 #include "app-layer-frames.h"
37 
38 #include "detect.h"
39 #include "detect-dsize.h"
40 #include "detect-engine.h"
41 #include "detect-engine-build.h"
42 #include "detect-engine-frame.h"
43 #include "detect-engine-profile.h"
44 
45 #include "detect-engine-alert.h"
46 #include "detect-engine-siggroup.h"
47 #include "detect-engine-address.h"
48 #include "detect-engine-proto.h"
49 #include "detect-engine-port.h"
50 #include "detect-engine-mpm.h"
51 #include "detect-engine-iponly.h"
54 #include "detect-engine-state.h"
55 #include "detect-engine-analyzer.h"
56 
57 #include "detect-engine-payload.h"
58 #include "detect-engine-event.h"
59 
60 #include "detect-filestore.h"
61 #include "detect-flowvar.h"
62 #include "detect-replace.h"
63 
64 #include "util-validate.h"
65 #include "util-detect.h"
66 #include "util-profiling.h"
67 
68 #include "action-globals.h"
69 
70 typedef struct DetectRunScratchpad {
72  const uint8_t flow_flags; /* flow/state flags: STREAM_* */
73  const bool app_decoder_events;
74  const SigGroupHead *sgh;
77 
78 /* prototypes */
79 static DetectRunScratchpad DetectRunSetup(const DetectEngineCtx *de_ctx,
80  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow);
81 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
82  DetectEngineThreadCtx *det_ctx, Flow * const pflow, Packet * const p);
83 static inline void DetectRunGetRuleGroup(const DetectEngineCtx *de_ctx,
84  Packet * const p, Flow * const pflow, DetectRunScratchpad *scratch);
85 static inline void DetectRunPrefilterPkt(ThreadVars *tv,
87  DetectRunScratchpad *scratch);
88 static inline void DetectRulePacketRules(ThreadVars * const tv,
89  DetectEngineCtx * const de_ctx, DetectEngineThreadCtx * const det_ctx,
90  Packet * const p, Flow * const pflow, const DetectRunScratchpad *scratch);
91 static void DetectRunTx(ThreadVars *tv, DetectEngineCtx *de_ctx,
92  DetectEngineThreadCtx *det_ctx, Packet *p,
93  Flow *f, DetectRunScratchpad *scratch);
94 static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
95  Packet *p, Flow *f, DetectRunScratchpad *scratch);
96 static inline void DetectRunPostRules(ThreadVars *tv, DetectEngineCtx *de_ctx,
97  DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow,
98  DetectRunScratchpad *scratch);
99 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
100  Packet *p, Flow * const pflow);
101 
102 /** \internal
103  */
104 static void DetectRun(ThreadVars *th_v,
106  Packet *p)
107 {
108  SCEnter();
109  SCLogDebug("p->pcap_cnt %" PRIu64 " direction %s pkt_src %s", p->pcap_cnt,
110  p->flow ? (FlowGetPacketDirection(p->flow, p) == TOSERVER ? "toserver" : "toclient")
111  : "noflow",
112  PktSrcToString(p->pkt_src));
113 
114  /* bail early if packet should not be inspected */
115  if (p->flags & PKT_NOPACKET_INSPECTION) {
116  /* nothing to do */
117  SCReturn;
118  }
119 
120  /* Load the Packet's flow early, even though it might not be needed.
121  * Mark as a constant pointer, although the flow itself can change. */
122  Flow * const pflow = p->flow;
123 
124  DetectRunScratchpad scratch = DetectRunSetup(de_ctx, det_ctx, p, pflow);
125 
126  /* run the IPonly engine */
127  DetectRunInspectIPOnly(th_v, de_ctx, det_ctx, pflow, p);
128 
129  /* get our rule group */
130  DetectRunGetRuleGroup(de_ctx, p, pflow, &scratch);
131  /* if we didn't get a sig group head, we
132  * have nothing to do.... */
133  if (scratch.sgh == NULL) {
134  SCLogDebug("no sgh for this packet, nothing to match against");
135  goto end;
136  }
137 
138  /* run the prefilters for packets */
139  DetectRunPrefilterPkt(th_v, de_ctx, det_ctx, p, &scratch);
140 
142  /* inspect the rules against the packet */
143  DetectRulePacketRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
145 
146  /* run tx/state inspection. Don't call for ICMP error msgs. */
147  if (pflow && pflow->alstate && likely(pflow->proto == p->proto)) {
148  if (p->proto == IPPROTO_TCP) {
149  const TcpSession *ssn = p->flow->protoctx;
150  if (ssn && (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) {
151  // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX);
152  DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch);
153  // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_TX);
154  }
155  // no update to transactions
156  if (!PKT_IS_PSEUDOPKT(p) && p->app_update_direction == 0 &&
157  ((PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATED) == 0) ||
158  (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATED) == 0))) {
159  goto end;
160  }
161  } else if (p->proto == IPPROTO_UDP) {
162  DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch);
163  }
164 
166  DetectRunTx(th_v, de_ctx, det_ctx, p, pflow, &scratch);
168  /* see if we need to increment the inspect_id and reset the de_state */
171  pflow, pflow->alparser, pflow->alstate, scratch.flow_flags, (scratch.sgh == NULL));
173  }
174 
175 end:
176  DetectRunPostRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
177 
178  DetectRunCleanup(det_ctx, p, pflow);
179  SCReturn;
180 }
181 
182 static void DetectRunPostMatch(ThreadVars *tv,
183  DetectEngineThreadCtx *det_ctx, Packet *p,
184  const Signature *s)
185 {
186  /* run the packet match functions */
188  if (smd != NULL) {
190 
191  SCLogDebug("running match functions, sm %p", smd);
192 
193  while (1) {
195  (void)sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx);
196  KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
197  if (smd->is_last)
198  break;
199  smd++;
200  }
201  }
202 }
203 
204 /**
205  * \brief Get the SigGroupHead for a packet.
206  *
207  * \param de_ctx detection engine context
208  * \param p packet
209  *
210  * \retval sgh the SigGroupHead or NULL if non applies to the packet
211  */
213  const Packet *p)
214 {
215  SCEnter();
216  SigGroupHead *sgh = NULL;
217 
218  /* if the packet proto is 0 (not set), we're inspecting it against
219  * the decoder events sgh we have. */
220  if (p->proto == 0 && p->events.cnt > 0) {
221  SCReturnPtr(de_ctx->decoder_event_sgh, "SigGroupHead");
222  } else if (p->proto == 0) {
223  if (!(PKT_IS_IPV4(p) || PKT_IS_IPV6(p))) {
224  /* not IP, so nothing to do */
225  SCReturnPtr(NULL, "SigGroupHead");
226  }
227  }
228 
229  /* select the flow_gh */
230  const int dir = (p->flowflags & FLOW_PKT_TOCLIENT) == 0;
231 
232  int proto = IP_GET_IPPROTO(p);
233  if (proto == IPPROTO_TCP) {
234  DetectPort *list = de_ctx->flow_gh[dir].tcp;
235  SCLogDebug("tcp toserver %p, tcp toclient %p: going to use %p", de_ctx->flow_gh[1].tcp,
236  de_ctx->flow_gh[0].tcp, de_ctx->flow_gh[dir].tcp);
237  const uint16_t port = dir ? p->dp : p->sp;
238  SCLogDebug("tcp port %u -> %u:%u", port, p->sp, p->dp);
239  DetectPort *sghport = DetectPortLookupGroup(list, port);
240  if (sghport != NULL)
241  sgh = sghport->sh;
242  SCLogDebug("TCP list %p, port %u, direction %s, sghport %p, sgh %p", list, port,
243  dir ? "toserver" : "toclient", sghport, sgh);
244  } else if (proto == IPPROTO_UDP) {
245  DetectPort *list = de_ctx->flow_gh[dir].udp;
246  uint16_t port = dir ? p->dp : p->sp;
247  DetectPort *sghport = DetectPortLookupGroup(list, port);
248  if (sghport != NULL)
249  sgh = sghport->sh;
250  SCLogDebug("UDP list %p, port %u, direction %s, sghport %p, sgh %p", list, port,
251  dir ? "toserver" : "toclient", sghport, sgh);
252  } else {
253  sgh = de_ctx->flow_gh[dir].sgh[proto];
254  }
255 
256  SCReturnPtr(sgh, "SigGroupHead");
257 }
258 
259 static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
260  DetectEngineThreadCtx *det_ctx)
261 {
262  SigIntId mpm, nonmpm;
263  SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array;
264  SigIntId *nonmpm_ptr = det_ctx->non_pf_id_array;
265  uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt;
266  uint32_t n_cnt = det_ctx->non_pf_id_cnt;
267  SigIntId *final_ptr;
268  uint32_t final_cnt;
269  SigIntId id;
270  SigIntId previous_id = (SigIntId)-1;
271  Signature **sig_array = de_ctx->sig_array;
272  Signature **match_array = det_ctx->match_array;
273  Signature *s;
274 
275  SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt);
276 
277  /* Load first values. */
278  if (likely(m_cnt)) {
279  mpm = *mpm_ptr;
280  } else {
281  /* mpm list is empty */
282  final_ptr = nonmpm_ptr;
283  final_cnt = n_cnt;
284  goto final;
285  }
286  if (likely(n_cnt)) {
287  nonmpm = *nonmpm_ptr;
288  } else {
289  /* non-mpm list is empty. */
290  final_ptr = mpm_ptr;
291  final_cnt = m_cnt;
292  goto final;
293  }
294  while (1) {
295  if (mpm < nonmpm) {
296  /* Take from mpm list */
297  id = mpm;
298 
299  s = sig_array[id];
300  /* As the mpm list can contain duplicates, check for that here. */
301  if (likely(id != previous_id)) {
302  *match_array++ = s;
303  previous_id = id;
304  }
305  if (unlikely(--m_cnt == 0)) {
306  /* mpm list is now empty */
307  final_ptr = nonmpm_ptr;
308  final_cnt = n_cnt;
309  goto final;
310  }
311  mpm_ptr++;
312  mpm = *mpm_ptr;
313  } else if (mpm > nonmpm) {
314  id = nonmpm;
315 
316  s = sig_array[id];
317  /* As the mpm list can contain duplicates, check for that here. */
318  if (likely(id != previous_id)) {
319  *match_array++ = s;
320  previous_id = id;
321  }
322  if (unlikely(--n_cnt == 0)) {
323  final_ptr = mpm_ptr;
324  final_cnt = m_cnt;
325  goto final;
326  }
327  nonmpm_ptr++;
328  nonmpm = *nonmpm_ptr;
329 
330  } else { /* implied mpm == nonmpm */
331  /* special case: if on both lists, it's a negated mpm pattern */
332 
333  /* mpm list may have dups, so skip past them here */
334  while (--m_cnt != 0) {
335  mpm_ptr++;
336  mpm = *mpm_ptr;
337  if (mpm != nonmpm)
338  break;
339  }
340  /* if mpm is done, update nonmpm_ptrs and jump to final */
341  if (unlikely(m_cnt == 0)) {
342  n_cnt--;
343 
344  /* mpm list is now empty */
345  final_ptr = ++nonmpm_ptr;
346  final_cnt = n_cnt;
347  goto final;
348  }
349  /* otherwise, if nonmpm is done jump to final for mpm
350  * mpm ptrs already updated */
351  if (unlikely(--n_cnt == 0)) {
352  final_ptr = mpm_ptr;
353  final_cnt = m_cnt;
354  goto final;
355  }
356 
357  /* not at end of the lists, update nonmpm. Mpm already
358  * updated in while loop above. */
359  nonmpm_ptr++;
360  nonmpm = *nonmpm_ptr;
361  }
362  }
363 
364  final: /* Only one list remaining. Just walk that list. */
365 
366  while (final_cnt-- > 0) {
367  id = *final_ptr++;
368  s = sig_array[id];
369 
370  /* As the mpm list can contain duplicates, check for that here. */
371  if (likely(id != previous_id)) {
372  *match_array++ = s;
373  previous_id = id;
374  }
375  }
376 
377  det_ctx->match_array_cnt = match_array - det_ctx->match_array;
378  DEBUG_VALIDATE_BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_pf_id_cnt) < det_ctx->match_array_cnt);
379  PMQ_RESET(&det_ctx->pmq);
380 }
381 
382 /** \internal
383  * \brief build non-prefilter list based on the rule group list we've set.
384  */
385 static inline void DetectPrefilterBuildNonPrefilterList(
386  DetectEngineThreadCtx *det_ctx, const SignatureMask mask, const AppProto alproto)
387 {
388  for (uint32_t x = 0; x < det_ctx->non_pf_store_cnt; x++) {
389  /* only if the mask matches this rule can possibly match,
390  * so build the non_mpm array only for match candidates */
391  const SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask;
392  const AppProto rule_alproto = det_ctx->non_pf_store_ptr[x].alproto;
393  if ((rule_mask & mask) == rule_mask &&
394  (rule_alproto == 0 || AppProtoEquals(rule_alproto, alproto))) {
395  det_ctx->non_pf_id_array[det_ctx->non_pf_id_cnt++] = det_ctx->non_pf_store_ptr[x].id;
396  }
397  }
398 }
399 
400 /** \internal
401  * \brief select non-mpm list
402  * Based on the packet properties, select the non-mpm list to use
403  * \todo move non_pf_store* into scratchpad */
404 static inline void
405 DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx, DetectRunScratchpad *scratch)
406 {
407  if ((p->proto == IPPROTO_TCP) && (p->tcph != NULL) && (p->tcph->th_flags & TH_SYN)) {
408  det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_syn_store_array;
409  det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_syn_store_cnt;
410  } else {
411  det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_other_store_array;
412  det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_other_store_cnt;
413  }
414  SCLogDebug("sgh non_pf ptr %p cnt %u (syn %p/%u, other %p/%u)",
415  det_ctx->non_pf_store_ptr, det_ctx->non_pf_store_cnt,
416  scratch->sgh->non_pf_syn_store_array, scratch->sgh->non_pf_syn_store_cnt,
418 }
419 
420 /** \internal
421  * \brief update flow's file tracking flags based on the detection engine
422  * A set of flags is prepared that is sent to the File API. The
423  File API may reject one or more based on the global force settings.
424  */
425 static inline void
426 DetectPostInspectFileFlagsUpdate(Flow *f, const SigGroupHead *sgh, uint8_t direction)
427 {
428  uint16_t flow_file_flags = FLOWFILE_INIT;
429 
430  if (sgh == NULL) {
431  SCLogDebug("requesting disabling all file features for flow");
432  flow_file_flags = FLOWFILE_NONE;
433  } else {
434  if (sgh->filestore_cnt == 0) {
435  SCLogDebug("requesting disabling filestore for flow");
436  flow_file_flags |= (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC);
437  }
438 #ifdef HAVE_MAGIC
439  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)) {
440  SCLogDebug("requesting disabling magic for flow");
441  flow_file_flags |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC);
442  }
443 #endif
444  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5)) {
445  SCLogDebug("requesting disabling md5 for flow");
446  flow_file_flags |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC);
447  }
448  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA1)) {
449  SCLogDebug("requesting disabling sha1 for flow");
450  flow_file_flags |= (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC);
451  }
452  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA256)) {
453  SCLogDebug("requesting disabling sha256 for flow");
454  flow_file_flags |= (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC);
455  }
456  if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE)) {
457  SCLogDebug("requesting disabling filesize for flow");
458  flow_file_flags |= (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC);
459  }
460  }
461  if (flow_file_flags != 0) {
462  FileUpdateFlowFileFlags(f, flow_file_flags, direction);
463  }
464 }
465 
466 static inline void
467 DetectRunPostGetFirstRuleGroup(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
468 {
469  if ((p->flowflags & FLOW_PKT_TOSERVER) && !(pflow->flags & FLOW_SGH_TOSERVER)) {
470  /* first time we see this toserver sgh, store it */
471  pflow->sgh_toserver = sgh;
472  pflow->flags |= FLOW_SGH_TOSERVER;
473 
474  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
475  if (pflow->protoctx != NULL) {
476  TcpSession *ssn = pflow->protoctx;
477  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.client");
479  }
480  }
481 
482  DetectPostInspectFileFlagsUpdate(pflow,
483  pflow->sgh_toserver, STREAM_TOSERVER);
484 
485  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && !(pflow->flags & FLOW_SGH_TOCLIENT)) {
486  pflow->sgh_toclient = sgh;
487  pflow->flags |= FLOW_SGH_TOCLIENT;
488 
489  if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
490  if (pflow->protoctx != NULL) {
491  TcpSession *ssn = pflow->protoctx;
492  SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.server");
494  }
495  }
496 
497  DetectPostInspectFileFlagsUpdate(pflow,
498  pflow->sgh_toclient, STREAM_TOCLIENT);
499  }
500 }
501 
502 static inline void DetectRunGetRuleGroup(
503  const DetectEngineCtx *de_ctx,
504  Packet * const p, Flow * const pflow,
505  DetectRunScratchpad *scratch)
506 {
507  const SigGroupHead *sgh = NULL;
508 
509  if (pflow) {
510  bool use_flow_sgh = false;
511  /* Get the stored sgh from the flow (if any). Make sure we're not using
512  * the sgh for icmp error packets part of the same stream. */
513  if (IP_GET_IPPROTO(p) == pflow->proto) { /* filter out icmp */
515  if ((p->flowflags & FLOW_PKT_TOSERVER) && (pflow->flags & FLOW_SGH_TOSERVER)) {
516  sgh = pflow->sgh_toserver;
517  SCLogDebug("sgh = pflow->sgh_toserver; => %p", sgh);
518  use_flow_sgh = true;
519  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && (pflow->flags & FLOW_SGH_TOCLIENT)) {
520  sgh = pflow->sgh_toclient;
521  SCLogDebug("sgh = pflow->sgh_toclient; => %p", sgh);
522  use_flow_sgh = true;
523  }
525  }
526 
527  if (!(use_flow_sgh)) {
531 
532  /* HACK: prevent the wrong sgh (or NULL) from being stored in the
533  * flow's sgh pointers */
535  ; /* no-op */
536  } else {
537  /* store the found sgh (or NULL) in the flow to save us
538  * from looking it up again for the next packet.
539  * Also run other tasks */
540  DetectRunPostGetFirstRuleGroup(p, pflow, sgh);
541  }
542  }
543  } else { /* p->flags & PKT_HAS_FLOW */
544  /* no flow */
545 
549  }
550 
551  scratch->sgh = sgh;
552 }
553 
554 static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
555  DetectEngineThreadCtx *det_ctx,
556  Flow * const pflow, Packet * const p)
557 {
558  if (pflow) {
559  /* set the iponly stuff */
560  if (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)
562  if (pflow->flags & FLOW_TOSERVER_IPONLY_SET)
564 
567  {
568  SCLogDebug("testing against \"ip-only\" signatures");
569 
571  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, p);
573 
574  /* save in the flow that we scanned this direction... */
575  FlowSetIPOnlyFlag(pflow, p->flowflags & FLOW_PKT_TOSERVER ? 1 : 0);
576  }
577  } else { /* p->flags & PKT_HAS_FLOW */
578  /* no flow */
579 
580  /* Even without flow we should match the packet src/dst */
582  IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, p);
584  }
585 }
586 
587 /** \internal
588  * \brief inspect the rule header: protocol, ports, etc
589  * \retval bool false if no match, true if match */
590 static inline bool DetectRunInspectRuleHeader(const Packet *p, const Flow *f, const Signature *s,
591  const uint32_t sflags, const uint8_t s_proto_flags)
592 {
593  /* check if this signature has a requirement for flowvars of some type
594  * and if so, if we actually have any in the flow. If not, the sig
595  * can't match and we skip it. */
596  if ((p->flags & PKT_HAS_FLOW) && (sflags & SIG_FLAG_REQUIRE_FLOWVAR)) {
597  DEBUG_VALIDATE_BUG_ON(f == NULL);
598 
599  /* no flowvars? skip this sig */
600  const bool fv = f->flowvar != NULL;
601  if (fv == false) {
602  SCLogDebug("skipping sig as the flow has no flowvars and sig "
603  "has SIG_FLAG_REQUIRE_FLOWVAR flag set.");
604  return false;
605  }
606  }
607 
608  if ((s_proto_flags & DETECT_PROTO_IPV4) && !PKT_IS_IPV4(p)) {
609  SCLogDebug("ip version didn't match");
610  return false;
611  }
612  if ((s_proto_flags & DETECT_PROTO_IPV6) && !PKT_IS_IPV6(p)) {
613  SCLogDebug("ip version didn't match");
614  return false;
615  }
616 
617  if (DetectProtoContainsProto(&s->proto, IP_GET_IPPROTO(p)) == 0) {
618  SCLogDebug("proto didn't match");
619  return false;
620  }
621 
622  /* check the source & dst port in the sig */
623  if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP || p->proto == IPPROTO_SCTP) {
624  if (!(sflags & SIG_FLAG_DP_ANY)) {
625  if (p->flags & PKT_IS_FRAGMENT)
626  return false;
627  const DetectPort *dport = DetectPortLookupGroup(s->dp, p->dp);
628  if (dport == NULL) {
629  SCLogDebug("dport didn't match.");
630  return false;
631  }
632  }
633  if (!(sflags & SIG_FLAG_SP_ANY)) {
634  if (p->flags & PKT_IS_FRAGMENT)
635  return false;
636  const DetectPort *sport = DetectPortLookupGroup(s->sp, p->sp);
637  if (sport == NULL) {
638  SCLogDebug("sport didn't match.");
639  return false;
640  }
641  }
642  } else if ((sflags & (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) != (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) {
643  SCLogDebug("port-less protocol and sig needs ports");
644  return false;
645  }
646 
647  /* check the destination address */
648  if (!(sflags & SIG_FLAG_DST_ANY)) {
649  if (PKT_IS_IPV4(p)) {
651  return false;
652  } else if (PKT_IS_IPV6(p)) {
654  return false;
655  }
656  }
657  /* check the source address */
658  if (!(sflags & SIG_FLAG_SRC_ANY)) {
659  if (PKT_IS_IPV4(p)) {
661  return false;
662  } else if (PKT_IS_IPV6(p)) {
664  return false;
665  }
666  }
667 
668  return true;
669 }
670 
671 /** \internal
672  * \brief run packet/stream prefilter engines
673  */
674 static inline void DetectRunPrefilterPkt(
675  ThreadVars *tv,
677  DetectEngineThreadCtx *det_ctx,
678  Packet *p,
679  DetectRunScratchpad *scratch
680 )
681 {
682  DetectPrefilterSetNonPrefilterList(p, det_ctx, scratch);
683 
684  /* create our prefilter mask */
685  PacketCreateMask(p, &scratch->pkt_mask, scratch->alproto, scratch->app_decoder_events);
686 
687  /* build and prefilter non_pf list against the mask of the packet */
689  det_ctx->non_pf_id_cnt = 0;
690  if (likely(det_ctx->non_pf_store_cnt > 0)) {
691  DetectPrefilterBuildNonPrefilterList(det_ctx, scratch->pkt_mask, scratch->alproto);
692  }
694 
695  /* run the prefilter engines */
696  Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags);
697  /* create match list if we have non-pf and/or pf */
698  if (det_ctx->non_pf_store_cnt || det_ctx->pmq.rule_id_array_cnt) {
699 #ifdef PROFILING
700  if (tv) {
701  StatsAddUI64(tv, det_ctx->counter_mpm_list, (uint64_t)det_ctx->pmq.rule_id_array_cnt);
702  }
703 #endif
705  DetectPrefilterMergeSort(de_ctx, det_ctx);
707  }
708 
709 #ifdef PROFILING
710  if (tv) {
712  (uint64_t)det_ctx->non_pf_store_cnt);
713  /* non mpm sigs after mask prefilter */
715  (uint64_t)det_ctx->non_pf_id_cnt);
716  }
717 #endif
718 }
719 
720 static inline void DetectRulePacketRules(
721  ThreadVars * const tv,
722  DetectEngineCtx * const de_ctx,
723  DetectEngineThreadCtx * const det_ctx,
724  Packet * const p,
725  Flow * const pflow,
726  const DetectRunScratchpad *scratch
727 )
728 {
729  const Signature *s = NULL;
730  const Signature *next_s = NULL;
731 
732  /* inspect the sigs against the packet */
733  /* Prefetch the next signature. */
734  SigIntId match_cnt = det_ctx->match_array_cnt;
735 #ifdef PROFILING
736  if (tv) {
738  (uint64_t)match_cnt);
739  }
740 #endif
741  Signature **match_array = det_ctx->match_array;
742 
743  SGH_PROFILING_RECORD(det_ctx, scratch->sgh);
744 #ifdef PROFILING
745  if (match_cnt >= de_ctx->profile_match_logging_threshold)
746  RulesDumpMatchArray(det_ctx, scratch->sgh, p);
747 #endif
748 
749  uint32_t sflags, next_sflags = 0;
750  if (match_cnt) {
751  next_s = *match_array++;
752  next_sflags = next_s->flags;
753  }
754  while (match_cnt--) {
756  uint8_t alert_flags = 0;
757 #ifdef PROFILE_RULES
758  bool smatch = false; /* signature match */
759 #endif
760  s = next_s;
761  sflags = next_sflags;
762  if (match_cnt) {
763  next_s = *match_array++;
764  next_sflags = next_s->flags;
765  }
766  const uint8_t s_proto_flags = s->proto.flags;
767 
768  SCLogDebug("inspecting signature id %"PRIu32"", s->id);
769 
770  if (s->app_inspect != NULL) {
771  goto next; // handle sig in DetectRunTx
772  }
773  if (s->frame_inspect != NULL) {
774  goto next; // handle sig in DetectRunFrame
775  }
776 
777  /* don't run mask check for stateful rules.
778  * There we depend on prefilter */
779  if ((s->mask & scratch->pkt_mask) != s->mask) {
780  SCLogDebug("mask mismatch %x & %x != %x", s->mask, scratch->pkt_mask, s->mask);
781  goto next;
782  }
783 
784  if (SigDsizePrefilter(p, s, sflags))
785  goto next;
786 
787  /* if the sig has alproto and the session as well they should match */
788  if (likely(sflags & SIG_FLAG_APPLAYER)) {
789  if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, scratch->alproto)) {
790  SCLogDebug("alproto mismatch");
791  goto next;
792  }
793  }
794 
795  if (DetectRunInspectRuleHeader(p, pflow, s, sflags, s_proto_flags) == false) {
796  goto next;
797  }
798 
799  if (DetectEnginePktInspectionRun(tv, det_ctx, s, pflow, p, &alert_flags) == false) {
800  goto next;
801  }
802 
803 #ifdef PROFILE_RULES
804  smatch = true;
805 #endif
806  DetectRunPostMatch(tv, det_ctx, p, s);
807 
808  AlertQueueAppend(det_ctx, s, p, 0, alert_flags);
809 next:
810  DetectVarProcessList(det_ctx, pflow, p);
811  DetectReplaceFree(det_ctx);
812  RULE_PROFILING_END(det_ctx, s, smatch, p);
813 
814  det_ctx->flags = 0;
815  continue;
816  }
817 }
818 
819 static DetectRunScratchpad DetectRunSetup(
820  const DetectEngineCtx *de_ctx,
821  DetectEngineThreadCtx *det_ctx,
822  Packet * const p, Flow * const pflow)
823 {
824  AppProto alproto = ALPROTO_UNKNOWN;
825  uint8_t flow_flags = 0; /* flow/state flags */
826  bool app_decoder_events = false;
827 
829 
830 #ifdef UNITTESTS
831  p->alerts.cnt = 0;
832  p->alerts.discarded = 0;
833  p->alerts.suppressed = 0;
834 #endif
835  det_ctx->filestore_cnt = 0;
836  det_ctx->base64_decoded_len = 0;
837  det_ctx->raw_stream_progress = 0;
838  det_ctx->match_array_cnt = 0;
839 
840  det_ctx->alert_queue_size = 0;
841  p->alerts.drop.action = 0;
842 
843 #ifdef DEBUG
844  if (p->flags & PKT_STREAM_ADD) {
845  det_ctx->pkt_stream_add_cnt++;
846  }
847 #endif
848 
849  /* grab the protocol state we will detect on */
850  if (p->flags & PKT_HAS_FLOW) {
851  DEBUG_VALIDATE_BUG_ON(pflow == NULL);
852 
853  if (p->flowflags & FLOW_PKT_TOSERVER) {
854  flow_flags = STREAM_TOSERVER;
855  SCLogDebug("flag STREAM_TOSERVER set");
856  } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
857  flow_flags = STREAM_TOCLIENT;
858  SCLogDebug("flag STREAM_TOCLIENT set");
859  }
860  SCLogDebug("p->flowflags 0x%02x", p->flowflags);
861 
862  if (p->flags & PKT_STREAM_EOF) {
863  flow_flags |= STREAM_EOF;
864  SCLogDebug("STREAM_EOF set");
865  }
866 
867  /* store tenant_id in the flow so that we can use it
868  * for creating pseudo packets */
869  if (p->tenant_id > 0 && pflow->tenant_id == 0) {
870  pflow->tenant_id = p->tenant_id;
871  }
872 
873  /* live ruleswap check for flow updates */
874  if (pflow->de_ctx_version == 0) {
875  /* first time this flow is inspected, set id */
876  pflow->de_ctx_version = de_ctx->version;
877  } else if (pflow->de_ctx_version != de_ctx->version) {
878  /* first time we inspect flow with this de_ctx, reset */
879  pflow->flags &= ~FLOW_SGH_TOSERVER;
880  pflow->flags &= ~FLOW_SGH_TOCLIENT;
881  pflow->sgh_toserver = NULL;
882  pflow->sgh_toclient = NULL;
883 
884  pflow->de_ctx_version = de_ctx->version;
885  GenericVarFree(pflow->flowvar);
886  pflow->flowvar = NULL;
887 
889  }
890 
891  /* Retrieve the app layer state and protocol and the tcp reassembled
892  * stream chunks. */
893  if ((p->proto == IPPROTO_TCP && (p->flags & PKT_STREAM_EST)) ||
894  (p->proto == IPPROTO_UDP) ||
896  {
897  /* update flow flags with knowledge on disruptions */
898  flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
899  alproto = FlowGetAppProtocol(pflow);
900  if (p->proto == IPPROTO_TCP && pflow->protoctx &&
903  }
904  SCLogDebug("alproto %u", alproto);
905  } else {
906  SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
907  }
908 
909  app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
910  }
911 
912  DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL, 0 };
914  return pad;
915 }
916 
917 static inline void DetectRunPostRules(
918  ThreadVars *tv,
920  DetectEngineThreadCtx *det_ctx,
921  Packet * const p,
922  Flow * const pflow,
923  DetectRunScratchpad *scratch)
924 {
925  /* so now let's iterate the alerts and remove the ones after a pass rule
926  * matched (if any). This is done inside PacketAlertFinalize() */
927  /* PR: installed "tag" keywords are handled after the threshold inspection */
928 
930  PacketAlertFinalize(de_ctx, det_ctx, p);
931  if (p->alerts.cnt > 0) {
932  StatsAddUI64(tv, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt);
933  }
934  if (p->alerts.discarded > 0) {
935  StatsAddUI64(tv, det_ctx->counter_alerts_overflow, (uint64_t)p->alerts.discarded);
936  }
937  if (p->alerts.suppressed > 0) {
938  StatsAddUI64(tv, det_ctx->counter_alerts_suppressed, (uint64_t)p->alerts.suppressed);
939  }
941 }
942 
943 static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
944  Packet *p, Flow * const pflow)
945 {
947  InspectionBufferClean(det_ctx);
948 
949  if (pflow != NULL) {
950  /* update inspected tracker for raw reassembly */
951  if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL &&
954  det_ctx->raw_stream_progress);
955  }
956  }
958  SCReturn;
959 }
960 
962 {
964  det_ctx->tx_candidates = SCCalloc(size, sizeof(RuleMatchCandidateTx));
965  if (det_ctx->tx_candidates == NULL) {
966  FatalError("failed to allocate %" PRIu64 " bytes",
967  (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
968  }
969  det_ctx->tx_candidates_size = size;
970  SCLogDebug("array initialized to %u elements (%"PRIu64" bytes)",
971  size, (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
972 }
973 
975 {
976  SCFree(det_ctx->tx_candidates);
977  det_ctx->tx_candidates_size = 0;
978 }
979 
980 /* if size >= cur_space */
981 static inline bool RuleMatchCandidateTxArrayHasSpace(const DetectEngineThreadCtx *det_ctx,
982  const uint32_t need)
983 {
984  if (det_ctx->tx_candidates_size >= need)
985  return 1;
986  return 0;
987 }
988 
989 /* realloc */
990 static int RuleMatchCandidateTxArrayExpand(DetectEngineThreadCtx *det_ctx, const uint32_t needed)
991 {
992  const uint32_t old_size = det_ctx->tx_candidates_size;
993  uint32_t new_size = needed;
994  void *ptmp = SCRealloc(det_ctx->tx_candidates, (new_size * sizeof(RuleMatchCandidateTx)));
995  if (ptmp == NULL) {
996  FatalError("failed to expand to %" PRIu64 " bytes",
997  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx)));
998  // TODO can this be handled more gracefully?
999  }
1000  det_ctx->tx_candidates = ptmp;
1001  det_ctx->tx_candidates_size = new_size;
1002  SCLogDebug("array expanded from %u to %u elements (%"PRIu64" bytes -> %"PRIu64" bytes)",
1003  old_size, new_size, (uint64_t)(old_size * sizeof(RuleMatchCandidateTx)),
1004  (uint64_t)(new_size * sizeof(RuleMatchCandidateTx))); (void)old_size;
1005  return 1;
1006 }
1007 
1008 /** \internal
1009  * \brief sort helper for sorting match candidates by id: ascending
1010  *
1011  * The id field is set from Signature::num, so we sort the candidates to match the signature
1012  * sort order (ascending), where candidates that have flags go first.
1013  */
1014 static int
1015 DetectRunTxSortHelper(const void *a, const void *b)
1016 {
1017  const RuleMatchCandidateTx *s0 = a;
1018  const RuleMatchCandidateTx *s1 = b;
1019  if (s1->id == s0->id) {
1020  if (s1->flags && !s0->flags)
1021  return 1;
1022  else if (!s1->flags && s0->flags)
1023  return -1;
1024  return 0;
1025  } else
1026  return s0->id > s1->id ? 1 : -1;
1027 }
1028 
1029 #if 0
1030 #define TRACE_SID_TXS(sid,txs,...) \
1031  do { \
1032  char _trace_buf[2048]; \
1033  snprintf(_trace_buf, sizeof(_trace_buf), __VA_ARGS__); \
1034  SCLogNotice("%p/%"PRIu64"/%u: %s", txs->tx_ptr, txs->tx_id, sid, _trace_buf); \
1035  } while(0)
1036 #else
1037 #define TRACE_SID_TXS(sid,txs,...)
1038 #endif
1039 
1040 /** \internal
1041  * \brief inspect a rule against a transaction
1042  *
1043  * Inspect a rule. New detection or continued stateful
1044  * detection.
1045  *
1046  * \param stored_flags pointer to stored flags or NULL.
1047  * If stored_flags is set it means we're continuing
1048  * inspection from an earlier run.
1049  *
1050  * \retval bool true sig matched, false didn't match
1051  */
1052 static bool DetectRunTxInspectRule(ThreadVars *tv,
1054  DetectEngineThreadCtx *det_ctx,
1055  Packet *p,
1056  Flow *f,
1057  const uint8_t in_flow_flags, // direction, EOF, etc
1058  void *alstate,
1059  DetectTransaction *tx,
1060  const Signature *s,
1061  uint32_t *stored_flags,
1062  RuleMatchCandidateTx *can,
1063  DetectRunScratchpad *scratch)
1064 {
1065  const uint8_t flow_flags = in_flow_flags;
1066  const int direction = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1067  uint32_t inspect_flags = stored_flags ? *stored_flags : 0;
1068  int total_matches = 0;
1069  uint16_t file_no_match = 0;
1070  bool retval = false;
1071  bool mpm_before_progress = false; // is mpm engine before progress?
1072  bool mpm_in_progress = false; // is mpm engine in a buffer we will revisit?
1073 
1074  TRACE_SID_TXS(s->id, tx, "starting %s", direction ? "toclient" : "toserver");
1075 
1076  /* for a new inspection we inspect pkt header and packet matches */
1077  if (likely(stored_flags == NULL)) {
1078  TRACE_SID_TXS(s->id, tx, "first inspect, run packet matches");
1079  if (DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags) == false) {
1080  TRACE_SID_TXS(s->id, tx, "DetectRunInspectRuleHeader() no match");
1081  return false;
1082  }
1083  if (DetectEnginePktInspectionRun(tv, det_ctx, s, f, p, NULL) == false) {
1084  TRACE_SID_TXS(s->id, tx, "DetectEnginePktInspectionRun no match");
1085  return false;
1086  }
1087  /* stream mpm and negated mpm sigs can end up here with wrong proto */
1088  if (!(AppProtoEquals(s->alproto, f->alproto) || s->alproto == ALPROTO_UNKNOWN)) {
1089  TRACE_SID_TXS(s->id, tx, "alproto mismatch");
1090  return false;
1091  }
1092  }
1093 
1094  const DetectEngineAppInspectionEngine *engine = s->app_inspect;
1095  do {
1096  TRACE_SID_TXS(s->id, tx, "engine %p inspect_flags %x", engine, inspect_flags);
1097  if (!(inspect_flags & BIT_U32(engine->id)) &&
1098  direction == engine->dir)
1099  {
1100  const bool skip_engine = (engine->alproto != 0 && engine->alproto != f->alproto);
1101  /* special case: file_data on 'alert tcp' will have engines
1102  * in the list that are not for us. */
1103  if (unlikely(skip_engine)) {
1104  engine = engine->next;
1105  continue;
1106  }
1107 
1108  /* engines are sorted per progress, except that the one with
1109  * mpm/prefilter enabled is first */
1110  if (tx->tx_progress < engine->progress) {
1111  SCLogDebug("tx progress %d < engine progress %d",
1112  tx->tx_progress, engine->progress);
1113  break;
1114  }
1115  if (engine->mpm) {
1116  if (tx->tx_progress > engine->progress) {
1117  TRACE_SID_TXS(s->id, tx,
1118  "engine->mpm: t->tx_progress %u > engine->progress %u, so set "
1119  "mpm_before_progress",
1120  tx->tx_progress, engine->progress);
1121  mpm_before_progress = true;
1122  } else if (tx->tx_progress == engine->progress) {
1123  TRACE_SID_TXS(s->id, tx,
1124  "engine->mpm: t->tx_progress %u == engine->progress %u, so set "
1125  "mpm_in_progress",
1126  tx->tx_progress, engine->progress);
1127  mpm_in_progress = true;
1128  }
1129  }
1130 
1131  /* run callback: but bypass stream callback if we can */
1132  uint8_t match;
1133  if (unlikely(engine->stream && can->stream_stored)) {
1134  match = can->stream_result;
1135  TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match);
1136  } else {
1137  KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list);
1138  DEBUG_VALIDATE_BUG_ON(engine->v2.Callback == NULL);
1139  match = engine->v2.Callback(
1140  de_ctx, det_ctx, engine, s, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
1141  TRACE_SID_TXS(s->id, tx, "engine %p match %d", engine, match);
1142  if (engine->stream) {
1143  can->stream_stored = true;
1144  can->stream_result = match;
1145  TRACE_SID_TXS(s->id, tx, "stream ran, store result %d for next tx (if any)", match);
1146  }
1147  }
1148  if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) {
1149  inspect_flags |= BIT_U32(engine->id);
1150  engine = engine->next;
1151  total_matches++;
1152  continue;
1153  } else if (match == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES) {
1154  /* if the file engine matched, but indicated more
1155  * files are still in progress, we don't set inspect
1156  * flags as these would end inspection for this tx */
1157  engine = engine->next;
1158  total_matches++;
1159  continue;
1160  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
1161  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1162  inspect_flags |= BIT_U32(engine->id);
1163  } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES) {
1164  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1165  inspect_flags |= BIT_U32(engine->id);
1166  file_no_match = 1;
1167  }
1168  /* implied DETECT_ENGINE_INSPECT_SIG_NO_MATCH */
1169  if (engine->mpm && mpm_before_progress) {
1170  inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
1171  inspect_flags |= BIT_U32(engine->id);
1172  }
1173  break;
1174  }
1175  engine = engine->next;
1176  } while (engine != NULL);
1177  TRACE_SID_TXS(s->id, tx, "inspect_flags %x, total_matches %u, engine %p",
1178  inspect_flags, total_matches, engine);
1179 
1180  if (engine == NULL && total_matches) {
1181  inspect_flags |= DE_STATE_FLAG_FULL_INSPECT;
1182  TRACE_SID_TXS(s->id, tx, "MATCH");
1183  retval = true;
1184  }
1185 
1186  if (stored_flags) {
1187  *stored_flags = inspect_flags;
1188  TRACE_SID_TXS(s->id, tx, "continue inspect flags %08x", inspect_flags);
1189  } else {
1190  // store... or? If tx is done we might not want to come back to this tx
1191 
1192  // also... if mpmid tracking is enabled, we won't do a sig again for this tx...
1193  TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags);
1194  if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) {
1195  if (file_no_match) {
1196  /* if we have a mismatch on a file sig, we need to keep state.
1197  * We may get another file on the same tx (for http and smtp
1198  * at least), so for a new file we need to re-eval the sig.
1199  * Thoughts / TODO:
1200  * - not for some protos that have 1 file per tx (e.g. nfs)
1201  * - maybe we only need this for file sigs that mix with
1202  * other matches? E.g. 'POST + filename', is different than
1203  * just 'filename'.
1204  */
1205  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1206  inspect_flags, flow_flags, file_no_match);
1207  }
1208  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) {
1209  TRACE_SID_TXS(s->id, tx, "no need to store match sig, "
1210  "mpm won't trigger for it anymore");
1211 
1212  if (inspect_flags & DE_STATE_FLAG_FILE_INSPECT) {
1213  TRACE_SID_TXS(s->id, tx, "except that for new files, "
1214  "we may have to revisit anyway");
1215  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1216  inspect_flags, flow_flags, file_no_match);
1217  }
1218  } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) {
1219  TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, "
1220  "mpm will revisit it");
1221  } else if (inspect_flags != 0 || file_no_match != 0) {
1222  TRACE_SID_TXS(s->id, tx, "storing state: flags %08x", inspect_flags);
1223  DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
1224  inspect_flags, flow_flags, file_no_match);
1225  }
1226  }
1227 
1228  return retval;
1229 }
1230 
1231 #define NO_TX \
1232  { \
1233  NULL, 0, NULL, NULL, 0, 0, 0, 0, 0, \
1234  }
1235 
1236 /** \internal
1237  * \brief get a DetectTransaction object
1238  * \retval struct filled with relevant info or all nulls/0s
1239  */
1240 static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto,
1241  void *alstate, const uint64_t tx_id, void *tx_ptr, const int tx_end_state,
1242  const uint8_t flow_flags)
1243 {
1244  AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx_ptr);
1245  if (unlikely(txd == NULL)) {
1246  DetectTransaction no_tx = NO_TX;
1247  return no_tx;
1248  }
1249  uint64_t detect_flags =
1250  (flow_flags & STREAM_TOSERVER) ? txd->detect_flags_ts : txd->detect_flags_tc;
1251  if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) {
1252  SCLogDebug("%"PRIu64" tx already fully inspected for %s. Flags %016"PRIx64,
1253  tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1254  detect_flags);
1255  DetectTransaction no_tx = NO_TX;
1256  return no_tx;
1257  }
1258  if (detect_flags & APP_LAYER_TX_SKIP_INSPECT_FLAG) {
1259  SCLogDebug("%" PRIu64 " tx should not be inspected in direction %s. Flags %016" PRIx64,
1260  tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
1261  DetectTransaction no_tx = NO_TX;
1262  return no_tx;
1263  }
1264 
1265  const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags);
1266  const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
1267  DetectEngineState *tx_de_state = txd->de_state;
1268  DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL;
1269  uint64_t prefilter_flags = detect_flags & APP_LAYER_TX_PREFILTER_MASK;
1271 
1272  DetectTransaction tx = {
1273  .tx_ptr = tx_ptr,
1274  .tx_id = tx_id,
1275  .tx_data_ptr = (struct AppLayerTxData *)txd,
1276  .de_state = tx_dir_state,
1277  .detect_flags = detect_flags,
1278  .prefilter_flags = prefilter_flags,
1279  .prefilter_flags_orig = prefilter_flags,
1280  .tx_progress = tx_progress,
1281  .tx_end_state = tx_end_state,
1282  };
1283  return tx;
1284 }
1285 
1286 static inline void StoreDetectFlags(DetectTransaction *tx, const uint8_t flow_flags,
1287  const uint8_t ipproto, const AppProto alproto, const uint64_t detect_flags)
1288 {
1290  if (likely(txd != NULL)) {
1291  if (flow_flags & STREAM_TOSERVER) {
1292  txd->detect_flags_ts = detect_flags;
1293  } else {
1294  txd->detect_flags_tc = detect_flags;
1295  }
1296  }
1297 }
1298 
1299 // Merge 'state' rules from the regular prefilter
1300 // updates array_idx on the way
1301 static inline void RuleMatchCandidateMergeStateRules(
1302  DetectEngineThreadCtx *det_ctx, uint32_t *array_idx)
1303 {
1304  // Now, we will merge 2 sorted lists :
1305  // the one in det_ctx->tx_candidates
1306  // and the one in det_ctx->match_array
1307  // For match_array, we take only the relevant elements where s->app_inspect != NULL
1308 
1309  // Basically, we iterate at the same time over the 2 lists
1310  // comparing and taking an element from either.
1311 
1312  // Trick is to do so in place in det_ctx->tx_candidates,
1313  // so as to minimize the number of moves in det_ctx->tx_candidates.
1314  // For this, the algorithm traverses the lists in reverse order.
1315  // Otherwise, if the first element of match_array was to be put before
1316  // all tx_candidates, we would need to shift all tx_candidates
1317 
1318  // Retain the number of elements sorted in tx_candidates before merge
1319  uint32_t j = *array_idx;
1320  // First loop only counting the number of elements to add
1321  for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) {
1322  const Signature *s = det_ctx->match_array[i];
1323  if (s->app_inspect != NULL) {
1324  (*array_idx)++;
1325  }
1326  }
1327  // Future number of elements in tx_candidates after merge
1328  uint32_t k = *array_idx;
1329 
1330  if (k == j) {
1331  // no new element from match_array to merge in tx_candidates
1332  return;
1333  }
1334 
1335  // variable i is for all elements of match_array (even not relevant ones)
1336  // variable j is for elements of tx_candidates before merge
1337  // variable k is for elements of tx_candidates after merge
1338  for (uint32_t i = det_ctx->match_array_cnt; i > 0;) {
1339  const Signature *s = det_ctx->match_array[i - 1];
1340  if (s->app_inspect == NULL) {
1341  // no relevant element, get the next one from match_array
1342  i--;
1343  continue;
1344  }
1345  // we have one element from match_array to merge in tx_candidates
1346  k--;
1347  if (j > 0) {
1348  // j > 0 means there is still at least one element in tx_candidates to merge
1349  const RuleMatchCandidateTx *s0 = &det_ctx->tx_candidates[j - 1];
1350  if (s->num <= s0->id) {
1351  // get next element from previous tx_candidates
1352  j--;
1353  // take the element from tx_candidates before merge
1354  det_ctx->tx_candidates[k].s = det_ctx->tx_candidates[j].s;
1355  det_ctx->tx_candidates[k].id = det_ctx->tx_candidates[j].id;
1356  det_ctx->tx_candidates[k].flags = det_ctx->tx_candidates[j].flags;
1357  det_ctx->tx_candidates[k].stream_reset = det_ctx->tx_candidates[j].stream_reset;
1358  continue;
1359  }
1360  } // otherwise
1361  // get next element from match_array
1362  i--;
1363  // take the element from match_array
1364  det_ctx->tx_candidates[k].s = s;
1365  det_ctx->tx_candidates[k].id = s->num;
1366  det_ctx->tx_candidates[k].flags = NULL;
1367  det_ctx->tx_candidates[k].stream_reset = 0;
1368  }
1369  // Even if k > 0 or j > 0, the loop is over. (Note that j == k now)
1370  // The remaining elements in tx_candidates up to k were already sorted
1371  // and come before any other element later in the list
1372 }
1373 
1374 static void DetectRunTx(ThreadVars *tv,
1376  DetectEngineThreadCtx *det_ctx,
1377  Packet *p,
1378  Flow *f,
1379  DetectRunScratchpad *scratch)
1380 {
1381  const uint8_t flow_flags = scratch->flow_flags;
1382  const SigGroupHead * const sgh = scratch->sgh;
1383  void * const alstate = f->alstate;
1384  const uint8_t ipproto = f->proto;
1385  const AppProto alproto = f->alproto;
1386 
1387  const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
1388  uint64_t tx_id_min = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags);
1389  const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags);
1390 
1391  AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
1392  AppLayerGetTxIterState state;
1393  memset(&state, 0, sizeof(state));
1394 
1395  while (1) {
1396  AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id_min, total_txs, &state);
1397  if (ires.tx_ptr == NULL)
1398  break;
1399 
1400  DetectTransaction tx = GetDetectTx(ipproto, alproto,
1401  alstate, ires.tx_id, ires.tx_ptr, tx_end_state, flow_flags);
1402  if (tx.tx_ptr == NULL) {
1403  SCLogDebug("%p/%"PRIu64" no transaction to inspect",
1404  tx.tx_ptr, tx_id_min);
1405 
1406  tx_id_min++; // next (if any) run look for +1
1407  goto next;
1408  }
1409  tx_id_min = tx.tx_id + 1; // next look for cur + 1
1410 
1411  bool do_sort = false; // do we need to sort the tx candidate list?
1412  uint32_t array_idx = 0;
1413  uint32_t total_rules = det_ctx->match_array_cnt;
1414  total_rules += (tx.de_state ? tx.de_state->cnt : 0);
1415 
1416  /* run prefilter engines and merge results into a candidates array */
1417  if (sgh->tx_engines) {
1419  DetectRunPrefilterTx(det_ctx, sgh, p, ipproto, flow_flags, alproto,
1420  alstate, &tx);
1422  SCLogDebug("%p/%"PRIu64" rules added from prefilter: %u candidates",
1423  tx.tx_ptr, tx.tx_id, det_ctx->pmq.rule_id_array_cnt);
1424 
1425  total_rules += det_ctx->pmq.rule_id_array_cnt;
1426  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1427  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1428  }
1429 
1430  for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) {
1431  const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]];
1432  const SigIntId id = s->num;
1433  det_ctx->tx_candidates[array_idx].s = s;
1434  det_ctx->tx_candidates[array_idx].id = id;
1435  det_ctx->tx_candidates[array_idx].flags = NULL;
1436  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1437  array_idx++;
1438  }
1439  PMQ_RESET(&det_ctx->pmq);
1440  } else {
1441  if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
1442  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1443  }
1444  }
1445 
1446  /* merge 'state' rules from the regular prefilter */
1447 #ifdef PROFILING
1448  uint32_t x = array_idx;
1449 #endif
1450  RuleMatchCandidateMergeStateRules(det_ctx, &array_idx);
1451 
1452  /* merge stored state into results */
1453  if (tx.de_state != NULL) {
1454  const uint32_t old = array_idx;
1455 
1456  /* if tx.de_state->flags has 'new file' set and sig below has
1457  * 'file inspected' flag, reset the file part of the state */
1458  const bool have_new_file = (tx.de_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_NEW);
1459  if (have_new_file) {
1460  SCLogDebug("%p/%"PRIu64" destate: need to consider new file",
1461  tx.tx_ptr, tx.tx_id);
1463  }
1464 
1465  SigIntId state_cnt = 0;
1466  DeStateStore *tx_store = tx.de_state->head;
1467  for (; tx_store != NULL; tx_store = tx_store->next) {
1468  SCLogDebug("tx_store %p", tx_store);
1469 
1470  SigIntId store_cnt = 0;
1471  for (store_cnt = 0;
1472  store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < tx.de_state->cnt;
1473  store_cnt++, state_cnt++)
1474  {
1475  DeStateStoreItem *item = &tx_store->store[store_cnt];
1476  SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags);
1477  if (have_new_file && (item->flags & DE_STATE_FLAG_FILE_INSPECT)) {
1478  /* remove part of the state. File inspect engine will now
1479  * be able to run again */
1481  SCLogDebug("rule id %u, post file reset inspect_flags %u", item->sid, item->flags);
1482  }
1483  det_ctx->tx_candidates[array_idx].s = de_ctx->sig_array[item->sid];
1484  det_ctx->tx_candidates[array_idx].id = item->sid;
1485  det_ctx->tx_candidates[array_idx].flags = &item->flags;
1486  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1487  array_idx++;
1488  }
1489  }
1490  do_sort |= (old && old != array_idx); // sort if continue list adds sids
1491  SCLogDebug("%p/%" PRIu64 " rules added from 'continue' list: %u", tx.tx_ptr, tx.tx_id,
1492  array_idx - old);
1493  }
1494  if (do_sort) {
1495  qsort(det_ctx->tx_candidates, array_idx, sizeof(RuleMatchCandidateTx),
1496  DetectRunTxSortHelper);
1497  }
1498 
1499 #ifdef PROFILING
1500  if (array_idx >= de_ctx->profile_match_logging_threshold)
1501  RulesDumpTxMatchArray(det_ctx, scratch->sgh, p, tx.tx_id, array_idx, x);
1502 #endif
1503  det_ctx->tx_id = tx.tx_id;
1504  det_ctx->tx_id_set = true;
1505  det_ctx->p = p;
1506 
1507 #ifdef DEBUG
1508  for (uint32_t i = 0; i < array_idx; i++) {
1509  RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i];
1510  const Signature *s = det_ctx->tx_candidates[i].s;
1511  SCLogDebug("%u: sid %u flags %p", i, s->id, can->flags);
1512  }
1513 #endif
1514  /* run rules: inspect the match candidates */
1515  for (uint32_t i = 0; i < array_idx; i++) {
1516  RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i];
1517  const Signature *s = det_ctx->tx_candidates[i].s;
1518  uint32_t *inspect_flags = det_ctx->tx_candidates[i].flags;
1519 
1520  /* deduplicate: rules_array is sorted, but not deduplicated:
1521  * both mpm and stored state could give us the same sid.
1522  * As they are back to back in that case we can check for it
1523  * here. We select the stored state one as that comes first
1524  * in the array. */
1525  while ((i + 1) < array_idx &&
1526  det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i + 1].s) {
1527  SCLogDebug("%p/%" PRIu64 " inspecting SKIP NEXT: sid %u (%u), flags %08x",
1528  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1529  i++;
1530  }
1531 
1532  SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x",
1533  tx.tx_ptr, tx.tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
1534 
1535  if (inspect_flags) {
1537  SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x ALREADY COMPLETE",
1538  tx.tx_ptr, tx.tx_id, s->id, s->num, *inspect_flags);
1539  continue;
1540  }
1541  }
1542 
1543  if (inspect_flags) {
1544  /* continue previous inspection */
1545  SCLogDebug("%p/%" PRIu64 " Continuing sid %u", tx.tx_ptr, tx.tx_id, s->id);
1546  } else {
1547  /* start new inspection */
1548  SCLogDebug("%p/%"PRIu64" Start sid %u", tx.tx_ptr, tx.tx_id, s->id);
1549  }
1550 
1551  /* call individual rule inspection */
1553  const int r = DetectRunTxInspectRule(tv, de_ctx, det_ctx, p, f, flow_flags,
1554  alstate, &tx, s, inspect_flags, can, scratch);
1555  if (r == 1) {
1556  /* match */
1557  DetectRunPostMatch(tv, det_ctx, p, s);
1558 
1559  const uint8_t alert_flags = (PACKET_ALERT_FLAG_STATE_MATCH | PACKET_ALERT_FLAG_TX);
1560  SCLogDebug("%p/%"PRIu64" sig %u (%u) matched", tx.tx_ptr, tx.tx_id, s->id, s->num);
1561  AlertQueueAppend(det_ctx, s, p, tx.tx_id, alert_flags);
1562  }
1563  DetectVarProcessList(det_ctx, p->flow, p);
1564  RULE_PROFILING_END(det_ctx, s, r, p);
1565  }
1566 
1567  det_ctx->tx_id = 0;
1568  det_ctx->tx_id_set = false;
1569  det_ctx->p = NULL;
1570 
1571  /* see if we have any updated state to store in the tx */
1572 
1573  uint64_t new_detect_flags = 0;
1574  /* this side of the tx is done */
1575  if (tx.tx_progress >= tx.tx_end_state) {
1576  new_detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
1577  SCLogDebug("%p/%"PRIu64" tx is done for direction %s. Flag %016"PRIx64,
1578  tx.tx_ptr, tx.tx_id,
1579  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1580  new_detect_flags);
1581  }
1582  if (tx.prefilter_flags != tx.prefilter_flags_orig) {
1583  new_detect_flags |= tx.prefilter_flags;
1585  SCLogDebug("%p/%"PRIu64" updated prefilter flags %016"PRIx64" "
1586  "(was: %016"PRIx64") for direction %s. Flag %016"PRIx64,
1588  flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
1589  new_detect_flags);
1590  }
1591  if (new_detect_flags != 0 &&
1592  (new_detect_flags | tx.detect_flags) != tx.detect_flags)
1593  {
1594  new_detect_flags |= tx.detect_flags;
1596  SCLogDebug("%p/%"PRIu64" Storing new flags %016"PRIx64" (was %016"PRIx64")",
1597  tx.tx_ptr, tx.tx_id, new_detect_flags, tx.detect_flags);
1598 
1599  StoreDetectFlags(&tx, flow_flags, ipproto, alproto, new_detect_flags);
1600  }
1601 next:
1602  InspectionBufferClean(det_ctx);
1603 
1604  if (!ires.has_next)
1605  break;
1606  }
1607 }
1608 
1609 static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
1610  Packet *p, Flow *f, DetectRunScratchpad *scratch)
1611 {
1612  const SigGroupHead *const sgh = scratch->sgh;
1613  const AppProto alproto = f->alproto;
1614 
1615  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
1616  if (frames_container == NULL) {
1617  return;
1618  }
1619  Frames *frames;
1620  if (PKT_IS_TOSERVER(p)) {
1621  frames = &frames_container->toserver;
1622  } else {
1623  frames = &frames_container->toclient;
1624  }
1625 
1626  for (uint32_t idx = 0; idx < frames->cnt; idx++) {
1627  SCLogDebug("frame %u", idx);
1628  Frame *frame = FrameGetByIndex(frames, idx);
1629  if (frame == NULL) {
1630  continue;
1631  }
1632 
1633  det_ctx->frame_inspect_progress = 0;
1634  uint32_t array_idx = 0;
1635  uint32_t total_rules = det_ctx->match_array_cnt;
1636 
1637  /* run prefilter engines and merge results into a candidates array */
1638  if (sgh->frame_engines) {
1639  // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_TX);
1640  DetectRunPrefilterFrame(det_ctx, sgh, p, frames, frame, alproto);
1641  // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_TX);
1642  SCLogDebug("%p/%" PRIi64 " rules added from prefilter: %u candidates", frame, frame->id,
1643  det_ctx->pmq.rule_id_array_cnt);
1644 
1645  total_rules += det_ctx->pmq.rule_id_array_cnt;
1646 
1647  if (!(RuleMatchCandidateTxArrayHasSpace(
1648  det_ctx, total_rules))) { // TODO is it safe to overload?
1649  RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
1650  }
1651 
1652  for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) {
1653  const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]];
1654  const SigIntId id = s->num;
1655  det_ctx->tx_candidates[array_idx].s = s;
1656  det_ctx->tx_candidates[array_idx].id = id;
1657  det_ctx->tx_candidates[array_idx].flags = NULL;
1658  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1659  array_idx++;
1660  }
1661  PMQ_RESET(&det_ctx->pmq);
1662  }
1663  /* merge 'state' rules from the regular prefilter */
1664  uint32_t x = array_idx;
1665  for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) {
1666  const Signature *s = det_ctx->match_array[i];
1667  if (s->frame_inspect != NULL) {
1668  const SigIntId id = s->num;
1669  det_ctx->tx_candidates[array_idx].s = s;
1670  det_ctx->tx_candidates[array_idx].id = id;
1671  det_ctx->tx_candidates[array_idx].flags = NULL;
1672  det_ctx->tx_candidates[array_idx].stream_reset = 0;
1673  array_idx++;
1674 
1675  SCLogDebug("%p/%" PRIi64 " rule %u (%u) added from 'match' list", frame, frame->id,
1676  s->id, id);
1677  }
1678  }
1679  SCLogDebug("%p/%" PRIi64 " rules added from 'match' list: %u", frame, frame->id,
1680  array_idx - x);
1681  (void)x;
1682 
1683  /* run rules: inspect the match candidates */
1684  for (uint32_t i = 0; i < array_idx; i++) {
1685  const Signature *s = det_ctx->tx_candidates[i].s;
1686 
1687  /* deduplicate: rules_array is sorted, but not deduplicated.
1688  * As they are back to back in that case we can check for it
1689  * here. We select the stored state one as that comes first
1690  * in the array. */
1691  while ((i + 1) < array_idx &&
1692  det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i + 1].s) {
1693  i++;
1694  }
1695  SCLogDebug("%p/%" PRIi64 " inspecting: sid %u (%u)", frame, frame->id, s->id, s->num);
1696 
1697  /* start new inspection */
1698  SCLogDebug("%p/%" PRIi64 " Start sid %u", frame, frame->id, s->id);
1699 
1700  /* call individual rule inspection */
1702  bool r = DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags);
1703  if (r == true) {
1704  r = DetectRunFrameInspectRule(tv, det_ctx, s, f, p, frames, frame);
1705  if (r == true) {
1706  /* match */
1707  DetectRunPostMatch(tv, det_ctx, p, s);
1708 
1709  const uint8_t alert_flags =
1712  det_ctx->frame_id = frame->id;
1713  SCLogDebug(
1714  "%p/%" PRIi64 " sig %u (%u) matched", frame, frame->id, s->id, s->num);
1715  AlertQueueAppend(det_ctx, s, p, frame->tx_id, alert_flags);
1716  }
1717  }
1718  DetectVarProcessList(det_ctx, p->flow, p);
1719  RULE_PROFILING_END(det_ctx, s, r, p);
1720  }
1721 
1722  /* update Frame::inspect_progress here instead of in the code above. The reason is that a
1723  * frame might be used more than once in buffers with transforms. */
1724  if (frame->inspect_progress < det_ctx->frame_inspect_progress) {
1725  frame->inspect_progress = det_ctx->frame_inspect_progress;
1726  SCLogDebug("frame->inspect_progress: %" PRIu64 " -> updated", frame->inspect_progress);
1727  } else {
1728  SCLogDebug(
1729  "frame->inspect_progress: %" PRIu64 " -> not updated", frame->inspect_progress);
1730  }
1731 
1732  SCLogDebug("%p/%" PRIi64 " rules inspected, running cleanup", frame, frame->id);
1733  InspectionBufferClean(det_ctx);
1734  }
1735 }
1736 
1737 static DetectEngineThreadCtx *GetTenantById(HashTable *h, uint32_t id)
1738 {
1739  /* technically we need to pass a DetectEngineThreadCtx struct with the
1740  * tenant_id member. But as that member is the first in the struct, we
1741  * can use the id directly. */
1742  return HashTableLookup(h, &id, 0);
1743 }
1744 
1745 static void DetectFlow(ThreadVars *tv,
1747  Packet *p)
1748 {
1749  Flow *const f = p->flow;
1750 
1751  if (p->flags & PKT_NOPACKET_INSPECTION) {
1752  /* hack: if we are in pass the entire flow mode, we need to still
1753  * update the inspect_id forward. So test for the condition here,
1754  * and call the update code if necessary. */
1755  const int pass = ((f->flags & FLOW_NOPACKET_INSPECTION));
1756  if (pass) {
1757  uint8_t flags = STREAM_FLAGS_FOR_PACKET(p);
1759  if (f->alstate) {
1761  }
1762  }
1763  SCLogDebug("p->pcap %"PRIu64": no detection on packet, "
1764  "PKT_NOPACKET_INSPECTION is set", p->pcap_cnt);
1765  return;
1766  }
1767 
1768  /* we check the flow drop here, and not the packet drop. This is
1769  * to allow stream engine "invalid" drop packets to still be
1770  * evaluated by the stream event rules. */
1771  if (f->flags & FLOW_ACTION_DROP) {
1773  SCReturn;
1774  }
1775 
1776  /* see if the packet matches one or more of the sigs */
1777  (void)DetectRun(tv, de_ctx, det_ctx, p);
1778 }
1779 
1780 
1781 static void DetectNoFlow(ThreadVars *tv,
1783  Packet *p)
1784 {
1785  /* No need to perform any detection on this packet, if the given flag is set.*/
1787  return;
1788  }
1789 
1790  /* see if the packet matches one or more of the sigs */
1791  DetectRun(tv, de_ctx, det_ctx, p);
1792  return;
1793 }
1794 
1795 /** \brief Detection engine thread wrapper.
1796  * \param tv thread vars
1797  * \param p packet to inspect
1798  * \param data thread specific data
1799  * \param pq packet queue
1800  * \retval TM_ECODE_FAILED error
1801  * \retval TM_ECODE_OK ok
1802  */
1803 TmEcode Detect(ThreadVars *tv, Packet *p, void *data)
1804 {
1806 
1807  DetectEngineCtx *de_ctx = NULL;
1808  DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data;
1809  if (det_ctx == NULL) {
1810  printf("ERROR: Detect has no thread ctx\n");
1811  goto error;
1812  }
1813 
1814  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
1815  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
1816  SCLogDebug("Detect Engine using new det_ctx - %p",
1817  det_ctx);
1818  }
1819 
1820  /* if in MT mode _and_ we have tenants registered, use
1821  * MT logic. */
1822  if (det_ctx->mt_det_ctxs_cnt > 0 && det_ctx->TenantGetId != NULL)
1823  {
1824  uint32_t tenant_id = p->tenant_id;
1825  if (tenant_id == 0)
1826  tenant_id = det_ctx->TenantGetId(det_ctx, p);
1827  if (tenant_id > 0 && tenant_id < det_ctx->mt_det_ctxs_cnt) {
1828  p->tenant_id = tenant_id;
1829  det_ctx = GetTenantById(det_ctx->mt_det_ctxs_hash, tenant_id);
1830  if (det_ctx == NULL)
1831  return TM_ECODE_OK;
1832  de_ctx = det_ctx->de_ctx;
1833  if (de_ctx == NULL)
1834  return TM_ECODE_OK;
1835 
1836  if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) {
1837  (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1);
1838  SCLogDebug("MT de_ctx %p det_ctx %p (tenant %u)", de_ctx, det_ctx, tenant_id);
1839  }
1840  } else {
1841  /* use default if no tenants are registered for this packet */
1842  de_ctx = det_ctx->de_ctx;
1843  }
1844  } else {
1845  de_ctx = det_ctx->de_ctx;
1846  }
1847 
1848  if (p->flow) {
1849  DetectFlow(tv, de_ctx, det_ctx, p);
1850  } else {
1851  DetectNoFlow(tv, de_ctx, det_ctx, p);
1852  }
1853 
1854 #ifdef PROFILE_RULES
1855  /* aggregate statistics */
1856  struct timeval ts;
1857  gettimeofday(&ts, NULL);
1858  if (ts.tv_sec != det_ctx->rule_perf_last_sync) {
1859  SCProfilingRuleThreatAggregate(det_ctx);
1860  det_ctx->rule_perf_last_sync = ts.tv_sec;
1861  }
1862 #endif
1863 
1864  return TM_ECODE_OK;
1865 error:
1866  return TM_ECODE_FAILED;
1867 }
1868 
1869 /** \brief disable file features we don't need
1870  * Called if we have no detection engine.
1871  */
1873 {
1874  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOSERVER);
1875  DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOCLIENT);
1876 }
1877 
1878 #ifdef UNITTESTS
1879 /**
1880  * \brief wrapper for old tests
1881  */
1884 {
1885  if (p->flow) {
1886  DetectFlow(tv, de_ctx, det_ctx, p);
1887  } else {
1888  DetectNoFlow(tv, de_ctx, det_ctx, p);
1889  }
1890 }
1891 #endif
1892 
1893 /*
1894  * TESTS
1895  */
1896 
1897 #ifdef UNITTESTS
1898 #include "tests/detect.c"
1899 #endif
1900 
PKT_IS_TOCLIENT
#define PKT_IS_TOCLIENT(p)
Definition: decode.h:252
DetectEngineAppInspectionEngine_::stream
bool stream
Definition: detect.h:428
SigGroupHead_::non_pf_syn_store_cnt
uint32_t non_pf_syn_store_cnt
Definition: detect.h:1456
PacketCheckAction
bool PacketCheckAction(const Packet *p, const uint8_t a)
Definition: packet.c:48
FLOWFILE_NO_MD5_TS
#define FLOWFILE_NO_MD5_TS
Definition: flow.h:130
DetectEngineThreadCtx_::non_pf_store_ptr
SignatureNonPrefilterStore * non_pf_store_ptr
Definition: detect.h:1196
RulesDumpTxMatchArray
void RulesDumpTxMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, const Packet *p, const uint64_t tx_id, const uint32_t rule_cnt, const uint32_t pkt_prefilter_cnt)
Definition: detect-engine-profile.c:34
SIG_GROUP_HEAD_HAVEFILEMD5
#define SIG_GROUP_HEAD_HAVEFILEMD5
Definition: detect.h:1322
FLOWFILE_NO_MD5_TC
#define FLOWFILE_NO_MD5_TC
Definition: flow.h:131
SigGroupHead_::tx_engines
PrefilterEngine * tx_engines
Definition: detect.h:1463
DetectEngineAppInspectionEngine_
Definition: detect.h:423
Packet_::proto
uint8_t proto
Definition: decode.h:452
DetectTransaction_::tx_data_ptr
struct AppLayerTxData * tx_data_ptr
Definition: detect-engine-prefilter.h:34
DetectEngineAppInspectionEngine_::mpm
bool mpm
Definition: detect.h:427
DE_STATE_CHUNK_SIZE
#define DE_STATE_CHUNK_SIZE
Definition: detect-engine-state.h:53
RuleMatchCandidateTx::stream_stored
bool stream_stored
Definition: detect.h:1079
FLOWFILE_NO_SIZE_TS
#define FLOWFILE_NO_SIZE_TS
Definition: flow.h:142
Frame::inspect_progress
uint64_t inspect_progress
Definition: app-layer-frames.h:55
DetectEngineThreadCtx_::alert_queue_size
uint16_t alert_queue_size
Definition: detect.h:1180
PROF_DETECT_GETSGH
@ PROF_DETECT_GETSGH
Definition: suricata-common.h:441
ts
uint64_t ts
Definition: source-erf-file.c:55
FLOWFILE_NO_SIZE_TC
#define FLOWFILE_NO_SIZE_TC
Definition: flow.h:143
detect-engine.h
detect-engine-proto.h
DetectEngineThreadCtx_::match_array_cnt
SigIntId match_array_cnt
Definition: detect.h:1191
Frame::tx_id
uint64_t tx_id
Definition: app-layer-frames.h:54
DetectEngineStateDirection_::flags
uint8_t flags
Definition: detect-engine-state.h:89
detect-dsize.h
Signature_::addr_src_match6
DetectMatchAddressIPv6 * addr_src_match6
Definition: detect.h:625
PKT_HAS_FLOW
#define PKT_HAS_FLOW
Definition: decode.h:1005
PACKET_ALERT_FLAG_TX
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:279
DetectEngineCtx_::decoder_event_sgh
struct SigGroupHead_ * decoder_event_sgh
Definition: detect.h:918
FlowGetPacketDirection
int FlowGetPacketDirection(const Flow *f, const Packet *p)
determine the direction of the packet compared to the flow
Definition: flow.c:315
DetectEngineCtx_::flow_gh
DetectEngineLookupFlow flow_gh[FLOW_STATES]
Definition: detect.h:865
DE_STATE_FLAG_SIG_CANT_MATCH
#define DE_STATE_FLAG_SIG_CANT_MATCH
Definition: detect-engine-state.h:57
DetectEngineThreadCtx_::counter_match_list
uint16_t counter_match_list
Definition: detect.h:1150
DetectEngineAppInspectionEngine_::next
struct DetectEngineAppInspectionEngine_ * next
Definition: detect.h:442
DETECT_PROTO_IPV6
#define DETECT_PROTO_IPV6
Definition: detect-engine-proto.h:34
detect-engine-siggroup.h
SigGroupHead_::flags
uint16_t flags
Definition: detect.h:1445
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1058
PKT_IS_IPV6
#define PKT_IS_IPV6(p)
Definition: decode.h:246
DetectEngineState_
Definition: detect-engine-state.h:93
Signature_::num
SigIntId num
Definition: detect.h:604
stream-tcp.h
Prefilter
void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t flags)
Definition: detect-engine-prefilter.c:143
PrefilterRuleStore_::rule_id_array_cnt
uint32_t rule_id_array_cnt
Definition: util-prefilter.h:40
detect-engine-event.h
SigGroupHead_
Container for matching data for a signature group.
Definition: detect.h:1444
SIG_GROUP_HEAD_HAVEFILESIZE
#define SIG_GROUP_HEAD_HAVEFILESIZE
Definition: detect.h:1323
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
detect.c
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
KEYWORD_PROFILING_SET_LIST
#define KEYWORD_PROFILING_SET_LIST(ctx, list)
Definition: util-profiling.h:46
DetectAddressMatchIPv4
int DetectAddressMatchIPv4(const DetectMatchAddressIPv4 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
Definition: detect-engine-address.c:1610
DetectEngineAppInspectionEngine_::Callback
InspectEngineFuncPtr Callback
Definition: detect.h:435
Signature_::alproto
AppProto alproto
Definition: detect.h:597
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
SignatureNonPrefilterStore_::id
SigIntId id
Definition: detect.h:1068
FLOW_SGH_TOCLIENT
#define FLOW_SGH_TOCLIENT
Definition: flow.h:71
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:598
AppLayerParserSetTransactionInspectId
void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *pstate, void *alstate, const uint8_t flags, bool tag_txs_as_inspected)
Definition: app-layer-parser.c:758
DetectEngineStateDirection_::cnt
SigIntId cnt
Definition: detect-engine-state.h:87
AppLayerGetTxIterator
AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, const AppProto alproto)
Definition: app-layer-parser.c:702
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
SigMatchData_::is_last
bool is_last
Definition: detect.h:357
Flow_::proto
uint8_t proto
Definition: flow.h:372
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:80
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:289
AppLayerParserGetStateProgress
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
Definition: app-layer-parser.c:1103
DetectEngineThreadCtx_::tx_id
uint64_t tx_id
Definition: detect.h:1174
SigMatchData_::ctx
SigMatchCtx * ctx
Definition: detect.h:358
action-globals.h
FLOWFILE_NO_MAGIC_TS
#define FLOWFILE_NO_MAGIC_TS
Definition: flow.h:123
FramesContainer::toserver
Frames toserver
Definition: app-layer-frames.h:74
Packet_::flags
uint32_t flags
Definition: decode.h:467
ICMPV4_DEST_UNREACH_IS_VALID
#define ICMPV4_DEST_UNREACH_IS_VALID(p)
Definition: decode-icmpv4.h:267
Frame
Definition: app-layer-frames.h:45
Flow_
Flow data structure.
Definition: flow.h:350
DetectTransaction_::tx_end_state
const int tx_end_state
Definition: detect-engine-prefilter.h:41
PROF_DETECT_ALERT
@ PROF_DETECT_ALERT
Definition: suricata-common.h:452
DetectEngineThreadCtx_::pmq
PrefilterRuleStore pmq
Definition: detect.h:1200
SIG_GROUP_HEAD_HAVEFILESHA1
#define SIG_GROUP_HEAD_HAVEFILESHA1
Definition: detect.h:1324
FLOW_TC_APP_UPDATED
#define FLOW_TC_APP_UPDATED
Definition: flow.h:116
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:835
AppLayerParserGetStateProgressCompletionStatus
int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto, uint8_t direction)
Definition: app-layer-parser.c:1132
DetectRunScratchpad
struct DetectRunScratchpad DetectRunScratchpad
PROF_DETECT_PF_TX
@ PROF_DETECT_PF_TX
Definition: suricata-common.h:447
Frames::cnt
uint16_t cnt
Definition: app-layer-frames.h:61
Frame::id
int64_t id
Definition: app-layer-frames.h:53
APP_LAYER_TX_SKIP_INSPECT_FLAG
#define APP_LAYER_TX_SKIP_INSPECT_FLAG
Definition: app-layer-parser.h:79
DetectEngineThreadCtx_::p
Packet * p
Definition: detect.h:1178
DetectEngineState_::dir_state
DetectEngineStateDirection dir_state[2]
Definition: detect-engine-state.h:94
FrameGetByIndex
Frame * FrameGetByIndex(Frames *frames, const uint32_t idx)
Definition: app-layer-frames.c:106
RuleMatchCandidateTx::id
SigIntId id
Definition: detect.h:1075
PROF_DETECT_CLEANUP
@ PROF_DETECT_CLEANUP
Definition: suricata-common.h:454
SIG_FLAG_DST_ANY
#define SIG_FLAG_DST_ANY
Definition: detect.h:235
NO_TX
#define NO_TX
Definition: detect.c:1231
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:222
HashTable_
Definition: util-hash.h:35
DetectEngineThreadCtx_::flags
uint16_t flags
Definition: detect.h:1169
Frames
Definition: app-layer-frames.h:60
FramesContainer
Definition: app-layer-frames.h:73
PacketAlerts_::drop
PacketAlert drop
Definition: decode.h:295
APP_LAYER_TX_INSPECTED_FLAG
#define APP_LAYER_TX_INSPECTED_FLAG
Definition: app-layer-parser.h:81
DetectRunScratchpad
Definition: detect.c:70
SIG_GROUP_HEAD_HAVERAWSTREAM
#define SIG_GROUP_HEAD_HAVERAWSTREAM
Definition: detect.h:1318
FLOW_ACTION_DROP
#define FLOW_ACTION_DROP
Definition: flow.h:66
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
FLOWFILE_NONE
#define FLOWFILE_NONE
Definition: flow.h:161
detect-engine-frame.h
SigMatchSignatures
void SigMatchSignatures(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
wrapper for old tests
Definition: detect.c:1882
Signature_::sm_arrays
SigMatchData * sm_arrays[DETECT_SM_LIST_MAX]
Definition: detect.h:645
proto
uint8_t proto
Definition: decode-template.h:0
DetectPort_::sh
struct SigGroupHead_ * sh
Definition: detect.h:225
PACKET_PROFILING_DETECT_END
#define PACKET_PROFILING_DETECT_END(p, id)
Definition: util-profiling.h:222
Detect
TmEcode Detect(ThreadVars *tv, Packet *p, void *data)
Detection engine thread wrapper.
Definition: detect.c:1803
FLOW_PKT_TOCLIENT_IPONLY_SET
#define FLOW_PKT_TOCLIENT_IPONLY_SET
Definition: flow.h:226
detect-engine-payload.h
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:461
SIG_FLAG_SRC_ANY
#define SIG_FLAG_SRC_ANY
Definition: detect.h:234
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
DetectRunFrameInspectRule
bool DetectRunFrameInspectRule(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p, const Frames *frames, const Frame *frame)
Definition: detect-engine-frame.c:227
Flow_::protoctx
void * protoctx
Definition: flow.h:440
SigMatchData_
Data needed for Match()
Definition: detect.h:355
DeStateStoreItem_::sid
SigIntId sid
Definition: detect-engine-state.h:75
RuleMatchCandidateTx::s
const Signature * s
Definition: detect.h:1085
KEYWORD_PROFILING_START
#define KEYWORD_PROFILING_START
Definition: util-profiling.h:50
DetectEngineThreadCtx_::counter_fnonmpm_list
uint16_t counter_fnonmpm_list
Definition: detect.h:1149
SigMatchData_::type
uint16_t type
Definition: detect.h:356
DetectEngineCtx_::version
uint32_t version
Definition: detect.h:914
DisableDetectFlowFileFlags
void DisableDetectFlowFileFlags(Flow *f)
disable file features we don't need Called if we have no detection engine.
Definition: detect.c:1872
AppLayerParserGetTransactionInspectId
uint64_t AppLayerParserGetTransactionInspectId(AppLayerParserState *pstate, uint8_t direction)
Definition: app-layer-parser.c:727
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:592
DetectTransaction_::tx_progress
const int tx_progress
Definition: detect-engine-prefilter.h:40
DetectEngineThreadCtx_::counter_nonmpm_list
uint16_t counter_nonmpm_list
Definition: detect.h:1148
detect-engine-prefilter.h
DetectRunPrefilterFrame
void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const Frames *frames, const Frame *frame, const AppProto alproto)
Definition: detect-engine-frame.c:73
Packet_::events
PacketEngineEvents events
Definition: decode.h:602
Signature_::frame_inspect
DetectEngineFrameInspectionEngine * frame_inspect
Definition: detect.h:641
DetectTransaction_::prefilter_flags_orig
const uint64_t prefilter_flags_orig
Definition: detect-engine-prefilter.h:39
PROF_DETECT_PF_SORT2
@ PROF_DETECT_PF_SORT2
Definition: suricata-common.h:450
pad
uint16_t pad
Definition: source-erf-file.c:61
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
PacketCreateMask
void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, bool app_decoder_events)
Definition: detect-engine-build.c:405
SIG_FLAG_APPLAYER
#define SIG_FLAG_APPLAYER
Definition: detect.h:241
KEYWORD_PROFILING_END
#define KEYWORD_PROFILING_END(ctx, type, m)
Definition: util-profiling.h:64
detect-flowvar.h
DetectEngineThreadCtx_::mt_det_ctxs_hash
HashTable * mt_det_ctxs_hash
Definition: detect.h:1108
PacketAlert_::action
uint8_t action
Definition: decode.h:265
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:292
DetectEnginePktInspectionRun
bool DetectEnginePktInspectionRun(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f, Packet *p, uint8_t *alert_flags)
Definition: detect-engine.c:1984
Flow_::sgh_toserver
const struct SigGroupHead_ * sgh_toserver
Definition: flow.h:482
PROF_DETECT_TX_UPDATE
@ PROF_DETECT_TX_UPDATE
Definition: suricata-common.h:453
DetectEngineAppInspectionEngine_::id
uint8_t id
Definition: detect.h:426
DetectEngineThreadCtx_::counter_alerts_suppressed
uint16_t counter_alerts_suppressed
Definition: detect.h:1145
DetectRunStoreStateTx
void DetectRunStoreStateTx(const SigGroupHead *sgh, Flow *f, void *tx, uint64_t tx_id, const Signature *s, uint32_t inspect_flags, uint8_t flow_flags, const uint16_t file_no_match)
Definition: detect-engine-state.c:221
DetectEngineAppInspectionEngine_::sm_list
uint16_t sm_list
Definition: detect.h:429
FLOWFILE_INIT
#define FLOWFILE_INIT
Definition: flow.h:120
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:474
DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES
Definition: detect-engine-state.h:44
FLOWFILE_NO_SHA1_TC
#define FLOWFILE_NO_SHA1_TC
Definition: flow.h:135
DETECT_SM_LIST_POSTMATCH
@ DETECT_SM_LIST_POSTMATCH
Definition: detect.h:121
FramesContainer::toclient
Frames toclient
Definition: app-layer-frames.h:75
DetectEngineThreadCtx_::tx_candidates
RuleMatchCandidateTx * tx_candidates
Definition: detect.h:1193
Signature_::addr_src_match4
DetectMatchAddressIPv4 * addr_src_match4
Definition: detect.h:622
decode.h
DetectEngineThreadCtx_::counter_alerts_overflow
uint16_t counter_alerts_overflow
Definition: detect.h:1143
TOSERVER
#define TOSERVER
Definition: flow.h:43
DETECT_ENGINE_THREAD_CTX_FRAME_ID_SET
#define DETECT_ENGINE_THREAD_CTX_FRAME_ID_SET
Definition: detect.h:307
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:251
DetectEngineThreadCtx_
Definition: detect.h:1091
DEBUG_VALIDATE_PACKET
#define DEBUG_VALIDATE_PACKET(p)
Definition: util-validate.h:103
DeStateStoreItem_::flags
uint32_t flags
Definition: detect-engine-state.h:74
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1000
FLOWFILE_NO_STORE_TS
#define FLOWFILE_NO_STORE_TS
Definition: flow.h:127
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:400
DetectEngineThreadCtx_::tx_candidates_size
uint32_t tx_candidates_size
Definition: detect.h:1194
FLOW_PKT_TOSERVER_IPONLY_SET
#define FLOW_PKT_TOSERVER_IPONLY_SET
Definition: flow.h:225
PKT_IS_FRAGMENT
#define PKT_IS_FRAGMENT
Definition: decode.h:1029
STREAM_FLAGS_FOR_PACKET
#define STREAM_FLAGS_FOR_PACKET(p)
Definition: stream.h:30
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
PMQ_RESET
#define PMQ_RESET(pmq)
Definition: util-prefilter.h:46
detect-engine-mpm.h
DetectEngineLookupFlow_::sgh
struct SigGroupHead_ * sgh[256]
Definition: detect.h:773
HashTableLookup
void * HashTableLookup(HashTable *ht, void *data, uint16_t datalen)
Definition: util-hash.c:191
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
DeStateStore_::next
struct DeStateStore_ * next
Definition: detect-engine-state.h:80
SignatureNonPrefilterStore_::alproto
AppProto alproto
Definition: detect.h:1070
Packet_::sp
Port sp
Definition: decode.h:437
FLOWFILE_NO_SHA256_TS
#define FLOWFILE_NO_SHA256_TS
Definition: flow.h:138
DETECT_ENGINE_INSPECT_SIG_MATCH
#define DETECT_ENGINE_INSPECT_SIG_MATCH
Definition: detect-engine-state.h:39
PKT_DETECT_HAS_STREAMDATA
#define PKT_DETECT_HAS_STREAMDATA
Definition: decode.h:1044
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1465
detect-engine-port.h
PktSrcToString
const char * PktSrcToString(enum PktSrcEnum pkt_src)
Definition: decode.c:739
DetectEngineThreadCtx_::non_pf_id_cnt
uint32_t non_pf_id_cnt
Definition: detect.h:1104
DetectPort_
Port structure for detection engine.
Definition: detect.h:214
PacketAlerts_::discarded
uint16_t discarded
Definition: decode.h:290
app-layer-parser.h
Signature_::app_inspect
DetectEngineAppInspectionEngine * app_inspect
Definition: detect.h:639
util-detect.h
detect-engine-profile.h
Flow_::sgh_toclient
const struct SigGroupHead_ * sgh_toclient
Definition: flow.h:479
DetectEngineThreadCtx_::base64_decoded_len
int base64_decoded_len
Definition: detect.h:1134
SIG_FLAG_REQUIRE_FLOWVAR
#define SIG_FLAG_REQUIRE_FLOWVAR
Definition: detect.h:258
RuleMatchCandidateTx::stream_result
uint8_t stream_result
Definition: detect.h:1080
util-profiling.h
DetectTransaction_::tx_id
const uint64_t tx_id
Definition: detect-engine-prefilter.h:33
DetectEngineLookupFlow_::udp
DetectPort * udp
Definition: detect.h:772
DetectEngineThreadCtx_::raw_stream_progress
uint64_t raw_stream_progress
Definition: detect.h:1117
FileUpdateFlowFileFlags
void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction)
set a flow's file flags
Definition: util-file.c:1118
SCReturn
#define SCReturn
Definition: util-debug.h:273
Signature_::flags
uint32_t flags
Definition: detect.h:593
AlertQueueAppend
void AlertQueueAppend(DetectEngineThreadCtx *det_ctx, const Signature *s, Packet *p, uint64_t tx_id, uint8_t alert_flags)
Append signature to local packet alert queue for later preprocessing.
Definition: detect-engine-alert.c:283
RuleMatchCandidateTxArrayFree
void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx)
Definition: detect.c:974
AppLayerGetTxIterState
Definition: app-layer-parser.h:144
DetectTransaction_::prefilter_flags
uint64_t prefilter_flags
Definition: detect-engine-prefilter.h:37
Packet_
Definition: decode.h:430
detect-engine-build.h
AppLayerFramesGetContainer
FramesContainer * AppLayerFramesGetContainer(Flow *f)
Definition: app-layer-parser.c:182
DetectEngineThreadCtx_::frame_id
int64_t frame_id
Definition: detect.h:1175
APP_LAYER_TX_PREFILTER_MASK
#define APP_LAYER_TX_PREFILTER_MASK
Definition: app-layer-parser.h:84
detect-engine-alert.h
conf.h
DetectRunScratchpad::pkt_mask
SignatureMask pkt_mask
Definition: detect.c:75
FlowSetIPOnlyFlag
void FlowSetIPOnlyFlag(Flow *f, int direction)
Set the IPOnly scanned flag for 'direction'.
Definition: flow.c:162
FLOW_TOSERVER_IPONLY_SET
#define FLOW_TOSERVER_IPONLY_SET
Definition: flow.h:56
DetectEngineThreadCtx_::filestore_cnt
uint16_t filestore_cnt
Definition: detect.h:1138
PROF_DETECT_IPONLY
@ PROF_DETECT_IPONLY
Definition: suricata-common.h:442
RULE_PROFILING_END
#define RULE_PROFILING_END(a, b, c, p)
Definition: util-profiling.h:431
TmEcode
TmEcode
Definition: tm-threads-common.h:83
RULE_PROFILING_START
#define RULE_PROFILING_START(p)
Definition: util-profiling.h:430
PKT_STREAM_EOF
#define PKT_STREAM_EOF
Definition: decode.h:1004
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:287
DetectTransaction_::de_state
DetectEngineStateDirection * de_state
Definition: detect-engine-prefilter.h:35
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
detect-filestore.h
SigTableElmt_::Match
int(* Match)(DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1260
detect-replace.h
RulesDumpMatchArray
void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, const Packet *p)
Definition: detect-engine-profile.c:79
Signature_::addr_dst_match6_cnt
uint16_t addr_dst_match6_cnt
Definition: detect.h:619
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:223
SigGroupHead_::frame_engines
PrefilterEngine * frame_engines
Definition: detect.h:1464
DetectEngineAppInspectionEngine_::v2
struct DetectEngineAppInspectionEngine_::@85 v2
Signature_::sp
DetectPort * sp
Definition: detect.h:633
DetectRunScratchpad::sgh
const SigGroupHead * sgh
Definition: detect.c:74
DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
Definition: detect-engine-state.h:40
DETECT_ENGINE_STATE_FLAG_FILE_NEW
#define DETECT_ENGINE_STATE_FLAG_FILE_NEW
Definition: detect-engine-state.h:71
Flow_::flowvar
GenericVar * flowvar
Definition: flow.h:485
RuleMatchCandidateTx::flags
uint32_t * flags
Definition: detect.h:1076
DetectEngineAppInspectionEngine_::alproto
AppProto alproto
Definition: detect.h:424
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
detect-engine-analyzer.h
FLOWFILE_NO_SHA256_TC
#define FLOWFILE_NO_SHA256_TC
Definition: flow.h:139
DetectEngineStateDirection_::head
DeStateStore * head
Definition: detect-engine-state.h:84
DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES
#define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES
Definition: detect-engine-state.h:50
DetectProto_::flags
uint8_t flags
Definition: detect-engine-proto.h:38
AppLayerTxData
struct AppLayerTxData AppLayerTxData
Definition: detect.h:1354
DETECT_PROTO_IPV4
#define DETECT_PROTO_IPV4
Definition: detect-engine-proto.h:33
app-layer-frames.h
DetectAddressMatchIPv6
int DetectAddressMatchIPv6(const DetectMatchAddressIPv6 *addrs, uint16_t addrs_cnt, const Address *a)
Match a packets address against a signatures addrs array.
Definition: detect-engine-address.c:1643
Packet_::tenant_id
uint32_t tenant_id
Definition: decode.h:634
Packet_::flow
struct Flow_ * flow
Definition: decode.h:469
DetectEngineStateResetTxs
void DetectEngineStateResetTxs(Flow *f)
Reset de state for active tx' To be used on detect engine reload.
Definition: detect-engine-state.c:267
FLOWFILE_NO_MAGIC_TC
#define FLOWFILE_NO_MAGIC_TC
Definition: flow.h:124
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
Signature_::proto
DetectProto proto
Definition: detect.h:611
SigGroupHead_::non_pf_other_store_array
SignatureNonPrefilterStore * non_pf_other_store_array
Definition: detect.h:1457
FLOW_TOCLIENT_IPONLY_SET
#define FLOW_TOCLIENT_IPONLY_SET
Definition: flow.h:58
suricata-common.h
SIG_FLAG_SP_ANY
#define SIG_FLAG_SP_ANY
Definition: detect.h:236
APP_LAYER_TX_RESERVED_FLAGS
#define APP_LAYER_TX_RESERVED_FLAGS
Definition: app-layer-parser.h:69
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:557
SigGroupHead_::non_pf_other_store_cnt
uint32_t non_pf_other_store_cnt
Definition: detect.h:1455
packet.h
DeStateStore_
Definition: detect-engine-state.h:78
sigmatch_table
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect-parse.c:127
ACTION_DROP
#define ACTION_DROP
Definition: action-globals.h:30
DetectTransaction_
Definition: detect-engine-prefilter.h:31
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:464
PROF_DETECT_SETUP
@ PROF_DETECT_SETUP
Definition: suricata-common.h:440
DetectEngineStateDirection_
Definition: detect-engine-state.h:83
DetectEngineThreadCtx_::frame_inspect_progress
uint64_t frame_inspect_progress
Definition: detect.h:1176
DetectEngineCtx_::profile_match_logging_threshold
uint32_t profile_match_logging_threshold
Definition: detect.h:956
FLOW_TS_APP_UPDATED
#define FLOW_TS_APP_UPDATED
Definition: flow.h:115
AppLayerParserGetTxData
AppLayerTxData * AppLayerParserGetTxData(uint8_t ipproto, AppProto alproto, void *tx)
Definition: app-layer-parser.c:1195
FatalError
#define FatalError(...)
Definition: util-debug.h:502
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:295
FLOWFILE_NO_STORE_TC
#define FLOWFILE_NO_STORE_TC
Definition: flow.h:128
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
DetectEngineThreadCtx_::non_pf_id_array
SigIntId * non_pf_id_array
Definition: detect.h:1103
DetectEngineAppInspectionEngine_::progress
int16_t progress
Definition: detect.h:431
PKT_IS_ICMPV4
#define PKT_IS_ICMPV4(p)
Definition: decode.h:249
util-validate.h
DetectTransaction_::tx_ptr
void * tx_ptr
Definition: detect-engine-prefilter.h:32
Signature_::addr_src_match6_cnt
uint16_t addr_src_match6_cnt
Definition: detect.h:620
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1511
StatsAddUI64
void StatsAddUI64(ThreadVars *tv, uint16_t id, uint64_t x)
Adds a value of type uint64_t to the local counter.
Definition: counters.c:146
FLOWFILE_NO_SHA1_TS
#define FLOWFILE_NO_SHA1_TS
Definition: flow.h:134
PROF_DETECT_RULES
@ PROF_DETECT_RULES
Definition: suricata-common.h:443
DetectRunScratchpad::app_decoder_events
const bool app_decoder_events
Definition: detect.c:73
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:294
PROF_DETECT_NONMPMLIST
@ PROF_DETECT_NONMPMLIST
Definition: suricata-common.h:451
Signature_::dp
DetectPort * dp
Definition: detect.h:633
PacketAlerts_::suppressed
uint16_t suppressed
Definition: decode.h:291
PACKET_PROFILING_DETECT_START
#define PACKET_PROFILING_DETECT_START(p, id)
Definition: util-profiling.h:215
PACKET_ALERT_FLAG_FRAME
#define PACKET_ALERT_FLAG_FRAME
Definition: decode.h:283
DetectEngineThreadCtx_::non_pf_store_cnt
uint32_t non_pf_store_cnt
Definition: detect.h:1197
FLOW_SGH_TOSERVER
#define FLOW_SGH_TOSERVER
Definition: flow.h:69
DetectEngineThreadCtx_::counter_alerts
uint16_t counter_alerts
Definition: detect.h:1141
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Packet_::pkt_src
uint8_t pkt_src
Definition: decode.h:583
SGH_PROFILING_RECORD
#define SGH_PROFILING_RECORD(det_ctx, sgh)
Definition: util-profiling.h:253
PacketAlertFinalize
void PacketAlertFinalize(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
Check the threshold of the sigs that match, set actions, break on pass action This function iterate t...
Definition: detect-engine-alert.c:366
Signature_::addr_dst_match6
DetectMatchAddressIPv6 * addr_dst_match6
Definition: detect.h:624
Flow_::alstate
void * alstate
Definition: flow.h:475
Signature_::id
uint32_t id
Definition: detect.h:627
DeStateStore_::store
DeStateStoreItem store[DE_STATE_CHUNK_SIZE]
Definition: detect-engine-state.h:79
Flow_::flags
uint32_t flags
Definition: flow.h:420
RuleMatchCandidateTx::stream_reset
uint32_t stream_reset
Definition: detect.h:1082
detect-engine-iponly.h
Signature_
Signature container.
Definition: detect.h:592
IP_GET_IPPROTO
#define IP_GET_IPPROTO(p)
Definition: decode.h:257
InspectionBufferClean
void InspectionBufferClean(DetectEngineThreadCtx *det_ctx)
Definition: detect-engine.c:1498
SignatureMask
#define SignatureMask
Definition: detect.h:305
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
TRACE_SID_TXS
#define TRACE_SID_TXS(sid, txs,...)
Definition: detect.c:1037
FLOW_PKT_ESTABLISHED
#define FLOW_PKT_ESTABLISHED
Definition: flow.h:224
DetectEngineThreadCtx_::tx_id_set
bool tx_id_set
Definition: detect.h:1172
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
DetectRunScratchpad::flow_flags
const uint8_t flow_flags
Definition: detect.c:72
DetectEngineThreadCtx_::de_ctx
DetectEngineCtx * de_ctx
Definition: detect.h:1215
suricata.h
IPOnlyMatchPacket
void IPOnlyMatchPacket(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const DetectEngineIPOnlyCtx *io_ctx, Packet *p)
Match a packet against the IP Only detection engine contexts.
Definition: detect-engine-iponly.c:1004
DetectEngineCtx_::sig_array
Signature ** sig_array
Definition: detect.h:852
Packet_::dst
Address dst
Definition: decode.h:435
SigGroupHead_::non_pf_syn_store_array
SignatureNonPrefilterStore * non_pf_syn_store_array
Definition: detect.h:1459
PROF_DETECT_TX
@ PROF_DETECT_TX
Definition: suricata-common.h:444
DetectEngineAppInspectionEngine_::dir
uint8_t dir
Definition: detect.h:425
PKT_NOPACKET_INSPECTION
#define PKT_NOPACKET_INSPECTION
Definition: decode.h:990
SignatureNonPrefilterStore_::mask
SignatureMask mask
Definition: detect.h:1069
DetectRunScratchpad::alproto
const AppProto alproto
Definition: detect.c:71
DE_STATE_FLAG_FULL_INSPECT
#define DE_STATE_FLAG_FULL_INSPECT
Definition: detect-engine-state.h:56
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
DetectEngineThreadCtx_::mt_det_ctxs_cnt
uint32_t mt_det_ctxs_cnt
Definition: detect.h:1106
AppLayerGetTxIteratorFunc
AppLayerGetTxIterTuple(* AppLayerGetTxIteratorFunc)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, AppLayerGetTxIterState *state)
tx iterator prototype
Definition: app-layer-parser.h:153
likely
#define likely(expr)
Definition: util-optimize.h:32
IPPROTO_SCTP
#define IPPROTO_SCTP
Definition: decode.h:933
DE_STATE_FLAG_FILE_INSPECT
#define DE_STATE_FLAG_FILE_INSPECT
Definition: detect-engine-state.h:61
FLOW_NOPACKET_INSPECTION
#define FLOW_NOPACKET_INSPECTION
Definition: flow.h:61
DetectEngineCtx_::io_ctx
DetectEngineIPOnlyCtx io_ctx
Definition: detect.h:876
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
Signature_::addr_dst_match4
DetectMatchAddressIPv4 * addr_dst_match4
Definition: detect.h:621
FlowGetDisruptionFlags
uint8_t FlowGetDisruptionFlags(const Flow *f, uint8_t flags)
get 'disruption' flags: GAP/DEPTH/PASS
Definition: flow.c:1157
TcpSession_
Definition: stream-tcp-private.h:283
flow.h
Signature_::addr_src_match4_cnt
uint16_t addr_src_match4_cnt
Definition: detect.h:618
SigIntId
#define SigIntId
Definition: suricata-common.h:315
DeStateStoreItem_
Definition: detect-engine-state.h:73
GenericVarFree
void GenericVarFree(GenericVar *gv)
Definition: util-var.c:47
Signature_::addr_dst_match4_cnt
uint16_t addr_dst_match4_cnt
Definition: detect.h:617
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:449
Packet_::dp
Port dp
Definition: decode.h:445
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SIG_GROUP_HEAD_HAVEFILESHA256
#define SIG_GROUP_HEAD_HAVEFILESHA256
Definition: detect.h:1325
SigGroupHead_::filestore_cnt
uint16_t filestore_cnt
Definition: detect.h:1450
AppLayerParserGetTxCnt
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
Definition: app-layer-parser.c:1118
PKT_IS_IPV4
#define PKT_IS_IPV4(p)
Definition: decode.h:245
PACKET_ALERT_FLAG_STATE_MATCH
#define PACKET_ALERT_FLAG_STATE_MATCH
Definition: decode.h:275
AppLayerParserHasDecoderEvents
bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
Definition: app-layer-parser.c:1527
DetectEngineLookupFlow_::tcp
DetectPort * tcp
Definition: detect.h:771
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:104
DetectTransaction_::detect_flags
const uint64_t detect_flags
Definition: detect-engine-prefilter.h:36
detect-engine-address.h
Flow_::tenant_id
uint32_t tenant_id
Definition: flow.h:415
Packet_::src
Address src
Definition: decode.h:434
DetectEngineThreadCtx_::counter_mpm_list
uint16_t counter_mpm_list
Definition: detect.h:1147
SigMatchSignaturesGetSgh
const SigGroupHead * SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx, const Packet *p)
Get the SigGroupHead for a packet.
Definition: detect.c:212
DetectRunPrefilterTx
void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, const uint8_t ipproto, const uint8_t flow_flags, const AppProto alproto, void *alstate, DetectTransaction *tx)
run prefilter engines on a transaction
Definition: detect-engine-prefilter.c:93
RuleMatchCandidateTxArrayInit
void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size)
Definition: detect.c:961
SIG_FLAG_DP_ANY
#define SIG_FLAG_DP_ANY
Definition: detect.h:237
DetectPortLookupGroup
DetectPort * DetectPortLookupGroup(DetectPort *dp, uint16_t port)
Function that find the group matching address in a group head.
Definition: detect-engine-port.c:674
PKT_STREAM_EST
#define PKT_STREAM_EST
Definition: decode.h:1002
detect-engine-threshold.h
Signature_::mask
SignatureMask mask
Definition: detect.h:603
app-layer.h
RuleMatchCandidateTx
Definition: detect.h:1074
DetectEngineThreadCtx_::TenantGetId
uint32_t(* TenantGetId)(const void *, const Packet *p)
Definition: detect.h:1113
PrefilterRuleStore_::rule_id_array
SigIntId * rule_id_array
Definition: util-prefilter.h:38
PacketEngineEvents_::cnt
uint8_t cnt
Definition: decode.h:308
Flow_::de_ctx_version
uint32_t de_ctx_version
Definition: flow.h:463
DetectProtoContainsProto
int DetectProtoContainsProto(const DetectProto *dp, int proto)
see if a DetectProto contains a certain proto
Definition: detect-engine-proto.c:135
DetectEngineThreadCtx_::match_array
Signature ** match_array
Definition: detect.h:1186