suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2026 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "packet.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 #include "action-globals.h"
37 
38 #include "flow-util.h"
39 
40 #include "threadvars.h"
41 #include "tm-threads.h"
42 
43 #include "util-pool.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device-private.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-cache.h"
53 #include "stream-tcp-reassemble.h"
54 #include "stream-tcp-inline.h"
55 #include "stream-tcp-list.h"
56 #include "stream-tcp-util.h"
57 
58 #include "stream.h"
59 
60 #include "util-debug.h"
61 #include "app-layer-protos.h"
62 #include "app-layer.h"
63 #include "app-layer-events.h"
64 #include "app-layer-parser.h"
65 #include "app-layer-frames.h"
66 
67 #include "detect-engine-state.h"
68 
69 #include "util-profiling.h"
70 #include "util-validate.h"
71 #include "util-exception-policy.h"
72 
73 #ifdef DEBUG
74 static SCMutex segment_pool_memuse_mutex;
75 static uint64_t segment_pool_memuse = 0;
76 static uint64_t segment_pool_memcnt = 0;
77 #endif
78 #ifdef QA_SIMULATION
79 thread_local uint64_t t_pcapcnt = UINT64_MAX;
80 #endif
81 
83 /* init only, protect initializing and growing pool */
84 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
85 
86 /* Memory use counter */
87 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
88 
89 static int g_tcp_session_dump_enabled = 0;
90 
91 inline bool IsTcpSessionDumpingEnabled(void)
92 {
93  return g_tcp_session_dump_enabled == 1;
94 }
95 
97 {
98  g_tcp_session_dump_enabled = 1;
99 }
100 
101 /* prototypes */
103 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
104 
106 {
107  SC_ATOMIC_INIT(ra_memuse);
108 }
109 
110 /**
111  * \brief Function to Increment the memory usage counter for the TCP reassembly
112  * segments
113  *
114  * \param size Size of the TCP segment and its payload length memory allocated
115  */
116 static void StreamTcpReassembleIncrMemuse(uint64_t size)
117 {
118  (void) SC_ATOMIC_ADD(ra_memuse, size);
119  SCLogDebug("REASSEMBLY %" PRIu64 ", incr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
120  size);
121 }
122 
123 /**
124  * \brief Function to Decrease the memory usage counter for the TCP reassembly
125  * segments
126  *
127  * \param size Size of the TCP segment and its payload length memory allocated
128  */
129 static void StreamTcpReassembleDecrMemuse(uint64_t size)
130 {
131 #ifdef UNITTESTS
132  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
133  if (RunmodeIsUnittests()) {
134  BUG_ON(presize > UINT_MAX);
135  }
136 #endif
137 
138  (void) SC_ATOMIC_SUB(ra_memuse, size);
139 
140 #ifdef UNITTESTS
141  if (RunmodeIsUnittests()) {
142  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
143  BUG_ON(postsize > presize);
144  }
145 #endif
146  SCLogDebug("REASSEMBLY %" PRIu64 ", decr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
147  size);
148 }
149 
151 {
152  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
153  return smemuse;
154 }
155 
156 /**
157  * \brief Function to Check the reassembly memory usage counter against the
158  * allowed max memory usage for TCP segments.
159  *
160  * \param size Size of the TCP segment and its payload length memory allocated
161  * \retval 1 if in bounds
162  * \retval 0 if not in bounds
163  */
165 {
166 #ifdef QA_SIMULATION
167  if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX &&
168  g_eps_stream_reassembly_memcap == t_pcapcnt))) {
169  SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt);
170  return 0;
171  }
172 #endif
173  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
174  if (memcapcopy == 0 ||
175  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
176  return 1;
177  return 0;
178 }
179 
180 /**
181  * \brief Update memcap value
182  *
183  * \param size new memcap value
184  */
186 {
187  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
188  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
189  return 1;
190  }
191 
192  return 0;
193 }
194 
195 /**
196  * \brief Return memcap value
197  *
198  * \return memcap memcap value
199  */
201 {
202  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
203  return memcapcopy;
204 }
205 
206 /* memory functions for the streaming buffer API */
207 
208 /*
209  void *(*Calloc)(size_t n, size_t size);
210 */
211 static void *ReassembleCalloc(size_t n, size_t size)
212 {
213  if (StreamTcpReassembleCheckMemcap(n * size) == 0) {
215  return NULL;
216  }
217  void *ptr = SCCalloc(n, size);
218  if (ptr == NULL) {
220  return NULL;
221  }
222  StreamTcpReassembleIncrMemuse(n * size);
223  return ptr;
224 }
225 
226 /*
227  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
228 */
229 void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
230 {
231  if (size > orig_size) {
232  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) {
233  SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap));
235  return NULL;
236  }
237  }
238  void *nptr = SCRealloc(optr, size);
239  if (nptr == NULL) {
240  SCLogDebug("realloc fail");
242  return NULL;
243  }
244  if (size > orig_size) {
245  StreamTcpReassembleIncrMemuse(size - orig_size);
246  } else {
247  StreamTcpReassembleDecrMemuse(orig_size - size);
248  }
249  return nptr;
250 }
251 
252 /*
253  void (*Free)(void *ptr, size_t size);
254 */
255 static void ReassembleFree(void *ptr, size_t size)
256 {
257  SCFree(ptr);
258  StreamTcpReassembleDecrMemuse(size);
259 }
260 
261 /** \brief alloc a tcp segment pool entry */
262 static void *TcpSegmentPoolAlloc(void)
263 {
264  SCLogDebug("segment alloc");
265  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
266  return NULL;
267  }
268 
269  TcpSegment *seg = SCMalloc(sizeof(TcpSegment));
270  if (unlikely(seg == NULL))
271  return NULL;
272 
274  uint32_t memuse =
275  sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
276  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
277  SCFree(seg);
278  return NULL;
279  }
280 
282  if (seg->pcap_hdr_storage == NULL) {
283  SCLogDebug("Unable to allocate memory for "
284  "TcpSegmentPcapHdrStorage");
285  SCFree(seg);
286  return NULL;
287  }
288 
289  seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
290  seg->pcap_hdr_storage->pkt_hdr = SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
291  if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
292  SCLogDebug("Unable to allocate memory for "
293  "packet header data within "
294  "TcpSegmentPcapHdrStorage");
295  SCFree(seg->pcap_hdr_storage);
296  SCFree(seg);
297  return NULL;
298  }
299  } else {
300  seg->pcap_hdr_storage = NULL;
301  }
302 
303  return seg;
304 }
305 
306 static int TcpSegmentPoolInit(void *data)
307 {
308  TcpSegment *seg = (TcpSegment *) data;
309  TcpSegmentPcapHdrStorage *pcap_hdr;
310 
311  pcap_hdr = seg->pcap_hdr_storage;
312 
313  /* do this before the can bail, so TcpSegmentPoolCleanup
314  * won't have uninitialized memory to consider. */
315  memset(seg, 0, sizeof (TcpSegment));
316 
318  uint32_t memuse =
319  sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
320  seg->pcap_hdr_storage = pcap_hdr;
321  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
322  /* the pool frees the segment without calling
323  * TcpSegmentPoolCleanup(), so free the storage here */
324  if (pcap_hdr != NULL) {
325  SCFree(pcap_hdr->pkt_hdr);
326  SCFree(pcap_hdr);
327  seg->pcap_hdr_storage = NULL;
328  }
329  return 0;
330  }
331  StreamTcpReassembleIncrMemuse(memuse);
332  } else {
333  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
334  return 0;
335  }
336  }
337 
338 #ifdef DEBUG
339  SCMutexLock(&segment_pool_memuse_mutex);
340  segment_pool_memuse += sizeof(TcpSegment);
341  segment_pool_memcnt++;
342  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
343  SCMutexUnlock(&segment_pool_memuse_mutex);
344 #endif
345 
346  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
347  return 1;
348 }
349 
350 /** \brief clean up a tcp segment pool entry */
351 static void TcpSegmentPoolCleanup(void *ptr)
352 {
353  if (ptr == NULL)
354  return;
355 
356  TcpSegment *seg = (TcpSegment *)ptr;
357  if (seg->pcap_hdr_storage) {
358  if (seg->pcap_hdr_storage->pkt_hdr) {
360  StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen);
361  }
362  SCFree(seg->pcap_hdr_storage);
363  seg->pcap_hdr_storage = NULL;
364  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage));
365  }
366 
367  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
368 
369 #ifdef DEBUG
370  SCMutexLock(&segment_pool_memuse_mutex);
371  segment_pool_memuse -= sizeof(TcpSegment);
372  segment_pool_memcnt--;
373  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
374  SCMutexUnlock(&segment_pool_memuse_mutex);
375 #endif
376 }
377 
378 /**
379  * \brief Function to return the segment back to the pool.
380  *
381  * \param seg Segment which will be returned back to the pool.
382  */
384 {
385  if (seg == NULL)
386  return;
387 
388  if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
389  seg->pcap_hdr_storage->pktlen = 0;
390  }
391 
393 }
394 
395 /**
396  * \brief return all segments in this stream into the pool(s)
397  *
398  * \param stream the stream to cleanup
399  */
401 {
402  TcpSegment *seg = NULL, *safe = NULL;
403  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
404  {
405  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
407  }
408 }
409 
410 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
411 {
412  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
413  return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq);
414  }
415  return STREAM_BASE_OFFSET(stream);
416 }
417 
418 // may contain gaps
419 uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
420 {
421  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
422  if (!eof && !StreamTcpInlineMode()) {
423  right_edge = MIN(GetAbsLastAck(stream), right_edge);
424  }
425  return right_edge;
426 }
427 
428 uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
429 {
430  uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
431  if (!eof && !StreamTcpInlineMode()) {
432  right_edge = MIN(GetAbsLastAck(stream), right_edge);
433  }
434  return right_edge;
435 }
436 
437 #ifdef UNITTESTS
438 /** \internal
439  * \brief check if segments falls before stream 'offset' */
440 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
441 {
442  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
443  return 1;
444  return 0;
445 }
446 #endif
447 
448 /** \param f locked flow */
450 {
451  if (f->protoctx == NULL)
452  return;
453 
458  if (f->alparser) {
459  const uint16_t eof_flags = APP_LAYER_PARSER_EOF_TS | APP_LAYER_PARSER_EOF_TC;
460  const bool was_eof = SCAppLayerParserStateIssetFlag(f->alparser, eof_flags) == eof_flags;
462  /* The app layer is at EOF from here on: no further app-layer
463  * update will reach the tx loggers, while the flow may end up
464  * dropped (flow end skips FLOW_ACTION_DROP flows), so the txs
465  * still pending would never be logged. Ask the next output
466  * stage to run the loggers once. Only on the transition, so
467  * the flush does not repeat per packet. */
468  if (!was_eof) {
470  }
471  }
472 }
473 
474 /** \param f locked flow */
476 {
477  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
478  return 0;
479 
482 }
483 
484 static int StreamTcpReassemblyConfig(bool quiet)
485 {
486  uint32_t segment_prealloc = 2048;
487  SCConfNode *seg = SCConfGetNode("stream.reassembly.segment-prealloc");
488  if (seg) {
489  uint32_t prealloc = 0;
490  if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) {
491  SCLogError("segment-prealloc of "
492  "%s is invalid",
493  seg->val);
494  return -1;
495  }
496  segment_prealloc = prealloc;
497  }
498  if (!quiet)
499  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
500  stream_config.prealloc_segments = segment_prealloc;
501 
502  int overlap_diff_data = 0;
503  (void)SCConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
504  if (overlap_diff_data) {
506  }
507  if (StreamTcpInlineMode()) {
509  }
510 
511  uint16_t max_regions = 8;
512  SCConfNode *mr = SCConfGetNode("stream.reassembly.max-regions");
513  if (mr) {
514  uint16_t max_r = 0;
515  if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) {
516  SCLogError("max-regions %s is invalid", mr->val);
517  return -1;
518  }
519  max_regions = max_r;
520  }
521  if (!quiet)
522  SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions);
523 
524  stream_config.prealloc_segments = segment_prealloc;
526  stream_config.sbcnf.max_regions = max_regions;
528  stream_config.sbcnf.Calloc = ReassembleCalloc;
530  stream_config.sbcnf.Free = ReassembleFree;
531 
532  return 0;
533 }
534 
535 int StreamTcpReassembleInit(bool quiet)
536 {
537  /* init the memcap/use tracker */
539 
540  if (StreamTcpReassemblyConfig(quiet) < 0)
541  return -1;
542 
543 #ifdef DEBUG
544  SCMutexInit(&segment_pool_memuse_mutex, NULL);
545 #endif
546  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
548  return 0;
549 }
550 
551 void StreamTcpReassembleFree(bool quiet)
552 {
553  SCMutexLock(&segment_thread_pool_mutex);
554  if (segment_thread_pool != NULL) {
556  segment_thread_pool = NULL;
557  }
558  SCMutexUnlock(&segment_thread_pool_mutex);
559  SCMutexDestroy(&segment_thread_pool_mutex);
560 
561 #ifdef DEBUG
562  if (segment_pool_memuse > 0)
563  SCLogDebug("segment_pool_memuse %" PRIu64 " segment_pool_memcnt %" PRIu64 "",
564  segment_pool_memuse, segment_pool_memcnt);
565  SCMutexDestroy(&segment_pool_memuse_mutex);
566 #endif
567 }
568 
570 {
571  SCEnter();
573  if (unlikely(ra_ctx == NULL))
574  return NULL;
575 
576  ra_ctx->app_tctx = AppLayerGetCtxThread();
577 
578  SCMutexLock(&segment_thread_pool_mutex);
579  if (segment_thread_pool == NULL) {
580  segment_thread_pool = PoolThreadInit(1, /* thread */
581  0, /* unlimited */
582  stream_config.prealloc_segments, sizeof(TcpSegment), TcpSegmentPoolAlloc,
583  TcpSegmentPoolInit, TcpSegmentPoolCleanup);
584  ra_ctx->segment_thread_pool_id = 0;
585  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
587  ra_ctx->segment_thread_pool_id);
588  } else {
589  /* grow segment_thread_pool until we have an element for our thread id */
591  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
593  ra_ctx->segment_thread_pool_id);
594  }
595  SCMutexUnlock(&segment_thread_pool_mutex);
596  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
597  SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
599  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
600  }
601 
602  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
603 }
604 
606 {
607  SCEnter();
609 
610  if (ra_ctx) {
612  SCFree(ra_ctx);
613  }
614  SCReturn;
615 }
616 
617 static void StreamTcpReassembleExceptionPolicyStatsIncr(
618  ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, enum ExceptionPolicy policy)
619 {
620  StatsCounterId id = ra_ctx->counter_tcp_reas_eps.eps_id[policy];
621  if (likely(tv && id.id > 0)) {
622  StatsCounterIncr(&tv->stats, id);
623  }
624 }
625 
626 /**
627  * \brief check if stream in pkt direction has depth reached
628  *
629  * \param p packet with *LOCKED* flow
630  *
631  * \retval true stream has depth reached
632  * \retval false stream does not have depth reached
633  */
635 {
636  if (p->flow != NULL && p->flow->protoctx != NULL) {
638  TcpStream *stream;
639  if (p->flowflags & FLOW_PKT_TOSERVER) {
640  stream = &ssn->client;
641  } else {
642  stream = &ssn->server;
643  }
644 
645  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) != 0;
646  }
647 
648  return false;
649 }
650 
651 /**
652  * \internal
653  * \brief Function to check the reassembly depth value against the
654  * allowed max depth of the stream reassembly for TCP streams.
655  *
656  * \param stream stream direction
657  * \param seq sequence number where "size" starts
658  * \param size size of the segment that is added
659  *
660  * \retval size Part of the size that fits in the depth, 0 if none
661  */
662 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
663  uint32_t seq, uint32_t size)
664 {
665  SCEnter();
666 
667  /* if the configured depth value is 0, it means there is no limit on
668  reassembly depth. Otherwise carry on my boy ;) */
669  if (ssn->reassembly_depth == 0) {
670  SCReturnUInt(size);
671  }
672 
673  /* if the final flag is set, we're not accepting anymore */
675  SCReturnUInt(0);
676  }
677 
678  uint64_t seg_depth;
679  if (SEQ_GT(stream->base_seq, seq)) {
680  if (SEQ_LEQ(seq+size, stream->base_seq)) {
681  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
682  stream->base_seq, seq, seq+size);
683  SCReturnUInt(0);
684  }
685 
686  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
687  } else {
688  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
689  }
690 
691  /* if the base_seq has moved past the depth window we stop
692  * checking and just reject the rest of the packets including
693  * retransmissions. Saves us the hassle of dealing with sequence
694  * wraps as well */
695  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
696  stream->base_seq, seg_depth,
698 
699  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
700  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
702  SCReturnUInt(0);
703  }
704  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
705  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
706 #if 0
707  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
708  (stream->base_seq_offset + stream->base_seq + size),
709  (stream->isn + ssn->reassembly_depth));
710 #endif
711  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
712  /* packet (partly?) fits the depth window */
713 
714  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
715  /* complete fit */
716  SCReturnUInt(size);
717  } else {
719  /* partial fit, return only what fits */
720  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
721  DEBUG_VALIDATE_BUG_ON(part > size);
722  if (part > size)
723  part = size;
724  SCReturnUInt(part);
725  }
726  }
727 
728  SCReturnUInt(0);
729 }
730 
732 {
733  if (RB_EMPTY(&stream->sb.sbb_tree)) {
734  if (stream->sb.region.stream_offset != 0)
735  return 0;
736 
737  return stream->sb.region.buf_offset;
738  }
739 
740  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
741  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
742  return stream->sb.sbb_size;
743 }
744 
745 /**
746  * \brief Insert a TCP packet data into the stream reassembly engine.
747  *
748  * \retval 0 good segment, as far as we checked.
749  * \retval -1 insert failure due to memcap
750  *
751  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
752  * or it wasn't added because of reassembly depth.
753  *
754  */
756  TcpSession *ssn, TcpStream *stream, Packet *p)
757 {
758  SCEnter();
759 
760  if (ssn->data_first_seen_dir == 0) {
761  if (PKT_IS_TOSERVER(p)) {
762  ssn->data_first_seen_dir = STREAM_TOSERVER;
763  } else {
764  ssn->data_first_seen_dir = STREAM_TOCLIENT;
765  }
766  }
767 
768  /* If the OS policy is not set then set the OS policy for this stream */
769  if (stream->os_policy == 0) {
770  StreamTcpSetOSPolicy(stream, p);
771  }
772 
775  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
776  SCReturnInt(0);
777  }
778 
779  uint16_t *urg_offset;
780  if (PKT_IS_TOSERVER(p)) {
781  urg_offset = &ssn->urg_offset_ts;
782  } else {
783  urg_offset = &ssn->urg_offset_tc;
784  }
785 
786  const TCPHdr *tcph = PacketGetTCP(p);
787  /* segment sequence number, offset by previously accepted
788  * URG OOB data. */
789  uint32_t seg_seq = TCP_GET_RAW_SEQ(tcph) - (*urg_offset);
790  uint8_t urg_data = 0;
791 
792  /* if stream_config.urgent_policy == TCP_STREAM_URGENT_DROP, we won't get here */
793  if (tcph->th_flags & TH_URG) {
794  const uint16_t urg_ptr = SCNtohs(tcph->th_urp);
795  if (urg_ptr > 0 && urg_ptr <= p->payload_len &&
798  /* track up to 64k out of band URG bytes. Fall back to inline
799  * when that budget is exceeded. */
800  if ((*urg_offset) < UINT16_MAX) {
802  (*urg_offset)++;
803 
804  if ((*urg_offset) == UINT16_MAX) {
806  }
807  } else {
808  /* OOB limit DROP is handled here */
811  SCReturnInt(0);
812  }
813  }
814  urg_data = 1; /* only treat last 1 byte as out of band. */
817  }
818 
819  /* depending on hitting the OOB limit, update urg_data or not */
821  (*urg_offset) == UINT16_MAX &&
823  urg_data = 0;
824  } else {
825  if (urg_ptr == 1 && p->payload_len == 1) {
826  SCLogDebug("no non-URG data");
827  SCReturnInt(0);
828  }
829  }
830  }
831  }
832 
833  const uint16_t payload_len = p->payload_len - urg_data;
834 
835  /* If we have reached the defined depth for either of the stream, then stop
836  reassembling the TCP session */
837  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, seg_seq, payload_len);
838  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
839 
842  /* increment stream depth counter */
845  }
846  if (size == 0) {
847  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
848  SCReturnInt(0);
849  }
850 
852  if (size > payload_len)
853  size = payload_len;
854 
855  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
856  if (seg == NULL) {
857  SCLogDebug("segment_pool is empty");
860  SCReturnInt(-1);
861  }
862 
863  DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
864  TCP_SEG_LEN(seg) = (uint16_t)size;
865  /* set SEQUENCE number, adjusted to any URG pointer offset */
866  seg->seq = seg_seq;
867 
868  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
869  if (TCP_HAS_TFO(p) && p->payload_len && (tcph->th_flags & TH_SYN))
870  seg->seq += 1;
871 
872  /* proto detection skipped, but now we do get data. Set event. */
873  if (RB_EMPTY(&stream->seg_tree) &&
875 
877  }
878 
879  int r = StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, p->payload, payload_len);
880  if (r < 0) {
881  if (r == -SC_ENOMEM) {
883  }
884  SCLogDebug("StreamTcpReassembleInsertSegment failed");
885  SCReturnInt(-1);
886  }
887  SCReturnInt(0);
888 }
889 
890 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
891  Packet *p)
892 {
893  uint8_t flag = 0;
894 
896  flag |= STREAM_START;
897  }
898 
899  if (ssn->state == TCP_CLOSED) {
900  flag |= STREAM_EOF;
901  }
902 
904  flag |= STREAM_MIDSTREAM;
905  }
906 
907  if (p->flags & PKT_PSEUDO_STREAM_END) {
908  flag |= STREAM_EOF;
909  }
910 
911  if (&ssn->client == stream) {
912  flag |= STREAM_TOSERVER;
913  } else {
914  flag |= STREAM_TOCLIENT;
915  }
917  flag |= STREAM_DEPTH;
918  }
919  return flag;
920 }
921 
922 /**
923  * \brief Check the minimum size limits for reassembly.
924  *
925  * \retval false don't reassemble yet
926  * \retval true do reassemble
927  */
928 static bool StreamTcpReassembleRawCheckLimit(
929  const TcpSession *ssn, const TcpStream *stream, const Packet *p)
930 {
931  SCEnter();
932 
933  /* if any of these flags is set we always inspect immediately */
934 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
935  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
936  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
937  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
938 
941  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
942  "is set, so not expecting any new data segments");
943  }
945  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
946  }
948  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
949  "so no new segments will be considered");
950  }
951  SCReturnBool(true);
952  }
953 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
954 
955  /* some states mean we reassemble no matter how much data we have */
956  if (ssn->state > TCP_TIME_WAIT)
957  SCReturnBool(true);
958 
960  SCReturnBool(true);
961 
962  const uint64_t last_ack_abs = GetAbsLastAck(stream);
963  int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream);
964  int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size
966 
967  /* check if we have enough data to do raw reassembly */
968  if (chunk_size <= diff) {
969  SCReturnBool(true);
970  } else {
971  SCLogDebug("%s min chunk len not yet reached: "
972  "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < "
973  "%" PRIi64,
974  PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq,
975  (stream->last_ack - stream->base_seq), chunk_size);
976  SCReturnBool(false);
977  }
978 
979  SCReturnBool(false);
980 }
981 
982 /**
983  * \brief see what if any work the TCP session still needs
984  */
985 uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
986 {
987  const TcpStream *stream = NULL;
988 #ifdef DEBUG
989  const char *dirstr = NULL;
990 #endif
991  if (direction == STREAM_TOSERVER) {
992  stream = &ssn->client;
993 #ifdef DEBUG
994  dirstr = "client";
995 #endif
996  } else {
997  stream = &ssn->server;
998 #ifdef DEBUG
999  dirstr = "server";
1000 #endif
1001  }
1002  int use_app = 1;
1003  int use_raw = 1;
1004 
1006  // app is dead
1007  use_app = 0;
1008  }
1009 
1010  if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
1011  // raw is dead
1012  use_raw = 0;
1013  }
1014  if (use_raw) {
1015  const uint64_t right_edge =
1016  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1017  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1018  " (use: %s). Stream right edge: %" PRIu64,
1019  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1020  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1021  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
1022  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1024  }
1025  }
1026  if (use_app) {
1027  const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
1028  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1029  " (use: %s). Stream right edge: %" PRIu64,
1030  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1031  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1032  if (right_edge > STREAM_APP_PROGRESS(stream)) {
1033  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1035  }
1036  }
1037 
1038  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
1040 }
1041 
1042 #ifdef DEBUG
1043 static uint64_t GetStreamSize(TcpStream *stream)
1044 {
1045  if (stream) {
1046  uint64_t size = 0;
1047  uint32_t cnt = 0;
1048  uint64_t last_ack_abs = GetAbsLastAck(stream);
1049  uint64_t last_re = 0;
1050 
1051  SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset);
1052 
1053  TcpSegment *seg;
1054  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
1055  const uint64_t seg_abs =
1056  STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq);
1057  if (last_re != 0 && last_re < seg_abs) {
1058  const char *gacked = NULL;
1059  if (last_ack_abs >= seg_abs) {
1060  gacked = "fully ack'd";
1061  } else if (last_ack_abs > last_re) {
1062  gacked = "partly ack'd";
1063  } else {
1064  gacked = "not yet ack'd";
1065  }
1066  SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked);
1067  }
1068 
1069  const char *acked = NULL;
1070  if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) {
1071  acked = "fully ack'd";
1072  } else if (last_ack_abs > seg_abs) {
1073  acked = "partly ack'd";
1074  } else {
1075  acked = "not yet ack'd";
1076  }
1077 
1078  SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64
1079  ") ack:%s",
1080  cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg),
1081  seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked);
1082  last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg);
1083  cnt++;
1084  size += (uint64_t)TCP_SEG_LEN(seg);
1085  }
1086 
1087  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
1088  return size;
1089  }
1090  return (uint64_t)0;
1091 }
1092 
1093 static void GetSessionSize(TcpSession *ssn, Packet *p)
1094 {
1095  uint64_t size = 0;
1096  if (ssn) {
1097  size = GetStreamSize(&ssn->client);
1098  size += GetStreamSize(&ssn->server);
1099 
1100  SCLogDebug("size %" PRIu64 ", packet %" PRIu64, size, PcapPacketCntGet(p));
1101  }
1102 }
1103 #endif
1104 
1105 static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk)
1106 {
1107  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
1108  /* only if the stream has been ack'd, consider a gap for sure
1109  * otherwise there may still be a chance of pkts coming in */
1110  return nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
1111  GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len);
1112 }
1113 
1114 /** \internal
1115  *
1116  * Get buffer, or first part of the buffer if data gaps exist.
1117  *
1118  * \brief get stream data from offset
1119  * \param offset stream offset
1120  * \param check_for_gap check if there is a gap ahead. Optional as it is only
1121  * needed for app-layer incomplete support.
1122  * \retval bool pkt loss ahead */
1123 static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1124  uint64_t offset, const bool check_for_gap)
1125 {
1126  const uint8_t *mydata;
1127  uint32_t mydata_len;
1128  bool gap_ahead = false;
1129 
1130  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1131  SCLogDebug("getting one blob");
1132 
1133  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1134 
1135  *data = mydata;
1136  *data_len = mydata_len;
1137  } else {
1138  SCLogDebug("block mode");
1139  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1140  StreamingBufferBlock *blk = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1141  if (blk == NULL) {
1142  *data = NULL;
1143  *data_len = 0;
1144  return false;
1145  }
1146  SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len);
1147 
1148  /* block at expected offset */
1149  if (blk->offset == offset) {
1150  SCLogDebug("blk at offset");
1151 
1152  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
1153  DEBUG_VALIDATE_BUG_ON(blk->len != *data_len);
1154 
1155  gap_ahead = check_for_gap && GapAhead(stream, blk);
1156 
1157  /* block past out offset */
1158  } else if (blk->offset > offset) {
1159  SCLogDebug("gap, want data at offset %"PRIu64", "
1160  "got data at %"PRIu64". GAP of size %"PRIu64,
1161  offset, blk->offset, blk->offset - offset);
1162  *data = NULL;
1163  *data_len = (uint32_t)(blk->offset - offset);
1164 
1165  /* block starts before offset, but ends after */
1166  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
1167  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
1168  offset, blk->offset, blk->len);
1169  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
1170  SCLogDebug("data %p, data_len %u", *data, *data_len);
1171 
1172  gap_ahead = check_for_gap && GapAhead(stream, blk);
1173 
1174  } else {
1175  *data = NULL;
1176  *data_len = 0;
1177  }
1178  }
1179  return gap_ahead;
1180 }
1181 
1182 /** \internal
1183  * \brief check to see if we should declare a GAP
1184  * Call this when the app layer didn't get data at the requested
1185  * offset.
1186  */
1187 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
1188 {
1189  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1190  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1191  const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded;
1192 
1193  SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs);
1194  SCLogDebug("next_seq %u", stream->next_seq);
1195 
1196  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1197  * has been ack'd. This looks like a GAP. */
1198  if (last_ack_abs > app_progress) {
1199  /* however, we can accept ACKs a bit too liberally. If last_ack
1200  * is beyond next_seq, we only consider it a gap now if we do
1201  * already have data beyond the gap. */
1202  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1203  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1204  SCLogDebug("packet %" PRIu64 ": no GAP. "
1205  "next_seq %u < last_ack %u, but no data in list",
1206  PcapPacketCntGet(p), stream->next_seq, stream->last_ack);
1207  return false;
1208  }
1209  const uint64_t next_seq_abs =
1210  STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1211  const StreamingBufferBlock *blk = stream->sb.head;
1212  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1213  /* ack'd data after the gap */
1214  SCLogDebug("packet %" PRIu64 ": GAP. "
1215  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1216  PcapPacketCntGet(p), stream->next_seq, stream->last_ack);
1217  return true;
1218  }
1219  }
1220 
1221  SCLogDebug("packet %" PRIu64 ": GAP! "
1222  "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", "
1223  "but we have no data.",
1224  PcapPacketCntGet(p), last_ack_abs, app_progress);
1225  return true;
1226  }
1227  SCLogDebug("packet %" PRIu64 ": no GAP. "
1228  "last_ack_abs %" PRIu64 " <= app_progress %" PRIu64,
1229  PcapPacketCntGet(p), last_ack_abs, app_progress);
1230  return false;
1231 }
1232 
1233 static inline uint32_t AdjustToAcked(const Packet *p,
1234  const TcpSession *ssn, const TcpStream *stream,
1235  const uint64_t app_progress, const uint32_t data_len)
1236 {
1237  uint32_t adjusted = data_len;
1238 
1239  /* get window of data that is acked */
1240  if (!StreamTcpInlineMode()) {
1241  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1242  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1243  (ssn->state == TCP_CLOSED &&
1244  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1245  (p->flags & PKT_PSEUDO_STREAM_END))) {
1246  // fall through, we use all available data
1247  } else {
1248  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1249  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1250 
1251  /* see if the buffer contains unack'd data as well */
1252  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1253  adjusted = (uint32_t)(last_ack_abs - app_progress);
1254  DEBUG_VALIDATE_BUG_ON(adjusted > data_len);
1255  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1256  "data is considered", adjusted);
1257  }
1258  }
1259  }
1260  return adjusted;
1261 }
1262 
1263 /** \internal
1264  * \brief get stream buffer and update the app-layer
1265  * \param stream pointer to pointer as app-layer can switch flow dir
1266  * \retval 0 success
1267  */
1268 static int ReassembleUpdateAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn,
1269  TcpStream **stream, Packet *p, enum StreamUpdateDir app_update_dir)
1270 {
1271  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1272 
1273  SCLogDebug("app progress %"PRIu64, app_progress);
1274 #ifdef DEBUG
1275  uint64_t last_ack_abs = GetAbsLastAck(*stream);
1276  SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs,
1277  (*stream)->base_seq);
1278 #endif
1279  const uint8_t *mydata;
1280  uint32_t mydata_len;
1281  bool last_was_gap = false;
1282 
1283  while (1) {
1284  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1285  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1286  bool gap_ahead =
1287  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead);
1288  SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len);
1289  if (last_was_gap && mydata_len == 0) {
1290  break;
1291  }
1292  last_was_gap = false;
1293 
1294  /* make sure to only deal with ACK'd data */
1295  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1296  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1297  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1298  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1299 
1300  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len,
1301  StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, app_update_dir);
1302  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1303 
1305  (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP;
1308 
1309  /* AppLayerHandleTCPData has likely updated progress. */
1310  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1311  app_progress = STREAM_APP_PROGRESS(*stream);
1312 
1313  /* a GAP also consumes 'data required'. TODO perhaps we can use
1314  * this to skip post GAP data until the start of a next record. */
1315  if ((*stream)->data_required > 0) {
1316  if ((*stream)->data_required > mydata_len) {
1317  (*stream)->data_required -= mydata_len;
1318  } else {
1319  (*stream)->data_required = 0;
1320  }
1321  }
1322  if (r < 0)
1323  return 0;
1324  if (no_progress_update)
1325  break;
1326  last_was_gap = true;
1327  continue;
1328 
1329  } else if (flags & STREAM_DEPTH) {
1330  SCLogDebug("DEPTH");
1331  // we're just called once with this flag, so make sure we pass it on
1332  if (mydata == NULL && mydata_len > 0) {
1333  mydata_len = 0;
1334  }
1335  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1336  SCLogDebug("GAP?1");
1337  /* Possibly a gap, but no new data. */
1338  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1339  SCReturnInt(0);
1340 
1341  mydata = NULL;
1342  mydata_len = 0;
1343  SCLogDebug("%" PRIu64 " got %p/%u", PcapPacketCntGet(p), mydata, mydata_len);
1344  break;
1345  }
1346  DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1347 
1348  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1349  *stream, &(*stream)->sb, mydata_len, app_progress);
1350 
1351  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1352  SCLogDebug("GAP?2");
1353  if (mydata_len < (*stream)->data_required) {
1354  SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead));
1355  if (gap_ahead) {
1356  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1357  (*stream)->data_required, mydata_len);
1358  (*stream)->app_progress_rel += mydata_len;
1359  (*stream)->data_required -= mydata_len;
1360  // TODO send incomplete data to app-layer with special flag
1361  // indicating its all there is for this rec?
1362  } else {
1363  SCReturnInt(0);
1364  }
1365  app_progress = STREAM_APP_PROGRESS(*stream);
1366  continue;
1367  }
1368  }
1369  (*stream)->data_required = 0;
1370 
1371  SCLogDebug("parser");
1372  /* update the app-layer */
1373  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata,
1374  mydata_len, flags, app_update_dir);
1375  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1377  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1378  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1379  break;
1380  app_progress = new_app_progress;
1381  if (flags & STREAM_DEPTH)
1382  break;
1383  }
1384 
1385  SCReturnInt(0);
1386 }
1387 
1388 /**
1389  * \brief Update the stream reassembly upon receiving a packet.
1390  *
1391  * For IDS mode, the stream is in the opposite direction of the packet,
1392  * as the ACK-packet is ACK'ing the stream.
1393  *
1394  * One of the utilities call by this function AppLayerHandleTCPData(),
1395  * has a feature where it will call this very same function for the
1396  * stream opposing the stream it is called with. This shouldn't cause
1397  * any issues, since processing of each stream is independent of the
1398  * other stream.
1399  */
1401  TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
1402 {
1403  SCEnter();
1404 
1405  /* this function can be directly called by app layer protocol
1406  * detection. */
1409  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1410  SCReturnInt(0);
1411  }
1412 
1413 #ifdef DEBUG
1414  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1415  GetSessionSize(ssn, p);
1416 #endif
1417  /* if no segments are in the list or all are already processed,
1418  * and state is beyond established, we send an empty msg */
1419  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1420  {
1421  /* send an empty EOF msg if we have no segments but TCP state
1422  * is beyond ESTABLISHED */
1423  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1424  SCLogDebug("sending empty eof message");
1425  /* send EOF to app layer */
1426  uint8_t stream_flags = StreamGetAppLayerFlags(ssn, stream, p);
1428  tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, stream_flags, app_update_dir);
1429  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1430 
1431  SCReturnInt(0);
1432  }
1433  }
1434 
1435  /* with all that out of the way, lets update the app-layer */
1436  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, app_update_dir);
1437 }
1438 
1439 /** \internal
1440  * \brief get stream data from offset
1441  * \param offset stream offset */
1442 static int GetRawBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1443  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1444 {
1445  const uint8_t *mydata;
1446  uint32_t mydata_len;
1447  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1448  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1449 
1450  /* No gaps in the stream, data must exist in the streaming buffer array */
1451  uint64_t roffset = offset;
1452  if (offset) {
1453  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1454  } else {
1455  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1456  }
1457 
1458  *data = mydata;
1459  *data_len = mydata_len;
1460  *data_offset = roffset;
1461  } else {
1462  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1463  *iter == NULL ? "starting" : "continuing", offset);
1464  if (*iter == NULL) {
1465  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1466  *iter = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1467  SCLogDebug("*iter %p", *iter);
1468  }
1469  if (*iter == NULL) {
1470  SCLogDebug("no data");
1471  *data = NULL;
1472  *data_len = 0;
1473  *data_offset = 0;
1474  return 0;
1475  }
1476  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1477 
1478  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1479  SCLogDebug("mydata %p", mydata);
1480 
1481  if ((*iter)->offset < offset) {
1482  uint64_t delta = offset - (*iter)->offset;
1483  if (delta < mydata_len) {
1484  *data = mydata + delta;
1485  *data_len = (uint32_t)(mydata_len - delta);
1486  *data_offset = offset;
1487  } else {
1488  SCLogDebug("no data (yet)");
1489  *data = NULL;
1490  *data_len = 0;
1491  *data_offset = 0;
1492  }
1493 
1494  } else {
1495  *data = mydata;
1496  *data_len = mydata_len;
1497  *data_offset = (*iter)->offset;
1498  }
1499 
1500  *iter = SBB_RB_NEXT(*iter);
1501  SCLogDebug("*iter %p", *iter);
1502  }
1503  return 0;
1504 }
1505 
1506 /** \brief does the stream engine have data to inspect?
1507  *
1508  * Returns true if there is data to inspect. In IDS case this is
1509  * about ACK'd data in the packet's direction.
1510  *
1511  * In the IPS case this is about the packet itself.
1512  */
1514 {
1515  TcpStream *stream;
1516  if (PKT_IS_TOSERVER(p)) {
1517  stream = &ssn->client;
1518  } else {
1519  stream = &ssn->server;
1520  }
1521 
1522  if (RB_EMPTY(&stream->seg_tree)) {
1523  return false;
1524  }
1525 
1528  return false;
1529 
1530  if (!StreamTcpInlineMode()) {
1531  const uint64_t segs_re_abs =
1532  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1533  if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) {
1534  return false;
1535  }
1536  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1537  return true;
1538  }
1539  } else {
1540  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1541  return true;
1542  }
1543  }
1544  return false;
1545 }
1546 
1547 /** \brief update stream engine after detection
1548  *
1549  * Tasked with progressing the 'progress' for Raw reassembly.
1550  * 2 main scenario's:
1551  * 1. progress is != 0, so we use this
1552  * 2. progress is 0, meaning the detect engine didn't touch
1553  * raw at all. In this case we need to look into progressing
1554  * raw anyway.
1555  *
1556  * Additionally, this function is tasked with disabling raw
1557  * reassembly if the app-layer requested to disable it.
1558  */
1559 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
1560 {
1561  TcpStream *stream;
1562  if (PKT_IS_TOSERVER(p)) {
1563  stream = &ssn->client;
1564  } else {
1565  stream = &ssn->server;
1566  }
1567 
1568  if (progress > STREAM_RAW_PROGRESS(stream)) {
1569  DEBUG_VALIDATE_BUG_ON(progress - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1570  uint32_t slide = (uint32_t)(progress - STREAM_RAW_PROGRESS(stream));
1571  stream->raw_progress_rel += slide;
1573 
1574  } else if (progress == 0) {
1575  uint64_t target;
1577  target = STREAM_APP_PROGRESS(stream);
1578  } else {
1579  target = GetAbsLastAck(stream);
1580  }
1581  if (target > STREAM_RAW_PROGRESS(stream)) {
1582  DEBUG_VALIDATE_BUG_ON(target - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1583  uint32_t slide = (uint32_t)(target - STREAM_RAW_PROGRESS(stream));
1584  stream->raw_progress_rel += slide;
1585  }
1587 
1588  } else {
1589  SCLogDebug("pcap_cnt %" PRIu64 ": progress %" PRIu64 " app %" PRIu64 " raw %" PRIu64
1590  " tcp win %" PRIu32,
1591  PcapPacketCntGet(p), progress, STREAM_APP_PROGRESS(stream),
1592  STREAM_RAW_PROGRESS(stream), stream->window);
1593  }
1594 
1595  /* if we were told to accept no more raw data, we can mark raw as
1596  * disabled now. */
1599  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1600  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1601  }
1602 
1603  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1604 }
1605 
1606 /** \internal
1607  * \brief get a buffer around the current packet and run the callback on it
1608  *
1609  * The inline/IPS scanning method takes the current payload and wraps it in
1610  * data from other segments.
1611  *
1612  * How much data is inspected is controlled by the available data, chunk_size
1613  * and the payload size of the packet.
1614  *
1615  * Large packets: if payload size is close to the chunk_size, where close is
1616  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1617  * used: payload_len + 33% of the chunk_size.
1618  * If the payload size if equal to or bigger than the chunk_size, we use
1619  * payload len + 33% of the chunk size.
1620  */
1621 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1622  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1623 {
1624  SCEnter();
1625  int r = 0;
1626 
1627  TcpStream *stream;
1628  if (PKT_IS_TOSERVER(p)) {
1629  stream = &ssn->client;
1630  } else {
1631  stream = &ssn->server;
1632  }
1633 
1634  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1636  {
1637  *progress_out = STREAM_RAW_PROGRESS(stream);
1638  return 0;
1639  }
1640 
1641  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1644  if ((chunk_size <= p->payload_len) || (((chunk_size / 3) * 2) < p->payload_len)) {
1645  chunk_size = p->payload_len + (chunk_size / 3);
1646  SCLogDebug(
1647  "packet payload len %u, so chunk_size adjusted to %u", p->payload_len, chunk_size);
1648  }
1649 
1650  const TCPHdr *tcph = PacketGetTCP(p);
1651  /* use packet SEQ, but account for TFO needing a + 1 */
1652  const uint32_t tfo_add = (TCP_HAS_TFO(p));
1653  const uint32_t pkt_seq = TCP_GET_RAW_SEQ(tcph) + tfo_add;
1654  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (pkt_seq - stream->base_seq);
1655  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1656  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1657  packet_leftedge_abs, packet_rightedge_abs);
1658 
1659  const uint8_t *mydata = NULL;
1660  uint32_t mydata_len = 0;
1661  uint64_t mydata_offset = 0;
1662  /* simply return progress from the block we inspected. */
1663  bool return_progress = false;
1664 
1665  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1666  /* continues block */
1667  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1668  return_progress = true;
1669 
1670  } else {
1671  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1672  /* find our block */
1673  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1674  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1675  if (sbb) {
1676  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1677  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1678  mydata_offset = sbb->offset;
1679  }
1680  }
1681 
1682  /* this can only happen if the segment insert of our current 'p' failed */
1683  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1684  if ((mydata == NULL || mydata_len == 0) || /* no data */
1685  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1686  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1687  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1688  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1689  {
1690  /* no data, or data is incomplete or wrong: use packet data */
1691  mydata = p->payload;
1692  mydata_len = p->payload_len;
1693  mydata_offset = packet_leftedge_abs;
1694  //mydata_rightedge_abs = packet_rightedge_abs;
1695  } else {
1696  /* adjust buffer to match chunk_size */
1697  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1698  if (mydata_len > chunk_size) {
1699  uint32_t excess = mydata_len - chunk_size;
1700  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1701 
1702  if (mydata_rightedge_abs == packet_rightedge_abs) {
1703  mydata += excess;
1704  mydata_len -= excess;
1705  mydata_offset += excess;
1706  SCLogDebug("cutting front of the buffer with %u", excess);
1707  } else if (mydata_offset == packet_leftedge_abs) {
1708  mydata_len -= excess;
1709  SCLogDebug("cutting tail of the buffer with %u", excess);
1710  } else {
1711  DEBUG_VALIDATE_BUG_ON(mydata_offset > packet_leftedge_abs);
1712  uint32_t abs_before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1713  DEBUG_VALIDATE_BUG_ON(packet_rightedge_abs > mydata_rightedge_abs);
1714  uint32_t abs_after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1715  uint32_t before = abs_before;
1716  uint32_t after = abs_after;
1717  SCLogDebug("before %u after %u", before, after);
1718 
1719  if (after >= (chunk_size - p->payload_len) / 2) {
1720  // more trailing data than we need
1721 
1722  if (before >= (chunk_size - p->payload_len) / 2) {
1723  // also more heading data, divide evenly
1724  before = after = (chunk_size - p->payload_len) / 2;
1725  } else {
1726  // heading data is less than requested, give the
1727  // rest to the trailing data
1728  after = (chunk_size - p->payload_len) - before;
1729  }
1730  } else {
1731  // less trailing data than requested
1732 
1733  if (before >= (chunk_size - p->payload_len) / 2) {
1734  before = (chunk_size - p->payload_len) - after;
1735  } else {
1736  // both smaller than their requested size
1737  }
1738  }
1739 
1740  /* adjust the buffer */
1741  DEBUG_VALIDATE_BUG_ON(before > abs_before);
1742  uint32_t skip = abs_before - before;
1743  DEBUG_VALIDATE_BUG_ON(after > abs_after);
1744  uint32_t cut = abs_after - after;
1745  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1746  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1747  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1748  mydata += skip;
1749  mydata_len -= (skip + cut);
1750  mydata_offset += skip;
1751  }
1752  }
1753  }
1754 
1755  /* run the callback */
1756  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1757  DEBUG_VALIDATE_BUG_ON(r < 0);
1758 
1759  if (return_progress) {
1760  *progress_out = (mydata_offset + mydata_len);
1761  } else {
1762  /* several blocks of data, so we need to be a bit more careful:
1763  * - if last_ack is beyond last progress, move progress forward to last_ack
1764  * - if our block matches or starts before last ack, return right edge of
1765  * our block.
1766  */
1767  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1768  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1769 
1770  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1771  if (mydata_offset > last_ack_abs) {
1772  /* gap between us and last ack, set progress to last ack */
1773  *progress_out = last_ack_abs;
1774  } else {
1775  *progress_out = (mydata_offset + mydata_len);
1776  }
1777  } else {
1778  *progress_out = STREAM_RAW_PROGRESS(stream);
1779  }
1780  }
1781  return r;
1782 }
1783 
1784 /** \brief access 'raw' reassembly data.
1785  *
1786  * Access data as tracked by 'raw' tracker. Data is made available to
1787  * callback that is passed to this function.
1788  *
1789  * In the case of IDS the callback may be run multiple times if data
1790  * contains gaps. It will then be run for each block of data that is
1791  * continuous.
1792  *
1793  * The callback should give on of 2 return values:
1794  * - 0 ok
1795  * - 1 done
1796  * The value 1 will break the loop if there is a block list that is
1797  * inspected.
1798  *
1799  * This function will return the 'progress' value that has been
1800  * consumed until now.
1801  *
1802  * \param ssn tcp session
1803  * \param stream tcp stream
1804  * \param Callback the function pointer to the callback function
1805  * \param cb_data callback data
1806  * \param[in] progress_in progress to work from
1807  * \param[in] re right edge of data to consider
1808  * \param[out] progress_out absolute progress value of the data this
1809  * call handled.
1810  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1811  */
1812 static int StreamReassembleRawDo(const TcpSession *ssn, const TcpStream *stream,
1813  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1814  const uint64_t re, uint64_t *progress_out, bool eof)
1815 {
1816  SCEnter();
1817  int r = 0;
1818 
1819  StreamingBufferBlock *iter = NULL;
1820  uint64_t progress = progress_in;
1821 
1822  /* loop through available buffers. On no packet loss we'll have a single
1823  * iteration. On missing data we'll walk the blocks */
1824  while (1) {
1825  const uint8_t *mydata;
1826  uint32_t mydata_len;
1827  uint64_t mydata_offset = 0;
1828 
1829  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1830  if (mydata_len == 0) {
1831  SCLogDebug("no data");
1832  break;
1833  }
1834  //PrintRawDataFp(stdout, mydata, mydata_len);
1835 
1836  SCLogDebug("raw progress %"PRIu64, progress);
1837  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1838  stream, &stream->sb, mydata_len, progress);
1839 
1840  if (eof) {
1841  // inspect all remaining data, ack'd or not
1842  } else {
1843  if (re < progress) {
1844  SCLogDebug("nothing to do");
1845  goto end;
1846  }
1847 
1848  SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress);
1849  SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len,
1850  re);
1851 
1852  /* see if the buffer contains unack'd data as well */
1853  if (progress + mydata_len > re) {
1854 #ifdef DEBUG_VALIDATION
1855  uint32_t check = mydata_len;
1856 #endif
1857  mydata_len = (uint32_t)(re - progress);
1858  DEBUG_VALIDATE_BUG_ON(check < mydata_len);
1859  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1860  "data is considered", mydata_len);
1861  }
1862  }
1863  if (mydata_len == 0)
1864  break;
1865 
1866  SCLogDebug("data %p len %u", mydata, mydata_len);
1867 
1868  /* we have data. */
1869  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1870  DEBUG_VALIDATE_BUG_ON(r < 0);
1871 
1872  if (mydata_offset == progress) {
1873  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1874  progress, mydata_len, progress_in + mydata_len);
1875 
1876  progress += mydata_len;
1877  SCLogDebug("raw progress now %"PRIu64, progress);
1878 
1879  /* data is beyond the progress we'd like, and before last ack. Gap. */
1880  } else if (mydata_offset > progress && mydata_offset < re) {
1881  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1882  SCLogDebug("re %" PRIu64, re);
1883  progress = mydata_offset;
1884  SCLogDebug("raw progress now %"PRIu64, progress);
1885 
1886  /* data is beyond the progress we'd like, and also beyond the last ack:
1887  * there is a gap and we can't expect it to get filled anymore. */
1888  } else if (mydata_offset > progress && mydata_offset == re) {
1889  SCLogDebug("mydata_offset %" PRIu64 ", progress %" PRIu64 ", re %" PRIu64,
1890  mydata_offset, progress, re);
1891  progress = re;
1892  } else {
1893  SCLogDebug("not increasing progress, data gap => mydata_offset "
1894  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1895  }
1896 
1897  if (iter == NULL || r == 1)
1898  break;
1899  }
1900 end:
1901  *progress_out = progress;
1902  return r;
1903 }
1904 
1906  void *cb_data, const uint64_t offset, const bool eof)
1907 {
1908  /* take app progress as the right edge of used data. */
1909  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1910  SCLogDebug("app_progress %" PRIu64, app_progress);
1911 
1912  uint64_t unused = 0;
1913  return StreamReassembleRawDo(
1914  ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof);
1915 }
1916 
1918  StreamReassembleRawFunc Callback, void *cb_data,
1919  uint64_t *progress_out, bool respect_inspect_depth)
1920 {
1921  /* handle inline separately as the logic is very different */
1922  if (StreamTcpInlineMode()) {
1923  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1924  }
1925 
1926  TcpStream *stream;
1927  if (PKT_IS_TOSERVER(p)) {
1928  stream = &ssn->client;
1929  } else {
1930  stream = &ssn->server;
1931  }
1932 
1934  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1935  {
1936  *progress_out = STREAM_RAW_PROGRESS(stream);
1937  return 0;
1938  }
1939 
1940  uint64_t progress = STREAM_RAW_PROGRESS(stream);
1941  /* if the app layer triggered a flush, and we're supposed to
1942  * use a minimal inspect depth, we actually take the app progress
1943  * as that is the right edge of the data. Then we take the window
1944  * of 'min_inspect_depth' before that. */
1945 
1946  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s "
1947  "stream->min_inspect_depth %u",
1948  respect_inspect_depth ? "true" : "false",
1949  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1950  stream->min_inspect_depth);
1951 
1952  if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) &&
1953  stream->min_inspect_depth) {
1954  progress = STREAM_APP_PROGRESS(stream);
1955  if (stream->min_inspect_depth >= progress) {
1956  progress = 0;
1957  } else {
1958  progress -= stream->min_inspect_depth;
1959  }
1960 
1961  SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream),
1962  STREAM_RAW_PROGRESS(stream));
1963 
1964  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1965  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress "
1966  "%" PRIu64,
1967  progress);
1968  }
1969 
1970  SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth,
1971  stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW"
1972  : "(no trigger)");
1973 
1974  /* absolute right edge of ack'd data */
1975  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1976  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1977 
1978  return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs,
1979  progress_out, (p->flags & PKT_PSEUDO_STREAM_END));
1980 }
1981 
1982 int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream,
1983  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1984  uint64_t *progress_out, const bool eof)
1985 {
1986  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1987  return 0;
1988 
1989  /* absolute right edge of ack'd data */
1990  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1991  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1992 
1993  return StreamReassembleRawDo(
1994  ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof);
1995 }
1996 
1997 /** \internal
1998  * \brief update app layer based on received ACK
1999  *
2000  * \retval r 0 on success, -1 on error
2001  */
2002 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
2003  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
2004 {
2005  SCEnter();
2006 
2007  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
2008  SCReturnInt(-1);
2009 
2010  SCReturnInt(0);
2011 }
2012 
2014  TcpSession *ssn, TcpStream *stream, Packet *p)
2015 {
2016  SCEnter();
2017 
2018  DEBUG_VALIDATE_BUG_ON(!PacketIsTCP(p));
2019  const TCPHdr *tcph = PacketGetTCP(p);
2020 
2021  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
2022  ssn, stream, p, p->payload_len);
2023 
2024  /* default IDS: update opposing side (triggered by ACK) */
2026  /* inline and stream end and flow timeout packets trigger same dir handling */
2027  if (StreamTcpInlineMode()) {
2028  dir = UPDATE_DIR_PACKET;
2029  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
2030  dir = UPDATE_DIR_PACKET;
2031  } else if (tcph->th_flags & TH_RST) { // accepted rst
2032  dir = UPDATE_DIR_PACKET;
2033  } else if ((tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
2034  if (tcph->th_flags & TH_ACK) {
2035  dir = UPDATE_DIR_BOTH;
2036  } else {
2037  dir = UPDATE_DIR_PACKET;
2038  }
2039  } else if (ssn->state == TCP_CLOSED) {
2040  dir = UPDATE_DIR_BOTH;
2041  } else if ((ssn->flags & STREAMTCP_FLAG_ASYNC) != 0) {
2042  dir = UPDATE_DIR_PACKET;
2043  SCLogDebug("%" PRIu64 ": ASYNC: UPDATE_DIR_PACKET", PcapPacketCntGet(p));
2044  }
2045 
2046  /* handle ack received */
2047  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
2048  /* we need to update the opposing stream in
2049  * StreamTcpReassembleHandleSegmentUpdateACK */
2050  TcpStream *opposing_stream = NULL;
2051  if (stream == &ssn->client) {
2052  opposing_stream = &ssn->server;
2053  } else {
2054  opposing_stream = &ssn->client;
2055  }
2056 
2057  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2058 
2059  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
2060  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
2061  SCReturnInt(-1);
2062  }
2063 
2064  /* StreamTcpReassembleHandleSegmentUpdateACK
2065  * may swap content of ssn->server and ssn->client structures.
2066  * We have to continue with initial content of the stream in such case */
2067  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2068  if (reversed_before_ack_handling != reversed_after_ack_handling) {
2069  SCLogDebug("TCP streams were swapped");
2070  stream = opposing_stream;
2071  }
2072  }
2073  /* if this segment contains data, insert it */
2074  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) &&
2075  (tcph->th_flags & TH_RST) == 0) {
2076  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
2077 
2078  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
2079  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
2080  /* failure can only be because of memcap hit, so see if this should lead to a drop */
2083  StreamTcpReassembleExceptionPolicyStatsIncr(
2085  SCReturnInt(-1);
2086  }
2087 
2088  SCLogDebug("packet %" PRIu64 " set PKT_STREAM_ADD", PcapPacketCntGet(p));
2089  p->flags |= PKT_STREAM_ADD;
2090  } else {
2091  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
2092  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
2093  ssn, stream, p->payload_len,
2094  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
2095  }
2096 
2097  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
2098  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
2099  * was *just* set. In this case we trigger the AppLayer Truncate
2100  * logic, to inform the applayer no more data in this direction is
2101  * to be expected. */
2102  if ((stream->flags &
2105  {
2106  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
2107  if (dir != UPDATE_DIR_PACKET) {
2108  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
2109  "can trigger Truncate");
2110  dir = UPDATE_DIR_PACKET;
2111  }
2112  }
2113 
2114  /* in stream inline mode even if we have no data we call the reassembly
2115  * functions to handle EOF */
2116  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
2117  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
2118  StreamTcpInlineMode()?"true":"false",
2119  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
2120  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
2121  SCReturnInt(-1);
2122  }
2123  }
2124 
2125  SCReturnInt(0);
2126 }
2127 
2128 /**
2129  * \brief get a segment from the pool
2130  *
2131  * \retval seg Segment from the pool or NULL
2132  */
2134 {
2136  if (seg) {
2138  return seg;
2139  }
2140 
2141  seg = (TcpSegment *)PoolThreadGetById(
2142  segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id);
2143  SCLogDebug("seg we return is %p", seg);
2144  if (seg == NULL) {
2145  /* Increment the counter to show that we are not able to serve the
2146  segment request due to memcap limit */
2148  } else {
2149  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2151  }
2152 
2153  return seg;
2154 }
2155 
2156 /**
2157  * \brief Trigger RAW stream inspection
2158  *
2159  * Used by AppLayerTriggerRawStreamInspection to trigger RAW stream
2160  * inspection from the applayer, for example upon completion of a
2161  * HTTP request.
2162  *
2163  * It sets a flag in the stream so that the next Raw call will return
2164  * the data.
2165  *
2166  * \param ssn TcpSession
2167  */
2169 {
2170 #ifdef DEBUG
2171  BUG_ON(ssn == NULL);
2172 #endif
2173 
2174  if (ssn != NULL) {
2175  if (direction == STREAM_TOSERVER) {
2177  } else {
2179  }
2180 
2181  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
2182  }
2183 }
2184 
2185 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
2186 {
2187 #ifdef DEBUG
2188  BUG_ON(ssn == NULL);
2189 #endif
2190 
2191  if (ssn != NULL) {
2192  if (direction == STREAM_TOSERVER) {
2193  ssn->client.min_inspect_depth = depth;
2194  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
2195  } else {
2196  ssn->server.min_inspect_depth = depth;
2197  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
2198  }
2199  }
2200 }
2201 
2202 #ifdef UNITTESTS
2203 /** unit tests and it's support functions below */
2204 
2205 #define SET_ISN(stream, setseq) \
2206  (stream)->isn = (setseq); \
2207  (stream)->base_seq = (setseq) + 1
2208 
2209 /** \brief The Function to create the packet with given payload, which is used
2210  * to test the reassembly of the engine.
2211  *
2212  * \param payload The variable used to store the payload contents of the
2213  * current packet.
2214  * \param value The value which current payload will have for this packet
2215  * \param payload_len The length of the filed payload for current packet.
2216  * \param len Length of the payload array
2217  */
2218 
2219 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2220  uint8_t payload_len, uint8_t len)
2221 {
2222  uint8_t i;
2223  for (i = 0; i < payload_len; i++)
2224  payload[i] = value;
2225  for (; i < len; i++)
2226  payload = NULL;
2227 }
2228 
2229 /** \brief The Function Checks the reassembled stream contents against predefined
2230  * stream contents according to OS policy used.
2231  *
2232  * \param stream_policy Predefined value of stream for different OS policies
2233  * \param stream Reassembled stream returned from the reassembly functions
2234  */
2235 
2236 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2237 {
2238  if (StreamingBufferCompareRawData(&stream->sb,
2239  data, data_len) == 0)
2240  {
2241  SCReturnInt(0);
2242  }
2243  SCLogInfo("OK");
2244  PrintRawDataFp(stdout, data, data_len);
2245  return 1;
2246 }
2247 
2248 #define MISSED_START(isn) \
2249  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2250  TcpSession ssn; \
2251  ThreadVars tv; \
2252  memset(&tv, 0, sizeof(tv)); \
2253  \
2254  StreamTcpUTInit(&ra_ctx); \
2255  \
2256  StreamTcpUTSetupSession(&ssn); \
2257  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2258  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2259  \
2260  TcpStream *stream = &ssn.client;
2261 
2262 #define MISSED_END \
2263  StreamTcpUTClearSession(&ssn); \
2264  StreamTcpUTDeinit(ra_ctx); \
2265  PASS
2266 
2267 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2268  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2269  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2270 
2271 #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \
2272  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen));
2273 
2274 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len);
2275 
2276 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
2277 {
2278  int cnt = 0;
2279  uint64_t last_re = 0;
2280  StreamingBufferBlock *sbb = NULL;
2281  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2282  {
2283  if (sbb->offset != last_re) {
2284  // gap before us
2285  if (cnt == pos && last_re == offset && len == sbb->offset - last_re) {
2286  return 1;
2287  }
2288  cnt++;
2289  }
2290  last_re = sbb->offset + sbb->len;
2291  cnt++;
2292  }
2293  return 0;
2294 }
2295 
2297  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len);
2298 
2300  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
2301 {
2302  int cnt = 0;
2303  uint64_t last_re = 0;
2304  StreamingBufferBlock *sbb = NULL;
2305  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2306  {
2307  if (sbb->offset != last_re) {
2308  // gap before us
2309  cnt++;
2310  }
2311 
2312  if (cnt == pos && sbb->offset == offset) {
2313  const uint8_t *buf = NULL;
2314  uint32_t buf_len = 0;
2315  StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len);
2316 
2317  if (len == buf_len) {
2318  return (memcmp(data, buf, len) == 0);
2319  }
2320  }
2321 
2322  last_re = sbb->offset + sbb->len;
2323  cnt++;
2324  }
2325  return 0;
2326 }
2327 
2328 /**
2329  * \test Test the handling of packets missed by both IDS and the end host.
2330  * The packet is missed in the starting of the stream.
2331  *
2332  * \retval On success it returns 1 and on failure 0.
2333  */
2334 
2335 static int StreamTcpReassembleTest25 (void)
2336 {
2337  MISSED_START(6);
2338  MISSED_ADD_PAYLOAD(10, "BB", 2);
2339  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2340  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BB", 2) == 1);
2341  MISSED_ADD_PAYLOAD(12, "CC", 2);
2342  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2343  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BBCC", 4) == 1);
2344  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2345  MISSED_END;
2346  PASS;
2347 }
2348 
2349 /**
2350  * \test Test the handling of packets missed by both IDS and the end host.
2351  * The packet is missed in the middle of the stream.
2352  *
2353  * \retval On success it returns 1 and on failure 0.
2354  */
2355 
2356 static int StreamTcpReassembleTest26 (void)
2357 {
2358  MISSED_START(9);
2359  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2360  MISSED_ADD_PAYLOAD(15, "CC", 2);
2361  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2362  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2363  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2364  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2365  MISSED_END;
2366 }
2367 
2368 /**
2369  * \test Test the handling of packets missed by both IDS and the end host.
2370  * The packet is missed in the end of the stream.
2371  *
2372  * \retval On success it returns 1 and on failure 0.
2373  */
2374 
2375 static int StreamTcpReassembleTest27 (void)
2376 {
2377  MISSED_START(9);
2378  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2379  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2380  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2381  MISSED_END;
2382 }
2383 
2384 /**
2385  * \test Test the handling of packets missed by IDS, but the end host has
2386  * received it and send the acknowledgment of it. The packet is missed
2387  * in the starting of the stream.
2388  *
2389  * \retval On success it returns 1 and on failure 0.
2390  */
2391 
2392 static int StreamTcpReassembleTest28 (void)
2393 {
2394  MISSED_START(6);
2395  MISSED_ADD_PAYLOAD(10, "AAA", 3);
2396  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2397  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAA", 3) == 1);
2398  MISSED_ADD_PAYLOAD(13, "BB", 2);
2399  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2400  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABB", 5) == 1);
2402  MISSED_ADD_PAYLOAD(15, "CC", 2);
2403  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2404  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABBCC", 7) == 1);
2405  MISSED_END;
2406 }
2407 
2408 /**
2409  * \test Test the handling of packets missed by IDS, but the end host has
2410  * received it and send the acknowledgment of it. The packet is missed
2411  * in the middle of the stream.
2412  *
2413  * \retval On success it returns 1 and on failure 0.
2414  */
2415 
2416 static int StreamTcpReassembleTest29 (void)
2417 {
2418  MISSED_START(9);
2419  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2421  MISSED_ADD_PAYLOAD(15, "CC", 2);
2422  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2423  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2424  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2425  MISSED_END;
2426 }
2427 
2428 static int StreamTcpReassembleTest33(void)
2429 {
2430  TcpSession ssn;
2432  FAIL_IF(unlikely(p == NULL));
2433  Flow f;
2434  TCPHdr tcph;
2435  TcpReassemblyThreadCtx *ra_ctx = NULL;
2437  uint8_t packet[1460] = "";
2438 
2439  StreamTcpUTInit(&ra_ctx);
2441 
2442  memset(&f, 0, sizeof (Flow));
2443  memset(&tcph, 0, sizeof (TCPHdr));
2444  ThreadVars tv;
2445  memset(&tv, 0, sizeof (ThreadVars));
2446  FLOW_INITIALIZE(&f);
2447  f.protoctx = &ssn;
2448  f.proto = IPPROTO_TCP;
2449  p->src.family = AF_INET;
2450  p->dst.family = AF_INET;
2451  p->proto = IPPROTO_TCP;
2452  p->flow = &f;
2453  tcph.th_win = 5480;
2454  tcph.th_flags = TH_PUSH | TH_ACK;
2455  UTHSetTCPHdr(p, &tcph);
2457  p->payload = packet;
2458 
2459  tcph.th_seq = htonl(10);
2460  tcph.th_ack = htonl(31);
2461  p->payload_len = 10;
2462 
2463  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2464 
2465  tcph.th_seq = htonl(20);
2466  tcph.th_ack = htonl(31);
2467  p->payload_len = 10;
2468 
2469  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2470 
2471  tcph.th_seq = htonl(40);
2472  tcph.th_ack = htonl(31);
2473  p->payload_len = 10;
2474 
2475  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2476 
2477  tcph.th_seq = htonl(5);
2478  tcph.th_ack = htonl(31);
2479  p->payload_len = 30;
2480 
2481  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2482 
2484  FLOW_DESTROY(&f);
2485  PacketFree(p);
2486  StreamTcpUTDeinit(ra_ctx);
2487  PASS;
2488 }
2489 
2490 static int StreamTcpReassembleTest34(void)
2491 {
2492  TcpSession ssn;
2494  FAIL_IF(unlikely(p == NULL));
2495  Flow f;
2496  TCPHdr tcph;
2497  TcpReassemblyThreadCtx *ra_ctx = NULL;
2499  uint8_t packet[1460] = "";
2500 
2501  StreamTcpUTInit(&ra_ctx);
2503  memset(&f, 0, sizeof (Flow));
2504  memset(&tcph, 0, sizeof (TCPHdr));
2505  ThreadVars tv;
2506  memset(&tv, 0, sizeof (ThreadVars));
2507  FLOW_INITIALIZE(&f);
2508  f.protoctx = &ssn;
2509  f.proto = IPPROTO_TCP;
2510  p->src.family = AF_INET;
2511  p->dst.family = AF_INET;
2512  p->proto = IPPROTO_TCP;
2513  p->flow = &f;
2514  tcph.th_win = 5480;
2515  tcph.th_flags = TH_PUSH | TH_ACK;
2516  UTHSetTCPHdr(p, &tcph);
2518  p->payload = packet;
2519  SET_ISN(&ssn.client, 857961230);
2520 
2521  tcph.th_seq = htonl(857961230);
2522  tcph.th_ack = htonl(31);
2523  p->payload_len = 304;
2524 
2525  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2526 
2527  tcph.th_seq = htonl(857961534);
2528  tcph.th_ack = htonl(31);
2529  p->payload_len = 1460;
2530 
2531  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2532 
2533  tcph.th_seq = htonl(857963582);
2534  tcph.th_ack = htonl(31);
2535  p->payload_len = 1460;
2536 
2537  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2538 
2539  tcph.th_seq = htonl(857960946);
2540  tcph.th_ack = htonl(31);
2541  p->payload_len = 1460;
2542 
2543  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2544 
2546  FLOW_DESTROY(&f);
2547  StreamTcpUTDeinit(ra_ctx);
2548  PacketFree(p);
2549  PASS;
2550 }
2551 
2552 /**
2553  * \test Test to make sure that we don't return the segments until the app
2554  * layer proto has been detected and after that remove the processed
2555  * segments.
2556  *
2557  * \retval On success it returns 1 and on failure 0.
2558  */
2559 
2560 static int StreamTcpReassembleTest39 (void)
2561 {
2563  FAIL_IF(unlikely(p == NULL));
2564  Flow f;
2565  ThreadVars tv;
2566  StreamTcpThread stt;
2567  TCPHdr tcph;
2568  PacketQueueNoLock pq;
2569  memset(&pq,0,sizeof(PacketQueueNoLock));
2570  memset (&f, 0, sizeof(Flow));
2571  memset(&tv, 0, sizeof (ThreadVars));
2572  memset(&stt, 0, sizeof (stt));
2573  memset(&tcph, 0, sizeof (TCPHdr));
2574 
2575  FLOW_INITIALIZE(&f);
2576  f.flags = FLOW_IPV4;
2577  f.proto = IPPROTO_TCP;
2578  p->flow = &f;
2579  UTHSetTCPHdr(p, &tcph);
2580 
2581  StreamTcpUTInit(&stt.ra_ctx);
2582 
2583  /* handshake */
2584  tcph.th_win = htons(5480);
2585  tcph.th_flags = TH_SYN;
2587  p->payload_len = 0;
2588  p->payload = NULL;
2589  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2590 
2592  FAIL_IF_NULL(ssn);
2593 
2600  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2601  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2602  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2603  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2607 
2608  /* handshake */
2609  tcph.th_ack = htonl(1);
2610  tcph.th_flags = TH_SYN | TH_ACK;
2612  p->payload_len = 0;
2613  p->payload = NULL;
2614  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2621  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2622  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2623  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2624  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2628 
2629  /* handshake */
2630  tcph.th_ack = htonl(1);
2631  tcph.th_seq = htonl(1);
2632  tcph.th_flags = TH_ACK;
2634  p->payload_len = 0;
2635  p->payload = NULL;
2636  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2643  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2644  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2645  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2646  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2650 
2651  /* partial request */
2652  uint8_t request1[] = { 0x47, 0x45, };
2653  tcph.th_ack = htonl(1);
2654  tcph.th_seq = htonl(1);
2655  tcph.th_flags = TH_PUSH | TH_ACK;
2657  p->payload_len = sizeof(request1);
2658  p->payload = request1;
2659  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2666  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2667  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2668  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2669  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2671  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2673  FAIL_IF_NOT(ssn->data_first_seen_dir == STREAM_TOSERVER);
2674 
2675  /* response ack against partial request */
2676  tcph.th_ack = htonl(3);
2677  tcph.th_seq = htonl(1);
2678  tcph.th_flags = TH_ACK;
2680  p->payload_len = 0;
2681  p->payload = NULL;
2682  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2689  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2690  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2691  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2692  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2694  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2696  FAIL_IF_NOT(ssn->data_first_seen_dir == STREAM_TOSERVER);
2697 
2698  /* complete partial request */
2699  uint8_t request2[] = {
2700  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2701  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2702  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2703  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2704  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2705  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2706  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2707  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2708  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2709  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2710  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2711  tcph.th_ack = htonl(1);
2712  tcph.th_seq = htonl(3);
2713  tcph.th_flags = TH_PUSH | TH_ACK;
2715  p->payload_len = sizeof(request2);
2716  p->payload = request2;
2717  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2724  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2725  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2726  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2727  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2729  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2730  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2732  FAIL_IF_NOT(ssn->data_first_seen_dir == STREAM_TOSERVER);
2733 
2734  /* response - request ack */
2735  uint8_t response[] = {
2736  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2737  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2738  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2739  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2740  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2741  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2742  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2743  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2744  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2745  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2746  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2747  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2748  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2749  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2750  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2751  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2752  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2753  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2754  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2755  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2756  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2757  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2758  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2759  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2760  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2761  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2762  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2763  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2764  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2765  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2766  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2767  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2768  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2769  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2770  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2771  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2772  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2773  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2774  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2775  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2776  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2777  tcph.th_ack = htonl(88);
2778  tcph.th_seq = htonl(1);
2779  tcph.th_flags = TH_PUSH | TH_ACK;
2781  p->payload_len = sizeof(response);
2782  p->payload = response;
2783 
2784  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2791  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2792  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2793  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2794  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2798  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2799  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2800 
2801  /* response ack from request */
2802  tcph.th_ack = htonl(328);
2803  tcph.th_seq = htonl(88);
2804  tcph.th_flags = TH_ACK;
2806  p->payload_len = 0;
2807  p->payload = NULL;
2808  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2815  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2816  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2817  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2818  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2822  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2823  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2824  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2825 
2826  /* response - acking */
2827  tcph.th_ack = htonl(88);
2828  tcph.th_seq = htonl(328);
2829  tcph.th_flags = TH_PUSH | TH_ACK;
2831  p->payload_len = 0;
2832  p->payload = NULL;
2833  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2840  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2841  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2842  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2843  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2847  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2848  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2849  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2850 
2851  /* response ack from request */
2852  tcph.th_ack = htonl(328);
2853  tcph.th_seq = htonl(88);
2854  tcph.th_flags = TH_ACK;
2856  p->payload_len = 0;
2857  p->payload = NULL;
2858  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2865  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2866  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2867  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2868  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2871  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2873  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2874 
2875  /* response - acking the request again*/
2876  tcph.th_ack = htonl(88);
2877  tcph.th_seq = htonl(328);
2878  tcph.th_flags = TH_PUSH | TH_ACK;
2880  p->payload_len = 0;
2881  p->payload = NULL;
2882  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2889  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2890  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2891  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2892  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2895  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2897  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2898 
2899  /*** New Request ***/
2900 
2901  /* partial request */
2902  tcph.th_ack = htonl(328);
2903  tcph.th_seq = htonl(88);
2904  tcph.th_flags = TH_PUSH | TH_ACK;
2906  p->payload_len = sizeof(request1);
2907  p->payload = request1;
2908  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2915  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2916  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2917  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2918  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2921  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2922  FAIL_IF_NOT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2924  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2925 
2926  /* response ack against partial request */
2927  tcph.th_ack = htonl(90);
2928  tcph.th_seq = htonl(328);
2929  tcph.th_flags = TH_ACK;
2931  p->payload_len = 0;
2932  p->payload = NULL;
2933  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2934  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2941  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2942  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2943  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2944  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2947  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2948  FAIL_IF_NOT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2950  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2951 
2952  /* complete request */
2953  tcph.th_ack = htonl(328);
2954  tcph.th_seq = htonl(90);
2955  tcph.th_flags = TH_PUSH | TH_ACK;
2957  p->payload_len = sizeof(request2);
2958  p->payload = request2;
2959  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2966  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2967  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2968  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2969  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2972  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2973  FAIL_IF_NOT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2974  FAIL_IF_NOT(
2975  TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2977  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2978 
2979  /* response ack against second partial request */
2980  tcph.th_ack = htonl(175);
2981  tcph.th_seq = htonl(328);
2982  tcph.th_flags = TH_ACK;
2984  p->payload_len = 0;
2985  p->payload = NULL;
2986 
2987  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2994  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2995  FAIL_IF_NOT(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2996  FAIL_IF_NOT(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2997  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
3000  FAIL_IF_NOT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
3001  FAIL_IF_NOT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
3002  FAIL_IF_NOT(
3003  TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
3005  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
3006 
3007  /* response acking a request */
3008  tcph.th_ack = htonl(175);
3009  tcph.th_seq = htonl(328);
3010  tcph.th_flags = TH_ACK;
3012  p->payload_len = 0;
3013  p->payload = NULL;
3014  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3020 
3021  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3022  StreamTcpPruneSession(&f, STREAM_TOCLIENT);
3023 
3024  /* request acking a response */
3025  tcph.th_ack = htonl(328);
3026  tcph.th_seq = htonl(175);
3027  tcph.th_flags = TH_ACK;
3029  p->payload_len = 0;
3030  p->payload = NULL;
3031  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3032 
3033  PacketFree(p);
3035  FLOW_DESTROY(&f);
3037  PASS;
3038 }
3039 
3040 /**
3041  * \test Test to make sure that we sent all the segments from the initial
3042  * segments to app layer until we have detected the app layer proto.
3043  *
3044  * \retval On success it returns 1 and on failure 0.
3045  */
3046 
3047 static int StreamTcpReassembleTest40 (void)
3048 {
3050  FAIL_IF_NULL(p);
3051  Flow *f = NULL;
3052  TCPHdr tcph;
3053  TcpSession ssn;
3054  memset(&tcph, 0, sizeof (TCPHdr));
3055  ThreadVars tv;
3056  memset(&tv, 0, sizeof (ThreadVars));
3057 
3058  StreamTcpInitConfig(true);
3060 
3062  FAIL_IF_NULL(ra_ctx);
3063 
3064  uint8_t httpbuf1[] = "P";
3065  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3066  uint8_t httpbuf3[] = "O";
3067  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3068  uint8_t httpbuf4[] = "S";
3069  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3070  uint8_t httpbuf5[] = "T \r\n";
3071  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3072 
3073  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3074  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3075 
3076  SET_ISN(&ssn.server, 9);
3077  ssn.server.last_ack = 10;
3078  SET_ISN(&ssn.client, 9);
3079  ssn.client.isn = 9;
3080 
3081  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3082  FAIL_IF_NULL(f);
3083  f->protoctx = &ssn;
3084  f->proto = IPPROTO_TCP;
3085  p->flow = f;
3086 
3087  tcph.th_win = htons(5480);
3088  tcph.th_seq = htonl(10);
3089  tcph.th_ack = htonl(10);
3090  tcph.th_flags = TH_ACK|TH_PUSH;
3091  UTHSetTCPHdr(p, &tcph);
3093  p->payload = httpbuf1;
3094  p->payload_len = httplen1;
3096  TcpStream *s = &ssn.client;
3097  SCLogDebug("1 -- start");
3098  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3099 
3101  p->payload = httpbuf2;
3102  p->payload_len = httplen2;
3103  tcph.th_seq = htonl(10);
3104  tcph.th_ack = htonl(11);
3105  s = &ssn.server;
3106  ssn.server.last_ack = 11;
3107  SCLogDebug("2 -- start");
3108  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3109 
3111  p->payload = httpbuf3;
3112  p->payload_len = httplen3;
3113  tcph.th_seq = htonl(11);
3114  tcph.th_ack = htonl(55);
3115  s = &ssn.client;
3116  ssn.client.last_ack = 55;
3117  SCLogDebug("3 -- start");
3118  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3119 
3121  p->payload = httpbuf2;
3122  p->payload_len = httplen2;
3123  tcph.th_seq = htonl(55);
3124  tcph.th_ack = htonl(12);
3125  s = &ssn.server;
3126  ssn.server.last_ack = 12;
3127  SCLogDebug("4 -- start");
3128  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3129 
3130  /* check is have the segment in the list and flagged or not */
3131  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3132  FAIL_IF_NULL(seg);
3133  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
3134 
3136  p->payload = httpbuf4;
3137  p->payload_len = httplen4;
3138  tcph.th_seq = htonl(12);
3139  tcph.th_ack = htonl(100);
3140  s = &ssn.client;
3141  ssn.client.last_ack = 100;
3142  SCLogDebug("5 -- start");
3143  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3144 
3146  p->payload = httpbuf2;
3147  p->payload_len = httplen2;
3148  tcph.th_seq = htonl(100);
3149  tcph.th_ack = htonl(13);
3150  s = &ssn.server;
3151  ssn.server.last_ack = 13;
3152  SCLogDebug("6 -- start");
3153  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3154 
3156  p->payload = httpbuf5;
3157  p->payload_len = httplen5;
3158  tcph.th_seq = htonl(13);
3159  tcph.th_ack = htonl(145);
3160  s = &ssn.client;
3161  ssn.client.last_ack = 145;
3162  SCLogDebug("7 -- start");
3163  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3164 
3166  p->payload = httpbuf2;
3167  p->payload_len = httplen2;
3168  tcph.th_seq = htonl(145);
3169  tcph.th_ack = htonl(16);
3170  s = &ssn.server;
3171  ssn.server.last_ack = 16;
3172  SCLogDebug("8 -- start");
3173  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3175 
3177  PacketFree(p);
3178  UTHFreeFlow(f);
3180  StreamTcpFreeConfig(true);
3181  PASS;
3182 }
3183 
3184 /** \test Test the memcap incrementing/decrementing and memcap check */
3185 static int StreamTcpReassembleTest44(void)
3186 {
3187  StreamTcpInitConfig(true);
3188  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
3189  StreamTcpReassembleIncrMemuse(500);
3190  FAIL_IF_NOT(SC_ATOMIC_GET(ra_memuse) == (memuse + 500));
3191  StreamTcpReassembleDecrMemuse(500);
3192  FAIL_IF_NOT(SC_ATOMIC_GET(ra_memuse) == memuse);
3195  (1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) == 0);
3196  StreamTcpFreeConfig(true);
3197  FAIL_IF_NOT(SC_ATOMIC_GET(ra_memuse) == 0);
3198  PASS;
3199 }
3200 
3201 /** \test Test that one TcpSegmentPoolAlloc(), TcpSegmentPoolInit(),
3202  * TcpSegmentPoolCleanup() cycle leaves tcp.reassembly_memuse unchanged
3203  * when TCP session dumping is enabled, so the pcap hdr storage is
3204  * counted once and subtracted once. */
3205 static int StreamTcpReassembleTestMemusePcapHdrStorage(void)
3206 {
3207  StreamTcpInitConfig(true);
3208 
3209  const int dump_enabled = g_tcp_session_dump_enabled;
3210  g_tcp_session_dump_enabled = 1;
3211 
3212  const uint64_t memuse = StreamTcpReassembleMemuseGlobalCounter();
3213 
3214  /* what PoolGet() does when it has to create a new element */
3215  TcpSegment *seg = TcpSegmentPoolAlloc();
3216  FAIL_IF_NULL(seg);
3217  FAIL_IF_NOT(TcpSegmentPoolInit(seg) == 1);
3219 
3220  /* and what PoolReturn() does when it cannot hand the element back */
3221  TcpSegmentPoolCleanup(seg);
3222  SCFree(seg);
3223 
3225 
3226  g_tcp_session_dump_enabled = dump_enabled;
3227  StreamTcpFreeConfig(true);
3228  PASS;
3229 }
3230 
3231 /** \test Test that a failed memcap check in TcpSegmentPoolInit() frees the
3232  * pcap hdr storage that TcpSegmentPoolAlloc() allocated. The pool
3233  * frees the segment without calling TcpSegmentPoolCleanup() when
3234  * Init() fails. */
3235 static int StreamTcpReassembleTestPcapHdrStorageInitFail(void)
3236 {
3237  StreamTcpInitConfig(true);
3238 
3239  const int dump_enabled = g_tcp_session_dump_enabled;
3240  g_tcp_session_dump_enabled = 1;
3241 
3242  const uint64_t memuse = StreamTcpReassembleMemuseGlobalCounter();
3243  const uint64_t memcap = SC_ATOMIC_GET(stream_config.reassembly_memcap);
3244 
3245  TcpSegment *seg = TcpSegmentPoolAlloc();
3246  const bool alloc_ok = seg != NULL && seg->pcap_hdr_storage != NULL;
3247  int init_ret = -1;
3248  bool storage_freed = false;
3249  /* leave no room for the segment so the Init() memcap check fails */
3250  if (alloc_ok && StreamTcpReassembleSetMemcap(memuse + 1) == 1) {
3251  init_ret = TcpSegmentPoolInit(seg);
3252  storage_freed = seg->pcap_hdr_storage == NULL;
3253  }
3254  SCFree(seg);
3255  const uint64_t memuse_after = StreamTcpReassembleMemuseGlobalCounter();
3256 
3257  /* restore the globals before checking, so a failure here does not
3258  * leave a tiny memcap and dumping enabled for later tests */
3259  SC_ATOMIC_SET(stream_config.reassembly_memcap, memcap);
3260  g_tcp_session_dump_enabled = dump_enabled;
3261  StreamTcpFreeConfig(true);
3262 
3263  FAIL_IF_NOT(alloc_ok);
3264  FAIL_IF_NOT(init_ret == 0);
3265  FAIL_IF_NOT(storage_freed);
3266  FAIL_IF_NOT(memuse_after == memuse);
3267  PASS;
3268 }
3269 
3270 /**
3271  * \test Test to make sure that reassembly_depth is enforced.
3272  *
3273  * \retval On success it returns 1 and on failure 0.
3274  */
3275 
3276 static int StreamTcpReassembleTest45 (void)
3277 {
3278  TcpReassemblyThreadCtx *ra_ctx = NULL;
3279  TcpSession ssn;
3280  ThreadVars tv;
3281  memset(&tv, 0, sizeof(tv));
3282  uint8_t payload[100] = {0};
3283  uint16_t payload_size = 100;
3284 
3285  StreamTcpUTInit(&ra_ctx);
3287 
3289  ssn.reassembly_depth = 100;
3292 
3293  FAIL_IF_NOT(
3294  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size) == 0);
3296 
3297  FAIL_IF_NOT(
3298  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size) == 0);
3300 
3304  StreamTcpUTDeinit(ra_ctx);
3305  PASS;
3306 }
3307 
3308 /**
3309  * \test Test the unlimited config value of reassembly depth.
3310  *
3311  * \retval On success it returns 1 and on failure 0.
3312  */
3313 
3314 static int StreamTcpReassembleTest46 (void)
3315 {
3316  TcpReassemblyThreadCtx *ra_ctx = NULL;
3317  TcpSession ssn;
3318  ThreadVars tv;
3319  memset(&tv, 0, sizeof(tv));
3320  uint8_t payload[100] = {0};
3321  uint16_t payload_size = 100;
3322 
3323  StreamTcpUTInit(&ra_ctx);
3325 
3329 
3330  FAIL_IF_NOT(
3331  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size) == 0);
3333 
3334  FAIL_IF_NOT(
3335  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size) == 0);
3337 
3341  StreamTcpUTDeinit(ra_ctx);
3342  PASS;
3343 }
3344 
3345 /**
3346  * \test Test to make sure we detect the sequence wrap around and continue
3347  * stream reassembly properly.
3348  *
3349  * \retval On success it returns 1 and on failure 0.
3350  */
3351 
3352 static int StreamTcpReassembleTest47 (void)
3353 {
3355  FAIL_IF(unlikely(p == NULL));
3356  Flow *f = NULL;
3357  TCPHdr tcph;
3358  TcpSession ssn;
3359  ThreadVars tv;
3360  memset(&tcph, 0, sizeof (TCPHdr));
3361  UTHSetTCPHdr(p, &tcph);
3362  memset(&tv, 0, sizeof (ThreadVars));
3363  StreamTcpInitConfig(true);
3366 
3367  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3368  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3369 
3370  SET_ISN(&ssn.server, 572799781UL);
3371  ssn.server.last_ack = 572799782UL;
3372 
3373  SET_ISN(&ssn.client, 4294967289UL);
3374  ssn.client.last_ack = 21;
3375 
3376  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3377  FAIL_IF_NULL(f);
3378  f->protoctx = &ssn;
3379  f->proto = IPPROTO_TCP;
3380  p->flow = f;
3381 
3382  tcph.th_win = htons(5480);
3384  TcpStream *s = NULL;
3385  uint8_t cnt = 0;
3386 
3387  for (cnt=0; cnt < httplen1; cnt++) {
3388  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3389  tcph.th_ack = htonl(572799782UL);
3390  tcph.th_flags = TH_ACK | TH_PUSH;
3392  p->payload = &httpbuf1[cnt];
3393  p->payload_len = 1;
3394  s = &ssn.client;
3395 
3396  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3397 
3399  p->payload = NULL;
3400  p->payload_len = 0;
3401  tcph.th_seq = htonl(572799782UL);
3402  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3403  tcph.th_flags = TH_ACK;
3404  s = &ssn.server;
3405 
3406  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3407  }
3408 
3410 
3412  PacketFree(p);
3413  UTHFreeFlow(f);
3415  StreamTcpFreeConfig(true);
3416  PASS;
3417 }
3418 
3419 /** \test 3 in order segments in inline reassembly */
3420 static int StreamTcpReassembleInlineTest01(void)
3421 {
3422  TcpReassemblyThreadCtx *ra_ctx = NULL;
3423  ThreadVars tv;
3424  TcpSession ssn;
3425  Flow f;
3426 
3427  memset(&tv, 0x00, sizeof(tv));
3428 
3429  StreamTcpUTInit(&ra_ctx);
3433  FLOW_INITIALIZE(&f);
3434 
3435  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3436  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3437  FAIL_IF_NULL(p);
3438  p->l4.hdrs.tcph->th_seq = htonl(12);
3439  p->flow = &f;
3440 
3441  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3442  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3443  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3444  ssn.client.next_seq = 17;
3445  FLOW_DESTROY(&f);
3446  UTHFreePacket(p);
3448  StreamTcpUTDeinit(ra_ctx);
3449  PASS;
3450 }
3451 
3452 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3453  * test the sliding window reassembly.
3454  */
3455 static int StreamTcpReassembleInlineTest02(void)
3456 {
3457  TcpReassemblyThreadCtx *ra_ctx = NULL;
3458  ThreadVars tv;
3459  TcpSession ssn;
3460  Flow f;
3461 
3462  memset(&tv, 0x00, sizeof(tv));
3463 
3464  StreamTcpUTInit(&ra_ctx);
3468  FLOW_INITIALIZE(&f);
3469 
3470  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3471  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3472  FAIL_IF_NULL(p);
3473  p->l4.hdrs.tcph->th_seq = htonl(12);
3474  p->flow = &f;
3475 
3476  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3477  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3478  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3479  ssn.client.next_seq = 17;
3480  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3481  ssn.client.next_seq = 22;
3482  FLOW_DESTROY(&f);
3483  UTHFreePacket(p);
3485  StreamTcpUTDeinit(ra_ctx);
3486  PASS;
3487 }
3488 
3489 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3490  * test the sliding window reassembly with a small window size so that we
3491  * cutting off at the start (left edge)
3492  */
3493 static int StreamTcpReassembleInlineTest03(void)
3494 {
3495  TcpReassemblyThreadCtx *ra_ctx = NULL;
3496  ThreadVars tv;
3497  TcpSession ssn;
3498  Flow f;
3499 
3500  memset(&tv, 0x00, sizeof(tv));
3501 
3502  StreamTcpUTInit(&ra_ctx);
3506  FLOW_INITIALIZE(&f);
3507 
3509 
3510  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3511  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3512  FAIL_IF_NULL(p);
3513  p->l4.hdrs.tcph->th_seq = htonl(12);
3514  p->flow = &f;
3516 
3517  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3518  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3519  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3520  ssn.client.next_seq = 17;
3521  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3522  ssn.client.next_seq = 22;
3523 
3524  p->l4.hdrs.tcph->th_seq = htonl(17);
3525  FLOW_DESTROY(&f);
3526  UTHFreePacket(p);
3528  StreamTcpUTDeinit(ra_ctx);
3529  PASS;
3530 }
3531 
3532 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3533  * test the sliding window reassembly with a small window size so that we
3534  * cutting off at the start (left edge) with small packet overlap.
3535  */
3536 static int StreamTcpReassembleInlineTest04(void)
3537 {
3538  TcpReassemblyThreadCtx *ra_ctx = NULL;
3539  ThreadVars tv;
3540  TcpSession ssn;
3541  Flow f;
3542 
3543  memset(&tv, 0x00, sizeof(tv));
3544 
3545  StreamTcpUTInit(&ra_ctx);
3549  FLOW_INITIALIZE(&f);
3550 
3552 
3553  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3554  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3555  FAIL_IF_NULL(p);
3556  p->l4.hdrs.tcph->th_seq = htonl(12);
3557  p->flow = &f;
3559 
3560  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3561  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3562  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3563  ssn.client.next_seq = 17;
3564  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3565  ssn.client.next_seq = 22;
3566 
3567  p->l4.hdrs.tcph->th_seq = htonl(17);
3568  FLOW_DESTROY(&f);
3569  UTHFreePacket(p);
3571  StreamTcpUTDeinit(ra_ctx);
3572  PASS;
3573 }
3574 
3575 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3576  * test the sliding window reassembly with a small window size so that we
3577  * cutting off at the start (left edge). Test if the first segment is
3578  * removed from the list.
3579  */
3580 static int StreamTcpReassembleInlineTest08(void)
3581 {
3582  TcpReassemblyThreadCtx *ra_ctx = NULL;
3583  ThreadVars tv;
3584  memset(&tv, 0x00, sizeof(tv));
3585  TcpSession ssn;
3586  Flow f;
3587  StreamTcpUTInit(&ra_ctx);
3591  FLOW_INITIALIZE(&f);
3592 
3594  f.protoctx = &ssn;
3595 
3596  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3597  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3598  FAIL_IF_NULL(p);
3599  p->l4.hdrs.tcph->th_seq = htonl(12);
3600  p->flow = &f;
3602 
3603  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3604  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3605  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3606  ssn.client.next_seq = 17;
3607  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3608  ssn.client.next_seq = 22;
3609  p->l4.hdrs.tcph->th_seq = htonl(17);
3610  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3611 
3612  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3613  FAIL_IF_NULL(seg);
3614  FAIL_IF_NOT(seg->seq == 2);
3615 
3616  FLOW_DESTROY(&f);
3617  UTHFreePacket(p);
3619  StreamTcpUTDeinit(ra_ctx);
3620  PASS;
3621 }
3622 
3623 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3624  * test the sliding window reassembly with a small window size so that we
3625  * cutting off at the start (left edge). Test if the first segment is
3626  * removed from the list.
3627  */
3628 static int StreamTcpReassembleInlineTest09(void)
3629 {
3630  TcpReassemblyThreadCtx *ra_ctx = NULL;
3631  ThreadVars tv;
3632  TcpSession ssn;
3633  Flow f;
3634 
3635  memset(&tv, 0x00, sizeof(tv));
3636 
3637  StreamTcpUTInit(&ra_ctx);
3641  FLOW_INITIALIZE(&f);
3642 
3644 
3645  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3646  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3647  FAIL_IF_NULL(p);
3648  p->l4.hdrs.tcph->th_seq = htonl(17);
3649  p->flow = &f;
3651 
3652  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3653  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3654  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3655  ssn.client.next_seq = 12;
3656  ssn.client.last_ack = 10;
3657 
3658  /* close the GAP and see if we properly reassemble and update base_seq */
3659  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3660  ssn.client.next_seq = 22;
3661 
3662  p->l4.hdrs.tcph->th_seq = htonl(12);
3663 
3664  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3665  FAIL_IF_NULL(seg);
3666  FAIL_IF_NOT(seg->seq == 2);
3667 
3668  FLOW_DESTROY(&f);
3669  UTHFreePacket(p);
3671  StreamTcpUTDeinit(ra_ctx);
3672  PASS;
3673 }
3674 
3675 /** \test App Layer reassembly.
3676  */
3677 static int StreamTcpReassembleInlineTest10(void)
3678 {
3679  TcpReassemblyThreadCtx *ra_ctx = NULL;
3680  ThreadVars tv;
3681  TcpSession ssn;
3682  Packet *p = NULL;
3683 
3684  memset(&tv, 0x00, sizeof(tv));
3685 
3686  StreamTcpUTInit(&ra_ctx);
3690  ssn.server.last_ack = 2;
3692  ssn.client.last_ack = 2;
3693  ssn.data_first_seen_dir = STREAM_TOSERVER;
3694 
3695  Flow *f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3696  FAIL_IF_NULL(f);
3697  f->protoctx = &ssn;
3698  f->proto = IPPROTO_TCP;
3699 
3700  uint8_t stream_payload1[] = "GE";
3701  uint8_t stream_payload2[] = "T /";
3702  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3703 
3704  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3705  FAIL_IF_NULL(p);
3706  p->l4.hdrs.tcph->th_seq = htonl(7);
3707  p->flow = f;
3709 
3710  FAIL_IF(StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) ==
3711  -1);
3712  ssn.client.next_seq = 4;
3713 
3715 
3716  /* ssn.server.ra_app_base_seq should be isn here. */
3718 
3719  FAIL_IF(StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) ==
3720  -1);
3721  FAIL_IF(StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) ==
3722  -1);
3723  ssn.client.next_seq = 19;
3724 
3726 
3728 
3729  UTHFreePacket(p);
3731  StreamTcpUTDeinit(ra_ctx);
3732  UTHFreeFlow(f);
3733  PASS;
3734 }
3735 
3736 /** \test test insert with overlap
3737  */
3738 static int StreamTcpReassembleInsertTest01(void)
3739 {
3740  TcpReassemblyThreadCtx *ra_ctx = NULL;
3741  ThreadVars tv;
3742  TcpSession ssn;
3743  Flow f;
3744 
3745  memset(&tv, 0x00, sizeof(tv));
3746 
3747  StreamTcpUTInit(&ra_ctx);
3751  FLOW_INITIALIZE(&f);
3752 
3753  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3754  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3755  FAIL_IF_NULL(p);
3756  p->l4.hdrs.tcph->th_seq = htonl(12);
3757  p->flow = &f;
3758 
3759  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3760  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3761  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3762  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3763  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3764  ssn.client.next_seq = 21;
3765 
3766  FLOW_DESTROY(&f);
3767  UTHFreePacket(p);
3769  StreamTcpUTDeinit(ra_ctx);
3770  PASS;
3771 }
3772 
3773 /** \test test insert with overlaps
3774  */
3775 static int StreamTcpReassembleInsertTest02(void)
3776 {
3777  TcpReassemblyThreadCtx *ra_ctx = NULL;
3778  ThreadVars tv;
3779  TcpSession ssn;
3780 
3781  memset(&tv, 0x00, sizeof(tv));
3782 
3783  StreamTcpUTInit(&ra_ctx);
3786 
3787  int i;
3788  for (i = 2; i < 10; i++) {
3789  int len;
3790  len = i % 2;
3791  if (len == 0)
3792  len = 1;
3793  int seq;
3794  seq = i * 10;
3795  if (seq < 2)
3796  seq = 2;
3797 
3798  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1);
3799  }
3800  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1);
3801 
3803  StreamTcpUTDeinit(ra_ctx);
3804  PASS;
3805 }
3806 
3807 /** \test test insert with overlaps
3808  */
3809 static int StreamTcpReassembleInsertTest03(void)
3810 {
3811  TcpReassemblyThreadCtx *ra_ctx = NULL;
3812  ThreadVars tv;
3813  TcpSession ssn;
3814 
3815  memset(&tv, 0x00, sizeof(tv));
3816 
3817  StreamTcpUTInit(&ra_ctx);
3820 
3821  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1);
3822 
3823  int i;
3824  for (i = 2; i < 10; i++) {
3825  int len;
3826  len = i % 2;
3827  if (len == 0)
3828  len = 1;
3829  int seq;
3830  seq = i * 10;
3831  if (seq < 2)
3832  seq = 2;
3833 
3834  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1);
3835  }
3837  StreamTcpUTDeinit(ra_ctx);
3838  PASS;
3839 }
3840 
3842 #endif /* UNITTESTS */
3843 
3844 /** \brief The Function Register the Unit tests to test the reassembly engine
3845  * for various OS policies.
3846  */
3847 
3849 {
3850 #ifdef UNITTESTS
3851  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3852  StreamTcpReassembleTest25);
3853  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3854  StreamTcpReassembleTest26);
3856  "StreamTcpReassembleTest27 -- Gap at after Reassembly Test", StreamTcpReassembleTest27);
3857  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3858  StreamTcpReassembleTest28);
3859  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3860  StreamTcpReassembleTest29);
3861  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3862  StreamTcpReassembleTest33);
3863  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3864  StreamTcpReassembleTest34);
3865  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3866  StreamTcpReassembleTest39);
3867  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3868  StreamTcpReassembleTest40);
3869  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3870  StreamTcpReassembleTest44);
3871  UtRegisterTest("StreamTcpReassembleTestMemusePcapHdrStorage -- pcap hdr storage memuse",
3872  StreamTcpReassembleTestMemusePcapHdrStorage);
3873  UtRegisterTest("StreamTcpReassembleTestPcapHdrStorageInitFail -- pcap hdr storage init fail",
3874  StreamTcpReassembleTestPcapHdrStorageInitFail);
3875  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3876  StreamTcpReassembleTest45);
3877  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3878  StreamTcpReassembleTest46);
3879  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3880  StreamTcpReassembleTest47);
3881 
3882  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3883  StreamTcpReassembleInlineTest01);
3884  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3885  StreamTcpReassembleInlineTest02);
3886  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3887  StreamTcpReassembleInlineTest03);
3888  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3889  StreamTcpReassembleInlineTest04);
3890  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3891  StreamTcpReassembleInlineTest08);
3892  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3893  StreamTcpReassembleInlineTest09);
3894 
3895  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3896  StreamTcpReassembleInlineTest10);
3897 
3898  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3899  StreamTcpReassembleInsertTest01);
3900  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3901  StreamTcpReassembleInsertTest02);
3902  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3903  StreamTcpReassembleInsertTest03);
3904 
3908  StreamTcpReassembleRawRegisterTests();
3909 #endif /* UNITTESTS */
3910 }
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
Definition: stream-tcp.h:142
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:277
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:40
TCP_GET_RAW_SEQ
#define TCP_GET_RAW_SEQ(tcph)
Definition: decode-tcp.h:80
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:58
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:56
util-device-private.h
UTHCheckGapAtPosition
int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
Definition: stream-tcp-reassemble.c:2276
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
StatsRegisterGlobalCounter
StatsCounterGlobalId StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1093
Packet_::proto
uint8_t proto
Definition: decode.h:550
TcpStream_
Definition: stream-tcp-private.h:106
len
uint8_t len
Definition: app-layer-dnp3.h:2
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:138
SCAppLayerParserStateIssetFlag
uint16_t SCAppLayerParserStateIssetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:2079
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:50
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:186
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:113
TcpReassemblyThreadCtx_::counter_tcp_reas_eps
ExceptionPolicyCounters counter_tcp_reas_eps
Definition: stream-tcp-reassemble.h:69
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:2133
TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool
StatsCounterId counter_tcp_segment_from_pool
Definition: stream-tcp-reassemble.h:72
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:383
Flow_::flags
uint64_t flags
Definition: flow.h:410
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:96
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1917
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:97
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:91
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:94
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
TCP_STREAM_URGENT_OOB
@ TCP_STREAM_URGENT_OOB
Definition: stream-tcp.h:48
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:69
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
stream-tcp.h
StreamTcpInlineMode
bool StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:7301
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
StatsCounterId counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:67
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:101
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:136
PcapPacketCntGet
uint64_t PcapPacketCntGet(const Packet *p)
Definition: decode.c:1244
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:2205
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:282
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StreamTcpThread_
Definition: stream-tcp.h:91
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
StatsCounterId counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:77
Flow_::proto
uint8_t proto
Definition: flow.h:382
Packet_::payload
uint8_t * payload
Definition: decode.h:640
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:2185
STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
@ STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
Definition: decode-events.h:324
action-globals.h
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:62
Packet_::flags
uint32_t flags
Definition: decode.h:582
SCAppLayerParserStateSetFlag
void SCAppLayerParserStateSetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:2071
ssn
TcpSession ssn
Definition: fuzz_dataset.c:31
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:110
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:74
Flow_
Flow data structure.
Definition: flow.h:360
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:77
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
TCPHdr_::th_win
uint16_t th_win
Definition: decode-tcp.h:156
TcpSegmentPcapHdrStorage
struct TcpSegmentPcapHdrStorage_ TcpSegmentPcapHdrStorage
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
StringParseUint16
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:295
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:156
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1400
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:40
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:88
PKT_DROP_REASON_STREAM_REASSEMBLY
@ PKT_DROP_REASON_STREAM_REASSEMBLY
Definition: decode.h:417
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1810
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir app_update_dir)
handle TCP data for the app-layer.
Definition: app-layer.c:713
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:236
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:155
MIN
#define MIN(x, y)
Definition: suricata-common.h:413
SCConfGetBool
int SCConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:523
TcpSession_::urg_offset_ts
uint16_t urg_offset_ts
Definition: stream-tcp-private.h:298
FLOW_APP_LAYER_FLUSH_PENDING
#define FLOW_APP_LAYER_FLUSH_PENDING
Definition: flow.h:132
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:75
UTHSetTCPHdr
void UTHSetTCPHdr(Packet *p, TCPHdr *tcph)
Definition: util-unittest-helper.c:261
StreamTcpThreadCacheGetSegment
TcpSegment * StreamTcpThreadCacheGetSegment(void)
Definition: stream-tcp-cache.c:179
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:122
p
Packet * p
Definition: fuzz_dataset.c:30
StatsCounterId
Definition: counters.h:30
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:49
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:151
TCP_STREAM_URGENT_GAP
@ TCP_STREAM_URGENT_GAP
Definition: stream-tcp.h:49
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1827
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:564
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:227
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:136
Flow_::protoctx
void * protoctx
Definition: flow.h:440
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:710
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:99
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:641
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:223
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:665
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
AppLayerProfilingStore
#define AppLayerProfilingStore(app_tctx, p)
Definition: app-layer.h:128
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2267
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
SCAppLayerDecoderEventsSetEventRaw
void SCAppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:97
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:535
PKT_DROP_REASON_STREAM_URG
@ PKT_DROP_REASON_STREAM_URG
Definition: decode.h:418
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:428
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:170
STREAMTCP_FLAG_LOSSY_BE_LIBERAL
#define STREAMTCP_FLAG_LOSSY_BE_LIBERAL
Definition: stream-tcp-private.h:194
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:191
TCPHdr_::th_ack
uint32_t th_ack
Definition: decode-tcp.h:153
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:301
StreamTcpReassembleTriggerRawInspection
void StreamTcpReassembleTriggerRawInspection(TcpSession *ssn, int direction)
Trigger RAW stream inspection.
Definition: stream-tcp-reassemble.c:2168
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:302
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:281
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:115
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:485
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:569
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:43
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:496
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:494
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:38
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:302
StreamTcpReassembleDepthReached
bool StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:634
TCPHdr_::th_flags
uint8_t th_flags
Definition: decode-tcp.h:155
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:475
StreamTcpReassembleRealloc
void * StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
Definition: stream-tcp-reassemble.c:229
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:256
UTHCheckDataAtPosition
int UTHCheckDataAtPosition(TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
Definition: stream-tcp-reassemble.c:2299
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:104
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:282
STREAMTCP_FLAG_ASYNC
#define STREAMTCP_FLAG_ASYNC
Definition: stream-tcp-private.h:182
TCPHdr_::th_seq
uint32_t th_seq
Definition: decode-tcp.h:152
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:188
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1314
TcpStreamCnf_::urgent_policy
enum TcpStreamUrgentHandling urgent_policy
Definition: stream-tcp.h:82
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:120
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1322
util-exception-policy.h
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
TcpSession_::urg_offset_tc
uint16_t urg_offset_tc
Definition: stream-tcp-private.h:299
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:96
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:542
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:131
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
SCEnter
#define SCEnter(...)
Definition: util-debug.h:284
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:5703
StreamNeedsReassembly
uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:985
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StatsCounterIncr
void StatsCounterIncr(StatsThreadContext *stats, StatsCounterId id)
Increments the local counter.
Definition: counters.c:163
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1513
PacketFree
void PacketFree(Packet *p)
Return a malloced packet.
Definition: decode.c:231
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:292
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:137
TCPHdr_::th_urp
uint16_t th_urp
Definition: decode-tcp.h:158
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:268
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:105
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2219
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:901
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:112
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:164
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:322
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:341
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:359
SCReturn
#define SCReturn
Definition: util-debug.h:286
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:310
Packet_
Definition: decode.h:528
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:234
stream-tcp-private.h
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:290
conf.h
Packet_::l4
struct PacketL4 l4
Definition: decode.h:636
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:308
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1850
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:117
STREAM_REASSEMBLY_DEPTH_REACHED
@ STREAM_REASSEMBLY_DEPTH_REACHED
Definition: decode-events.h:320
STREAMTCP_STREAM_FLAG_HAS_GAP
#define STREAMTCP_STREAM_FLAG_HAS_GAP
Definition: stream-tcp-private.h:217
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:605
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:300
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:731
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(void)
Creates a new app layer thread context.
Definition: app-layer.c:1117
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:400
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:237
TcpStreamCnf_::reassembly_memcap_policy
enum ExceptionPolicy reassembly_memcap_policy
Definition: stream-tcp.h:80
StreamTcpThreadCacheCleanup
void StreamTcpThreadCacheCleanup(void)
Definition: stream-tcp-cache.c:134
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:293
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:551
TH_URG
#define TH_URG
Definition: decode-tcp.h:39
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:232
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:128
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:103
VALIDATE
#define VALIDATE(e)
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
TcpReassemblyThreadCtx_::counter_tcp_urgent_oob
StatsCounterId counter_tcp_urgent_oob
Definition: stream-tcp-reassemble.h:88
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:499
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:57
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a TCP packet data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:755
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *), void(*Cleanup)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:64
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
TcpSegmentPcapHdrStorage_
Definition: stream-tcp-private.h:65
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1717
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
app-layer-frames.h
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:180
Packet_::flow
struct Flow_ * flow
Definition: decode.h:584
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:260
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:864
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
SCNtohs
#define SCNtohs(x)
Definition: suricata-common.h:436
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:449
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:259
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:649
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:124
packet.h
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:36
TcpStreamCnf_::urgent_oob_limit_policy
enum TcpStreamUrgentHandling urgent_oob_limit_policy
Definition: stream-tcp.h:83
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2248
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:135
TcpSegmentPcapHdrStorage_::alloclen
uint32_t alloclen
Definition: stream-tcp-private.h:68
ACTION_DROP
#define ACTION_DROP
Definition: action-globals.h:30
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:567
StreamTcpThreadCacheReturnSegment
void StreamTcpThreadCacheReturnSegment(TcpSegment *seg)
Definition: stream-tcp-cache.c:53
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:236
TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache
StatsCounterId counter_tcp_segment_from_cache
Definition: stream-tcp-reassemble.h:71
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:304
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Definition: stream-tcp-reassemble.c:2013
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:161
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:34
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TCP_STREAM_URGENT_DROP
@ TCP_STREAM_URGENT_DROP
Definition: stream-tcp.h:47
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:77
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:114
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:270
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1559
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
PacketL4::L4Hdrs::tcph
TCPHdr * tcph
Definition: decode.h:493
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:303
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
StatsCounterId counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:75
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:183
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:274
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:458
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:470
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayerGetCtxThread().
Definition: app-layer.c:1138
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:257
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:318
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:317
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:1028
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:71
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:53
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:150
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:314
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2262
stream-tcp-cache.h
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:158
PacketL4::hdrs
union PacketL4::L4Hdrs hdrs
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:116
MISSED_ADD_PAYLOAD
#define MISSED_ADD_PAYLOAD(seq, seg, seglen)
Definition: stream-tcp-reassemble.c:2271
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:61
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:68
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:596
StreamDataRightEdge
uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:419
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:169
PacketDrop
void PacketDrop(Packet *p, const uint8_t action, enum PacketDropReason r)
issue drop action
Definition: packet.c:35
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:219
suricata.h
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap(void)
Return memcap value.
Definition: stream-tcp-reassemble.c:200
Address_::family
char family
Definition: decode.h:131
Packet_::dst
Address dst
Definition: decode.h:533
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:7314
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:258
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
TCP_STREAM_URGENT_INLINE
@ TCP_STREAM_URGENT_INLINE
Definition: stream-tcp.h:46
likely
#define likely(expr)
Definition: util-optimize.h:32
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3848
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:102
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
segment_thread_pool
PoolThread * segment_thread_pool
Definition: stream-tcp-reassemble.c:82
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:194
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TCPSEG_PKT_HDR_DEFAULT_SIZE
#define TCPSEG_PKT_HDR_DEFAULT_SIZE
Definition: stream-tcp-private.h:58
TcpSession_
Definition: stream-tcp-private.h:290
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:295
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:185
flow.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:250
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:459
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:457
StreamReassembleLog
int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, uint64_t *progress_out, const bool eof)
Definition: stream-tcp-reassemble.c:1982
ExceptionPolicy
ExceptionPolicy
Definition: util-exception-policy-types.h:26
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:111
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:225
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:144
util-pool.h
ThreadVars_::stats
StatsThreadContext stats
Definition: threadvars.h:120
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:288
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:121
SCConfNode_
Definition: conf.h:37
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
SCConfNode_::val
char * val
Definition: conf.h:39
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamReassembleForFrame
int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof)
Definition: stream-tcp-reassemble.c:1905
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1681
SCMutex
#define SCMutex
Definition: threads-debug.h:114
ExceptionPolicyCounters_::eps_id
StatsCounterId eps_id[EXCEPTION_POLICY_MAX]
Definition: util-exception-policy-types.h:56
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:76
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:109
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:119
TCPHdr_
Definition: decode-tcp.h:149
Packet_::src
Address src
Definition: decode.h:532
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:70
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:232
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:146
app-layer.h
f
Flow f
Definition: fuzz_dataset.c:32