suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2024 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "packet.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 #include "action-globals.h"
37 
38 #include "flow-util.h"
39 
40 #include "threadvars.h"
41 #include "tm-threads.h"
42 
43 #include "util-pool.h"
44 #include "util-unittest.h"
45 #include "util-print.h"
46 #include "util-host-os-info.h"
47 #include "util-unittest-helper.h"
48 #include "util-byte.h"
49 #include "util-device-private.h"
50 
51 #include "stream-tcp.h"
52 #include "stream-tcp-private.h"
53 #include "stream-tcp-cache.h"
54 #include "stream-tcp-reassemble.h"
55 #include "stream-tcp-inline.h"
56 #include "stream-tcp-list.h"
57 #include "stream-tcp-util.h"
58 
59 #include "stream.h"
60 
61 #include "util-debug.h"
62 #include "app-layer-protos.h"
63 #include "app-layer.h"
64 #include "app-layer-events.h"
65 #include "app-layer-parser.h"
66 #include "app-layer-frames.h"
67 
68 #include "detect-engine-state.h"
69 
70 #include "util-profiling.h"
71 #include "util-validate.h"
72 #include "util-exception-policy.h"
73 
74 #ifdef DEBUG
75 static SCMutex segment_pool_memuse_mutex;
76 static uint64_t segment_pool_memuse = 0;
77 static uint64_t segment_pool_memcnt = 0;
78 #endif
79 
80 thread_local uint64_t t_pcapcnt = UINT64_MAX;
81 
83 /* init only, protect initializing and growing pool */
84 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
85 
86 /* Memory use counter */
87 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
88 
89 static int g_tcp_session_dump_enabled = 0;
90 
91 inline bool IsTcpSessionDumpingEnabled(void)
92 {
93  return g_tcp_session_dump_enabled == 1;
94 }
95 
97 {
98  g_tcp_session_dump_enabled = 1;
99 }
100 
101 /* prototypes */
103 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
104 
106 {
107  SC_ATOMIC_INIT(ra_memuse);
108 }
109 
110 /**
111  * \brief Function to Increment the memory usage counter for the TCP reassembly
112  * segments
113  *
114  * \param size Size of the TCP segment and its payload length memory allocated
115  */
116 static void StreamTcpReassembleIncrMemuse(uint64_t size)
117 {
118  (void) SC_ATOMIC_ADD(ra_memuse, size);
119  SCLogDebug("REASSEMBLY %" PRIu64 ", incr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
120  size);
121 }
122 
123 /**
124  * \brief Function to Decrease the memory usage counter for the TCP reassembly
125  * segments
126  *
127  * \param size Size of the TCP segment and its payload length memory allocated
128  */
129 static void StreamTcpReassembleDecrMemuse(uint64_t size)
130 {
131 #ifdef UNITTESTS
132  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
133  if (RunmodeIsUnittests()) {
134  BUG_ON(presize > UINT_MAX);
135  }
136 #endif
137 
138  (void) SC_ATOMIC_SUB(ra_memuse, size);
139 
140 #ifdef UNITTESTS
141  if (RunmodeIsUnittests()) {
142  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
143  BUG_ON(postsize > presize);
144  }
145 #endif
146  SCLogDebug("REASSEMBLY %" PRIu64 ", decr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
147  size);
148 }
149 
151 {
152  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
153  return smemuse;
154 }
155 
156 /**
157  * \brief Function to Check the reassembly memory usage counter against the
158  * allowed max memory usage for TCP segments.
159  *
160  * \param size Size of the TCP segment and its payload length memory allocated
161  * \retval 1 if in bounds
162  * \retval 0 if not in bounds
163  */
165 {
166 #ifdef DEBUG
167  if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX &&
168  g_eps_stream_reassembly_memcap == t_pcapcnt))) {
169  SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt);
170  return 0;
171  }
172 #endif
173  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
174  if (memcapcopy == 0 ||
175  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
176  return 1;
177  return 0;
178 }
179 
180 /**
181  * \brief Update memcap value
182  *
183  * \param size new memcap value
184  */
186 {
187  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
188  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
189  return 1;
190  }
191 
192  return 0;
193 }
194 
195 /**
196  * \brief Return memcap value
197  *
198  * \return memcap memcap value
199  */
201 {
202  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
203  return memcapcopy;
204 }
205 
206 /* memory functions for the streaming buffer API */
207 
208 /*
209  void *(*Calloc)(size_t n, size_t size);
210 */
211 static void *ReassembleCalloc(size_t n, size_t size)
212 {
213  if (StreamTcpReassembleCheckMemcap(n * size) == 0) {
215  return NULL;
216  }
217  void *ptr = SCCalloc(n, size);
218  if (ptr == NULL) {
220  return NULL;
221  }
222  StreamTcpReassembleIncrMemuse(n * size);
223  return ptr;
224 }
225 
226 /*
227  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
228 */
229 void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
230 {
231  if (size > orig_size) {
232  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) {
233  SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap));
235  return NULL;
236  }
237  }
238  void *nptr = SCRealloc(optr, size);
239  if (nptr == NULL) {
240  SCLogDebug("realloc fail");
242  return NULL;
243  }
244  if (size > orig_size) {
245  StreamTcpReassembleIncrMemuse(size - orig_size);
246  } else {
247  StreamTcpReassembleDecrMemuse(orig_size - size);
248  }
249  return nptr;
250 }
251 
252 /*
253  void (*Free)(void *ptr, size_t size);
254 */
255 static void ReassembleFree(void *ptr, size_t size)
256 {
257  SCFree(ptr);
258  StreamTcpReassembleDecrMemuse(size);
259 }
260 
261 /** \brief alloc a tcp segment pool entry */
262 static void *TcpSegmentPoolAlloc(void)
263 {
264  SCLogDebug("segment alloc");
265  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
266  return NULL;
267  }
268 
269  TcpSegment *seg = NULL;
270 
271  seg = SCMalloc(sizeof (TcpSegment));
272  if (unlikely(seg == NULL))
273  return NULL;
274 
276  uint32_t memuse =
277  sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
278  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
279  SCFree(seg);
280  return NULL;
281  }
282 
284  if (seg->pcap_hdr_storage == NULL) {
285  SCLogError("Unable to allocate memory for "
286  "TcpSegmentPcapHdrStorage");
287  SCFree(seg);
288  return NULL;
289  } else {
290  seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
291  seg->pcap_hdr_storage->pkt_hdr =
292  SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
293  if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
294  SCLogError("Unable to allocate memory for "
295  "packet header data within "
296  "TcpSegmentPcapHdrStorage");
297  SCFree(seg->pcap_hdr_storage);
298  SCFree(seg);
299  return NULL;
300  }
301  }
302 
303  StreamTcpReassembleIncrMemuse(memuse);
304  } else {
305  seg->pcap_hdr_storage = NULL;
306  }
307 
308  return seg;
309 }
310 
311 static int TcpSegmentPoolInit(void *data, void *initdata)
312 {
313  TcpSegment *seg = (TcpSegment *) data;
314  TcpSegmentPcapHdrStorage *pcap_hdr;
315 
316  pcap_hdr = seg->pcap_hdr_storage;
317 
318  /* do this before the can bail, so TcpSegmentPoolCleanup
319  * won't have uninitialized memory to consider. */
320  memset(seg, 0, sizeof (TcpSegment));
321 
323  uint32_t memuse =
324  sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
325  seg->pcap_hdr_storage = pcap_hdr;
326  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
327  return 0;
328  }
329  StreamTcpReassembleIncrMemuse(memuse);
330  } else {
331  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
332  return 0;
333  }
334  }
335 
336 #ifdef DEBUG
337  SCMutexLock(&segment_pool_memuse_mutex);
338  segment_pool_memuse += sizeof(TcpSegment);
339  segment_pool_memcnt++;
340  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
341  SCMutexUnlock(&segment_pool_memuse_mutex);
342 #endif
343 
344  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
345  return 1;
346 }
347 
348 /** \brief clean up a tcp segment pool entry */
349 static void TcpSegmentPoolCleanup(void *ptr)
350 {
351  if (ptr == NULL)
352  return;
353 
354  TcpSegment *seg = (TcpSegment *)ptr;
355  if (seg && seg->pcap_hdr_storage) {
356  if (seg->pcap_hdr_storage->pkt_hdr) {
358  StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen);
359  }
360  SCFree(seg->pcap_hdr_storage);
361  seg->pcap_hdr_storage = NULL;
362  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage));
363  }
364 
365  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
366 
367 #ifdef DEBUG
368  SCMutexLock(&segment_pool_memuse_mutex);
369  segment_pool_memuse -= sizeof(TcpSegment);
370  segment_pool_memcnt--;
371  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
372  SCMutexUnlock(&segment_pool_memuse_mutex);
373 #endif
374 }
375 
376 /**
377  * \brief Function to return the segment back to the pool.
378  *
379  * \param seg Segment which will be returned back to the pool.
380  */
382 {
383  if (seg == NULL)
384  return;
385 
386  if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
387  seg->pcap_hdr_storage->pktlen = 0;
388  }
389 
391 }
392 
393 /**
394  * \brief return all segments in this stream into the pool(s)
395  *
396  * \param stream the stream to cleanup
397  */
399 {
400  TcpSegment *seg = NULL, *safe = NULL;
401  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
402  {
403  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
405  }
406 }
407 
408 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
409 {
410  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
411  return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq);
412  }
413  return STREAM_BASE_OFFSET(stream);
414 }
415 
416 // may contain gaps
417 uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
418 {
419  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
420  if (!eof && !StreamTcpInlineMode()) {
421  right_edge = MIN(GetAbsLastAck(stream), right_edge);
422  }
423  return right_edge;
424 }
425 
426 uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
427 {
428  uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
429  if (!eof && !StreamTcpInlineMode()) {
430  right_edge = MIN(GetAbsLastAck(stream), right_edge);
431  }
432  return right_edge;
433 }
434 
435 #ifdef UNITTESTS
436 /** \internal
437  * \brief check if segments falls before stream 'offset' */
438 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
439 {
440  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
441  return 1;
442  return 0;
443 }
444 #endif
445 
446 /** \param f locked flow */
448 {
449  if (f->protoctx == NULL)
450  return;
451 
452  TcpSession *ssn = (TcpSession *)f->protoctx;
456  if (f->alparser) {
459  }
460 }
461 
462 /** \param f locked flow */
464 {
465  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
466  return 0;
467 
468  TcpSession *ssn = (TcpSession *)f->protoctx;
470 }
471 
472 static int StreamTcpReassemblyConfig(bool quiet)
473 {
474  uint32_t segment_prealloc = 2048;
475  SCConfNode *seg = SCConfGetNode("stream.reassembly.segment-prealloc");
476  if (seg) {
477  uint32_t prealloc = 0;
478  if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) {
479  SCLogError("segment-prealloc of "
480  "%s is invalid",
481  seg->val);
482  return -1;
483  }
484  segment_prealloc = prealloc;
485  }
486  if (!quiet)
487  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
488  stream_config.prealloc_segments = segment_prealloc;
489 
490  int overlap_diff_data = 0;
491  (void)SCConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
492  if (overlap_diff_data) {
494  }
495  if (StreamTcpInlineMode()) {
497  }
498 
499  uint16_t max_regions = 8;
500  SCConfNode *mr = SCConfGetNode("stream.reassembly.max-regions");
501  if (mr) {
502  uint16_t max_r = 0;
503  if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) {
504  SCLogError("max-regions %s is invalid", mr->val);
505  return -1;
506  }
507  max_regions = max_r;
508  }
509  if (!quiet)
510  SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions);
511 
512  stream_config.prealloc_segments = segment_prealloc;
514  stream_config.sbcnf.max_regions = max_regions;
516  stream_config.sbcnf.Calloc = ReassembleCalloc;
518  stream_config.sbcnf.Free = ReassembleFree;
519 
520  return 0;
521 }
522 
523 int StreamTcpReassembleInit(bool quiet)
524 {
525  /* init the memcap/use tracker */
527 
528  if (StreamTcpReassemblyConfig(quiet) < 0)
529  return -1;
530 
531 #ifdef DEBUG
532  SCMutexInit(&segment_pool_memuse_mutex, NULL);
533 #endif
534  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
536  return 0;
537 }
538 
539 void StreamTcpReassembleFree(bool quiet)
540 {
541  SCMutexLock(&segment_thread_pool_mutex);
542  if (segment_thread_pool != NULL) {
544  segment_thread_pool = NULL;
545  }
546  SCMutexUnlock(&segment_thread_pool_mutex);
547  SCMutexDestroy(&segment_thread_pool_mutex);
548 
549 #ifdef DEBUG
550  if (segment_pool_memuse > 0)
551  SCLogDebug("segment_pool_memuse %" PRIu64 " segment_pool_memcnt %" PRIu64 "",
552  segment_pool_memuse, segment_pool_memcnt);
553  SCMutexDestroy(&segment_pool_memuse_mutex);
554 #endif
555 }
556 
558 {
559  SCEnter();
561  if (unlikely(ra_ctx == NULL))
562  return NULL;
563 
564  ra_ctx->app_tctx = AppLayerGetCtxThread();
565 
566  SCMutexLock(&segment_thread_pool_mutex);
567  if (segment_thread_pool == NULL) {
568  segment_thread_pool = PoolThreadInit(1, /* thread */
569  0, /* unlimited */
571  sizeof(TcpSegment),
572  TcpSegmentPoolAlloc,
573  TcpSegmentPoolInit, NULL,
574  TcpSegmentPoolCleanup, NULL);
575  ra_ctx->segment_thread_pool_id = 0;
576  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
578  ra_ctx->segment_thread_pool_id);
579  } else {
580  /* grow segment_thread_pool until we have an element for our thread id */
582  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
584  ra_ctx->segment_thread_pool_id);
585  }
586  SCMutexUnlock(&segment_thread_pool_mutex);
587  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
588  SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
590  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
591  }
592 
593  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
594 }
595 
597 {
598  SCEnter();
600 
601  if (ra_ctx) {
603  SCFree(ra_ctx);
604  }
605  SCReturn;
606 }
607 
608 static void StreamTcpReassembleExceptionPolicyStatsIncr(
609  ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, enum ExceptionPolicy policy)
610 {
611  uint16_t id = ra_ctx->counter_tcp_reas_eps.eps_id[policy];
612  if (likely(tv && id > 0)) {
613  StatsIncr(tv, id);
614  }
615 }
616 
617 /**
618  * \brief check if stream in pkt direction has depth reached
619  *
620  * \param p packet with *LOCKED* flow
621  *
622  * \retval true stream has depth reached
623  * \retval false stream does not have depth reached
624  */
626 {
627  if (p->flow != NULL && p->flow->protoctx != NULL) {
628  TcpSession *ssn = p->flow->protoctx;
629  TcpStream *stream;
630  if (p->flowflags & FLOW_PKT_TOSERVER) {
631  stream = &ssn->client;
632  } else {
633  stream = &ssn->server;
634  }
635 
636  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? true : false;
637  }
638 
639  return false;
640 }
641 
642 /**
643  * \internal
644  * \brief Function to check the reassembly depth value against the
645  * allowed max depth of the stream reassembly for TCP streams.
646  *
647  * \param stream stream direction
648  * \param seq sequence number where "size" starts
649  * \param size size of the segment that is added
650  *
651  * \retval size Part of the size that fits in the depth, 0 if none
652  */
653 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
654  uint32_t seq, uint32_t size)
655 {
656  SCEnter();
657 
658  /* if the configured depth value is 0, it means there is no limit on
659  reassembly depth. Otherwise carry on my boy ;) */
660  if (ssn->reassembly_depth == 0) {
661  SCReturnUInt(size);
662  }
663 
664  /* if the final flag is set, we're not accepting anymore */
666  SCReturnUInt(0);
667  }
668 
669  uint64_t seg_depth;
670  if (SEQ_GT(stream->base_seq, seq)) {
671  if (SEQ_LEQ(seq+size, stream->base_seq)) {
672  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
673  stream->base_seq, seq, seq+size);
674  SCReturnUInt(0);
675  }
676 
677  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
678  } else {
679  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
680  }
681 
682  /* if the base_seq has moved past the depth window we stop
683  * checking and just reject the rest of the packets including
684  * retransmissions. Saves us the hassle of dealing with sequence
685  * wraps as well */
686  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
687  stream->base_seq, seg_depth,
688  ssn->reassembly_depth);
689 
690  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
691  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
693  SCReturnUInt(0);
694  }
695  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
696  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
697 #if 0
698  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
699  (stream->base_seq_offset + stream->base_seq + size),
700  (stream->isn + ssn->reassembly_depth));
701 #endif
702  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
703  /* packet (partly?) fits the depth window */
704 
705  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
706  /* complete fit */
707  SCReturnUInt(size);
708  } else {
710  /* partial fit, return only what fits */
711  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
712  DEBUG_VALIDATE_BUG_ON(part > size);
713  if (part > size)
714  part = size;
715  SCReturnUInt(part);
716  }
717  }
718 
719  SCReturnUInt(0);
720 }
721 
723 {
724  if (RB_EMPTY(&stream->sb.sbb_tree)) {
725  if (stream->sb.region.stream_offset != 0)
726  return 0;
727 
728  return stream->sb.region.buf_offset;
729  }
730 
731  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
732  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
733  return stream->sb.sbb_size;
734 }
735 
736 /**
737  * \brief Insert a TCP packet data into the stream reassembly engine.
738  *
739  * \retval 0 good segment, as far as we checked.
740  * \retval -1 insert failure due to memcap
741  *
742  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
743  * or it wasn't added because of reassembly depth.
744  *
745  */
747  TcpSession *ssn, TcpStream *stream, Packet *p)
748 {
749  SCEnter();
750 
751  if (ssn->data_first_seen_dir == 0) {
752  if (PKT_IS_TOSERVER(p)) {
753  ssn->data_first_seen_dir = STREAM_TOSERVER;
754  } else {
755  ssn->data_first_seen_dir = STREAM_TOCLIENT;
756  }
757  }
758 
759  /* If the OS policy is not set then set the OS policy for this stream */
760  if (stream->os_policy == 0) {
761  StreamTcpSetOSPolicy(stream, p);
762  }
763 
766  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
767  SCReturnInt(0);
768  }
769 
770  uint16_t *urg_offset;
771  if (PKT_IS_TOSERVER(p)) {
772  urg_offset = &ssn->urg_offset_ts;
773  } else {
774  urg_offset = &ssn->urg_offset_tc;
775  }
776 
777  const TCPHdr *tcph = PacketGetTCP(p);
778  /* segment sequence number, offset by previously accepted
779  * URG OOB data. */
780  uint32_t seg_seq = TCP_GET_RAW_SEQ(tcph) - (*urg_offset);
781  uint8_t urg_data = 0;
782 
783  /* if stream_config.urgent_policy == TCP_STREAM_URGENT_DROP, we won't get here */
784  if (tcph->th_flags & TH_URG) {
785  const uint16_t urg_ptr = SCNtohs(tcph->th_urp);
786  if (urg_ptr > 0 && urg_ptr <= p->payload_len &&
789  /* track up to 64k out of band URG bytes. Fall back to inline
790  * when that budget is exceeded. */
791  if ((*urg_offset) < UINT16_MAX) {
793  (*urg_offset)++;
794 
795  if ((*urg_offset) == UINT16_MAX) {
797  }
798  } else {
799  /* OOB limit DROP is handled here */
802  SCReturnInt(0);
803  }
804  }
805  urg_data = 1; /* only treat last 1 byte as out of band. */
808  }
809 
810  /* depending on hitting the OOB limit, update urg_data or not */
812  (*urg_offset) == UINT16_MAX &&
814  urg_data = 0;
815  } else {
816  if (urg_ptr == 1 && p->payload_len == 1) {
817  SCLogDebug("no non-URG data");
818  SCReturnInt(0);
819  }
820  }
821  }
822  }
823 
824  const uint16_t payload_len = p->payload_len - urg_data;
825 
826  /* If we have reached the defined depth for either of the stream, then stop
827  reassembling the TCP session */
828  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, seg_seq, payload_len);
829  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
830 
833  /* increment stream depth counter */
836  }
837  if (size == 0) {
838  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
839  SCReturnInt(0);
840  }
841 
843  if (size > payload_len)
844  size = payload_len;
845 
846  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
847  if (seg == NULL) {
848  SCLogDebug("segment_pool is empty");
851  SCReturnInt(-1);
852  }
853 
854  DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
855  TCP_SEG_LEN(seg) = (uint16_t)size;
856  /* set SEQUENCE number, adjusted to any URG pointer offset */
857  seg->seq = seg_seq;
858 
859  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
860  if (TCP_HAS_TFO(p) && p->payload_len && (tcph->th_flags & TH_SYN))
861  seg->seq += 1;
862 
863  /* proto detection skipped, but now we do get data. Set event. */
864  if (RB_EMPTY(&stream->seg_tree) &&
866 
869  }
870 
871  int r = StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, p->payload, payload_len);
872  if (r < 0) {
873  if (r == -SC_ENOMEM) {
875  }
876  SCLogDebug("StreamTcpReassembleInsertSegment failed");
877  SCReturnInt(-1);
878  }
879  SCReturnInt(0);
880 }
881 
882 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
883  Packet *p)
884 {
885  uint8_t flag = 0;
886 
888  flag |= STREAM_START;
889  }
890 
891  if (ssn->state == TCP_CLOSED) {
892  flag |= STREAM_EOF;
893  }
894 
895  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
896  flag |= STREAM_MIDSTREAM;
897  }
898 
899  if (p->flags & PKT_PSEUDO_STREAM_END) {
900  flag |= STREAM_EOF;
901  }
902 
903  if (&ssn->client == stream) {
904  flag |= STREAM_TOSERVER;
905  } else {
906  flag |= STREAM_TOCLIENT;
907  }
909  flag |= STREAM_DEPTH;
910  }
911  return flag;
912 }
913 
914 /**
915  * \brief Check the minimum size limits for reassembly.
916  *
917  * \retval false don't reassemble yet
918  * \retval true do reassemble
919  */
920 static bool StreamTcpReassembleRawCheckLimit(
921  const TcpSession *ssn, const TcpStream *stream, const Packet *p)
922 {
923  SCEnter();
924 
925  /* if any of these flags is set we always inspect immediately */
926 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
927  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
928  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
929  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
930 
933  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
934  "is set, so not expecting any new data segments");
935  }
937  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
938  }
940  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
941  "so no new segments will be considered");
942  }
943  SCReturnBool(true);
944  }
945 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
946 
947  /* some states mean we reassemble no matter how much data we have */
948  if (ssn->state > TCP_TIME_WAIT)
949  SCReturnBool(true);
950 
951  if (p->flags & PKT_PSEUDO_STREAM_END)
952  SCReturnBool(true);
953 
954  const uint64_t last_ack_abs = GetAbsLastAck(stream);
955  int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream);
956  int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size
958 
959  /* check if we have enough data to do raw reassembly */
960  if (chunk_size <= diff) {
961  SCReturnBool(true);
962  } else {
963  SCLogDebug("%s min chunk len not yet reached: "
964  "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < "
965  "%" PRIi64,
966  PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq,
967  (stream->last_ack - stream->base_seq), chunk_size);
968  SCReturnBool(false);
969  }
970 
971  SCReturnBool(false);
972 }
973 
974 /**
975  * \brief see what if any work the TCP session still needs
976  */
977 uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
978 {
979  const TcpStream *stream = NULL;
980 #ifdef DEBUG
981  const char *dirstr = NULL;
982 #endif
983  if (direction == STREAM_TOSERVER) {
984  stream = &ssn->client;
985 #ifdef DEBUG
986  dirstr = "client";
987 #endif
988  } else {
989  stream = &ssn->server;
990 #ifdef DEBUG
991  dirstr = "server";
992 #endif
993  }
994  int use_app = 1;
995  int use_raw = 1;
996 
998  // app is dead
999  use_app = 0;
1000  }
1001 
1002  if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
1003  // raw is dead
1004  use_raw = 0;
1005  }
1006  if (use_raw) {
1007  const uint64_t right_edge =
1008  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1009  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1010  " (use: %s). Stream right edge: %" PRIu64,
1011  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1012  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1013  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
1014  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1016  }
1017  }
1018  if (use_app) {
1019  const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
1020  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1021  " (use: %s). Stream right edge: %" PRIu64,
1022  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1023  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1024  if (right_edge > STREAM_APP_PROGRESS(stream)) {
1025  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1027  }
1028  }
1029 
1030  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
1032 }
1033 
1034 #ifdef DEBUG
1035 static uint64_t GetStreamSize(TcpStream *stream)
1036 {
1037  if (stream) {
1038  uint64_t size = 0;
1039  uint32_t cnt = 0;
1040  uint64_t last_ack_abs = GetAbsLastAck(stream);
1041  uint64_t last_re = 0;
1042 
1043  SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset);
1044 
1045  TcpSegment *seg;
1046  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
1047  const uint64_t seg_abs =
1048  STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq);
1049  if (last_re != 0 && last_re < seg_abs) {
1050  const char *gacked = NULL;
1051  if (last_ack_abs >= seg_abs) {
1052  gacked = "fully ack'd";
1053  } else if (last_ack_abs > last_re) {
1054  gacked = "partly ack'd";
1055  } else {
1056  gacked = "not yet ack'd";
1057  }
1058  SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked);
1059  }
1060 
1061  const char *acked = NULL;
1062  if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) {
1063  acked = "fully ack'd";
1064  } else if (last_ack_abs > seg_abs) {
1065  acked = "partly ack'd";
1066  } else {
1067  acked = "not yet ack'd";
1068  }
1069 
1070  SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64
1071  ") ack:%s",
1072  cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg),
1073  seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked);
1074  last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg);
1075  cnt++;
1076  size += (uint64_t)TCP_SEG_LEN(seg);
1077  }
1078 
1079  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
1080  return size;
1081  }
1082  return (uint64_t)0;
1083 }
1084 
1085 static void GetSessionSize(TcpSession *ssn, Packet *p)
1086 {
1087  uint64_t size = 0;
1088  if (ssn) {
1089  size = GetStreamSize(&ssn->client);
1090  size += GetStreamSize(&ssn->server);
1091 
1092  //if (size > 900000)
1093  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1094  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1095  }
1096 }
1097 #endif
1098 
1099 static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk)
1100 {
1101  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
1102  /* only if the stream has been ack'd, consider a gap for sure
1103  * otherwise there may still be a chance of pkts coming in */
1104  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
1105  GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len)) {
1106  return true;
1107  }
1108  return false;
1109 }
1110 
1111 /** \internal
1112  *
1113  * Get buffer, or first part of the buffer if data gaps exist.
1114  *
1115  * \brief get stream data from offset
1116  * \param offset stream offset
1117  * \param check_for_gap check if there is a gap ahead. Optional as it is only
1118  * needed for app-layer incomplete support.
1119  * \retval bool pkt loss ahead */
1120 static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1121  uint64_t offset, const bool check_for_gap)
1122 {
1123  const uint8_t *mydata;
1124  uint32_t mydata_len;
1125  bool gap_ahead = false;
1126 
1127  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1128  SCLogDebug("getting one blob");
1129 
1130  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1131 
1132  *data = mydata;
1133  *data_len = mydata_len;
1134  } else {
1135  SCLogDebug("block mode");
1136  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1137  StreamingBufferBlock *blk = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1138  if (blk == NULL) {
1139  *data = NULL;
1140  *data_len = 0;
1141  return false;
1142  }
1143  SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len);
1144 
1145  /* block at expected offset */
1146  if (blk->offset == offset) {
1147  SCLogDebug("blk at offset");
1148 
1149  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
1150  BUG_ON(blk->len != *data_len);
1151 
1152  gap_ahead = check_for_gap && GapAhead(stream, blk);
1153 
1154  /* block past out offset */
1155  } else if (blk->offset > offset) {
1156  SCLogDebug("gap, want data at offset %"PRIu64", "
1157  "got data at %"PRIu64". GAP of size %"PRIu64,
1158  offset, blk->offset, blk->offset - offset);
1159  *data = NULL;
1160  *data_len = (uint32_t)(blk->offset - offset);
1161 
1162  /* block starts before offset, but ends after */
1163  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
1164  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
1165  offset, blk->offset, blk->len);
1166  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
1167  SCLogDebug("data %p, data_len %u", *data, *data_len);
1168 
1169  gap_ahead = check_for_gap && GapAhead(stream, blk);
1170 
1171  } else {
1172  *data = NULL;
1173  *data_len = 0;
1174  }
1175  }
1176  return gap_ahead;
1177 }
1178 
1179 /** \internal
1180  * \brief check to see if we should declare a GAP
1181  * Call this when the app layer didn't get data at the requested
1182  * offset.
1183  */
1184 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
1185 {
1186  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1187  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1188  const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded;
1189 
1190  SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs);
1191  SCLogDebug("next_seq %u", stream->next_seq);
1192 
1193  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1194  * has been ack'd. This looks like a GAP. */
1195  if (last_ack_abs > app_progress) {
1196  /* however, we can accept ACKs a bit too liberally. If last_ack
1197  * is beyond next_seq, we only consider it a gap now if we do
1198  * already have data beyond the gap. */
1199  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1200  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1201  SCLogDebug("packet %" PRIu64 ": no GAP. "
1202  "next_seq %u < last_ack %u, but no data in list",
1203  p->pcap_cnt, stream->next_seq, stream->last_ack);
1204  return false;
1205  }
1206  const uint64_t next_seq_abs =
1207  STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1208  const StreamingBufferBlock *blk = stream->sb.head;
1209  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1210  /* ack'd data after the gap */
1211  SCLogDebug("packet %" PRIu64 ": GAP. "
1212  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1213  p->pcap_cnt, stream->next_seq, stream->last_ack);
1214  return true;
1215  }
1216  }
1217 
1218  SCLogDebug("packet %" PRIu64 ": GAP! "
1219  "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", "
1220  "but we have no data.",
1221  p->pcap_cnt, last_ack_abs, app_progress);
1222  return true;
1223  }
1224  SCLogDebug("packet %"PRIu64": no GAP. "
1225  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1226  p->pcap_cnt, last_ack_abs, app_progress);
1227  return false;
1228 }
1229 
1230 static inline uint32_t AdjustToAcked(const Packet *p,
1231  const TcpSession *ssn, const TcpStream *stream,
1232  const uint64_t app_progress, const uint32_t data_len)
1233 {
1234  uint32_t adjusted = data_len;
1235 
1236  /* get window of data that is acked */
1237  if (!StreamTcpInlineMode()) {
1238  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1239  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1240  (ssn->state == TCP_CLOSED &&
1241  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1242  (p->flags & PKT_PSEUDO_STREAM_END))) {
1243  // fall through, we use all available data
1244  } else {
1245  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1246  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1247 
1248  /* see if the buffer contains unack'd data as well */
1249  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1250  uint32_t check = data_len;
1251  adjusted = (uint32_t)(last_ack_abs - app_progress);
1252  BUG_ON(adjusted > check);
1253  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1254  "data is considered", adjusted);
1255  }
1256  }
1257  }
1258  return adjusted;
1259 }
1260 
1261 /** \internal
1262  * \brief get stream buffer and update the app-layer
1263  * \param stream pointer to pointer as app-layer can switch flow dir
1264  * \retval 0 success
1265  */
1266 static int ReassembleUpdateAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn,
1267  TcpStream **stream, Packet *p, enum StreamUpdateDir app_update_dir)
1268 {
1269  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1270 
1271  SCLogDebug("app progress %"PRIu64, app_progress);
1272 #ifdef DEBUG
1273  uint64_t last_ack_abs = GetAbsLastAck(*stream);
1274  SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs,
1275  (*stream)->base_seq);
1276 #endif
1277  const uint8_t *mydata;
1278  uint32_t mydata_len;
1279  bool last_was_gap = false;
1280 
1281  while (1) {
1282  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1283  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1284  bool gap_ahead =
1285  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead);
1286  SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len);
1287  if (last_was_gap && mydata_len == 0) {
1288  break;
1289  }
1290  last_was_gap = false;
1291 
1292  /* make sure to only deal with ACK'd data */
1293  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1294  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1295  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1296  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1297 
1298  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len,
1299  StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, app_update_dir);
1300  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1301 
1303  (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP;
1306 
1307  /* AppLayerHandleTCPData has likely updated progress. */
1308  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1309  app_progress = STREAM_APP_PROGRESS(*stream);
1310 
1311  /* a GAP also consumes 'data required'. TODO perhaps we can use
1312  * this to skip post GAP data until the start of a next record. */
1313  if ((*stream)->data_required > 0) {
1314  if ((*stream)->data_required > mydata_len) {
1315  (*stream)->data_required -= mydata_len;
1316  } else {
1317  (*stream)->data_required = 0;
1318  }
1319  }
1320  if (r < 0)
1321  return 0;
1322  if (no_progress_update)
1323  break;
1324  last_was_gap = true;
1325  continue;
1326 
1327  } else if (flags & STREAM_DEPTH) {
1328  SCLogDebug("DEPTH");
1329  // we're just called once with this flag, so make sure we pass it on
1330  if (mydata == NULL && mydata_len > 0) {
1331  mydata_len = 0;
1332  }
1333  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1334  SCLogDebug("GAP?1");
1335  /* Possibly a gap, but no new data. */
1336  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1337  SCReturnInt(0);
1338 
1339  mydata = NULL;
1340  mydata_len = 0;
1341  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1342  break;
1343  }
1344  DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1345 
1346  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1347  *stream, &(*stream)->sb, mydata_len, app_progress);
1348 
1349  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1350  SCLogDebug("GAP?2");
1351  if (mydata_len < (*stream)->data_required) {
1352  SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead));
1353  if (gap_ahead) {
1354  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1355  (*stream)->data_required, mydata_len);
1356  (*stream)->app_progress_rel += mydata_len;
1357  (*stream)->data_required -= mydata_len;
1358  // TODO send incomplete data to app-layer with special flag
1359  // indicating its all there is for this rec?
1360  } else {
1361  SCReturnInt(0);
1362  }
1363  app_progress = STREAM_APP_PROGRESS(*stream);
1364  continue;
1365  }
1366  }
1367  (*stream)->data_required = 0;
1368 
1369  SCLogDebug("parser");
1370  /* update the app-layer */
1371  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata,
1372  mydata_len, flags, app_update_dir);
1373  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1374  AppLayerFrameDump(p->flow);
1375  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1376  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1377  break;
1378  app_progress = new_app_progress;
1379  if (flags & STREAM_DEPTH)
1380  break;
1381  }
1382 
1383  SCReturnInt(0);
1384 }
1385 
1386 /**
1387  * \brief Update the stream reassembly upon receiving a packet.
1388  *
1389  * For IDS mode, the stream is in the opposite direction of the packet,
1390  * as the ACK-packet is ACK'ing the stream.
1391  *
1392  * One of the utilities call by this function AppLayerHandleTCPData(),
1393  * has a feature where it will call this very same function for the
1394  * stream opposing the stream it is called with. This shouldn't cause
1395  * any issues, since processing of each stream is independent of the
1396  * other stream.
1397  */
1399  TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
1400 {
1401  SCEnter();
1402 
1403  /* this function can be directly called by app layer protocol
1404  * detection. */
1407  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1408  SCReturnInt(0);
1409  }
1410 
1411 #ifdef DEBUG
1412  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1413  GetSessionSize(ssn, p);
1414 #endif
1415  /* if no segments are in the list or all are already processed,
1416  * and state is beyond established, we send an empty msg */
1417  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1418  {
1419  /* send an empty EOF msg if we have no segments but TCP state
1420  * is beyond ESTABLISHED */
1421  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1422  SCLogDebug("sending empty eof message");
1423  /* send EOF to app layer */
1424  uint8_t stream_flags = StreamGetAppLayerFlags(ssn, stream, p);
1425  DEBUG_VALIDATE_BUG_ON((stream_flags & STREAM_EOF) == 0);
1427  tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, stream_flags, app_update_dir);
1428  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1429 
1430  SCReturnInt(0);
1431  }
1432  }
1433 
1434  /* with all that out of the way, lets update the app-layer */
1435  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, app_update_dir);
1436 }
1437 
1438 /** \internal
1439  * \brief get stream data from offset
1440  * \param offset stream offset */
1441 static int GetRawBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1442  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1443 {
1444  const uint8_t *mydata;
1445  uint32_t mydata_len;
1446  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1447  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1448 
1449  /* No gaps in the stream, data must exist in the streaming buffer array */
1450  uint64_t roffset = offset;
1451  if (offset) {
1452  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1453  } else {
1454  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1455  }
1456 
1457  *data = mydata;
1458  *data_len = mydata_len;
1459  *data_offset = roffset;
1460  } else {
1461  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1462  *iter == NULL ? "starting" : "continuing", offset);
1463  if (*iter == NULL) {
1464  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1465  *iter = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1466  SCLogDebug("*iter %p", *iter);
1467  }
1468  if (*iter == NULL) {
1469  SCLogDebug("no data");
1470  *data = NULL;
1471  *data_len = 0;
1472  *data_offset = 0;
1473  return 0;
1474  }
1475  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1476 
1477  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1478  SCLogDebug("mydata %p", mydata);
1479 
1480  if ((*iter)->offset < offset) {
1481  uint64_t delta = offset - (*iter)->offset;
1482  if (delta < mydata_len) {
1483  *data = mydata + delta;
1484  *data_len = (uint32_t)(mydata_len - delta);
1485  *data_offset = offset;
1486  } else {
1487  SCLogDebug("no data (yet)");
1488  *data = NULL;
1489  *data_len = 0;
1490  *data_offset = 0;
1491  }
1492 
1493  } else {
1494  *data = mydata;
1495  *data_len = mydata_len;
1496  *data_offset = (*iter)->offset;
1497  }
1498 
1499  *iter = SBB_RB_NEXT(*iter);
1500  SCLogDebug("*iter %p", *iter);
1501  }
1502  return 0;
1503 }
1504 
1505 /** \brief does the stream engine have data to inspect?
1506  *
1507  * Returns true if there is data to inspect. In IDS case this is
1508  * about ACK'd data in the packet's direction.
1509  *
1510  * In the IPS case this is about the packet itself.
1511  */
1513 {
1514  TcpStream *stream;
1515  if (PKT_IS_TOSERVER(p)) {
1516  stream = &ssn->client;
1517  } else {
1518  stream = &ssn->server;
1519  }
1520 
1521  if (RB_EMPTY(&stream->seg_tree)) {
1522  return false;
1523  }
1524 
1527  return false;
1528 
1529  if (!StreamTcpInlineMode()) {
1530  const uint64_t segs_re_abs =
1531  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1532  if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) {
1533  return false;
1534  }
1535  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1536  return true;
1537  }
1538  } else {
1539  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1540  return true;
1541  }
1542  }
1543  return false;
1544 }
1545 
1546 /** \brief update stream engine after detection
1547  *
1548  * Tasked with progressing the 'progress' for Raw reassembly.
1549  * 2 main scenario's:
1550  * 1. progress is != 0, so we use this
1551  * 2. progress is 0, meaning the detect engine didn't touch
1552  * raw at all. In this case we need to look into progressing
1553  * raw anyway.
1554  *
1555  * Additionally, this function is tasked with disabling raw
1556  * reassembly if the app-layer requested to disable it.
1557  */
1558 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
1559 {
1560  TcpStream *stream;
1561  if (PKT_IS_TOSERVER(p)) {
1562  stream = &ssn->client;
1563  } else {
1564  stream = &ssn->server;
1565  }
1566 
1567  if (progress > STREAM_RAW_PROGRESS(stream)) {
1568  DEBUG_VALIDATE_BUG_ON(progress - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1569  uint32_t slide = (uint32_t)(progress - STREAM_RAW_PROGRESS(stream));
1570  stream->raw_progress_rel += slide;
1572 
1573  } else if (progress == 0) {
1574  uint64_t target;
1575  if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) {
1576  target = STREAM_APP_PROGRESS(stream);
1577  } else {
1578  target = GetAbsLastAck(stream);
1579  }
1580  if (target > STREAM_RAW_PROGRESS(stream)) {
1581  DEBUG_VALIDATE_BUG_ON(target - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1582  uint32_t slide = (uint32_t)(target - STREAM_RAW_PROGRESS(stream));
1583  stream->raw_progress_rel += slide;
1584  }
1586 
1587  } else {
1588  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1589  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1590  STREAM_RAW_PROGRESS(stream), stream->window);
1591  }
1592 
1593  /* if we were told to accept no more raw data, we can mark raw as
1594  * disabled now. */
1597  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1598  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1599  }
1600 
1601  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1602 }
1603 
1604 /** \internal
1605  * \brief get a buffer around the current packet and run the callback on it
1606  *
1607  * The inline/IPS scanning method takes the current payload and wraps it in
1608  * data from other segments.
1609  *
1610  * How much data is inspected is controlled by the available data, chunk_size
1611  * and the payload size of the packet.
1612  *
1613  * Large packets: if payload size is close to the chunk_size, where close is
1614  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1615  * used: payload_len + 33% of the chunk_size.
1616  * If the payload size if equal to or bigger than the chunk_size, we use
1617  * payload len + 33% of the chunk size.
1618  */
1619 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1620  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1621 {
1622  SCEnter();
1623  int r = 0;
1624 
1625  TcpStream *stream;
1626  if (PKT_IS_TOSERVER(p)) {
1627  stream = &ssn->client;
1628  } else {
1629  stream = &ssn->server;
1630  }
1631 
1632  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1634  {
1635  *progress_out = STREAM_RAW_PROGRESS(stream);
1636  return 0;
1637  }
1638 
1639  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1642  if ((chunk_size <= p->payload_len) || (((chunk_size / 3) * 2) < p->payload_len)) {
1643  chunk_size = p->payload_len + (chunk_size / 3);
1644  SCLogDebug(
1645  "packet payload len %u, so chunk_size adjusted to %u", p->payload_len, chunk_size);
1646  }
1647 
1648  const TCPHdr *tcph = PacketGetTCP(p);
1649  uint64_t packet_leftedge_abs =
1650  STREAM_BASE_OFFSET(stream) + (TCP_GET_RAW_SEQ(tcph) - stream->base_seq);
1651  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1652  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1653  packet_leftedge_abs, packet_rightedge_abs);
1654 
1655  const uint8_t *mydata = NULL;
1656  uint32_t mydata_len = 0;
1657  uint64_t mydata_offset = 0;
1658  /* simply return progress from the block we inspected. */
1659  bool return_progress = false;
1660 
1661  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1662  /* continues block */
1663  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1664  return_progress = true;
1665 
1666  } else {
1667  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1668  /* find our block */
1669  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1670  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1671  if (sbb) {
1672  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1673  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1674  mydata_offset = sbb->offset;
1675  }
1676  }
1677 
1678  /* this can only happen if the segment insert of our current 'p' failed */
1679  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1680  if ((mydata == NULL || mydata_len == 0) || /* no data */
1681  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1682  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1683  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1684  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1685  {
1686  /* no data, or data is incomplete or wrong: use packet data */
1687  mydata = p->payload;
1688  mydata_len = p->payload_len;
1689  mydata_offset = packet_leftedge_abs;
1690  //mydata_rightedge_abs = packet_rightedge_abs;
1691  } else {
1692  /* adjust buffer to match chunk_size */
1693  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1694  if (mydata_len > chunk_size) {
1695  uint32_t excess = mydata_len - chunk_size;
1696  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1697 
1698  if (mydata_rightedge_abs == packet_rightedge_abs) {
1699  mydata += excess;
1700  mydata_len -= excess;
1701  mydata_offset += excess;
1702  SCLogDebug("cutting front of the buffer with %u", excess);
1703  } else if (mydata_offset == packet_leftedge_abs) {
1704  mydata_len -= excess;
1705  SCLogDebug("cutting tail of the buffer with %u", excess);
1706  } else {
1707  DEBUG_VALIDATE_BUG_ON(mydata_offset > packet_leftedge_abs);
1708  uint32_t abs_before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1709  DEBUG_VALIDATE_BUG_ON(packet_rightedge_abs > mydata_rightedge_abs);
1710  uint32_t abs_after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1711  uint32_t before = abs_before;
1712  uint32_t after = abs_after;
1713  SCLogDebug("before %u after %u", before, after);
1714 
1715  if (after >= (chunk_size - p->payload_len) / 2) {
1716  // more trailing data than we need
1717 
1718  if (before >= (chunk_size - p->payload_len) / 2) {
1719  // also more heading data, divide evenly
1720  before = after = (chunk_size - p->payload_len) / 2;
1721  } else {
1722  // heading data is less than requested, give the
1723  // rest to the trailing data
1724  after = (chunk_size - p->payload_len) - before;
1725  }
1726  } else {
1727  // less trailing data than requested
1728 
1729  if (before >= (chunk_size - p->payload_len) / 2) {
1730  before = (chunk_size - p->payload_len) - after;
1731  } else {
1732  // both smaller than their requested size
1733  }
1734  }
1735 
1736  /* adjust the buffer */
1737  DEBUG_VALIDATE_BUG_ON(before > abs_before);
1738  uint32_t skip = abs_before - before;
1739  DEBUG_VALIDATE_BUG_ON(after > abs_after);
1740  uint32_t cut = abs_after - after;
1741  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1742  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1743  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1744  mydata += skip;
1745  mydata_len -= (skip + cut);
1746  mydata_offset += skip;
1747  }
1748  }
1749  }
1750 
1751  /* run the callback */
1752  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1753  BUG_ON(r < 0);
1754 
1755  if (return_progress) {
1756  *progress_out = (mydata_offset + mydata_len);
1757  } else {
1758  /* several blocks of data, so we need to be a bit more careful:
1759  * - if last_ack is beyond last progress, move progress forward to last_ack
1760  * - if our block matches or starts before last ack, return right edge of
1761  * our block.
1762  */
1763  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1764  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1765 
1766  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1767  if (mydata_offset > last_ack_abs) {
1768  /* gap between us and last ack, set progress to last ack */
1769  *progress_out = last_ack_abs;
1770  } else {
1771  *progress_out = (mydata_offset + mydata_len);
1772  }
1773  } else {
1774  *progress_out = STREAM_RAW_PROGRESS(stream);
1775  }
1776  }
1777  return r;
1778 }
1779 
1780 /** \brief access 'raw' reassembly data.
1781  *
1782  * Access data as tracked by 'raw' tracker. Data is made available to
1783  * callback that is passed to this function.
1784  *
1785  * In the case of IDS the callback may be run multiple times if data
1786  * contains gaps. It will then be run for each block of data that is
1787  * continuous.
1788  *
1789  * The callback should give on of 2 return values:
1790  * - 0 ok
1791  * - 1 done
1792  * The value 1 will break the loop if there is a block list that is
1793  * inspected.
1794  *
1795  * This function will return the 'progress' value that has been
1796  * consumed until now.
1797  *
1798  * \param ssn tcp session
1799  * \param stream tcp stream
1800  * \param Callback the function pointer to the callback function
1801  * \param cb_data callback data
1802  * \param[in] progress_in progress to work from
1803  * \param[in] re right edge of data to consider
1804  * \param[out] progress_out absolute progress value of the data this
1805  * call handled.
1806  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1807  */
1808 static int StreamReassembleRawDo(const TcpSession *ssn, const TcpStream *stream,
1809  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1810  const uint64_t re, uint64_t *progress_out, bool eof)
1811 {
1812  SCEnter();
1813  int r = 0;
1814 
1815  StreamingBufferBlock *iter = NULL;
1816  uint64_t progress = progress_in;
1817 
1818  /* loop through available buffers. On no packet loss we'll have a single
1819  * iteration. On missing data we'll walk the blocks */
1820  while (1) {
1821  const uint8_t *mydata;
1822  uint32_t mydata_len;
1823  uint64_t mydata_offset = 0;
1824 
1825  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1826  if (mydata_len == 0) {
1827  SCLogDebug("no data");
1828  break;
1829  }
1830  //PrintRawDataFp(stdout, mydata, mydata_len);
1831 
1832  SCLogDebug("raw progress %"PRIu64, progress);
1833  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1834  stream, &stream->sb, mydata_len, progress);
1835 
1836  if (eof) {
1837  // inspect all remaining data, ack'd or not
1838  } else {
1839  if (re < progress) {
1840  SCLogDebug("nothing to do");
1841  goto end;
1842  }
1843 
1844  SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress);
1845  SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len,
1846  re);
1847 
1848  /* see if the buffer contains unack'd data as well */
1849  if (progress + mydata_len > re) {
1850  uint32_t check = mydata_len;
1851  mydata_len = (uint32_t)(re - progress);
1852  BUG_ON(check < mydata_len);
1853  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1854  "data is considered", mydata_len);
1855  }
1856  }
1857  if (mydata_len == 0)
1858  break;
1859 
1860  SCLogDebug("data %p len %u", mydata, mydata_len);
1861 
1862  /* we have data. */
1863  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1864  BUG_ON(r < 0);
1865 
1866  if (mydata_offset == progress) {
1867  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1868  progress, mydata_len, progress_in + mydata_len);
1869 
1870  progress += mydata_len;
1871  SCLogDebug("raw progress now %"PRIu64, progress);
1872 
1873  /* data is beyond the progress we'd like, and before last ack. Gap. */
1874  } else if (mydata_offset > progress && mydata_offset < re) {
1875  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1876  SCLogDebug("re %" PRIu64, re);
1877  progress = mydata_offset;
1878  SCLogDebug("raw progress now %"PRIu64, progress);
1879 
1880  } else {
1881  SCLogDebug("not increasing progress, data gap => mydata_offset "
1882  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1883  }
1884 
1885  if (iter == NULL || r == 1)
1886  break;
1887  }
1888 end:
1889  *progress_out = progress;
1890  return r;
1891 }
1892 
1894  void *cb_data, const uint64_t offset, const bool eof)
1895 {
1896  /* take app progress as the right edge of used data. */
1897  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1898  SCLogDebug("app_progress %" PRIu64, app_progress);
1899 
1900  uint64_t unused = 0;
1901  return StreamReassembleRawDo(
1902  ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof);
1903 }
1904 
1906  StreamReassembleRawFunc Callback, void *cb_data,
1907  uint64_t *progress_out, bool respect_inspect_depth)
1908 {
1909  /* handle inline separately as the logic is very different */
1910  if (StreamTcpInlineMode()) {
1911  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1912  }
1913 
1914  TcpStream *stream;
1915  if (PKT_IS_TOSERVER(p)) {
1916  stream = &ssn->client;
1917  } else {
1918  stream = &ssn->server;
1919  }
1920 
1922  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1923  {
1924  *progress_out = STREAM_RAW_PROGRESS(stream);
1925  return 0;
1926  }
1927 
1928  uint64_t progress = STREAM_RAW_PROGRESS(stream);
1929  /* if the app layer triggered a flush, and we're supposed to
1930  * use a minimal inspect depth, we actually take the app progress
1931  * as that is the right edge of the data. Then we take the window
1932  * of 'min_inspect_depth' before that. */
1933 
1934  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s "
1935  "stream->min_inspect_depth %u",
1936  respect_inspect_depth ? "true" : "false",
1937  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1938  stream->min_inspect_depth);
1939 
1940  if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) &&
1941  stream->min_inspect_depth) {
1942  progress = STREAM_APP_PROGRESS(stream);
1943  if (stream->min_inspect_depth >= progress) {
1944  progress = 0;
1945  } else {
1946  progress -= stream->min_inspect_depth;
1947  }
1948 
1949  SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream),
1950  STREAM_RAW_PROGRESS(stream));
1951 
1952  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1953  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress "
1954  "%" PRIu64,
1955  progress);
1956  }
1957 
1958  SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth,
1959  stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW"
1960  : "(no trigger)");
1961 
1962  /* absolute right edge of ack'd data */
1963  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1964  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1965 
1966  return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs,
1967  progress_out, (p->flags & PKT_PSEUDO_STREAM_END));
1968 }
1969 
1970 int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream,
1971  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1972  uint64_t *progress_out, const bool eof)
1973 {
1974  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1975  return 0;
1976 
1977  /* absolute right edge of ack'd data */
1978  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1979  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1980 
1981  return StreamReassembleRawDo(
1982  ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof);
1983 }
1984 
1985 /** \internal
1986  * \brief update app layer based on received ACK
1987  *
1988  * \retval r 0 on success, -1 on error
1989  */
1990 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1991  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1992 {
1993  SCEnter();
1994 
1995  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1996  SCReturnInt(-1);
1997 
1998  SCReturnInt(0);
1999 }
2000 
2002  TcpSession *ssn, TcpStream *stream, Packet *p)
2003 {
2004  SCEnter();
2005 
2006  DEBUG_VALIDATE_BUG_ON(!PacketIsTCP(p));
2007  const TCPHdr *tcph = PacketGetTCP(p);
2008 
2009  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
2010  ssn, stream, p, p->payload_len);
2011 
2012  /* default IDS: update opposing side (triggered by ACK) */
2014  /* inline and stream end and flow timeout packets trigger same dir handling */
2015  if (StreamTcpInlineMode()) {
2016  dir = UPDATE_DIR_PACKET;
2017  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
2018  dir = UPDATE_DIR_PACKET;
2019  } else if (tcph->th_flags & TH_RST) { // accepted rst
2020  dir = UPDATE_DIR_PACKET;
2021  } else if ((tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
2022  if (tcph->th_flags & TH_ACK) {
2023  dir = UPDATE_DIR_BOTH;
2024  } else {
2025  dir = UPDATE_DIR_PACKET;
2026  }
2027  } else if (ssn->state == TCP_CLOSED) {
2028  dir = UPDATE_DIR_BOTH;
2029  } else if ((ssn->flags & STREAMTCP_FLAG_ASYNC) != 0) {
2030  dir = UPDATE_DIR_PACKET;
2031  SCLogDebug("%" PRIu64 ": ASYNC: UPDATE_DIR_PACKET", p->pcap_cnt);
2032  }
2033 
2034  /* handle ack received */
2035  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
2036  /* we need to update the opposing stream in
2037  * StreamTcpReassembleHandleSegmentUpdateACK */
2038  TcpStream *opposing_stream = NULL;
2039  if (stream == &ssn->client) {
2040  opposing_stream = &ssn->server;
2041  } else {
2042  opposing_stream = &ssn->client;
2043  }
2044 
2045  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2046 
2047  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
2048  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
2049  SCReturnInt(-1);
2050  }
2051 
2052  /* StreamTcpReassembleHandleSegmentUpdateACK
2053  * may swap content of ssn->server and ssn->client structures.
2054  * We have to continue with initial content of the stream in such case */
2055  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2056  if (reversed_before_ack_handling != reversed_after_ack_handling) {
2057  SCLogDebug("TCP streams were swapped");
2058  stream = opposing_stream;
2059  }
2060  }
2061  /* if this segment contains data, insert it */
2062  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) &&
2063  (tcph->th_flags & TH_RST) == 0) {
2064  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
2065 
2066  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
2067  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
2068  /* failure can only be because of memcap hit, so see if this should lead to a drop */
2071  StreamTcpReassembleExceptionPolicyStatsIncr(
2073  SCReturnInt(-1);
2074  }
2075 
2076  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
2077  p->flags |= PKT_STREAM_ADD;
2078  } else {
2079  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
2080  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
2081  ssn, stream, p->payload_len,
2082  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
2083  }
2084 
2085  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
2086  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
2087  * was *just* set. In this case we trigger the AppLayer Truncate
2088  * logic, to inform the applayer no more data in this direction is
2089  * to be expected. */
2090  if ((stream->flags &
2093  {
2094  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
2095  if (dir != UPDATE_DIR_PACKET) {
2096  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
2097  "can trigger Truncate");
2098  dir = UPDATE_DIR_PACKET;
2099  }
2100  }
2101 
2102  /* in stream inline mode even if we have no data we call the reassembly
2103  * functions to handle EOF */
2104  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
2105  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
2106  StreamTcpInlineMode()?"true":"false",
2107  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
2108  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
2109  SCReturnInt(-1);
2110  }
2111  }
2112 
2113  SCReturnInt(0);
2114 }
2115 
2116 /**
2117  * \brief get a segment from the pool
2118  *
2119  * \retval seg Segment from the pool or NULL
2120  */
2122 {
2124  if (seg) {
2126  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2127  return seg;
2128  }
2129 
2130  seg = (TcpSegment *)PoolThreadGetById(
2131  segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id);
2132  SCLogDebug("seg we return is %p", seg);
2133  if (seg == NULL) {
2134  /* Increment the counter to show that we are not able to serve the
2135  segment request due to memcap limit */
2137  } else {
2138  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2140  }
2141 
2142  return seg;
2143 }
2144 
2145 /**
2146  * \brief Trigger RAW stream inspection
2147  *
2148  * Used by AppLayerTriggerRawStreamInspection to trigger RAW stream
2149  * inspection from the applayer, for example upon completion of a
2150  * HTTP request.
2151  *
2152  * It sets a flag in the stream so that the next Raw call will return
2153  * the data.
2154  *
2155  * \param ssn TcpSession
2156  */
2158 {
2159 #ifdef DEBUG
2160  BUG_ON(ssn == NULL);
2161 #endif
2162 
2163  if (ssn != NULL) {
2164  if (direction == STREAM_TOSERVER) {
2166  } else {
2168  }
2169 
2170  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
2171  }
2172 }
2173 
2174 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
2175 {
2176 #ifdef DEBUG
2177  BUG_ON(ssn == NULL);
2178 #endif
2179 
2180  if (ssn != NULL) {
2181  if (direction == STREAM_TOSERVER) {
2182  ssn->client.min_inspect_depth = depth;
2183  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
2184  } else {
2185  ssn->server.min_inspect_depth = depth;
2186  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
2187  }
2188  }
2189 }
2190 
2191 #ifdef UNITTESTS
2192 /** unit tests and it's support functions below */
2193 
2194 #define SET_ISN(stream, setseq) \
2195  (stream)->isn = (setseq); \
2196  (stream)->base_seq = (setseq) + 1
2197 
2198 /** \brief The Function to create the packet with given payload, which is used
2199  * to test the reassembly of the engine.
2200  *
2201  * \param payload The variable used to store the payload contents of the
2202  * current packet.
2203  * \param value The value which current payload will have for this packet
2204  * \param payload_len The length of the filed payload for current packet.
2205  * \param len Length of the payload array
2206  */
2207 
2208 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2209  uint8_t payload_len, uint8_t len)
2210 {
2211  uint8_t i;
2212  for (i = 0; i < payload_len; i++)
2213  payload[i] = value;
2214  for (; i < len; i++)
2215  payload = NULL;
2216 }
2217 
2218 /** \brief The Function Checks the reassembled stream contents against predefined
2219  * stream contents according to OS policy used.
2220  *
2221  * \param stream_policy Predefined value of stream for different OS policies
2222  * \param stream Reassembled stream returned from the reassembly functions
2223  */
2224 
2225 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2226 {
2227  if (StreamingBufferCompareRawData(&stream->sb,
2228  data, data_len) == 0)
2229  {
2230  SCReturnInt(0);
2231  }
2232  SCLogInfo("OK");
2233  PrintRawDataFp(stdout, data, data_len);
2234  return 1;
2235 }
2236 
2237 #define MISSED_START(isn) \
2238  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2239  TcpSession ssn; \
2240  ThreadVars tv; \
2241  memset(&tv, 0, sizeof(tv)); \
2242  \
2243  StreamTcpUTInit(&ra_ctx); \
2244  \
2245  StreamTcpUTSetupSession(&ssn); \
2246  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2247  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2248  \
2249  TcpStream *stream = &ssn.client;
2250 
2251 #define MISSED_END \
2252  StreamTcpUTClearSession(&ssn); \
2253  StreamTcpUTDeinit(ra_ctx); \
2254  PASS
2255 
2256 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2257  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2258  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2259 
2260 #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \
2261  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen));
2262 
2263 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len);
2264 
2265 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
2266 {
2267  int cnt = 0;
2268  uint64_t last_re = 0;
2269  StreamingBufferBlock *sbb = NULL;
2270  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2271  {
2272  if (sbb->offset != last_re) {
2273  // gap before us
2274  if (cnt == pos && last_re == offset && len == sbb->offset - last_re) {
2275  return 1;
2276  }
2277  cnt++;
2278  }
2279  last_re = sbb->offset + sbb->len;
2280  cnt++;
2281  }
2282  return 0;
2283 }
2284 
2286  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len);
2287 
2289  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
2290 {
2291  int cnt = 0;
2292  uint64_t last_re = 0;
2293  StreamingBufferBlock *sbb = NULL;
2294  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2295  {
2296  if (sbb->offset != last_re) {
2297  // gap before us
2298  cnt++;
2299  }
2300 
2301  if (cnt == pos && sbb->offset == offset) {
2302  const uint8_t *buf = NULL;
2303  uint32_t buf_len = 0;
2304  StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len);
2305 
2306  if (len == buf_len) {
2307  return (memcmp(data, buf, len) == 0);
2308  }
2309  }
2310 
2311  last_re = sbb->offset + sbb->len;
2312  cnt++;
2313  }
2314  return 0;
2315 }
2316 
2317 /**
2318  * \test Test the handling of packets missed by both IDS and the end host.
2319  * The packet is missed in the starting of the stream.
2320  *
2321  * \retval On success it returns 1 and on failure 0.
2322  */
2323 
2324 static int StreamTcpReassembleTest25 (void)
2325 {
2326  MISSED_START(6);
2327  MISSED_ADD_PAYLOAD(10, "BB", 2);
2328  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2329  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BB", 2) == 1);
2330  MISSED_ADD_PAYLOAD(12, "CC", 2);
2331  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2332  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BBCC", 4) == 1);
2333  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2334  MISSED_END;
2335  PASS;
2336 }
2337 
2338 /**
2339  * \test Test the handling of packets missed by both IDS and the end host.
2340  * The packet is missed in the middle of the stream.
2341  *
2342  * \retval On success it returns 1 and on failure 0.
2343  */
2344 
2345 static int StreamTcpReassembleTest26 (void)
2346 {
2347  MISSED_START(9);
2348  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2349  MISSED_ADD_PAYLOAD(15, "CC", 2);
2350  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2351  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2352  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2353  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2354  MISSED_END;
2355 }
2356 
2357 /**
2358  * \test Test the handling of packets missed by both IDS and the end host.
2359  * The packet is missed in the end of the stream.
2360  *
2361  * \retval On success it returns 1 and on failure 0.
2362  */
2363 
2364 static int StreamTcpReassembleTest27 (void)
2365 {
2366  MISSED_START(9);
2367  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2368  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2369  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2370  MISSED_END;
2371 }
2372 
2373 /**
2374  * \test Test the handling of packets missed by IDS, but the end host has
2375  * received it and send the acknowledgment of it. The packet is missed
2376  * in the starting of the stream.
2377  *
2378  * \retval On success it returns 1 and on failure 0.
2379  */
2380 
2381 static int StreamTcpReassembleTest28 (void)
2382 {
2383  MISSED_START(6);
2384  MISSED_ADD_PAYLOAD(10, "AAA", 3);
2385  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2386  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAA", 3) == 1);
2387  MISSED_ADD_PAYLOAD(13, "BB", 2);
2388  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2389  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABB", 5) == 1);
2390  ssn.state = TCP_TIME_WAIT;
2391  MISSED_ADD_PAYLOAD(15, "CC", 2);
2392  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2393  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABBCC", 7) == 1);
2394  MISSED_END;
2395 }
2396 
2397 /**
2398  * \test Test the handling of packets missed by IDS, but the end host has
2399  * received it and send the acknowledgment of it. The packet is missed
2400  * in the middle of the stream.
2401  *
2402  * \retval On success it returns 1 and on failure 0.
2403  */
2404 
2405 static int StreamTcpReassembleTest29 (void)
2406 {
2407  MISSED_START(9);
2408  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2409  ssn.state = TCP_TIME_WAIT;
2410  MISSED_ADD_PAYLOAD(15, "CC", 2);
2411  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2412  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2413  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2414  MISSED_END;
2415 }
2416 
2417 static int StreamTcpReassembleTest33(void)
2418 {
2419  TcpSession ssn;
2420  Packet *p = PacketGetFromAlloc();
2421  FAIL_IF(unlikely(p == NULL));
2422  Flow f;
2423  TCPHdr tcph;
2424  TcpReassemblyThreadCtx *ra_ctx = NULL;
2426  uint8_t packet[1460] = "";
2427 
2428  StreamTcpUTInit(&ra_ctx);
2430 
2431  memset(&f, 0, sizeof (Flow));
2432  memset(&tcph, 0, sizeof (TCPHdr));
2433  ThreadVars tv;
2434  memset(&tv, 0, sizeof (ThreadVars));
2435  FLOW_INITIALIZE(&f);
2436  f.protoctx = &ssn;
2437  f.proto = IPPROTO_TCP;
2438  p->src.family = AF_INET;
2439  p->dst.family = AF_INET;
2440  p->proto = IPPROTO_TCP;
2441  p->flow = &f;
2442  tcph.th_win = 5480;
2443  tcph.th_flags = TH_PUSH | TH_ACK;
2444  UTHSetTCPHdr(p, &tcph);
2446  p->payload = packet;
2447 
2448  tcph.th_seq = htonl(10);
2449  tcph.th_ack = htonl(31);
2450  p->payload_len = 10;
2451 
2452  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2453 
2454  tcph.th_seq = htonl(20);
2455  tcph.th_ack = htonl(31);
2456  p->payload_len = 10;
2457 
2458  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2459 
2460  tcph.th_seq = htonl(40);
2461  tcph.th_ack = htonl(31);
2462  p->payload_len = 10;
2463 
2464  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2465 
2466  tcph.th_seq = htonl(5);
2467  tcph.th_ack = htonl(31);
2468  p->payload_len = 30;
2469 
2470  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2471 
2473  StreamTcpUTDeinit(ra_ctx);
2474  SCFree(p);
2475  PASS;
2476 }
2477 
2478 static int StreamTcpReassembleTest34(void)
2479 {
2480  TcpSession ssn;
2481  Packet *p = PacketGetFromAlloc();
2482  FAIL_IF(unlikely(p == NULL));
2483  Flow f;
2484  TCPHdr tcph;
2485  TcpReassemblyThreadCtx *ra_ctx = NULL;
2487  uint8_t packet[1460] = "";
2488 
2489  StreamTcpUTInit(&ra_ctx);
2491  memset(&f, 0, sizeof (Flow));
2492  memset(&tcph, 0, sizeof (TCPHdr));
2493  ThreadVars tv;
2494  memset(&tv, 0, sizeof (ThreadVars));
2495  FLOW_INITIALIZE(&f);
2496  f.protoctx = &ssn;
2497  f.proto = IPPROTO_TCP;
2498  p->src.family = AF_INET;
2499  p->dst.family = AF_INET;
2500  p->proto = IPPROTO_TCP;
2501  p->flow = &f;
2502  tcph.th_win = 5480;
2503  tcph.th_flags = TH_PUSH | TH_ACK;
2504  UTHSetTCPHdr(p, &tcph);
2506  p->payload = packet;
2507  SET_ISN(&ssn.client, 857961230);
2508 
2509  tcph.th_seq = htonl(857961230);
2510  tcph.th_ack = htonl(31);
2511  p->payload_len = 304;
2512 
2513  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2514 
2515  tcph.th_seq = htonl(857961534);
2516  tcph.th_ack = htonl(31);
2517  p->payload_len = 1460;
2518 
2519  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2520 
2521  tcph.th_seq = htonl(857963582);
2522  tcph.th_ack = htonl(31);
2523  p->payload_len = 1460;
2524 
2525  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2526 
2527  tcph.th_seq = htonl(857960946);
2528  tcph.th_ack = htonl(31);
2529  p->payload_len = 1460;
2530 
2531  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2532 
2534  StreamTcpUTDeinit(ra_ctx);
2535  SCFree(p);
2536  PASS;
2537 }
2538 
2539 /**
2540  * \test Test to make sure that we don't return the segments until the app
2541  * layer proto has been detected and after that remove the processed
2542  * segments.
2543  *
2544  * \retval On success it returns 1 and on failure 0.
2545  */
2546 
2547 static int StreamTcpReassembleTest39 (void)
2548 {
2549  Packet *p = PacketGetFromAlloc();
2550  FAIL_IF(unlikely(p == NULL));
2551  Flow f;
2552  ThreadVars tv;
2553  StreamTcpThread stt;
2554  TCPHdr tcph;
2555  PacketQueueNoLock pq;
2556  memset(&pq,0,sizeof(PacketQueueNoLock));
2557  memset (&f, 0, sizeof(Flow));
2558  memset(&tv, 0, sizeof (ThreadVars));
2559  memset(&stt, 0, sizeof (stt));
2560  memset(&tcph, 0, sizeof (TCPHdr));
2561 
2562  FLOW_INITIALIZE(&f);
2563  f.flags = FLOW_IPV4;
2564  f.proto = IPPROTO_TCP;
2565  p->flow = &f;
2566  UTHSetTCPHdr(p, &tcph);
2567 
2568  StreamTcpUTInit(&stt.ra_ctx);
2569 
2570  /* handshake */
2571  tcph.th_win = htons(5480);
2572  tcph.th_flags = TH_SYN;
2574  p->payload_len = 0;
2575  p->payload = NULL;
2576  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2577 
2578  TcpSession *ssn = (TcpSession *)f.protoctx;
2579  FAIL_IF_NULL(ssn);
2580 
2587  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2588  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2589  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2590  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2591  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2592  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2593  FAIL_IF(ssn->data_first_seen_dir != 0);
2594 
2595  /* handshake */
2596  tcph.th_ack = htonl(1);
2597  tcph.th_flags = TH_SYN | TH_ACK;
2599  p->payload_len = 0;
2600  p->payload = NULL;
2601  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2608  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2609  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2610  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2611  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2612  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2613  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2614  FAIL_IF(ssn->data_first_seen_dir != 0);
2615 
2616  /* handshake */
2617  tcph.th_ack = htonl(1);
2618  tcph.th_seq = htonl(1);
2619  tcph.th_flags = TH_ACK;
2621  p->payload_len = 0;
2622  p->payload = NULL;
2623  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2630  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2631  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2632  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2633  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2634  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2635  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2636  FAIL_IF(ssn->data_first_seen_dir != 0);
2637 
2638  /* partial request */
2639  uint8_t request1[] = { 0x47, 0x45, };
2640  tcph.th_ack = htonl(1);
2641  tcph.th_seq = htonl(1);
2642  tcph.th_flags = TH_PUSH | TH_ACK;
2644  p->payload_len = sizeof(request1);
2645  p->payload = request1;
2646  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2653  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2654  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2655  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2656  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2657  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2658  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2659  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2660  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2661 
2662  /* response ack against partial request */
2663  tcph.th_ack = htonl(3);
2664  tcph.th_seq = htonl(1);
2665  tcph.th_flags = TH_ACK;
2667  p->payload_len = 0;
2668  p->payload = NULL;
2669  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2676  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2677  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2678  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2679  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2680  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2681  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2682  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2683  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2684 
2685  /* complete partial request */
2686  uint8_t request2[] = {
2687  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2688  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2689  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2690  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2691  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2692  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2693  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2694  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2695  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2696  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2697  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2698  tcph.th_ack = htonl(1);
2699  tcph.th_seq = htonl(3);
2700  tcph.th_flags = TH_PUSH | TH_ACK;
2702  p->payload_len = sizeof(request2);
2703  p->payload = request2;
2704  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2711  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2712  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2713  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2714  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2715  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2716  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2717  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2718  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2719  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2720 
2721  /* response - request ack */
2722  uint8_t response[] = {
2723  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2724  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2725  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2726  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2727  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2728  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2729  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2730  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2731  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2732  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2733  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2734  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2735  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2736  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2737  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2738  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2739  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2740  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2741  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2742  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2743  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2744  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2745  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2746  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2747  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2748  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2749  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2750  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2751  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2752  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2753  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2754  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2755  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2756  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2757  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2758  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2759  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2760  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2761  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2762  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2763  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2764  tcph.th_ack = htonl(88);
2765  tcph.th_seq = htonl(1);
2766  tcph.th_flags = TH_PUSH | TH_ACK;
2768  p->payload_len = sizeof(response);
2769  p->payload = response;
2770 
2771  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2778  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2779  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2780  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2781  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2783  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2784  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2785  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2786  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2787 
2788  /* response ack from request */
2789  tcph.th_ack = htonl(328);
2790  tcph.th_seq = htonl(88);
2791  tcph.th_flags = TH_ACK;
2793  p->payload_len = 0;
2794  p->payload = NULL;
2795  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2802  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2803  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2804  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2805  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2807  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2808  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2809  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2810  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2811  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2812 
2813  /* response - acking */
2814  tcph.th_ack = htonl(88);
2815  tcph.th_seq = htonl(328);
2816  tcph.th_flags = TH_PUSH | TH_ACK;
2818  p->payload_len = 0;
2819  p->payload = NULL;
2820  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2827  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2828  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2829  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2830  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2832  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2833  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2834  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2835  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2836  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2837 
2838  /* response ack from request */
2839  tcph.th_ack = htonl(328);
2840  tcph.th_seq = htonl(88);
2841  tcph.th_flags = TH_ACK;
2843  p->payload_len = 0;
2844  p->payload = NULL;
2845  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2852  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2853  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2854  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2855  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2857  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2858  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2859  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2860  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2861 
2862  /* response - acking the request again*/
2863  tcph.th_ack = htonl(88);
2864  tcph.th_seq = htonl(328);
2865  tcph.th_flags = TH_PUSH | TH_ACK;
2867  p->payload_len = 0;
2868  p->payload = NULL;
2869  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2876  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2877  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2878  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2879  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2881  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2882  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2883  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2884  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2885 
2886  /*** New Request ***/
2887 
2888  /* partial request */
2889  tcph.th_ack = htonl(328);
2890  tcph.th_seq = htonl(88);
2891  tcph.th_flags = TH_PUSH | TH_ACK;
2893  p->payload_len = sizeof(request1);
2894  p->payload = request1;
2895  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2902  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2903  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2904  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2905  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2907  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2908  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2909  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2910  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2911  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2912 
2913  /* response ack against partial request */
2914  tcph.th_ack = htonl(90);
2915  tcph.th_seq = htonl(328);
2916  tcph.th_flags = TH_ACK;
2918  p->payload_len = 0;
2919  p->payload = NULL;
2920  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2921  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2928  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2929  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2930  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2931  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2933  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2934  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2935  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2936  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2937  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2938 
2939  /* complete request */
2940  tcph.th_ack = htonl(328);
2941  tcph.th_seq = htonl(90);
2942  tcph.th_flags = TH_PUSH | TH_ACK;
2944  p->payload_len = sizeof(request2);
2945  p->payload = request2;
2946  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2953  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2954  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2955  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2956  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2958  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2959  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2960  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2961  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2962  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2963  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2964 
2965  /* response ack against second partial request */
2966  tcph.th_ack = htonl(175);
2967  tcph.th_seq = htonl(328);
2968  tcph.th_flags = TH_ACK;
2970  p->payload_len = 0;
2971  p->payload = NULL;
2972 
2973  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2980  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2981  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2982  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2983  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2985  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2986  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2987  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2988  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2989  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2990  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2991 
2992  /* response acking a request */
2993  tcph.th_ack = htonl(175);
2994  tcph.th_seq = htonl(328);
2995  tcph.th_flags = TH_ACK;
2997  p->payload_len = 0;
2998  p->payload = NULL;
2999  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3005 
3006  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3007  StreamTcpPruneSession(&f, STREAM_TOCLIENT);
3008 
3009  /* request acking a response */
3010  tcph.th_ack = htonl(328);
3011  tcph.th_seq = htonl(175);
3012  tcph.th_flags = TH_ACK;
3014  p->payload_len = 0;
3015  p->payload = NULL;
3016  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3017 
3018  StreamTcpSessionClear(ssn);
3020  SCFree(p);
3021  PASS;
3022 }
3023 
3024 /**
3025  * \test Test to make sure that we sent all the segments from the initial
3026  * segments to app layer until we have detected the app layer proto.
3027  *
3028  * \retval On success it returns 1 and on failure 0.
3029  */
3030 
3031 static int StreamTcpReassembleTest40 (void)
3032 {
3033  Packet *p = PacketGetFromAlloc();
3034  FAIL_IF_NULL(p);
3035  Flow *f = NULL;
3036  TCPHdr tcph;
3037  TcpSession ssn;
3038  memset(&tcph, 0, sizeof (TCPHdr));
3039  ThreadVars tv;
3040  memset(&tv, 0, sizeof (ThreadVars));
3041 
3042  StreamTcpInitConfig(true);
3044 
3046  FAIL_IF_NULL(ra_ctx);
3047 
3048  uint8_t httpbuf1[] = "P";
3049  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3050  uint8_t httpbuf3[] = "O";
3051  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3052  uint8_t httpbuf4[] = "S";
3053  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3054  uint8_t httpbuf5[] = "T \r\n";
3055  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3056 
3057  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3058  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3059 
3060  SET_ISN(&ssn.server, 9);
3061  ssn.server.last_ack = 10;
3062  SET_ISN(&ssn.client, 9);
3063  ssn.client.isn = 9;
3064 
3065  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3066  FAIL_IF_NULL(f);
3067  f->protoctx = &ssn;
3068  f->proto = IPPROTO_TCP;
3069  p->flow = f;
3070 
3071  tcph.th_win = htons(5480);
3072  tcph.th_seq = htonl(10);
3073  tcph.th_ack = htonl(10);
3074  tcph.th_flags = TH_ACK|TH_PUSH;
3075  UTHSetTCPHdr(p, &tcph);
3077  p->payload = httpbuf1;
3078  p->payload_len = httplen1;
3079  ssn.state = TCP_ESTABLISHED;
3080  TcpStream *s = &ssn.client;
3081  SCLogDebug("1 -- start");
3082  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3083 
3085  p->payload = httpbuf2;
3086  p->payload_len = httplen2;
3087  tcph.th_seq = htonl(10);
3088  tcph.th_ack = htonl(11);
3089  s = &ssn.server;
3090  ssn.server.last_ack = 11;
3091  SCLogDebug("2 -- start");
3092  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3093 
3095  p->payload = httpbuf3;
3096  p->payload_len = httplen3;
3097  tcph.th_seq = htonl(11);
3098  tcph.th_ack = htonl(55);
3099  s = &ssn.client;
3100  ssn.client.last_ack = 55;
3101  SCLogDebug("3 -- start");
3102  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3103 
3105  p->payload = httpbuf2;
3106  p->payload_len = httplen2;
3107  tcph.th_seq = htonl(55);
3108  tcph.th_ack = htonl(12);
3109  s = &ssn.server;
3110  ssn.server.last_ack = 12;
3111  SCLogDebug("4 -- start");
3112  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3113 
3114  /* check is have the segment in the list and flagged or not */
3115  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3116  FAIL_IF_NULL(seg);
3117  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
3118 
3120  p->payload = httpbuf4;
3121  p->payload_len = httplen4;
3122  tcph.th_seq = htonl(12);
3123  tcph.th_ack = htonl(100);
3124  s = &ssn.client;
3125  ssn.client.last_ack = 100;
3126  SCLogDebug("5 -- start");
3127  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3128 
3130  p->payload = httpbuf2;
3131  p->payload_len = httplen2;
3132  tcph.th_seq = htonl(100);
3133  tcph.th_ack = htonl(13);
3134  s = &ssn.server;
3135  ssn.server.last_ack = 13;
3136  SCLogDebug("6 -- start");
3137  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3138 
3140  p->payload = httpbuf5;
3141  p->payload_len = httplen5;
3142  tcph.th_seq = htonl(13);
3143  tcph.th_ack = htonl(145);
3144  s = &ssn.client;
3145  ssn.client.last_ack = 145;
3146  SCLogDebug("7 -- start");
3147  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3148 
3150  p->payload = httpbuf2;
3151  p->payload_len = httplen2;
3152  tcph.th_seq = htonl(145);
3153  tcph.th_ack = htonl(16);
3154  s = &ssn.server;
3155  ssn.server.last_ack = 16;
3156  SCLogDebug("8 -- start");
3157  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3158  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3159 
3162  StreamTcpFreeConfig(true);
3163  SCFree(p);
3164  UTHFreeFlow(f);
3165  PASS;
3166 }
3167 
3168 /** \test Test the memcap incrementing/decrementing and memcap check */
3169 static int StreamTcpReassembleTest44(void)
3170 {
3171  StreamTcpInitConfig(true);
3172  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
3173  StreamTcpReassembleIncrMemuse(500);
3174  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
3175  StreamTcpReassembleDecrMemuse(500);
3176  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
3178  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
3179  StreamTcpFreeConfig(true);
3180  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
3181  PASS;
3182 }
3183 
3184 /**
3185  * \test Test to make sure that reassembly_depth is enforced.
3186  *
3187  * \retval On success it returns 1 and on failure 0.
3188  */
3189 
3190 static int StreamTcpReassembleTest45 (void)
3191 {
3192  TcpReassemblyThreadCtx *ra_ctx = NULL;
3193  TcpSession ssn;
3194  ThreadVars tv;
3195  memset(&tv, 0, sizeof(tv));
3196  uint8_t payload[100] = {0};
3197  uint16_t payload_size = 100;
3198 
3199  StreamTcpUTInit(&ra_ctx);
3201 
3203  ssn.reassembly_depth = 100;
3204  StreamTcpUTSetupStream(&ssn.server, 100);
3205  StreamTcpUTSetupStream(&ssn.client, 100);
3206 
3207  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3208  FAIL_IF(r != 0);
3210 
3211  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3212  FAIL_IF(r != 0);
3214 
3218  StreamTcpUTDeinit(ra_ctx);
3219  PASS;
3220 }
3221 
3222 /**
3223  * \test Test the unlimited config value of reassembly depth.
3224  *
3225  * \retval On success it returns 1 and on failure 0.
3226  */
3227 
3228 static int StreamTcpReassembleTest46 (void)
3229 {
3230  int result = 0;
3231  TcpReassemblyThreadCtx *ra_ctx = NULL;
3232  TcpSession ssn;
3233  ThreadVars tv;
3234  memset(&tv, 0, sizeof(tv));
3235  uint8_t payload[100] = {0};
3236  uint16_t payload_size = 100;
3237 
3238  StreamTcpUTInit(&ra_ctx);
3240 
3242  StreamTcpUTSetupStream(&ssn.server, 100);
3243  StreamTcpUTSetupStream(&ssn.client, 100);
3244 
3245  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3246  if (r != 0)
3247  goto end;
3249  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3250  goto end;
3251  }
3252 
3253  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3254  if (r != 0)
3255  goto end;
3257  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3258  goto end;
3259  }
3260 
3261  result = 1;
3262 end:
3266  StreamTcpUTDeinit(ra_ctx);
3267  return result;
3268 }
3269 
3270 /**
3271  * \test Test to make sure we detect the sequence wrap around and continue
3272  * stream reassembly properly.
3273  *
3274  * \retval On success it returns 1 and on failure 0.
3275  */
3276 
3277 static int StreamTcpReassembleTest47 (void)
3278 {
3279  Packet *p = PacketGetFromAlloc();
3280  FAIL_IF(unlikely(p == NULL));
3281  Flow *f = NULL;
3282  TCPHdr tcph;
3283  TcpSession ssn;
3284  ThreadVars tv;
3285  memset(&tcph, 0, sizeof (TCPHdr));
3286  UTHSetTCPHdr(p, &tcph);
3287  memset(&tv, 0, sizeof (ThreadVars));
3288  StreamTcpInitConfig(true);
3291 
3292  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3293  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3294 
3295  SET_ISN(&ssn.server, 572799781UL);
3296  ssn.server.last_ack = 572799782UL;
3297 
3298  SET_ISN(&ssn.client, 4294967289UL);
3299  ssn.client.last_ack = 21;
3300 
3301  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3302  FAIL_IF(f == NULL);
3303  f->protoctx = &ssn;
3304  f->proto = IPPROTO_TCP;
3305  p->flow = f;
3306 
3307  tcph.th_win = htons(5480);
3308  ssn.state = TCP_ESTABLISHED;
3309  TcpStream *s = NULL;
3310  uint8_t cnt = 0;
3311 
3312  for (cnt=0; cnt < httplen1; cnt++) {
3313  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3314  tcph.th_ack = htonl(572799782UL);
3315  tcph.th_flags = TH_ACK | TH_PUSH;
3317  p->payload = &httpbuf1[cnt];
3318  p->payload_len = 1;
3319  s = &ssn.client;
3320 
3321  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3322 
3324  p->payload = NULL;
3325  p->payload_len = 0;
3326  tcph.th_seq = htonl(572799782UL);
3327  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3328  tcph.th_flags = TH_ACK;
3329  s = &ssn.server;
3330 
3331  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3332  }
3333 
3334  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3335 
3338  StreamTcpFreeConfig(true);
3339  SCFree(p);
3340  UTHFreeFlow(f);
3341  PASS;
3342 }
3343 
3344 /** \test 3 in order segments in inline reassembly */
3345 static int StreamTcpReassembleInlineTest01(void)
3346 {
3347  int ret = 0;
3348  TcpReassemblyThreadCtx *ra_ctx = NULL;
3349  ThreadVars tv;
3350  TcpSession ssn;
3351  Flow f;
3352 
3353  memset(&tv, 0x00, sizeof(tv));
3354 
3355  StreamTcpUTInit(&ra_ctx);
3358  StreamTcpUTSetupStream(&ssn.client, 1);
3359  FLOW_INITIALIZE(&f);
3360 
3361  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3362  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3363  if (p == NULL) {
3364  printf("couldn't get a packet: ");
3365  goto end;
3366  }
3367  p->l4.hdrs.tcph->th_seq = htonl(12);
3368  p->flow = &f;
3369 
3370  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3371  printf("failed to add segment 1: ");
3372  goto end;
3373  }
3374  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3375  printf("failed to add segment 2: ");
3376  goto end;
3377  }
3378  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3379  printf("failed to add segment 3: ");
3380  goto end;
3381  }
3382  ssn.client.next_seq = 17;
3383  ret = 1;
3384 end:
3385  FLOW_DESTROY(&f);
3386  UTHFreePacket(p);
3388  StreamTcpUTDeinit(ra_ctx);
3389  return ret;
3390 }
3391 
3392 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3393  * test the sliding window reassembly.
3394  */
3395 static int StreamTcpReassembleInlineTest02(void)
3396 {
3397  int ret = 0;
3398  TcpReassemblyThreadCtx *ra_ctx = NULL;
3399  ThreadVars tv;
3400  TcpSession ssn;
3401  Flow f;
3402 
3403  memset(&tv, 0x00, sizeof(tv));
3404 
3405  StreamTcpUTInit(&ra_ctx);
3408  StreamTcpUTSetupStream(&ssn.client, 1);
3409  FLOW_INITIALIZE(&f);
3410 
3411  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3412  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3413  if (p == NULL) {
3414  printf("couldn't get a packet: ");
3415  goto end;
3416  }
3417  p->l4.hdrs.tcph->th_seq = htonl(12);
3418  p->flow = &f;
3419 
3420  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3421  printf("failed to add segment 1: ");
3422  goto end;
3423  }
3424  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3425  printf("failed to add segment 2: ");
3426  goto end;
3427  }
3428  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3429  printf("failed to add segment 3: ");
3430  goto end;
3431  }
3432  ssn.client.next_seq = 17;
3433  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3434  printf("failed to add segment 4: ");
3435  goto end;
3436  }
3437  ssn.client.next_seq = 22;
3438  ret = 1;
3439 end:
3440  FLOW_DESTROY(&f);
3441  UTHFreePacket(p);
3443  StreamTcpUTDeinit(ra_ctx);
3444  return ret;
3445 }
3446 
3447 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3448  * test the sliding window reassembly with a small window size so that we
3449  * cutting off at the start (left edge)
3450  */
3451 static int StreamTcpReassembleInlineTest03(void)
3452 {
3453  int ret = 0;
3454  TcpReassemblyThreadCtx *ra_ctx = NULL;
3455  ThreadVars tv;
3456  TcpSession ssn;
3457  Flow f;
3458 
3459  memset(&tv, 0x00, sizeof(tv));
3460 
3461  StreamTcpUTInit(&ra_ctx);
3464  StreamTcpUTSetupStream(&ssn.client, 1);
3465  FLOW_INITIALIZE(&f);
3466 
3468 
3469  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3470  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3471  if (p == NULL) {
3472  printf("couldn't get a packet: ");
3473  goto end;
3474  }
3475  p->l4.hdrs.tcph->th_seq = htonl(12);
3476  p->flow = &f;
3478 
3479  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3480  printf("failed to add segment 1: ");
3481  goto end;
3482  }
3483  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3484  printf("failed to add segment 2: ");
3485  goto end;
3486  }
3487  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3488  printf("failed to add segment 3: ");
3489  goto end;
3490  }
3491  ssn.client.next_seq = 17;
3492  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3493  printf("failed to add segment 4: ");
3494  goto end;
3495  }
3496  ssn.client.next_seq = 22;
3497 
3498  p->l4.hdrs.tcph->th_seq = htonl(17);
3499  ret = 1;
3500 end:
3501  FLOW_DESTROY(&f);
3502  UTHFreePacket(p);
3504  StreamTcpUTDeinit(ra_ctx);
3505  return ret;
3506 }
3507 
3508 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3509  * test the sliding window reassembly with a small window size so that we
3510  * cutting off at the start (left edge) with small packet overlap.
3511  */
3512 static int StreamTcpReassembleInlineTest04(void)
3513 {
3514  int ret = 0;
3515  TcpReassemblyThreadCtx *ra_ctx = NULL;
3516  ThreadVars tv;
3517  TcpSession ssn;
3518  Flow f;
3519 
3520  memset(&tv, 0x00, sizeof(tv));
3521 
3522  StreamTcpUTInit(&ra_ctx);
3525  StreamTcpUTSetupStream(&ssn.client, 1);
3526  FLOW_INITIALIZE(&f);
3527 
3529 
3530  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3531  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3532  if (p == NULL) {
3533  printf("couldn't get a packet: ");
3534  goto end;
3535  }
3536  p->l4.hdrs.tcph->th_seq = htonl(12);
3537  p->flow = &f;
3539 
3540  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3541  printf("failed to add segment 1: ");
3542  goto end;
3543  }
3544  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3545  printf("failed to add segment 2: ");
3546  goto end;
3547  }
3548  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3549  printf("failed to add segment 3: ");
3550  goto end;
3551  }
3552  ssn.client.next_seq = 17;
3553  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3554  printf("failed to add segment 4: ");
3555  goto end;
3556  }
3557  ssn.client.next_seq = 22;
3558 
3559  p->l4.hdrs.tcph->th_seq = htonl(17);
3560  ret = 1;
3561 end:
3562  FLOW_DESTROY(&f);
3563  UTHFreePacket(p);
3565  StreamTcpUTDeinit(ra_ctx);
3566  return ret;
3567 }
3568 
3569 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3570  * test the sliding window reassembly with a small window size so that we
3571  * cutting off at the start (left edge). Test if the first segment is
3572  * removed from the list.
3573  */
3574 static int StreamTcpReassembleInlineTest08(void)
3575 {
3576  TcpReassemblyThreadCtx *ra_ctx = NULL;
3577  ThreadVars tv;
3578  memset(&tv, 0x00, sizeof(tv));
3579  TcpSession ssn;
3580  Flow f;
3581  StreamTcpUTInit(&ra_ctx);
3584  StreamTcpUTSetupStream(&ssn.client, 1);
3585  FLOW_INITIALIZE(&f);
3586 
3588  f.protoctx = &ssn;
3589 
3590  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3591  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3592  FAIL_IF(p == NULL);
3593  p->l4.hdrs.tcph->th_seq = htonl(12);
3594  p->flow = &f;
3596 
3597  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3598  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3599  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3600  ssn.client.next_seq = 17;
3601  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3602  ssn.client.next_seq = 22;
3603  p->l4.hdrs.tcph->th_seq = htonl(17);
3604  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3605 
3606  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3607  FAIL_IF_NULL(seg);
3608  FAIL_IF_NOT(seg->seq == 2);
3609 
3610  FLOW_DESTROY(&f);
3611  UTHFreePacket(p);
3613  StreamTcpUTDeinit(ra_ctx);
3614  PASS;
3615 }
3616 
3617 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3618  * test the sliding window reassembly with a small window size so that we
3619  * cutting off at the start (left edge). Test if the first segment is
3620  * removed from the list.
3621  */
3622 static int StreamTcpReassembleInlineTest09(void)
3623 {
3624  int ret = 0;
3625  TcpReassemblyThreadCtx *ra_ctx = NULL;
3626  ThreadVars tv;
3627  TcpSession ssn;
3628  Flow f;
3629 
3630  memset(&tv, 0x00, sizeof(tv));
3631 
3632  StreamTcpUTInit(&ra_ctx);
3635  StreamTcpUTSetupStream(&ssn.client, 1);
3636  FLOW_INITIALIZE(&f);
3637 
3639 
3640  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3641  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3642  if (p == NULL) {
3643  printf("couldn't get a packet: ");
3644  goto end;
3645  }
3646  p->l4.hdrs.tcph->th_seq = htonl(17);
3647  p->flow = &f;
3649 
3650  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3651  printf("failed to add segment 1: ");
3652  goto end;
3653  }
3654  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3655  printf("failed to add segment 2: ");
3656  goto end;
3657  }
3658  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3659  printf("failed to add segment 3: ");
3660  goto end;
3661  }
3662  ssn.client.next_seq = 12;
3663  ssn.client.last_ack = 10;
3664 
3665  /* close the GAP and see if we properly reassemble and update base_seq */
3666  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3667  printf("failed to add segment 4: ");
3668  goto end;
3669  }
3670  ssn.client.next_seq = 22;
3671 
3672  p->l4.hdrs.tcph->th_seq = htonl(12);
3673 
3674  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3675  FAIL_IF_NULL(seg);
3676  FAIL_IF_NOT(seg->seq == 2);
3677 
3678  ret = 1;
3679 end:
3680  FLOW_DESTROY(&f);
3681  UTHFreePacket(p);
3683  StreamTcpUTDeinit(ra_ctx);
3684  return ret;
3685 }
3686 
3687 /** \test App Layer reassembly.
3688  */
3689 static int StreamTcpReassembleInlineTest10(void)
3690 {
3691  int ret = 0;
3692  TcpReassemblyThreadCtx *ra_ctx = NULL;
3693  ThreadVars tv;
3694  TcpSession ssn;
3695  Flow *f = NULL;
3696  Packet *p = NULL;
3697 
3698  memset(&tv, 0x00, sizeof(tv));
3699 
3700  StreamTcpUTInit(&ra_ctx);
3703  StreamTcpUTSetupStream(&ssn.server, 1);
3704  ssn.server.last_ack = 2;
3705  StreamTcpUTSetupStream(&ssn.client, 1);
3706  ssn.client.last_ack = 2;
3707  ssn.data_first_seen_dir = STREAM_TOSERVER;
3708 
3709  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3710  if (f == NULL)
3711  goto end;
3712  f->protoctx = &ssn;
3713  f->proto = IPPROTO_TCP;
3714 
3715  uint8_t stream_payload1[] = "GE";
3716  uint8_t stream_payload2[] = "T /";
3717  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3718 
3719  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3720  if (p == NULL) {
3721  printf("couldn't get a packet: ");
3722  goto end;
3723  }
3724  p->l4.hdrs.tcph->th_seq = htonl(7);
3725  p->flow = f;
3727 
3728  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3729  printf("failed to add segment 1: ");
3730  goto end;
3731  }
3732  ssn.client.next_seq = 4;
3733 
3734  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3735  if (r < 0) {
3736  printf("StreamTcpReassembleAppLayer failed: ");
3737  goto end;
3738  }
3739 
3740  /* ssn.server.ra_app_base_seq should be isn here. */
3741  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3742  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3743  goto end;
3744  }
3745 
3746  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3747  printf("failed to add segment 2: ");
3748  goto end;
3749  }
3750  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3751  printf("failed to add segment 3: ");
3752  goto end;
3753  }
3754  ssn.client.next_seq = 19;
3755 
3756  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3757  if (r < 0) {
3758  printf("StreamTcpReassembleAppLayer failed: ");
3759  goto end;
3760  }
3761 
3762  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3763 
3764  ret = 1;
3765 end:
3766  UTHFreePacket(p);
3768  StreamTcpUTDeinit(ra_ctx);
3769  UTHFreeFlow(f);
3770  return ret;
3771 }
3772 
3773 /** \test test insert with overlap
3774  */
3775 static int StreamTcpReassembleInsertTest01(void)
3776 {
3777  TcpReassemblyThreadCtx *ra_ctx = NULL;
3778  ThreadVars tv;
3779  TcpSession ssn;
3780  Flow f;
3781 
3782  memset(&tv, 0x00, sizeof(tv));
3783 
3784  StreamTcpUTInit(&ra_ctx);
3786  StreamTcpUTSetupStream(&ssn.client, 1);
3788  FLOW_INITIALIZE(&f);
3789 
3790  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3791  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3792  FAIL_IF(p == NULL);
3793  p->l4.hdrs.tcph->th_seq = htonl(12);
3794  p->flow = &f;
3795 
3796  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3797  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3798  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3799  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3800  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3801  ssn.client.next_seq = 21;
3802 
3803  FLOW_DESTROY(&f);
3804  UTHFreePacket(p);
3806  StreamTcpUTDeinit(ra_ctx);
3807  PASS;
3808 }
3809 
3810 /** \test test insert with overlaps
3811  */
3812 static int StreamTcpReassembleInsertTest02(void)
3813 {
3814  int ret = 0;
3815  TcpReassemblyThreadCtx *ra_ctx = NULL;
3816  ThreadVars tv;
3817  TcpSession ssn;
3818 
3819  memset(&tv, 0x00, sizeof(tv));
3820 
3821  StreamTcpUTInit(&ra_ctx);
3823  StreamTcpUTSetupStream(&ssn.client, 1);
3824 
3825  int i;
3826  for (i = 2; i < 10; i++) {
3827  int len;
3828  len = i % 2;
3829  if (len == 0)
3830  len = 1;
3831  int seq;
3832  seq = i * 10;
3833  if (seq < 2)
3834  seq = 2;
3835 
3836  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3837  printf("failed to add segment 1: ");
3838  goto end;
3839  }
3840  }
3841  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3842  printf("failed to add segment 2: ");
3843  goto end;
3844  }
3845 
3846  ret = 1;
3847 end:
3849  StreamTcpUTDeinit(ra_ctx);
3850  return ret;
3851 }
3852 
3853 /** \test test insert with overlaps
3854  */
3855 static int StreamTcpReassembleInsertTest03(void)
3856 {
3857  int ret = 0;
3858  TcpReassemblyThreadCtx *ra_ctx = NULL;
3859  ThreadVars tv;
3860  TcpSession ssn;
3861 
3862  memset(&tv, 0x00, sizeof(tv));
3863 
3864  StreamTcpUTInit(&ra_ctx);
3866  StreamTcpUTSetupStream(&ssn.client, 1);
3867 
3868  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3869  printf("failed to add segment 2: ");
3870  goto end;
3871  }
3872 
3873  int i;
3874  for (i = 2; i < 10; i++) {
3875  int len;
3876  len = i % 2;
3877  if (len == 0)
3878  len = 1;
3879  int seq;
3880  seq = i * 10;
3881  if (seq < 2)
3882  seq = 2;
3883 
3884  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3885  printf("failed to add segment 2: ");
3886  goto end;
3887  }
3888  }
3889  ret = 1;
3890 end:
3892  StreamTcpUTDeinit(ra_ctx);
3893  return ret;
3894 }
3895 
3897 #endif /* UNITTESTS */
3898 
3899 /** \brief The Function Register the Unit tests to test the reassembly engine
3900  * for various OS policies.
3901  */
3902 
3904 {
3905 #ifdef UNITTESTS
3906  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3907  StreamTcpReassembleTest25);
3908  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3909  StreamTcpReassembleTest26);
3910  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3911  StreamTcpReassembleTest27);
3912  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3913  StreamTcpReassembleTest28);
3914  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3915  StreamTcpReassembleTest29);
3916  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3917  StreamTcpReassembleTest33);
3918  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3919  StreamTcpReassembleTest34);
3920  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3921  StreamTcpReassembleTest39);
3922  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3923  StreamTcpReassembleTest40);
3924  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3925  StreamTcpReassembleTest44);
3926  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3927  StreamTcpReassembleTest45);
3928  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3929  StreamTcpReassembleTest46);
3930  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3931  StreamTcpReassembleTest47);
3932 
3933  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3934  StreamTcpReassembleInlineTest01);
3935  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3936  StreamTcpReassembleInlineTest02);
3937  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3938  StreamTcpReassembleInlineTest03);
3939  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3940  StreamTcpReassembleInlineTest04);
3941  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3942  StreamTcpReassembleInlineTest08);
3943  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3944  StreamTcpReassembleInlineTest09);
3945 
3946  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3947  StreamTcpReassembleInlineTest10);
3948 
3949  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3950  StreamTcpReassembleInsertTest01);
3951  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3952  StreamTcpReassembleInsertTest02);
3953  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3954  StreamTcpReassembleInsertTest03);
3955 
3959  StreamTcpReassembleRawRegisterTests();
3960 #endif /* UNITTESTS */
3961 }
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
Definition: stream-tcp.h:143
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:270
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:40
TCP_GET_RAW_SEQ
#define TCP_GET_RAW_SEQ(tcph)
Definition: decode-tcp.h:80
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:58
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:56
util-device-private.h
UTHCheckGapAtPosition
int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
Definition: stream-tcp-reassemble.c:2265
util-byte.h
STREAM_REASSEMBLY_DEPTH_REACHED
@ STREAM_REASSEMBLY_DEPTH_REACHED
Definition: decode-events.h:302
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:514
TcpStream_
Definition: stream-tcp-private.h:106
len
uint8_t len
Definition: app-layer-dnp3.h:2
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:136
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:186
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:113
TcpReassemblyThreadCtx_::counter_tcp_reas_eps
ExceptionPolicyCounters counter_tcp_reas_eps
Definition: stream-tcp-reassemble.h:69
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:2121
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:166
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:381
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:97
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1905
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:97
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:91
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:94
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
TCP_STREAM_URGENT_OOB
@ TCP_STREAM_URGENT_OOB
Definition: stream-tcp.h:49
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:69
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
stream-tcp.h
StreamTcpInlineMode
bool StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:7099
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:136
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:2194
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:270
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:617
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1010
StreamTcpThread_
Definition: stream-tcp.h:92
Flow_::proto
uint8_t proto
Definition: flow.h:378
Packet_::payload
uint8_t * payload
Definition: decode.h:596
t_pcapcnt
thread_local uint64_t t_pcapcnt
Definition: stream-tcp-reassemble.c:80
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:2174
action-globals.h
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:62
Packet_::flags
uint32_t flags
Definition: decode.h:535
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:110
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:75
Flow_
Flow data structure.
Definition: flow.h:356
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:77
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:75
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
TCPHdr_::th_win
uint16_t th_win
Definition: decode-tcp.h:156
TcpSegmentPcapHdrStorage
struct TcpSegmentPcapHdrStorage_ TcpSegmentPcapHdrStorage
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
StringParseUint16
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:337
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:156
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1398
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:40
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:89
PKT_DROP_REASON_STREAM_REASSEMBLY
@ PKT_DROP_REASON_STREAM_REASSEMBLY
Definition: decode.h:387
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1813
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir app_update_dir)
handle TCP data for the app-layer.
Definition: app-layer.c:710
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:233
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:155
MIN
#define MIN(x, y)
Definition: suricata-common.h:408
SCConfGetBool
int SCConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:497
TcpSession_::urg_offset_ts
uint16_t urg_offset_ts
Definition: stream-tcp-private.h:291
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:75
UTHSetTCPHdr
void UTHSetTCPHdr(Packet *p, TCPHdr *tcph)
Definition: util-unittest-helper.c:136
StreamTcpThreadCacheGetSegment
TcpSegment * StreamTcpThreadCacheGetSegment(void)
Definition: stream-tcp-cache.c:179
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:52
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:155
TCP_STREAM_URGENT_GAP
@ TCP_STREAM_URGENT_GAP
Definition: stream-tcp.h:50
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:300
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1830
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:523
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:219
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:39
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:260
Flow_::protoctx
void * protoctx
Definition: flow.h:441
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:710
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:100
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:597
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:223
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:623
util-unittest.h
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
AppLayerProfilingStore
#define AppLayerProfilingStore(app_tctx, p)
Definition: app-layer.h:128
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2256
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:523
PKT_DROP_REASON_STREAM_URG
@ PKT_DROP_REASON_STREAM_URG
Definition: decode.h:388
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:426
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:170
STREAMTCP_FLAG_LOSSY_BE_LIBERAL
#define STREAMTCP_FLAG_LOSSY_BE_LIBERAL
Definition: stream-tcp-private.h:194
TCPHdr_::th_ack
uint32_t th_ack
Definition: decode-tcp.h:153
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:294
StreamTcpReassembleTriggerRawInspection
void StreamTcpReassembleTriggerRawInspection(TcpSession *ssn, int direction)
Trigger RAW stream inspection.
Definition: stream-tcp-reassemble.c:2157
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:290
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:278
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:115
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:478
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:557
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:488
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:488
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:38
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:295
StreamTcpReassembleDepthReached
bool StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:625
TCPHdr_::th_flags
uint8_t th_flags
Definition: decode-tcp.h:155
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:463
StreamTcpReassembleRealloc
void * StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
Definition: stream-tcp-reassemble.c:229
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:238
UTHCheckDataAtPosition
int UTHCheckDataAtPosition(TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
Definition: stream-tcp-reassemble.c:2288
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:104
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:279
STREAMTCP_FLAG_ASYNC
#define STREAMTCP_FLAG_ASYNC
Definition: stream-tcp-private.h:182
TCPHdr_::th_seq
uint32_t th_seq
Definition: decode-tcp.h:152
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1247
TcpStreamCnf_::urgent_policy
enum TcpStreamUrgentHandling urgent_policy
Definition: stream-tcp.h:83
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1255
util-exception-policy.h
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
TcpSession_::urg_offset_tc
uint16_t urg_offset_tc
Definition: stream-tcp-private.h:292
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:96
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:528
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:131
ExceptionPolicyCounters_::eps_id
uint16_t eps_id[EXCEPTION_POLICY_MAX]
Definition: util-exception-policy-types.h:53
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
SCEnter
#define SCEnter(...)
Definition: util-debug.h:272
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:193
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:5542
StreamNeedsReassembly
uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:977
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1512
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:285
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:137
TCPHdr_::th_urp
uint16_t th_urp
Definition: decode-tcp.h:158
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:105
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2208
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:886
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:112
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:164
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:317
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:341
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:351
SCReturn
#define SCReturn
Definition: util-debug.h:274
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:303
Packet_
Definition: decode.h:492
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:234
stream-tcp-private.h
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:40
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:278
TcpReassemblyThreadCtx_::counter_tcp_urgent_oob
uint16_t counter_tcp_urgent_oob
Definition: stream-tcp-reassemble.h:88
conf.h
Packet_::l4
struct PacketL4 l4
Definition: decode.h:592
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:301
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1853
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:117
STREAMTCP_STREAM_FLAG_HAS_GAP
#define STREAMTCP_STREAM_FLAG_HAS_GAP
Definition: stream-tcp-private.h:217
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:596
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:288
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:722
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(void)
Creates a new app layer thread context.
Definition: app-layer.c:1107
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:398
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:234
TcpStreamCnf_::reassembly_memcap_policy
enum ExceptionPolicy reassembly_memcap_policy
Definition: stream-tcp.h:81
StreamTcpThreadCacheCleanup
void StreamTcpThreadCacheCleanup(void)
Definition: stream-tcp-cache.c:134
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:257
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:539
TH_URG
#define TH_URG
Definition: decode-tcp.h:39
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:225
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:77
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:128
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:104
VALIDATE
#define VALIDATE(e)
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:493
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:57
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a TCP packet data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:746
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:64
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
TcpSegmentPcapHdrStorage_
Definition: stream-tcp-private.h:65
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1821
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1720
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
app-layer-frames.h
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:180
Packet_::flow
struct Flow_ * flow
Definition: decode.h:537
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:260
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:859
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
SCNtohs
#define SCNtohs(x)
Definition: suricata-common.h:431
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:447
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:259
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:634
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:124
packet.h
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:36
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:94
TcpStreamCnf_::urgent_oob_limit_policy
enum TcpStreamUrgentHandling urgent_oob_limit_policy
Definition: stream-tcp.h:84
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2237
STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
@ STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
Definition: decode-events.h:306
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:135
TcpSegmentPcapHdrStorage_::alloclen
uint32_t alloclen
Definition: stream-tcp-private.h:68
ACTION_DROP
#define ACTION_DROP
Definition: action-globals.h:30
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:526
StreamTcpThreadCacheReturnSegment
void StreamTcpThreadCacheReturnSegment(TcpSegment *seg)
Definition: stream-tcp-cache.c:53
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:236
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:297
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Definition: stream-tcp-reassemble.c:2001
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:161
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TCP_STREAM_URGENT_DROP
@ TCP_STREAM_URGENT_DROP
Definition: stream-tcp.h:48
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:78
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:114
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:232
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1558
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
PacketL4::L4Hdrs::tcph
TCPHdr * tcph
Definition: decode.h:460
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:296
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:181
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:262
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:451
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:473
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayerGetCtxThread().
Definition: app-layer.c:1128
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:257
Flow_::flags
uint32_t flags
Definition: flow.h:421
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:299
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:1022
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:71
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:53
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:150
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:307
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2251
stream-tcp-cache.h
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:158
PacketL4::hdrs
union PacketL4::L4Hdrs hdrs
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:117
MISSED_ADD_PAYLOAD
#define MISSED_ADD_PAYLOAD(seq, seg, seglen)
Definition: stream-tcp-reassemble.c:2260
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:61
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:69
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:583
StreamDataRightEdge
uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:417
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:173
PacketDrop
void PacketDrop(Packet *p, const uint8_t action, enum PacketDropReason r)
issue drop action
Definition: packet.c:33
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:219
suricata.h
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap(void)
Return memcap value.
Definition: stream-tcp-reassemble.c:200
Address_::family
char family
Definition: decode.h:113
Packet_::dst
Address dst
Definition: decode.h:497
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:7112
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:258
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool
uint16_t counter_tcp_segment_from_pool
Definition: stream-tcp-reassemble.h:72
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
TCP_STREAM_URGENT_INLINE
@ TCP_STREAM_URGENT_INLINE
Definition: stream-tcp.h:46
likely
#define likely(expr)
Definition: util-optimize.h:32
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3903
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:102
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:190
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
segment_thread_pool
PoolThread * segment_thread_pool
Definition: stream-tcp-reassemble.c:82
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:195
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TCPSEG_PKT_HDR_DEFAULT_SIZE
#define TCPSEG_PKT_HDR_DEFAULT_SIZE
Definition: stream-tcp-private.h:58
TcpSession_
Definition: stream-tcp-private.h:283
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:288
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:185
flow.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:238
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:452
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
StreamReassembleLog
int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, uint64_t *progress_out, const bool eof)
Definition: stream-tcp-reassemble.c:1970
ExceptionPolicy
ExceptionPolicy
Definition: util-exception-policy-types.h:25
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:112
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:225
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:148
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:276
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
SCConfNode_
Definition: conf.h:37
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
SCConfNode_::val
char * val
Definition: conf.h:39
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamReassembleForFrame
int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof)
Definition: stream-tcp-reassemble.c:1893
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:67
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1684
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:77
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:119
TCPHdr_
Definition: decode-tcp.h:149
Packet_::src
Address src
Definition: decode.h:496
TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache
uint16_t counter_tcp_segment_from_cache
Definition: stream-tcp-reassemble.h:71
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:70
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:232
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:146
app-layer.h