suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2024 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "packet.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 #include "action-globals.h"
37 
38 #include "flow-util.h"
39 
40 #include "threadvars.h"
41 #include "tm-threads.h"
42 
43 #include "util-pool.h"
44 #include "util-unittest.h"
45 #include "util-print.h"
46 #include "util-host-os-info.h"
47 #include "util-unittest-helper.h"
48 #include "util-byte.h"
49 #include "util-device.h"
50 
51 #include "stream-tcp.h"
52 #include "stream-tcp-private.h"
53 #include "stream-tcp-cache.h"
54 #include "stream-tcp-reassemble.h"
55 #include "stream-tcp-inline.h"
56 #include "stream-tcp-list.h"
57 #include "stream-tcp-util.h"
58 
59 #include "stream.h"
60 
61 #include "util-debug.h"
62 #include "app-layer-protos.h"
63 #include "app-layer.h"
64 #include "app-layer-events.h"
65 #include "app-layer-parser.h"
66 #include "app-layer-frames.h"
67 
68 #include "detect-engine-state.h"
69 
70 #include "util-profiling.h"
71 #include "util-validate.h"
72 #include "util-exception-policy.h"
73 
74 #ifdef DEBUG
75 static SCMutex segment_pool_memuse_mutex;
76 static uint64_t segment_pool_memuse = 0;
77 static uint64_t segment_pool_memcnt = 0;
78 #endif
79 
80 thread_local uint64_t t_pcapcnt = UINT64_MAX;
81 
83 /* init only, protect initializing and growing pool */
84 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
85 
86 /* Memory use counter */
87 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
88 
89 static int g_tcp_session_dump_enabled = 0;
90 
91 inline bool IsTcpSessionDumpingEnabled(void)
92 {
93  return g_tcp_session_dump_enabled == 1;
94 }
95 
97 {
98  g_tcp_session_dump_enabled = 1;
99 }
100 
101 /* prototypes */
103 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
104 
106 {
107  SC_ATOMIC_INIT(ra_memuse);
108 }
109 
110 /**
111  * \brief Function to Increment the memory usage counter for the TCP reassembly
112  * segments
113  *
114  * \param size Size of the TCP segment and its payload length memory allocated
115  */
116 static void StreamTcpReassembleIncrMemuse(uint64_t size)
117 {
118  (void) SC_ATOMIC_ADD(ra_memuse, size);
119  SCLogDebug("REASSEMBLY %" PRIu64 ", incr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
120  size);
121 }
122 
123 /**
124  * \brief Function to Decrease the memory usage counter for the TCP reassembly
125  * segments
126  *
127  * \param size Size of the TCP segment and its payload length memory allocated
128  */
129 static void StreamTcpReassembleDecrMemuse(uint64_t size)
130 {
131 #ifdef UNITTESTS
132  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
133  if (RunmodeIsUnittests()) {
134  BUG_ON(presize > UINT_MAX);
135  }
136 #endif
137 
138  (void) SC_ATOMIC_SUB(ra_memuse, size);
139 
140 #ifdef UNITTESTS
141  if (RunmodeIsUnittests()) {
142  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
143  BUG_ON(postsize > presize);
144  }
145 #endif
146  SCLogDebug("REASSEMBLY %" PRIu64 ", decr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
147  size);
148 }
149 
151 {
152  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
153  return smemuse;
154 }
155 
156 /**
157  * \brief Function to Check the reassembly memory usage counter against the
158  * allowed max memory usage for TCP segments.
159  *
160  * \param size Size of the TCP segment and its payload length memory allocated
161  * \retval 1 if in bounds
162  * \retval 0 if not in bounds
163  */
165 {
166 #ifdef DEBUG
167  if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX &&
168  g_eps_stream_reassembly_memcap == t_pcapcnt))) {
169  SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt);
170  return 0;
171  }
172 #endif
173  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
174  if (memcapcopy == 0 ||
175  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
176  return 1;
177  return 0;
178 }
179 
180 /**
181  * \brief Update memcap value
182  *
183  * \param size new memcap value
184  */
186 {
187  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
188  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
189  return 1;
190  }
191 
192  return 0;
193 }
194 
195 /**
196  * \brief Return memcap value
197  *
198  * \return memcap memcap value
199  */
201 {
202  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
203  return memcapcopy;
204 }
205 
206 /* memory functions for the streaming buffer API */
207 
208 /*
209  void *(*Calloc)(size_t n, size_t size);
210 */
211 static void *ReassembleCalloc(size_t n, size_t size)
212 {
213  if (StreamTcpReassembleCheckMemcap(n * size) == 0) {
215  return NULL;
216  }
217  void *ptr = SCCalloc(n, size);
218  if (ptr == NULL) {
220  return NULL;
221  }
222  StreamTcpReassembleIncrMemuse(n * size);
223  return ptr;
224 }
225 
226 /*
227  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
228 */
229 void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
230 {
231  if (size > orig_size) {
232  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) {
233  SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap));
235  return NULL;
236  }
237  }
238  void *nptr = SCRealloc(optr, size);
239  if (nptr == NULL) {
240  SCLogDebug("realloc fail");
242  return NULL;
243  }
244  if (size > orig_size) {
245  StreamTcpReassembleIncrMemuse(size - orig_size);
246  } else {
247  StreamTcpReassembleDecrMemuse(orig_size - size);
248  }
249  return nptr;
250 }
251 
252 /*
253  void (*Free)(void *ptr, size_t size);
254 */
255 static void ReassembleFree(void *ptr, size_t size)
256 {
257  SCFree(ptr);
258  StreamTcpReassembleDecrMemuse(size);
259 }
260 
261 /** \brief alloc a tcp segment pool entry */
262 static void *TcpSegmentPoolAlloc(void)
263 {
264  SCLogDebug("segment alloc");
265  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
266  return NULL;
267  }
268 
269  TcpSegment *seg = NULL;
270 
271  seg = SCMalloc(sizeof (TcpSegment));
272  if (unlikely(seg == NULL))
273  return NULL;
274 
276  uint32_t memuse =
277  sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
278  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
279  SCFree(seg);
280  return NULL;
281  }
282 
284  if (seg->pcap_hdr_storage == NULL) {
285  SCLogError("Unable to allocate memory for "
286  "TcpSegmentPcapHdrStorage");
287  SCFree(seg);
288  return NULL;
289  } else {
290  seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
291  seg->pcap_hdr_storage->pkt_hdr =
292  SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
293  if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
294  SCLogError("Unable to allocate memory for "
295  "packet header data within "
296  "TcpSegmentPcapHdrStorage");
297  SCFree(seg->pcap_hdr_storage);
298  SCFree(seg);
299  return NULL;
300  }
301  }
302 
303  StreamTcpReassembleIncrMemuse(memuse);
304  } else {
305  seg->pcap_hdr_storage = NULL;
306  }
307 
308  return seg;
309 }
310 
311 static int TcpSegmentPoolInit(void *data, void *initdata)
312 {
313  TcpSegment *seg = (TcpSegment *) data;
314  TcpSegmentPcapHdrStorage *pcap_hdr;
315 
316  pcap_hdr = seg->pcap_hdr_storage;
317 
318  /* do this before the can bail, so TcpSegmentPoolCleanup
319  * won't have uninitialized memory to consider. */
320  memset(seg, 0, sizeof (TcpSegment));
321 
323  uint32_t memuse =
324  sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
325  seg->pcap_hdr_storage = pcap_hdr;
326  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
327  return 0;
328  }
329  StreamTcpReassembleIncrMemuse(memuse);
330  } else {
331  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
332  return 0;
333  }
334  }
335 
336 #ifdef DEBUG
337  SCMutexLock(&segment_pool_memuse_mutex);
338  segment_pool_memuse += sizeof(TcpSegment);
339  segment_pool_memcnt++;
340  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
341  SCMutexUnlock(&segment_pool_memuse_mutex);
342 #endif
343 
344  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
345  return 1;
346 }
347 
348 /** \brief clean up a tcp segment pool entry */
349 static void TcpSegmentPoolCleanup(void *ptr)
350 {
351  if (ptr == NULL)
352  return;
353 
354  TcpSegment *seg = (TcpSegment *)ptr;
355  if (seg && seg->pcap_hdr_storage) {
356  if (seg->pcap_hdr_storage->pkt_hdr) {
358  StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen);
359  }
360  SCFree(seg->pcap_hdr_storage);
361  seg->pcap_hdr_storage = NULL;
362  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage));
363  }
364 
365  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
366 
367 #ifdef DEBUG
368  SCMutexLock(&segment_pool_memuse_mutex);
369  segment_pool_memuse -= sizeof(TcpSegment);
370  segment_pool_memcnt--;
371  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
372  SCMutexUnlock(&segment_pool_memuse_mutex);
373 #endif
374 }
375 
376 /**
377  * \brief Function to return the segment back to the pool.
378  *
379  * \param seg Segment which will be returned back to the pool.
380  */
382 {
383  if (seg == NULL)
384  return;
385 
386  if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
387  seg->pcap_hdr_storage->pktlen = 0;
388  }
389 
391 }
392 
393 /**
394  * \brief return all segments in this stream into the pool(s)
395  *
396  * \param stream the stream to cleanup
397  */
399 {
400  TcpSegment *seg = NULL, *safe = NULL;
401  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
402  {
403  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
405  }
406 }
407 
408 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
409 {
410  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
411  return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq);
412  }
413  return STREAM_BASE_OFFSET(stream);
414 }
415 
416 // may contain gaps
417 uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
418 {
419  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
420  if (!eof && !StreamTcpInlineMode()) {
421  right_edge = MIN(GetAbsLastAck(stream), right_edge);
422  }
423  return right_edge;
424 }
425 
426 uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
427 {
428  uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
429  if (!eof && !StreamTcpInlineMode()) {
430  right_edge = MIN(GetAbsLastAck(stream), right_edge);
431  }
432  return right_edge;
433 }
434 
435 #ifdef UNITTESTS
436 /** \internal
437  * \brief check if segments falls before stream 'offset' */
438 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
439 {
440  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
441  return 1;
442  return 0;
443 }
444 #endif
445 
446 /** \param f locked flow */
448 {
449  if (f->protoctx == NULL)
450  return;
451 
452  TcpSession *ssn = (TcpSession *)f->protoctx;
456  if (f->alparser) {
459  }
460 }
461 
462 /** \param f locked flow */
464 {
465  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
466  return 0;
467 
468  TcpSession *ssn = (TcpSession *)f->protoctx;
470 }
471 
472 static int StreamTcpReassemblyConfig(bool quiet)
473 {
474  uint32_t segment_prealloc = 2048;
475  SCConfNode *seg = SCConfGetNode("stream.reassembly.segment-prealloc");
476  if (seg) {
477  uint32_t prealloc = 0;
478  if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) {
479  SCLogError("segment-prealloc of "
480  "%s is invalid",
481  seg->val);
482  return -1;
483  }
484  segment_prealloc = prealloc;
485  }
486  if (!quiet)
487  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
488  stream_config.prealloc_segments = segment_prealloc;
489 
490  int overlap_diff_data = 0;
491  (void)SCConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
492  if (overlap_diff_data) {
494  }
495  if (StreamTcpInlineMode()) {
497  }
498 
499  uint16_t max_regions = 8;
500  SCConfNode *mr = SCConfGetNode("stream.reassembly.max-regions");
501  if (mr) {
502  uint16_t max_r = 0;
503  if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) {
504  SCLogError("max-regions %s is invalid", mr->val);
505  return -1;
506  }
507  max_regions = max_r;
508  }
509  if (!quiet)
510  SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions);
511 
512  stream_config.prealloc_segments = segment_prealloc;
514  stream_config.sbcnf.max_regions = max_regions;
516  stream_config.sbcnf.Calloc = ReassembleCalloc;
518  stream_config.sbcnf.Free = ReassembleFree;
519 
520  return 0;
521 }
522 
523 int StreamTcpReassembleInit(bool quiet)
524 {
525  /* init the memcap/use tracker */
527 
528  if (StreamTcpReassemblyConfig(quiet) < 0)
529  return -1;
530 
531 #ifdef DEBUG
532  SCMutexInit(&segment_pool_memuse_mutex, NULL);
533 #endif
534  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
536  return 0;
537 }
538 
539 void StreamTcpReassembleFree(bool quiet)
540 {
541  SCMutexLock(&segment_thread_pool_mutex);
542  if (segment_thread_pool != NULL) {
544  segment_thread_pool = NULL;
545  }
546  SCMutexUnlock(&segment_thread_pool_mutex);
547  SCMutexDestroy(&segment_thread_pool_mutex);
548 
549 #ifdef DEBUG
550  if (segment_pool_memuse > 0)
551  SCLogDebug("segment_pool_memuse %" PRIu64 " segment_pool_memcnt %" PRIu64 "",
552  segment_pool_memuse, segment_pool_memcnt);
553  SCMutexDestroy(&segment_pool_memuse_mutex);
554 #endif
555 }
556 
558 {
559  SCEnter();
561  if (unlikely(ra_ctx == NULL))
562  return NULL;
563 
564  ra_ctx->app_tctx = AppLayerGetCtxThread();
565 
566  SCMutexLock(&segment_thread_pool_mutex);
567  if (segment_thread_pool == NULL) {
568  segment_thread_pool = PoolThreadInit(1, /* thread */
569  0, /* unlimited */
571  sizeof(TcpSegment),
572  TcpSegmentPoolAlloc,
573  TcpSegmentPoolInit, NULL,
574  TcpSegmentPoolCleanup, NULL);
575  ra_ctx->segment_thread_pool_id = 0;
576  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
578  ra_ctx->segment_thread_pool_id);
579  } else {
580  /* grow segment_thread_pool until we have an element for our thread id */
582  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
584  ra_ctx->segment_thread_pool_id);
585  }
586  SCMutexUnlock(&segment_thread_pool_mutex);
587  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
588  SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
590  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
591  }
592 
593  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
594 }
595 
597 {
598  SCEnter();
600 
601  if (ra_ctx) {
603  SCFree(ra_ctx);
604  }
605  SCReturn;
606 }
607 
608 static void StreamTcpReassembleExceptionPolicyStatsIncr(
609  ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, enum ExceptionPolicy policy)
610 {
611  uint16_t id = ra_ctx->counter_tcp_reas_eps.eps_id[policy];
612  if (likely(tv && id > 0)) {
613  StatsIncr(tv, id);
614  }
615 }
616 
617 /**
618  * \brief check if stream in pkt direction has depth reached
619  *
620  * \param p packet with *LOCKED* flow
621  *
622  * \retval true stream has depth reached
623  * \retval false stream does not have depth reached
624  */
626 {
627  if (p->flow != NULL && p->flow->protoctx != NULL) {
628  TcpSession *ssn = p->flow->protoctx;
629  TcpStream *stream;
630  if (p->flowflags & FLOW_PKT_TOSERVER) {
631  stream = &ssn->client;
632  } else {
633  stream = &ssn->server;
634  }
635 
636  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? true : false;
637  }
638 
639  return false;
640 }
641 
642 /**
643  * \internal
644  * \brief Function to check the reassembly depth value against the
645  * allowed max depth of the stream reassembly for TCP streams.
646  *
647  * \param stream stream direction
648  * \param seq sequence number where "size" starts
649  * \param size size of the segment that is added
650  *
651  * \retval size Part of the size that fits in the depth, 0 if none
652  */
653 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
654  uint32_t seq, uint32_t size)
655 {
656  SCEnter();
657 
658  /* if the configured depth value is 0, it means there is no limit on
659  reassembly depth. Otherwise carry on my boy ;) */
660  if (ssn->reassembly_depth == 0) {
661  SCReturnUInt(size);
662  }
663 
664  /* if the final flag is set, we're not accepting anymore */
666  SCReturnUInt(0);
667  }
668 
669  uint64_t seg_depth;
670  if (SEQ_GT(stream->base_seq, seq)) {
671  if (SEQ_LEQ(seq+size, stream->base_seq)) {
672  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
673  stream->base_seq, seq, seq+size);
674  SCReturnUInt(0);
675  }
676 
677  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
678  } else {
679  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
680  }
681 
682  /* if the base_seq has moved past the depth window we stop
683  * checking and just reject the rest of the packets including
684  * retransmissions. Saves us the hassle of dealing with sequence
685  * wraps as well */
686  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
687  stream->base_seq, seg_depth,
688  ssn->reassembly_depth);
689 
690  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
691  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
693  SCReturnUInt(0);
694  }
695  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
696  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
697 #if 0
698  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
699  (stream->base_seq_offset + stream->base_seq + size),
700  (stream->isn + ssn->reassembly_depth));
701 #endif
702  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
703  /* packet (partly?) fits the depth window */
704 
705  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
706  /* complete fit */
707  SCReturnUInt(size);
708  } else {
710  /* partial fit, return only what fits */
711  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
712  DEBUG_VALIDATE_BUG_ON(part > size);
713  if (part > size)
714  part = size;
715  SCReturnUInt(part);
716  }
717  }
718 
719  SCReturnUInt(0);
720 }
721 
723 {
724  if (RB_EMPTY(&stream->sb.sbb_tree)) {
725  if (stream->sb.region.stream_offset != 0)
726  return 0;
727 
728  return stream->sb.region.buf_offset;
729  }
730 
731  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
732  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
733  return stream->sb.sbb_size;
734 }
735 
736 /**
737  * \brief Insert a TCP packet data into the stream reassembly engine.
738  *
739  * \retval 0 good segment, as far as we checked.
740  * \retval -1 insert failure due to memcap
741  *
742  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
743  * or it wasn't added because of reassembly depth.
744  *
745  */
747  TcpSession *ssn, TcpStream *stream, Packet *p)
748 {
749  SCEnter();
750 
751  if (ssn->data_first_seen_dir == 0) {
752  if (PKT_IS_TOSERVER(p)) {
753  ssn->data_first_seen_dir = STREAM_TOSERVER;
754  } else {
755  ssn->data_first_seen_dir = STREAM_TOCLIENT;
756  }
757  }
758 
759  /* If the OS policy is not set then set the OS policy for this stream */
760  if (stream->os_policy == 0) {
761  StreamTcpSetOSPolicy(stream, p);
762  }
763 
766  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
767  SCReturnInt(0);
768  }
769 
770  uint16_t *urg_offset;
771  if (PKT_IS_TOSERVER(p)) {
772  urg_offset = &ssn->urg_offset_ts;
773  } else {
774  urg_offset = &ssn->urg_offset_tc;
775  }
776 
777  const TCPHdr *tcph = PacketGetTCP(p);
778  /* segment sequence number, offset by previously accepted
779  * URG OOB data. */
780  uint32_t seg_seq = TCP_GET_RAW_SEQ(tcph) - (*urg_offset);
781  uint8_t urg_data = 0;
782 
783  /* if stream_config.urgent_policy == TCP_STREAM_URGENT_DROP, we won't get here */
784  if (tcph->th_flags & TH_URG) {
785  const uint16_t urg_ptr = SCNtohs(tcph->th_urp);
786  if (urg_ptr > 0 && urg_ptr <= p->payload_len &&
789  /* track up to 64k out of band URG bytes. Fall back to inline
790  * when that budget is exceeded. */
791  if ((*urg_offset) < UINT16_MAX) {
793  (*urg_offset)++;
794 
795  if ((*urg_offset) == UINT16_MAX) {
797  }
798  } else {
799  /* OOB limit DROP is handled here */
802  SCReturnInt(0);
803  }
804  }
805  urg_data = 1; /* only treat last 1 byte as out of band. */
808  }
809 
810  /* depending on hitting the OOB limit, update urg_data or not */
812  (*urg_offset) == UINT16_MAX &&
814  urg_data = 0;
815  } else {
816  if (urg_ptr == 1 && p->payload_len == 1) {
817  SCLogDebug("no non-URG data");
818  SCReturnInt(0);
819  }
820  }
821  }
822  }
823 
824  const uint16_t payload_len = p->payload_len - urg_data;
825 
826  /* If we have reached the defined depth for either of the stream, then stop
827  reassembling the TCP session */
828  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, seg_seq, payload_len);
829  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
830 
833  /* increment stream depth counter */
836  }
837  if (size == 0) {
838  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
839  SCReturnInt(0);
840  }
841 
843  if (size > payload_len)
844  size = payload_len;
845 
846  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
847  if (seg == NULL) {
848  SCLogDebug("segment_pool is empty");
851  SCReturnInt(-1);
852  }
853 
854  DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
855  TCP_SEG_LEN(seg) = (uint16_t)size;
856  /* set SEQUENCE number, adjusted to any URG pointer offset */
857  seg->seq = seg_seq;
858 
859  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
860  if (TCP_HAS_TFO(p) && p->payload_len && (tcph->th_flags & TH_SYN))
861  seg->seq += 1;
862 
863  /* proto detection skipped, but now we do get data. Set event. */
864  if (RB_EMPTY(&stream->seg_tree) &&
866 
869  }
870 
871  int r = StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, p->payload, payload_len);
872  if (r < 0) {
873  if (r == -SC_ENOMEM) {
875  }
876  SCLogDebug("StreamTcpReassembleInsertSegment failed");
877  SCReturnInt(-1);
878  }
879  SCReturnInt(0);
880 }
881 
882 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
883  Packet *p)
884 {
885  uint8_t flag = 0;
886 
888  flag |= STREAM_START;
889  }
890 
891  if (ssn->state == TCP_CLOSED) {
892  flag |= STREAM_EOF;
893  }
894 
895  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
896  flag |= STREAM_MIDSTREAM;
897  }
898 
899  if (p->flags & PKT_PSEUDO_STREAM_END) {
900  flag |= STREAM_EOF;
901  }
902 
903  if (&ssn->client == stream) {
904  flag |= STREAM_TOSERVER;
905  } else {
906  flag |= STREAM_TOCLIENT;
907  }
909  flag |= STREAM_DEPTH;
910  }
911  return flag;
912 }
913 
914 /**
915  * \brief Check the minimum size limits for reassembly.
916  *
917  * \retval false don't reassemble yet
918  * \retval true do reassemble
919  */
920 static bool StreamTcpReassembleRawCheckLimit(
921  const TcpSession *ssn, const TcpStream *stream, const Packet *p)
922 {
923  SCEnter();
924 
925  /* if any of these flags is set we always inspect immediately */
926 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
927  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
928  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
929  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
930 
933  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
934  "is set, so not expecting any new data segments");
935  }
937  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
938  }
940  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
941  "so no new segments will be considered");
942  }
943  SCReturnBool(true);
944  }
945 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
946 
947  /* some states mean we reassemble no matter how much data we have */
948  if (ssn->state > TCP_TIME_WAIT)
949  SCReturnBool(true);
950 
951  if (p->flags & PKT_PSEUDO_STREAM_END)
952  SCReturnBool(true);
953 
954  const uint64_t last_ack_abs = GetAbsLastAck(stream);
955  int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream);
956  int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size
958 
959  /* check if we have enough data to do raw reassembly */
960  if (chunk_size <= diff) {
961  SCReturnBool(true);
962  } else {
963  SCLogDebug("%s min chunk len not yet reached: "
964  "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < "
965  "%" PRIi64,
966  PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq,
967  (stream->last_ack - stream->base_seq), chunk_size);
968  SCReturnBool(false);
969  }
970 
971  SCReturnBool(false);
972 }
973 
974 /**
975  * \brief see what if any work the TCP session still needs
976  */
977 uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
978 {
979  const TcpStream *stream = NULL;
980 #ifdef DEBUG
981  const char *dirstr = NULL;
982 #endif
983  if (direction == STREAM_TOSERVER) {
984  stream = &ssn->client;
985 #ifdef DEBUG
986  dirstr = "client";
987 #endif
988  } else {
989  stream = &ssn->server;
990 #ifdef DEBUG
991  dirstr = "server";
992 #endif
993  }
994  int use_app = 1;
995  int use_raw = 1;
996 
998  // app is dead
999  use_app = 0;
1000  }
1001 
1002  if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
1003  // raw is dead
1004  use_raw = 0;
1005  }
1006  if (use_raw) {
1007  const uint64_t right_edge =
1008  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1009  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1010  " (use: %s). Stream right edge: %" PRIu64,
1011  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1012  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1013  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
1014  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1016  }
1017  }
1018  if (use_app) {
1019  const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
1020  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1021  " (use: %s). Stream right edge: %" PRIu64,
1022  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1023  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1024  if (right_edge > STREAM_APP_PROGRESS(stream)) {
1025  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1027  }
1028  }
1029 
1030  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
1032 }
1033 
1034 #ifdef DEBUG
1035 static uint64_t GetStreamSize(TcpStream *stream)
1036 {
1037  if (stream) {
1038  uint64_t size = 0;
1039  uint32_t cnt = 0;
1040  uint64_t last_ack_abs = GetAbsLastAck(stream);
1041  uint64_t last_re = 0;
1042 
1043  SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset);
1044 
1045  TcpSegment *seg;
1046  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
1047  const uint64_t seg_abs =
1048  STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq);
1049  if (last_re != 0 && last_re < seg_abs) {
1050  const char *gacked = NULL;
1051  if (last_ack_abs >= seg_abs) {
1052  gacked = "fully ack'd";
1053  } else if (last_ack_abs > last_re) {
1054  gacked = "partly ack'd";
1055  } else {
1056  gacked = "not yet ack'd";
1057  }
1058  SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked);
1059  }
1060 
1061  const char *acked = NULL;
1062  if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) {
1063  acked = "fully ack'd";
1064  } else if (last_ack_abs > seg_abs) {
1065  acked = "partly ack'd";
1066  } else {
1067  acked = "not yet ack'd";
1068  }
1069 
1070  SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64
1071  ") ack:%s",
1072  cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg),
1073  seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked);
1074  last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg);
1075  cnt++;
1076  size += (uint64_t)TCP_SEG_LEN(seg);
1077  }
1078 
1079  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
1080  return size;
1081  }
1082  return (uint64_t)0;
1083 }
1084 
1085 static void GetSessionSize(TcpSession *ssn, Packet *p)
1086 {
1087  uint64_t size = 0;
1088  if (ssn) {
1089  size = GetStreamSize(&ssn->client);
1090  size += GetStreamSize(&ssn->server);
1091 
1092  //if (size > 900000)
1093  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1094  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1095  }
1096 }
1097 #endif
1098 
1099 static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk)
1100 {
1101  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
1102  /* only if the stream has been ack'd, consider a gap for sure
1103  * otherwise there may still be a chance of pkts coming in */
1104  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
1105  GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len)) {
1106  return true;
1107  }
1108  return false;
1109 }
1110 
1111 /** \internal
1112  *
1113  * Get buffer, or first part of the buffer if data gaps exist.
1114  *
1115  * \brief get stream data from offset
1116  * \param offset stream offset
1117  * \param check_for_gap check if there is a gap ahead. Optional as it is only
1118  * needed for app-layer incomplete support.
1119  * \retval bool pkt loss ahead */
1120 static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1121  uint64_t offset, const bool check_for_gap)
1122 {
1123  const uint8_t *mydata;
1124  uint32_t mydata_len;
1125  bool gap_ahead = false;
1126 
1127  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1128  SCLogDebug("getting one blob");
1129 
1130  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1131 
1132  *data = mydata;
1133  *data_len = mydata_len;
1134  } else {
1135  SCLogDebug("block mode");
1136  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1137  StreamingBufferBlock *blk = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1138  if (blk == NULL) {
1139  *data = NULL;
1140  *data_len = 0;
1141  return false;
1142  }
1143  SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len);
1144 
1145  /* block at expected offset */
1146  if (blk->offset == offset) {
1147  SCLogDebug("blk at offset");
1148 
1149  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
1150  BUG_ON(blk->len != *data_len);
1151 
1152  gap_ahead = check_for_gap && GapAhead(stream, blk);
1153 
1154  /* block past out offset */
1155  } else if (blk->offset > offset) {
1156  SCLogDebug("gap, want data at offset %"PRIu64", "
1157  "got data at %"PRIu64". GAP of size %"PRIu64,
1158  offset, blk->offset, blk->offset - offset);
1159  *data = NULL;
1160  *data_len = (uint32_t)(blk->offset - offset);
1161 
1162  /* block starts before offset, but ends after */
1163  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
1164  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
1165  offset, blk->offset, blk->len);
1166  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
1167  SCLogDebug("data %p, data_len %u", *data, *data_len);
1168 
1169  gap_ahead = check_for_gap && GapAhead(stream, blk);
1170 
1171  } else {
1172  *data = NULL;
1173  *data_len = 0;
1174  }
1175  }
1176  return gap_ahead;
1177 }
1178 
1179 /** \internal
1180  * \brief check to see if we should declare a GAP
1181  * Call this when the app layer didn't get data at the requested
1182  * offset.
1183  */
1184 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
1185 {
1186  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1187  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1188  const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded;
1189 
1190  SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs);
1191  SCLogDebug("next_seq %u", stream->next_seq);
1192 
1193  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1194  * has been ack'd. This looks like a GAP. */
1195  if (last_ack_abs > app_progress) {
1196  /* however, we can accept ACKs a bit too liberally. If last_ack
1197  * is beyond next_seq, we only consider it a gap now if we do
1198  * already have data beyond the gap. */
1199  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1200  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1201  SCLogDebug("packet %" PRIu64 ": no GAP. "
1202  "next_seq %u < last_ack %u, but no data in list",
1203  p->pcap_cnt, stream->next_seq, stream->last_ack);
1204  return false;
1205  }
1206  const uint64_t next_seq_abs =
1207  STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1208  const StreamingBufferBlock *blk = stream->sb.head;
1209  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1210  /* ack'd data after the gap */
1211  SCLogDebug("packet %" PRIu64 ": GAP. "
1212  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1213  p->pcap_cnt, stream->next_seq, stream->last_ack);
1214  return true;
1215  }
1216  }
1217 
1218  SCLogDebug("packet %" PRIu64 ": GAP! "
1219  "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", "
1220  "but we have no data.",
1221  p->pcap_cnt, last_ack_abs, app_progress);
1222  return true;
1223  }
1224  SCLogDebug("packet %"PRIu64": no GAP. "
1225  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1226  p->pcap_cnt, last_ack_abs, app_progress);
1227  return false;
1228 }
1229 
1230 static inline uint32_t AdjustToAcked(const Packet *p,
1231  const TcpSession *ssn, const TcpStream *stream,
1232  const uint64_t app_progress, const uint32_t data_len)
1233 {
1234  uint32_t adjusted = data_len;
1235 
1236  /* get window of data that is acked */
1237  if (!StreamTcpInlineMode()) {
1238  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1239  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1240  (ssn->state == TCP_CLOSED &&
1241  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1242  (p->flags & PKT_PSEUDO_STREAM_END))) {
1243  // fall through, we use all available data
1244  } else {
1245  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1246  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1247 
1248  /* see if the buffer contains unack'd data as well */
1249  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1250  uint32_t check = data_len;
1251  adjusted = (uint32_t)(last_ack_abs - app_progress);
1252  BUG_ON(adjusted > check);
1253  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1254  "data is considered", adjusted);
1255  }
1256  }
1257  }
1258  return adjusted;
1259 }
1260 
1261 /** \internal
1262  * \brief get stream buffer and update the app-layer
1263  * \param stream pointer to pointer as app-layer can switch flow dir
1264  * \retval 0 success
1265  */
1266 static int ReassembleUpdateAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn,
1267  TcpStream **stream, Packet *p, enum StreamUpdateDir app_update_dir)
1268 {
1269  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1270 
1271  SCLogDebug("app progress %"PRIu64, app_progress);
1272 #ifdef DEBUG
1273  uint64_t last_ack_abs = GetAbsLastAck(*stream);
1274  SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs,
1275  (*stream)->base_seq);
1276 #endif
1277  const uint8_t *mydata;
1278  uint32_t mydata_len;
1279  bool last_was_gap = false;
1280 
1281  while (1) {
1282  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1283  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1284  bool gap_ahead =
1285  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead);
1286  SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len);
1287  if (last_was_gap && mydata_len == 0) {
1288  break;
1289  }
1290  last_was_gap = false;
1291 
1292  /* make sure to only deal with ACK'd data */
1293  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1294  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1295  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1296  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1297 
1298  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len,
1299  StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, app_update_dir);
1300  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1301 
1303  (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP;
1306 
1307  /* AppLayerHandleTCPData has likely updated progress. */
1308  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1309  app_progress = STREAM_APP_PROGRESS(*stream);
1310 
1311  /* a GAP also consumes 'data required'. TODO perhaps we can use
1312  * this to skip post GAP data until the start of a next record. */
1313  if ((*stream)->data_required > 0) {
1314  if ((*stream)->data_required > mydata_len) {
1315  (*stream)->data_required -= mydata_len;
1316  } else {
1317  (*stream)->data_required = 0;
1318  }
1319  }
1320  if (r < 0)
1321  return 0;
1322  if (no_progress_update)
1323  break;
1324  last_was_gap = true;
1325  continue;
1326 
1327  } else if (flags & STREAM_DEPTH) {
1328  SCLogDebug("DEPTH");
1329  // we're just called once with this flag, so make sure we pass it on
1330  if (mydata == NULL && mydata_len > 0) {
1331  mydata_len = 0;
1332  }
1333  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1334  SCLogDebug("GAP?1");
1335  /* Possibly a gap, but no new data. */
1336  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1337  SCReturnInt(0);
1338 
1339  mydata = NULL;
1340  mydata_len = 0;
1341  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1342  break;
1343  }
1344  DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1345 
1346  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1347  *stream, &(*stream)->sb, mydata_len, app_progress);
1348 
1349  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1350  SCLogDebug("GAP?2");
1351  if (mydata_len < (*stream)->data_required) {
1352  SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead));
1353  if (gap_ahead) {
1354  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1355  (*stream)->data_required, mydata_len);
1356  (*stream)->app_progress_rel += mydata_len;
1357  (*stream)->data_required -= mydata_len;
1358  // TODO send incomplete data to app-layer with special flag
1359  // indicating its all there is for this rec?
1360  } else {
1361  SCReturnInt(0);
1362  }
1363  app_progress = STREAM_APP_PROGRESS(*stream);
1364  continue;
1365  }
1366  }
1367  (*stream)->data_required = 0;
1368 
1369  SCLogDebug("parser");
1370  /* update the app-layer */
1371  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata,
1372  mydata_len, flags, app_update_dir);
1373  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1374  AppLayerFrameDump(p->flow);
1375  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1376  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1377  break;
1378  app_progress = new_app_progress;
1379  if (flags & STREAM_DEPTH)
1380  break;
1381  }
1382 
1383  SCReturnInt(0);
1384 }
1385 
1386 /**
1387  * \brief Update the stream reassembly upon receiving a packet.
1388  *
1389  * For IDS mode, the stream is in the opposite direction of the packet,
1390  * as the ACK-packet is ACK'ing the stream.
1391  *
1392  * One of the utilities call by this function AppLayerHandleTCPData(),
1393  * has a feature where it will call this very same function for the
1394  * stream opposing the stream it is called with. This shouldn't cause
1395  * any issues, since processing of each stream is independent of the
1396  * other stream.
1397  */
1399  TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
1400 {
1401  SCEnter();
1402 
1403  /* this function can be directly called by app layer protocol
1404  * detection. */
1407  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1408  SCReturnInt(0);
1409  }
1410 
1411 #ifdef DEBUG
1412  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1413  GetSessionSize(ssn, p);
1414 #endif
1415  /* if no segments are in the list or all are already processed,
1416  * and state is beyond established, we send an empty msg */
1417  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1418  {
1419  /* send an empty EOF msg if we have no segments but TCP state
1420  * is beyond ESTABLISHED */
1421  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1422  SCLogDebug("sending empty eof message");
1423  /* send EOF to app layer */
1424  uint8_t stream_flags = StreamGetAppLayerFlags(ssn, stream, p);
1425  DEBUG_VALIDATE_BUG_ON((stream_flags & STREAM_EOF) == 0);
1427  tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, stream_flags, app_update_dir);
1428  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1429 
1430  SCReturnInt(0);
1431  }
1432  }
1433 
1434  /* with all that out of the way, lets update the app-layer */
1435  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, app_update_dir);
1436 }
1437 
1438 /** \internal
1439  * \brief get stream data from offset
1440  * \param offset stream offset */
1441 static int GetRawBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1442  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1443 {
1444  const uint8_t *mydata;
1445  uint32_t mydata_len;
1446  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1447  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1448 
1449  /* No gaps in the stream, data must exist in the streaming buffer array */
1450  uint64_t roffset = offset;
1451  if (offset) {
1452  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1453  } else {
1454  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1455  }
1456 
1457  *data = mydata;
1458  *data_len = mydata_len;
1459  *data_offset = roffset;
1460  } else {
1461  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1462  *iter == NULL ? "starting" : "continuing", offset);
1463  if (*iter == NULL) {
1464  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1465  *iter = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1466  SCLogDebug("*iter %p", *iter);
1467  }
1468  if (*iter == NULL) {
1469  SCLogDebug("no data");
1470  *data = NULL;
1471  *data_len = 0;
1472  *data_offset = 0;
1473  return 0;
1474  }
1475  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1476 
1477  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1478  SCLogDebug("mydata %p", mydata);
1479 
1480  if ((*iter)->offset < offset) {
1481  uint64_t delta = offset - (*iter)->offset;
1482  if (delta < mydata_len) {
1483  *data = mydata + delta;
1484  *data_len = (uint32_t)(mydata_len - delta);
1485  *data_offset = offset;
1486  } else {
1487  SCLogDebug("no data (yet)");
1488  *data = NULL;
1489  *data_len = 0;
1490  *data_offset = 0;
1491  }
1492 
1493  } else {
1494  *data = mydata;
1495  *data_len = mydata_len;
1496  *data_offset = (*iter)->offset;
1497  }
1498 
1499  *iter = SBB_RB_NEXT(*iter);
1500  SCLogDebug("*iter %p", *iter);
1501  }
1502  return 0;
1503 }
1504 
1505 /** \brief does the stream engine have data to inspect?
1506  *
1507  * Returns true if there is data to inspect. In IDS case this is
1508  * about ACK'd data in the packet's direction.
1509  *
1510  * In the IPS case this is about the packet itself.
1511  */
1513 {
1514  TcpStream *stream;
1515  if (PKT_IS_TOSERVER(p)) {
1516  stream = &ssn->client;
1517  } else {
1518  stream = &ssn->server;
1519  }
1520 
1521  if (RB_EMPTY(&stream->seg_tree)) {
1522  return false;
1523  }
1524 
1527  return false;
1528 
1529  if (!StreamTcpInlineMode()) {
1530  const uint64_t segs_re_abs =
1531  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1532  if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) {
1533  return false;
1534  }
1535  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1536  return true;
1537  }
1538  } else {
1539  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1540  return true;
1541  }
1542  }
1543  return false;
1544 }
1545 
1546 /** \brief update stream engine after detection
1547  *
1548  * Tasked with progressing the 'progress' for Raw reassembly.
1549  * 2 main scenario's:
1550  * 1. progress is != 0, so we use this
1551  * 2. progress is 0, meaning the detect engine didn't touch
1552  * raw at all. In this case we need to look into progressing
1553  * raw anyway.
1554  *
1555  * Additionally, this function is tasked with disabling raw
1556  * reassembly if the app-layer requested to disable it.
1557  */
1558 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
1559 {
1560  TcpStream *stream;
1561  if (PKT_IS_TOSERVER(p)) {
1562  stream = &ssn->client;
1563  } else {
1564  stream = &ssn->server;
1565  }
1566 
1567  if (progress > STREAM_RAW_PROGRESS(stream)) {
1568  DEBUG_VALIDATE_BUG_ON(progress - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1569  uint32_t slide = (uint32_t)(progress - STREAM_RAW_PROGRESS(stream));
1570  stream->raw_progress_rel += slide;
1572 
1573  } else if (progress == 0) {
1574  uint64_t target;
1575  if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) {
1576  target = STREAM_APP_PROGRESS(stream);
1577  } else {
1578  target = GetAbsLastAck(stream);
1579  }
1580  if (target > STREAM_RAW_PROGRESS(stream)) {
1581  DEBUG_VALIDATE_BUG_ON(target - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1582  uint32_t slide = (uint32_t)(target - STREAM_RAW_PROGRESS(stream));
1583  stream->raw_progress_rel += slide;
1584  }
1586 
1587  } else {
1588  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1589  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1590  STREAM_RAW_PROGRESS(stream), stream->window);
1591  }
1592 
1593  /* if we were told to accept no more raw data, we can mark raw as
1594  * disabled now. */
1597  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1598  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1599  }
1600 
1601  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1602 }
1603 
1604 /** \internal
1605  * \brief get a buffer around the current packet and run the callback on it
1606  *
1607  * The inline/IPS scanning method takes the current payload and wraps it in
1608  * data from other segments.
1609  *
1610  * How much data is inspected is controlled by the available data, chunk_size
1611  * and the payload size of the packet.
1612  *
1613  * Large packets: if payload size is close to the chunk_size, where close is
1614  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1615  * used: payload_len + 33% of the chunk_size.
1616  * If the payload size if equal to or bigger than the chunk_size, we use
1617  * payload len + 33% of the chunk size.
1618  */
1619 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1620  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1621 {
1622  SCEnter();
1623  int r = 0;
1624 
1625  TcpStream *stream;
1626  if (PKT_IS_TOSERVER(p)) {
1627  stream = &ssn->client;
1628  } else {
1629  stream = &ssn->server;
1630  }
1631 
1632  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1634  {
1635  *progress_out = STREAM_RAW_PROGRESS(stream);
1636  return 0;
1637  }
1638 
1639  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1642  if ((chunk_size <= p->payload_len) || (((chunk_size / 3) * 2) < p->payload_len)) {
1643  chunk_size = p->payload_len + (chunk_size / 3);
1644  SCLogDebug(
1645  "packet payload len %u, so chunk_size adjusted to %u", p->payload_len, chunk_size);
1646  }
1647 
1648  const TCPHdr *tcph = PacketGetTCP(p);
1649  uint64_t packet_leftedge_abs =
1650  STREAM_BASE_OFFSET(stream) + (TCP_GET_RAW_SEQ(tcph) - stream->base_seq);
1651  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1652  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1653  packet_leftedge_abs, packet_rightedge_abs);
1654 
1655  const uint8_t *mydata = NULL;
1656  uint32_t mydata_len = 0;
1657  uint64_t mydata_offset = 0;
1658  /* simply return progress from the block we inspected. */
1659  bool return_progress = false;
1660 
1661  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1662  /* continues block */
1663  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1664  return_progress = true;
1665 
1666  } else {
1667  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1668  /* find our block */
1669  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1670  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1671  if (sbb) {
1672  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1673  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1674  mydata_offset = sbb->offset;
1675  }
1676  }
1677 
1678  /* this can only happen if the segment insert of our current 'p' failed */
1679  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1680  if ((mydata == NULL || mydata_len == 0) || /* no data */
1681  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1682  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1683  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1684  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1685  {
1686  /* no data, or data is incomplete or wrong: use packet data */
1687  mydata = p->payload;
1688  mydata_len = p->payload_len;
1689  mydata_offset = packet_leftedge_abs;
1690  //mydata_rightedge_abs = packet_rightedge_abs;
1691  } else {
1692  /* adjust buffer to match chunk_size */
1693  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1694  if (mydata_len > chunk_size) {
1695  uint32_t excess = mydata_len - chunk_size;
1696  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1697 
1698  if (mydata_rightedge_abs == packet_rightedge_abs) {
1699  mydata += excess;
1700  mydata_len -= excess;
1701  mydata_offset += excess;
1702  SCLogDebug("cutting front of the buffer with %u", excess);
1703  } else if (mydata_offset == packet_leftedge_abs) {
1704  mydata_len -= excess;
1705  SCLogDebug("cutting tail of the buffer with %u", excess);
1706  } else {
1707  DEBUG_VALIDATE_BUG_ON(mydata_offset > packet_leftedge_abs);
1708  uint32_t abs_before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1709  DEBUG_VALIDATE_BUG_ON(packet_rightedge_abs > mydata_rightedge_abs);
1710  uint32_t abs_after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1711  uint32_t before = abs_before;
1712  uint32_t after = abs_after;
1713  SCLogDebug("before %u after %u", before, after);
1714 
1715  if (after >= (chunk_size - p->payload_len) / 2) {
1716  // more trailing data than we need
1717 
1718  if (before >= (chunk_size - p->payload_len) / 2) {
1719  // also more heading data, divide evenly
1720  before = after = (chunk_size - p->payload_len) / 2;
1721  } else {
1722  // heading data is less than requested, give the
1723  // rest to the trailing data
1724  after = (chunk_size - p->payload_len) - before;
1725  }
1726  } else {
1727  // less trailing data than requested
1728 
1729  if (before >= (chunk_size - p->payload_len) / 2) {
1730  before = (chunk_size - p->payload_len) - after;
1731  } else {
1732  // both smaller than their requested size
1733  }
1734  }
1735 
1736  /* adjust the buffer */
1737  DEBUG_VALIDATE_BUG_ON(before > abs_before);
1738  uint32_t skip = abs_before - before;
1739  DEBUG_VALIDATE_BUG_ON(after > abs_after);
1740  uint32_t cut = abs_after - after;
1741  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1742  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1743  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1744  mydata += skip;
1745  mydata_len -= (skip + cut);
1746  mydata_offset += skip;
1747  }
1748  }
1749  }
1750 
1751  /* run the callback */
1752  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1753  BUG_ON(r < 0);
1754 
1755  if (return_progress) {
1756  *progress_out = (mydata_offset + mydata_len);
1757  } else {
1758  /* several blocks of data, so we need to be a bit more careful:
1759  * - if last_ack is beyond last progress, move progress forward to last_ack
1760  * - if our block matches or starts before last ack, return right edge of
1761  * our block.
1762  */
1763  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1764  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1765 
1766  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1767  if (mydata_offset > last_ack_abs) {
1768  /* gap between us and last ack, set progress to last ack */
1769  *progress_out = last_ack_abs;
1770  } else {
1771  *progress_out = (mydata_offset + mydata_len);
1772  }
1773  } else {
1774  *progress_out = STREAM_RAW_PROGRESS(stream);
1775  }
1776  }
1777  return r;
1778 }
1779 
1780 /** \brief access 'raw' reassembly data.
1781  *
1782  * Access data as tracked by 'raw' tracker. Data is made available to
1783  * callback that is passed to this function.
1784  *
1785  * In the case of IDS the callback may be run multiple times if data
1786  * contains gaps. It will then be run for each block of data that is
1787  * continuous.
1788  *
1789  * The callback should give on of 2 return values:
1790  * - 0 ok
1791  * - 1 done
1792  * The value 1 will break the loop if there is a block list that is
1793  * inspected.
1794  *
1795  * This function will return the 'progress' value that has been
1796  * consumed until now.
1797  *
1798  * \param ssn tcp session
1799  * \param stream tcp stream
1800  * \param Callback the function pointer to the callback function
1801  * \param cb_data callback data
1802  * \param[in] progress_in progress to work from
1803  * \param[in] re right edge of data to consider
1804  * \param[out] progress_out absolute progress value of the data this
1805  * call handled.
1806  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1807  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1808  *
1809  * `respect_inspect_depth` is used to avoid useless inspection of too
1810  * much data.
1811  */
1812 static int StreamReassembleRawDo(const TcpSession *ssn, const TcpStream *stream,
1813  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1814  const uint64_t re, uint64_t *progress_out, bool eof, bool respect_inspect_depth)
1815 {
1816  SCEnter();
1817  int r = 0;
1818 
1819  StreamingBufferBlock *iter = NULL;
1820  uint64_t progress = progress_in;
1821 
1822  /* loop through available buffers. On no packet loss we'll have a single
1823  * iteration. On missing data we'll walk the blocks */
1824  while (1) {
1825  const uint8_t *mydata;
1826  uint32_t mydata_len;
1827  uint64_t mydata_offset = 0;
1828 
1829  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1830  if (mydata_len == 0) {
1831  SCLogDebug("no data");
1832  break;
1833  }
1834  //PrintRawDataFp(stdout, mydata, mydata_len);
1835 
1836  SCLogDebug("raw progress %"PRIu64, progress);
1837  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1838  stream, &stream->sb, mydata_len, progress);
1839 
1840  if (eof) {
1841  // inspect all remaining data, ack'd or not
1842  } else {
1843  if (re < progress) {
1844  SCLogDebug("nothing to do");
1845  goto end;
1846  }
1847 
1848  SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress);
1849  SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len,
1850  re);
1851 
1852  /* see if the buffer contains unack'd data as well */
1853  if (progress + mydata_len > re) {
1854  uint32_t check = mydata_len;
1855  mydata_len = (uint32_t)(re - progress);
1856  BUG_ON(check < mydata_len);
1857  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1858  "data is considered", mydata_len);
1859  }
1860  }
1861  if (mydata_len == 0)
1862  break;
1863 
1864  SCLogDebug("data %p len %u", mydata, mydata_len);
1865 
1866  /* we have data. */
1867  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1868  BUG_ON(r < 0);
1869 
1870  if (mydata_offset == progress) {
1871  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1872  progress, mydata_len, progress_in + mydata_len);
1873 
1874  progress += mydata_len;
1875  SCLogDebug("raw progress now %"PRIu64, progress);
1876 
1877  /* data is beyond the progress we'd like, and before last ack. Gap. */
1878  } else if (mydata_offset > progress && mydata_offset < re) {
1879  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1880  SCLogDebug("re %" PRIu64, re);
1881  progress = mydata_offset;
1882  SCLogDebug("raw progress now %"PRIu64, progress);
1883 
1884  } else {
1885  SCLogDebug("not increasing progress, data gap => mydata_offset "
1886  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1887  }
1888 
1889  if (iter == NULL || r == 1)
1890  break;
1891  }
1892 end:
1893  *progress_out = progress;
1894  return r;
1895 }
1896 
1898  void *cb_data, const uint64_t offset, const bool eof)
1899 {
1900  /* take app progress as the right edge of used data. */
1901  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1902  SCLogDebug("app_progress %" PRIu64, app_progress);
1903 
1904  uint64_t unused = 0;
1905  return StreamReassembleRawDo(
1906  ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof, false);
1907 }
1908 
1910  StreamReassembleRawFunc Callback, void *cb_data,
1911  uint64_t *progress_out, bool respect_inspect_depth)
1912 {
1913  /* handle inline separately as the logic is very different */
1914  if (StreamTcpInlineMode()) {
1915  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1916  }
1917 
1918  TcpStream *stream;
1919  if (PKT_IS_TOSERVER(p)) {
1920  stream = &ssn->client;
1921  } else {
1922  stream = &ssn->server;
1923  }
1924 
1926  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1927  {
1928  *progress_out = STREAM_RAW_PROGRESS(stream);
1929  return 0;
1930  }
1931 
1932  uint64_t progress = STREAM_RAW_PROGRESS(stream);
1933  /* if the app layer triggered a flush, and we're supposed to
1934  * use a minimal inspect depth, we actually take the app progress
1935  * as that is the right edge of the data. Then we take the window
1936  * of 'min_inspect_depth' before that. */
1937 
1938  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s "
1939  "stream->min_inspect_depth %u",
1940  respect_inspect_depth ? "true" : "false",
1941  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1942  stream->min_inspect_depth);
1943 
1944  if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) &&
1945  stream->min_inspect_depth) {
1946  progress = STREAM_APP_PROGRESS(stream);
1947  if (stream->min_inspect_depth >= progress) {
1948  progress = 0;
1949  } else {
1950  progress -= stream->min_inspect_depth;
1951  }
1952 
1953  SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream),
1954  STREAM_RAW_PROGRESS(stream));
1955 
1956  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1957  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress "
1958  "%" PRIu64,
1959  progress);
1960  }
1961 
1962  SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth,
1963  stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW"
1964  : "(no trigger)");
1965 
1966  /* absolute right edge of ack'd data */
1967  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1968  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1969 
1970  return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs,
1971  progress_out, (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1972 }
1973 
1974 int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream,
1975  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1976  uint64_t *progress_out, const bool eof)
1977 {
1978  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1979  return 0;
1980 
1981  /* absolute right edge of ack'd data */
1982  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1983  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1984 
1985  return StreamReassembleRawDo(
1986  ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof, false);
1987 }
1988 
1989 /** \internal
1990  * \brief update app layer based on received ACK
1991  *
1992  * \retval r 0 on success, -1 on error
1993  */
1994 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1995  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1996 {
1997  SCEnter();
1998 
1999  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
2000  SCReturnInt(-1);
2001 
2002  SCReturnInt(0);
2003 }
2004 
2006  TcpSession *ssn, TcpStream *stream, Packet *p)
2007 {
2008  SCEnter();
2009 
2010  DEBUG_VALIDATE_BUG_ON(!PacketIsTCP(p));
2011  const TCPHdr *tcph = PacketGetTCP(p);
2012 
2013  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
2014  ssn, stream, p, p->payload_len);
2015 
2016  /* default IDS: update opposing side (triggered by ACK) */
2018  /* inline and stream end and flow timeout packets trigger same dir handling */
2019  if (StreamTcpInlineMode()) {
2020  dir = UPDATE_DIR_PACKET;
2021  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
2022  dir = UPDATE_DIR_PACKET;
2023  } else if (tcph->th_flags & TH_RST) { // accepted rst
2024  dir = UPDATE_DIR_PACKET;
2025  } else if ((tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
2026  if (tcph->th_flags & TH_ACK) {
2027  dir = UPDATE_DIR_BOTH;
2028  } else {
2029  dir = UPDATE_DIR_PACKET;
2030  }
2031  } else if (ssn->state == TCP_CLOSED) {
2032  dir = UPDATE_DIR_BOTH;
2033  } else if ((ssn->flags & STREAMTCP_FLAG_ASYNC) != 0) {
2034  dir = UPDATE_DIR_PACKET;
2035  SCLogDebug("%" PRIu64 ": ASYNC: UPDATE_DIR_PACKET", p->pcap_cnt);
2036  }
2037 
2038  /* handle ack received */
2039  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
2040  /* we need to update the opposing stream in
2041  * StreamTcpReassembleHandleSegmentUpdateACK */
2042  TcpStream *opposing_stream = NULL;
2043  if (stream == &ssn->client) {
2044  opposing_stream = &ssn->server;
2045  } else {
2046  opposing_stream = &ssn->client;
2047  }
2048 
2049  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2050 
2051  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
2052  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
2053  SCReturnInt(-1);
2054  }
2055 
2056  /* StreamTcpReassembleHandleSegmentUpdateACK
2057  * may swap content of ssn->server and ssn->client structures.
2058  * We have to continue with initial content of the stream in such case */
2059  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2060  if (reversed_before_ack_handling != reversed_after_ack_handling) {
2061  SCLogDebug("TCP streams were swapped");
2062  stream = opposing_stream;
2063  }
2064  }
2065  /* if this segment contains data, insert it */
2066  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) &&
2067  (tcph->th_flags & TH_RST) == 0) {
2068  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
2069 
2070  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
2071  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
2072  /* failure can only be because of memcap hit, so see if this should lead to a drop */
2075  StreamTcpReassembleExceptionPolicyStatsIncr(
2077  SCReturnInt(-1);
2078  }
2079 
2080  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
2081  p->flags |= PKT_STREAM_ADD;
2082  } else {
2083  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
2084  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
2085  ssn, stream, p->payload_len,
2086  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
2087  }
2088 
2089  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
2090  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
2091  * was *just* set. In this case we trigger the AppLayer Truncate
2092  * logic, to inform the applayer no more data in this direction is
2093  * to be expected. */
2094  if ((stream->flags &
2097  {
2098  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
2099  if (dir != UPDATE_DIR_PACKET) {
2100  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
2101  "can trigger Truncate");
2102  dir = UPDATE_DIR_PACKET;
2103  }
2104  }
2105 
2106  /* in stream inline mode even if we have no data we call the reassembly
2107  * functions to handle EOF */
2108  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
2109  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
2110  StreamTcpInlineMode()?"true":"false",
2111  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
2112  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
2113  SCReturnInt(-1);
2114  }
2115  }
2116 
2117  SCReturnInt(0);
2118 }
2119 
2120 /**
2121  * \brief get a segment from the pool
2122  *
2123  * \retval seg Segment from the pool or NULL
2124  */
2126 {
2128  if (seg) {
2130  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2131  return seg;
2132  }
2133 
2134  seg = (TcpSegment *)PoolThreadGetById(
2135  segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id);
2136  SCLogDebug("seg we return is %p", seg);
2137  if (seg == NULL) {
2138  /* Increment the counter to show that we are not able to serve the
2139  segment request due to memcap limit */
2141  } else {
2142  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2144  }
2145 
2146  return seg;
2147 }
2148 
2149 /**
2150  * \brief Trigger RAW stream reassembly
2151  *
2152  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
2153  * reassembly from the applayer, for example upon completion of a
2154  * HTTP request.
2155  *
2156  * It sets a flag in the stream so that the next Raw call will return
2157  * the data.
2158  *
2159  * \param ssn TcpSession
2160  */
2162 {
2163 #ifdef DEBUG
2164  BUG_ON(ssn == NULL);
2165 #endif
2166 
2167  if (ssn != NULL) {
2168  if (direction == STREAM_TOSERVER) {
2170  } else {
2172  }
2173 
2174  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
2175  }
2176 }
2177 
2178 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
2179 {
2180 #ifdef DEBUG
2181  BUG_ON(ssn == NULL);
2182 #endif
2183 
2184  if (ssn != NULL) {
2185  if (direction == STREAM_TOSERVER) {
2186  ssn->client.min_inspect_depth = depth;
2187  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
2188  } else {
2189  ssn->server.min_inspect_depth = depth;
2190  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
2191  }
2192  }
2193 }
2194 
2195 #ifdef UNITTESTS
2196 /** unit tests and it's support functions below */
2197 
2198 #define SET_ISN(stream, setseq) \
2199  (stream)->isn = (setseq); \
2200  (stream)->base_seq = (setseq) + 1
2201 
2202 /** \brief The Function to create the packet with given payload, which is used
2203  * to test the reassembly of the engine.
2204  *
2205  * \param payload The variable used to store the payload contents of the
2206  * current packet.
2207  * \param value The value which current payload will have for this packet
2208  * \param payload_len The length of the filed payload for current packet.
2209  * \param len Length of the payload array
2210  */
2211 
2212 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2213  uint8_t payload_len, uint8_t len)
2214 {
2215  uint8_t i;
2216  for (i = 0; i < payload_len; i++)
2217  payload[i] = value;
2218  for (; i < len; i++)
2219  payload = NULL;
2220 }
2221 
2222 /** \brief The Function Checks the reassembled stream contents against predefined
2223  * stream contents according to OS policy used.
2224  *
2225  * \param stream_policy Predefined value of stream for different OS policies
2226  * \param stream Reassembled stream returned from the reassembly functions
2227  */
2228 
2229 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2230 {
2231  if (StreamingBufferCompareRawData(&stream->sb,
2232  data, data_len) == 0)
2233  {
2234  SCReturnInt(0);
2235  }
2236  SCLogInfo("OK");
2237  PrintRawDataFp(stdout, data, data_len);
2238  return 1;
2239 }
2240 
2241 #define MISSED_START(isn) \
2242  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2243  TcpSession ssn; \
2244  ThreadVars tv; \
2245  memset(&tv, 0, sizeof(tv)); \
2246  \
2247  StreamTcpUTInit(&ra_ctx); \
2248  \
2249  StreamTcpUTSetupSession(&ssn); \
2250  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2251  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2252  \
2253  TcpStream *stream = &ssn.client;
2254 
2255 #define MISSED_END \
2256  StreamTcpUTClearSession(&ssn); \
2257  StreamTcpUTDeinit(ra_ctx); \
2258  PASS
2259 
2260 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2261  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2262  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2263 
2264 #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \
2265  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen));
2266 
2267 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len);
2268 
2269 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
2270 {
2271  int cnt = 0;
2272  uint64_t last_re = 0;
2273  StreamingBufferBlock *sbb = NULL;
2274  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2275  {
2276  if (sbb->offset != last_re) {
2277  // gap before us
2278  if (cnt == pos && last_re == offset && len == sbb->offset - last_re) {
2279  return 1;
2280  }
2281  cnt++;
2282  }
2283  last_re = sbb->offset + sbb->len;
2284  cnt++;
2285  }
2286  return 0;
2287 }
2288 
2290  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len);
2291 
2293  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
2294 {
2295  int cnt = 0;
2296  uint64_t last_re = 0;
2297  StreamingBufferBlock *sbb = NULL;
2298  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2299  {
2300  if (sbb->offset != last_re) {
2301  // gap before us
2302  cnt++;
2303  }
2304 
2305  if (cnt == pos && sbb->offset == offset) {
2306  const uint8_t *buf = NULL;
2307  uint32_t buf_len = 0;
2308  StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len);
2309 
2310  if (len == buf_len) {
2311  return (memcmp(data, buf, len) == 0);
2312  }
2313  }
2314 
2315  last_re = sbb->offset + sbb->len;
2316  cnt++;
2317  }
2318  return 0;
2319 }
2320 
2321 /**
2322  * \test Test the handling of packets missed by both IDS and the end host.
2323  * The packet is missed in the starting of the stream.
2324  *
2325  * \retval On success it returns 1 and on failure 0.
2326  */
2327 
2328 static int StreamTcpReassembleTest25 (void)
2329 {
2330  MISSED_START(6);
2331  MISSED_ADD_PAYLOAD(10, "BB", 2);
2332  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2333  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BB", 2) == 1);
2334  MISSED_ADD_PAYLOAD(12, "CC", 2);
2335  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2336  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BBCC", 4) == 1);
2337  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2338  MISSED_END;
2339  PASS;
2340 }
2341 
2342 /**
2343  * \test Test the handling of packets missed by both IDS and the end host.
2344  * The packet is missed in the middle of the stream.
2345  *
2346  * \retval On success it returns 1 and on failure 0.
2347  */
2348 
2349 static int StreamTcpReassembleTest26 (void)
2350 {
2351  MISSED_START(9);
2352  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2353  MISSED_ADD_PAYLOAD(15, "CC", 2);
2354  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2355  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2356  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2357  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2358  MISSED_END;
2359 }
2360 
2361 /**
2362  * \test Test the handling of packets missed by both IDS and the end host.
2363  * The packet is missed in the end of the stream.
2364  *
2365  * \retval On success it returns 1 and on failure 0.
2366  */
2367 
2368 static int StreamTcpReassembleTest27 (void)
2369 {
2370  MISSED_START(9);
2371  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2372  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2373  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2374  MISSED_END;
2375 }
2376 
2377 /**
2378  * \test Test the handling of packets missed by IDS, but the end host has
2379  * received it and send the acknowledgment of it. The packet is missed
2380  * in the starting of the stream.
2381  *
2382  * \retval On success it returns 1 and on failure 0.
2383  */
2384 
2385 static int StreamTcpReassembleTest28 (void)
2386 {
2387  MISSED_START(6);
2388  MISSED_ADD_PAYLOAD(10, "AAA", 3);
2389  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2390  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAA", 3) == 1);
2391  MISSED_ADD_PAYLOAD(13, "BB", 2);
2392  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2393  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABB", 5) == 1);
2394  ssn.state = TCP_TIME_WAIT;
2395  MISSED_ADD_PAYLOAD(15, "CC", 2);
2396  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2397  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABBCC", 7) == 1);
2398  MISSED_END;
2399 }
2400 
2401 /**
2402  * \test Test the handling of packets missed by IDS, but the end host has
2403  * received it and send the acknowledgment of it. The packet is missed
2404  * in the middle of the stream.
2405  *
2406  * \retval On success it returns 1 and on failure 0.
2407  */
2408 
2409 static int StreamTcpReassembleTest29 (void)
2410 {
2411  MISSED_START(9);
2412  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2413  ssn.state = TCP_TIME_WAIT;
2414  MISSED_ADD_PAYLOAD(15, "CC", 2);
2415  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2416  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2417  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2418  MISSED_END;
2419 }
2420 
2421 static int StreamTcpReassembleTest33(void)
2422 {
2423  TcpSession ssn;
2424  Packet *p = PacketGetFromAlloc();
2425  FAIL_IF(unlikely(p == NULL));
2426  Flow f;
2427  TCPHdr tcph;
2428  TcpReassemblyThreadCtx *ra_ctx = NULL;
2430  uint8_t packet[1460] = "";
2431 
2432  StreamTcpUTInit(&ra_ctx);
2434 
2435  memset(&f, 0, sizeof (Flow));
2436  memset(&tcph, 0, sizeof (TCPHdr));
2437  ThreadVars tv;
2438  memset(&tv, 0, sizeof (ThreadVars));
2439  FLOW_INITIALIZE(&f);
2440  f.protoctx = &ssn;
2441  f.proto = IPPROTO_TCP;
2442  p->src.family = AF_INET;
2443  p->dst.family = AF_INET;
2444  p->proto = IPPROTO_TCP;
2445  p->flow = &f;
2446  tcph.th_win = 5480;
2447  tcph.th_flags = TH_PUSH | TH_ACK;
2448  UTHSetTCPHdr(p, &tcph);
2450  p->payload = packet;
2451 
2452  tcph.th_seq = htonl(10);
2453  tcph.th_ack = htonl(31);
2454  p->payload_len = 10;
2455 
2456  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2457 
2458  tcph.th_seq = htonl(20);
2459  tcph.th_ack = htonl(31);
2460  p->payload_len = 10;
2461 
2462  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2463 
2464  tcph.th_seq = htonl(40);
2465  tcph.th_ack = htonl(31);
2466  p->payload_len = 10;
2467 
2468  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2469 
2470  tcph.th_seq = htonl(5);
2471  tcph.th_ack = htonl(31);
2472  p->payload_len = 30;
2473 
2474  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2475 
2477  StreamTcpUTDeinit(ra_ctx);
2478  SCFree(p);
2479  PASS;
2480 }
2481 
2482 static int StreamTcpReassembleTest34(void)
2483 {
2484  TcpSession ssn;
2485  Packet *p = PacketGetFromAlloc();
2486  FAIL_IF(unlikely(p == NULL));
2487  Flow f;
2488  TCPHdr tcph;
2489  TcpReassemblyThreadCtx *ra_ctx = NULL;
2491  uint8_t packet[1460] = "";
2492 
2493  StreamTcpUTInit(&ra_ctx);
2495  memset(&f, 0, sizeof (Flow));
2496  memset(&tcph, 0, sizeof (TCPHdr));
2497  ThreadVars tv;
2498  memset(&tv, 0, sizeof (ThreadVars));
2499  FLOW_INITIALIZE(&f);
2500  f.protoctx = &ssn;
2501  f.proto = IPPROTO_TCP;
2502  p->src.family = AF_INET;
2503  p->dst.family = AF_INET;
2504  p->proto = IPPROTO_TCP;
2505  p->flow = &f;
2506  tcph.th_win = 5480;
2507  tcph.th_flags = TH_PUSH | TH_ACK;
2508  UTHSetTCPHdr(p, &tcph);
2510  p->payload = packet;
2511  SET_ISN(&ssn.client, 857961230);
2512 
2513  tcph.th_seq = htonl(857961230);
2514  tcph.th_ack = htonl(31);
2515  p->payload_len = 304;
2516 
2517  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2518 
2519  tcph.th_seq = htonl(857961534);
2520  tcph.th_ack = htonl(31);
2521  p->payload_len = 1460;
2522 
2523  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2524 
2525  tcph.th_seq = htonl(857963582);
2526  tcph.th_ack = htonl(31);
2527  p->payload_len = 1460;
2528 
2529  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2530 
2531  tcph.th_seq = htonl(857960946);
2532  tcph.th_ack = htonl(31);
2533  p->payload_len = 1460;
2534 
2535  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2536 
2538  StreamTcpUTDeinit(ra_ctx);
2539  SCFree(p);
2540  PASS;
2541 }
2542 
2543 /**
2544  * \test Test to make sure that we don't return the segments until the app
2545  * layer proto has been detected and after that remove the processed
2546  * segments.
2547  *
2548  * \retval On success it returns 1 and on failure 0.
2549  */
2550 
2551 static int StreamTcpReassembleTest39 (void)
2552 {
2553  Packet *p = PacketGetFromAlloc();
2554  FAIL_IF(unlikely(p == NULL));
2555  Flow f;
2556  ThreadVars tv;
2557  StreamTcpThread stt;
2558  TCPHdr tcph;
2559  PacketQueueNoLock pq;
2560  memset(&pq,0,sizeof(PacketQueueNoLock));
2561  memset (&f, 0, sizeof(Flow));
2562  memset(&tv, 0, sizeof (ThreadVars));
2563  memset(&stt, 0, sizeof (stt));
2564  memset(&tcph, 0, sizeof (TCPHdr));
2565 
2566  FLOW_INITIALIZE(&f);
2567  f.flags = FLOW_IPV4;
2568  f.proto = IPPROTO_TCP;
2569  p->flow = &f;
2570  UTHSetTCPHdr(p, &tcph);
2571 
2572  StreamTcpUTInit(&stt.ra_ctx);
2573 
2574  /* handshake */
2575  tcph.th_win = htons(5480);
2576  tcph.th_flags = TH_SYN;
2578  p->payload_len = 0;
2579  p->payload = NULL;
2580  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2581 
2582  TcpSession *ssn = (TcpSession *)f.protoctx;
2583  FAIL_IF_NULL(ssn);
2584 
2591  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2592  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2593  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2594  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2595  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2596  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2597  FAIL_IF(ssn->data_first_seen_dir != 0);
2598 
2599  /* handshake */
2600  tcph.th_ack = htonl(1);
2601  tcph.th_flags = TH_SYN | TH_ACK;
2603  p->payload_len = 0;
2604  p->payload = NULL;
2605  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2612  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2613  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2614  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2615  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2616  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2617  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2618  FAIL_IF(ssn->data_first_seen_dir != 0);
2619 
2620  /* handshake */
2621  tcph.th_ack = htonl(1);
2622  tcph.th_seq = htonl(1);
2623  tcph.th_flags = TH_ACK;
2625  p->payload_len = 0;
2626  p->payload = NULL;
2627  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2634  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2635  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2636  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2637  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2638  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2639  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2640  FAIL_IF(ssn->data_first_seen_dir != 0);
2641 
2642  /* partial request */
2643  uint8_t request1[] = { 0x47, 0x45, };
2644  tcph.th_ack = htonl(1);
2645  tcph.th_seq = htonl(1);
2646  tcph.th_flags = TH_PUSH | TH_ACK;
2648  p->payload_len = sizeof(request1);
2649  p->payload = request1;
2650  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2657  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2658  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2659  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2660  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2661  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2662  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2663  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2664  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2665 
2666  /* response ack against partial request */
2667  tcph.th_ack = htonl(3);
2668  tcph.th_seq = htonl(1);
2669  tcph.th_flags = TH_ACK;
2671  p->payload_len = 0;
2672  p->payload = NULL;
2673  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2680  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2681  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2682  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2683  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2684  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2685  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2686  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2687  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2688 
2689  /* complete partial request */
2690  uint8_t request2[] = {
2691  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2692  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2693  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2694  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2695  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2696  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2697  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2698  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2699  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2700  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2701  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2702  tcph.th_ack = htonl(1);
2703  tcph.th_seq = htonl(3);
2704  tcph.th_flags = TH_PUSH | TH_ACK;
2706  p->payload_len = sizeof(request2);
2707  p->payload = request2;
2708  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2715  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2716  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2717  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2718  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2719  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2720  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2721  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2722  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2723  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2724 
2725  /* response - request ack */
2726  uint8_t response[] = {
2727  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2728  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2729  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2730  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2731  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2732  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2733  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2734  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2735  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2736  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2737  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2738  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2739  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2740  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2741  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2742  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2743  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2744  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2745  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2746  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2747  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2748  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2749  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2750  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2751  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2752  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2753  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2754  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2755  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2756  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2757  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2758  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2759  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2760  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2761  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2762  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2763  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2764  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2765  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2766  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2767  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2768  tcph.th_ack = htonl(88);
2769  tcph.th_seq = htonl(1);
2770  tcph.th_flags = TH_PUSH | TH_ACK;
2772  p->payload_len = sizeof(response);
2773  p->payload = response;
2774 
2775  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2782  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2783  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2784  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2785  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2787  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2788  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2789  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2790  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2791 
2792  /* response ack from request */
2793  tcph.th_ack = htonl(328);
2794  tcph.th_seq = htonl(88);
2795  tcph.th_flags = TH_ACK;
2797  p->payload_len = 0;
2798  p->payload = NULL;
2799  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2806  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2807  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2808  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2809  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2811  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2812  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2813  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2814  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2815  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2816 
2817  /* response - acking */
2818  tcph.th_ack = htonl(88);
2819  tcph.th_seq = htonl(328);
2820  tcph.th_flags = TH_PUSH | TH_ACK;
2822  p->payload_len = 0;
2823  p->payload = NULL;
2824  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2831  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2832  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2833  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2834  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2836  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2837  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2838  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2839  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2840  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2841 
2842  /* response ack from request */
2843  tcph.th_ack = htonl(328);
2844  tcph.th_seq = htonl(88);
2845  tcph.th_flags = TH_ACK;
2847  p->payload_len = 0;
2848  p->payload = NULL;
2849  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2856  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2857  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2858  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2859  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2861  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2862  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2863  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2864  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2865 
2866  /* response - acking the request again*/
2867  tcph.th_ack = htonl(88);
2868  tcph.th_seq = htonl(328);
2869  tcph.th_flags = TH_PUSH | TH_ACK;
2871  p->payload_len = 0;
2872  p->payload = NULL;
2873  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2880  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2881  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2882  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2883  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2885  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2886  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2887  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2888  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2889 
2890  /*** New Request ***/
2891 
2892  /* partial request */
2893  tcph.th_ack = htonl(328);
2894  tcph.th_seq = htonl(88);
2895  tcph.th_flags = TH_PUSH | TH_ACK;
2897  p->payload_len = sizeof(request1);
2898  p->payload = request1;
2899  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2906  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2907  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2908  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2909  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2911  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2912  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2913  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2914  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2915  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2916 
2917  /* response ack against partial request */
2918  tcph.th_ack = htonl(90);
2919  tcph.th_seq = htonl(328);
2920  tcph.th_flags = TH_ACK;
2922  p->payload_len = 0;
2923  p->payload = NULL;
2924  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2925  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2932  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2933  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2934  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2935  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2937  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2938  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2939  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2940  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2941  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2942 
2943  /* complete request */
2944  tcph.th_ack = htonl(328);
2945  tcph.th_seq = htonl(90);
2946  tcph.th_flags = TH_PUSH | TH_ACK;
2948  p->payload_len = sizeof(request2);
2949  p->payload = request2;
2950  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2957  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2958  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2959  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2960  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2962  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2963  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2964  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2965  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2966  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2967  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2968 
2969  /* response ack against second partial request */
2970  tcph.th_ack = htonl(175);
2971  tcph.th_seq = htonl(328);
2972  tcph.th_flags = TH_ACK;
2974  p->payload_len = 0;
2975  p->payload = NULL;
2976 
2977  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2984  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2985  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2986  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2987  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2989  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2990  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2991  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2992  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2993  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2994  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2995 
2996  /* response acking a request */
2997  tcph.th_ack = htonl(175);
2998  tcph.th_seq = htonl(328);
2999  tcph.th_flags = TH_ACK;
3001  p->payload_len = 0;
3002  p->payload = NULL;
3003  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3009 
3010  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3011  StreamTcpPruneSession(&f, STREAM_TOCLIENT);
3012 
3013  /* request acking a response */
3014  tcph.th_ack = htonl(328);
3015  tcph.th_seq = htonl(175);
3016  tcph.th_flags = TH_ACK;
3018  p->payload_len = 0;
3019  p->payload = NULL;
3020  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3021 
3022  StreamTcpSessionClear(ssn);
3024  SCFree(p);
3025  PASS;
3026 }
3027 
3028 /**
3029  * \test Test to make sure that we sent all the segments from the initial
3030  * segments to app layer until we have detected the app layer proto.
3031  *
3032  * \retval On success it returns 1 and on failure 0.
3033  */
3034 
3035 static int StreamTcpReassembleTest40 (void)
3036 {
3037  Packet *p = PacketGetFromAlloc();
3038  FAIL_IF_NULL(p);
3039  Flow *f = NULL;
3040  TCPHdr tcph;
3041  TcpSession ssn;
3042  memset(&tcph, 0, sizeof (TCPHdr));
3043  ThreadVars tv;
3044  memset(&tv, 0, sizeof (ThreadVars));
3045 
3046  StreamTcpInitConfig(true);
3048 
3050  FAIL_IF_NULL(ra_ctx);
3051 
3052  uint8_t httpbuf1[] = "P";
3053  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3054  uint8_t httpbuf3[] = "O";
3055  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3056  uint8_t httpbuf4[] = "S";
3057  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3058  uint8_t httpbuf5[] = "T \r\n";
3059  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3060 
3061  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3062  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3063 
3064  SET_ISN(&ssn.server, 9);
3065  ssn.server.last_ack = 10;
3066  SET_ISN(&ssn.client, 9);
3067  ssn.client.isn = 9;
3068 
3069  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3070  FAIL_IF_NULL(f);
3071  f->protoctx = &ssn;
3072  f->proto = IPPROTO_TCP;
3073  p->flow = f;
3074 
3075  tcph.th_win = htons(5480);
3076  tcph.th_seq = htonl(10);
3077  tcph.th_ack = htonl(10);
3078  tcph.th_flags = TH_ACK|TH_PUSH;
3079  UTHSetTCPHdr(p, &tcph);
3081  p->payload = httpbuf1;
3082  p->payload_len = httplen1;
3083  ssn.state = TCP_ESTABLISHED;
3084  TcpStream *s = &ssn.client;
3085  SCLogDebug("1 -- start");
3086  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3087 
3089  p->payload = httpbuf2;
3090  p->payload_len = httplen2;
3091  tcph.th_seq = htonl(10);
3092  tcph.th_ack = htonl(11);
3093  s = &ssn.server;
3094  ssn.server.last_ack = 11;
3095  SCLogDebug("2 -- start");
3096  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3097 
3099  p->payload = httpbuf3;
3100  p->payload_len = httplen3;
3101  tcph.th_seq = htonl(11);
3102  tcph.th_ack = htonl(55);
3103  s = &ssn.client;
3104  ssn.client.last_ack = 55;
3105  SCLogDebug("3 -- start");
3106  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3107 
3109  p->payload = httpbuf2;
3110  p->payload_len = httplen2;
3111  tcph.th_seq = htonl(55);
3112  tcph.th_ack = htonl(12);
3113  s = &ssn.server;
3114  ssn.server.last_ack = 12;
3115  SCLogDebug("4 -- start");
3116  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3117 
3118  /* check is have the segment in the list and flagged or not */
3119  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3120  FAIL_IF_NULL(seg);
3121  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
3122 
3124  p->payload = httpbuf4;
3125  p->payload_len = httplen4;
3126  tcph.th_seq = htonl(12);
3127  tcph.th_ack = htonl(100);
3128  s = &ssn.client;
3129  ssn.client.last_ack = 100;
3130  SCLogDebug("5 -- start");
3131  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3132 
3134  p->payload = httpbuf2;
3135  p->payload_len = httplen2;
3136  tcph.th_seq = htonl(100);
3137  tcph.th_ack = htonl(13);
3138  s = &ssn.server;
3139  ssn.server.last_ack = 13;
3140  SCLogDebug("6 -- start");
3141  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3142 
3144  p->payload = httpbuf5;
3145  p->payload_len = httplen5;
3146  tcph.th_seq = htonl(13);
3147  tcph.th_ack = htonl(145);
3148  s = &ssn.client;
3149  ssn.client.last_ack = 145;
3150  SCLogDebug("7 -- start");
3151  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3152 
3154  p->payload = httpbuf2;
3155  p->payload_len = httplen2;
3156  tcph.th_seq = htonl(145);
3157  tcph.th_ack = htonl(16);
3158  s = &ssn.server;
3159  ssn.server.last_ack = 16;
3160  SCLogDebug("8 -- start");
3161  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3162  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3163 
3166  StreamTcpFreeConfig(true);
3167  SCFree(p);
3168  UTHFreeFlow(f);
3169  PASS;
3170 }
3171 
3172 /** \test Test the memcap incrementing/decrementing and memcap check */
3173 static int StreamTcpReassembleTest44(void)
3174 {
3175  StreamTcpInitConfig(true);
3176  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
3177  StreamTcpReassembleIncrMemuse(500);
3178  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
3179  StreamTcpReassembleDecrMemuse(500);
3180  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
3182  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
3183  StreamTcpFreeConfig(true);
3184  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
3185  PASS;
3186 }
3187 
3188 /**
3189  * \test Test to make sure that reassembly_depth is enforced.
3190  *
3191  * \retval On success it returns 1 and on failure 0.
3192  */
3193 
3194 static int StreamTcpReassembleTest45 (void)
3195 {
3196  TcpReassemblyThreadCtx *ra_ctx = NULL;
3197  TcpSession ssn;
3198  ThreadVars tv;
3199  memset(&tv, 0, sizeof(tv));
3200  uint8_t payload[100] = {0};
3201  uint16_t payload_size = 100;
3202 
3203  StreamTcpUTInit(&ra_ctx);
3205 
3207  ssn.reassembly_depth = 100;
3208  StreamTcpUTSetupStream(&ssn.server, 100);
3209  StreamTcpUTSetupStream(&ssn.client, 100);
3210 
3211  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3212  FAIL_IF(r != 0);
3214 
3215  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3216  FAIL_IF(r != 0);
3218 
3222  StreamTcpUTDeinit(ra_ctx);
3223  PASS;
3224 }
3225 
3226 /**
3227  * \test Test the unlimited config value of reassembly depth.
3228  *
3229  * \retval On success it returns 1 and on failure 0.
3230  */
3231 
3232 static int StreamTcpReassembleTest46 (void)
3233 {
3234  int result = 0;
3235  TcpReassemblyThreadCtx *ra_ctx = NULL;
3236  TcpSession ssn;
3237  ThreadVars tv;
3238  memset(&tv, 0, sizeof(tv));
3239  uint8_t payload[100] = {0};
3240  uint16_t payload_size = 100;
3241 
3242  StreamTcpUTInit(&ra_ctx);
3244 
3246  StreamTcpUTSetupStream(&ssn.server, 100);
3247  StreamTcpUTSetupStream(&ssn.client, 100);
3248 
3249  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3250  if (r != 0)
3251  goto end;
3253  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3254  goto end;
3255  }
3256 
3257  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3258  if (r != 0)
3259  goto end;
3261  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3262  goto end;
3263  }
3264 
3265  result = 1;
3266 end:
3270  StreamTcpUTDeinit(ra_ctx);
3271  return result;
3272 }
3273 
3274 /**
3275  * \test Test to make sure we detect the sequence wrap around and continue
3276  * stream reassembly properly.
3277  *
3278  * \retval On success it returns 1 and on failure 0.
3279  */
3280 
3281 static int StreamTcpReassembleTest47 (void)
3282 {
3283  Packet *p = PacketGetFromAlloc();
3284  FAIL_IF(unlikely(p == NULL));
3285  Flow *f = NULL;
3286  TCPHdr tcph;
3287  TcpSession ssn;
3288  ThreadVars tv;
3289  memset(&tcph, 0, sizeof (TCPHdr));
3290  UTHSetTCPHdr(p, &tcph);
3291  memset(&tv, 0, sizeof (ThreadVars));
3292  StreamTcpInitConfig(true);
3295 
3296  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3297  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3298 
3299  SET_ISN(&ssn.server, 572799781UL);
3300  ssn.server.last_ack = 572799782UL;
3301 
3302  SET_ISN(&ssn.client, 4294967289UL);
3303  ssn.client.last_ack = 21;
3304 
3305  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3306  FAIL_IF(f == NULL);
3307  f->protoctx = &ssn;
3308  f->proto = IPPROTO_TCP;
3309  p->flow = f;
3310 
3311  tcph.th_win = htons(5480);
3312  ssn.state = TCP_ESTABLISHED;
3313  TcpStream *s = NULL;
3314  uint8_t cnt = 0;
3315 
3316  for (cnt=0; cnt < httplen1; cnt++) {
3317  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3318  tcph.th_ack = htonl(572799782UL);
3319  tcph.th_flags = TH_ACK | TH_PUSH;
3321  p->payload = &httpbuf1[cnt];
3322  p->payload_len = 1;
3323  s = &ssn.client;
3324 
3325  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3326 
3328  p->payload = NULL;
3329  p->payload_len = 0;
3330  tcph.th_seq = htonl(572799782UL);
3331  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3332  tcph.th_flags = TH_ACK;
3333  s = &ssn.server;
3334 
3335  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3336  }
3337 
3338  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3339 
3342  StreamTcpFreeConfig(true);
3343  SCFree(p);
3344  UTHFreeFlow(f);
3345  PASS;
3346 }
3347 
3348 /** \test 3 in order segments in inline reassembly */
3349 static int StreamTcpReassembleInlineTest01(void)
3350 {
3351  int ret = 0;
3352  TcpReassemblyThreadCtx *ra_ctx = NULL;
3353  ThreadVars tv;
3354  TcpSession ssn;
3355  Flow f;
3356 
3357  memset(&tv, 0x00, sizeof(tv));
3358 
3359  StreamTcpUTInit(&ra_ctx);
3362  StreamTcpUTSetupStream(&ssn.client, 1);
3363  FLOW_INITIALIZE(&f);
3364 
3365  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3366  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3367  if (p == NULL) {
3368  printf("couldn't get a packet: ");
3369  goto end;
3370  }
3371  p->l4.hdrs.tcph->th_seq = htonl(12);
3372  p->flow = &f;
3373 
3374  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3375  printf("failed to add segment 1: ");
3376  goto end;
3377  }
3378  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3379  printf("failed to add segment 2: ");
3380  goto end;
3381  }
3382  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3383  printf("failed to add segment 3: ");
3384  goto end;
3385  }
3386  ssn.client.next_seq = 17;
3387  ret = 1;
3388 end:
3389  FLOW_DESTROY(&f);
3390  UTHFreePacket(p);
3392  StreamTcpUTDeinit(ra_ctx);
3393  return ret;
3394 }
3395 
3396 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3397  * test the sliding window reassembly.
3398  */
3399 static int StreamTcpReassembleInlineTest02(void)
3400 {
3401  int ret = 0;
3402  TcpReassemblyThreadCtx *ra_ctx = NULL;
3403  ThreadVars tv;
3404  TcpSession ssn;
3405  Flow f;
3406 
3407  memset(&tv, 0x00, sizeof(tv));
3408 
3409  StreamTcpUTInit(&ra_ctx);
3412  StreamTcpUTSetupStream(&ssn.client, 1);
3413  FLOW_INITIALIZE(&f);
3414 
3415  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3416  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3417  if (p == NULL) {
3418  printf("couldn't get a packet: ");
3419  goto end;
3420  }
3421  p->l4.hdrs.tcph->th_seq = htonl(12);
3422  p->flow = &f;
3423 
3424  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3425  printf("failed to add segment 1: ");
3426  goto end;
3427  }
3428  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3429  printf("failed to add segment 2: ");
3430  goto end;
3431  }
3432  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3433  printf("failed to add segment 3: ");
3434  goto end;
3435  }
3436  ssn.client.next_seq = 17;
3437  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3438  printf("failed to add segment 4: ");
3439  goto end;
3440  }
3441  ssn.client.next_seq = 22;
3442  ret = 1;
3443 end:
3444  FLOW_DESTROY(&f);
3445  UTHFreePacket(p);
3447  StreamTcpUTDeinit(ra_ctx);
3448  return ret;
3449 }
3450 
3451 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3452  * test the sliding window reassembly with a small window size so that we
3453  * cutting off at the start (left edge)
3454  */
3455 static int StreamTcpReassembleInlineTest03(void)
3456 {
3457  int ret = 0;
3458  TcpReassemblyThreadCtx *ra_ctx = NULL;
3459  ThreadVars tv;
3460  TcpSession ssn;
3461  Flow f;
3462 
3463  memset(&tv, 0x00, sizeof(tv));
3464 
3465  StreamTcpUTInit(&ra_ctx);
3468  StreamTcpUTSetupStream(&ssn.client, 1);
3469  FLOW_INITIALIZE(&f);
3470 
3472 
3473  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3474  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3475  if (p == NULL) {
3476  printf("couldn't get a packet: ");
3477  goto end;
3478  }
3479  p->l4.hdrs.tcph->th_seq = htonl(12);
3480  p->flow = &f;
3482 
3483  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3484  printf("failed to add segment 1: ");
3485  goto end;
3486  }
3487  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3488  printf("failed to add segment 2: ");
3489  goto end;
3490  }
3491  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3492  printf("failed to add segment 3: ");
3493  goto end;
3494  }
3495  ssn.client.next_seq = 17;
3496  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3497  printf("failed to add segment 4: ");
3498  goto end;
3499  }
3500  ssn.client.next_seq = 22;
3501 
3502  p->l4.hdrs.tcph->th_seq = htonl(17);
3503  ret = 1;
3504 end:
3505  FLOW_DESTROY(&f);
3506  UTHFreePacket(p);
3508  StreamTcpUTDeinit(ra_ctx);
3509  return ret;
3510 }
3511 
3512 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3513  * test the sliding window reassembly with a small window size so that we
3514  * cutting off at the start (left edge) with small packet overlap.
3515  */
3516 static int StreamTcpReassembleInlineTest04(void)
3517 {
3518  int ret = 0;
3519  TcpReassemblyThreadCtx *ra_ctx = NULL;
3520  ThreadVars tv;
3521  TcpSession ssn;
3522  Flow f;
3523 
3524  memset(&tv, 0x00, sizeof(tv));
3525 
3526  StreamTcpUTInit(&ra_ctx);
3529  StreamTcpUTSetupStream(&ssn.client, 1);
3530  FLOW_INITIALIZE(&f);
3531 
3533 
3534  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3535  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3536  if (p == NULL) {
3537  printf("couldn't get a packet: ");
3538  goto end;
3539  }
3540  p->l4.hdrs.tcph->th_seq = htonl(12);
3541  p->flow = &f;
3543 
3544  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3545  printf("failed to add segment 1: ");
3546  goto end;
3547  }
3548  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3549  printf("failed to add segment 2: ");
3550  goto end;
3551  }
3552  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3553  printf("failed to add segment 3: ");
3554  goto end;
3555  }
3556  ssn.client.next_seq = 17;
3557  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3558  printf("failed to add segment 4: ");
3559  goto end;
3560  }
3561  ssn.client.next_seq = 22;
3562 
3563  p->l4.hdrs.tcph->th_seq = htonl(17);
3564  ret = 1;
3565 end:
3566  FLOW_DESTROY(&f);
3567  UTHFreePacket(p);
3569  StreamTcpUTDeinit(ra_ctx);
3570  return ret;
3571 }
3572 
3573 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3574  * test the sliding window reassembly with a small window size so that we
3575  * cutting off at the start (left edge). Test if the first segment is
3576  * removed from the list.
3577  */
3578 static int StreamTcpReassembleInlineTest08(void)
3579 {
3580  TcpReassemblyThreadCtx *ra_ctx = NULL;
3581  ThreadVars tv;
3582  memset(&tv, 0x00, sizeof(tv));
3583  TcpSession ssn;
3584  Flow f;
3585  StreamTcpUTInit(&ra_ctx);
3588  StreamTcpUTSetupStream(&ssn.client, 1);
3589  FLOW_INITIALIZE(&f);
3590 
3592  f.protoctx = &ssn;
3593 
3594  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3595  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3596  FAIL_IF(p == NULL);
3597  p->l4.hdrs.tcph->th_seq = htonl(12);
3598  p->flow = &f;
3600 
3601  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3602  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3603  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3604  ssn.client.next_seq = 17;
3605  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3606  ssn.client.next_seq = 22;
3607  p->l4.hdrs.tcph->th_seq = htonl(17);
3608  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3609 
3610  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3611  FAIL_IF_NULL(seg);
3612  FAIL_IF_NOT(seg->seq == 2);
3613 
3614  FLOW_DESTROY(&f);
3615  UTHFreePacket(p);
3617  StreamTcpUTDeinit(ra_ctx);
3618  PASS;
3619 }
3620 
3621 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3622  * test the sliding window reassembly with a small window size so that we
3623  * cutting off at the start (left edge). Test if the first segment is
3624  * removed from the list.
3625  */
3626 static int StreamTcpReassembleInlineTest09(void)
3627 {
3628  int ret = 0;
3629  TcpReassemblyThreadCtx *ra_ctx = NULL;
3630  ThreadVars tv;
3631  TcpSession ssn;
3632  Flow f;
3633 
3634  memset(&tv, 0x00, sizeof(tv));
3635 
3636  StreamTcpUTInit(&ra_ctx);
3639  StreamTcpUTSetupStream(&ssn.client, 1);
3640  FLOW_INITIALIZE(&f);
3641 
3643 
3644  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3645  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3646  if (p == NULL) {
3647  printf("couldn't get a packet: ");
3648  goto end;
3649  }
3650  p->l4.hdrs.tcph->th_seq = htonl(17);
3651  p->flow = &f;
3653 
3654  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3655  printf("failed to add segment 1: ");
3656  goto end;
3657  }
3658  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3659  printf("failed to add segment 2: ");
3660  goto end;
3661  }
3662  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3663  printf("failed to add segment 3: ");
3664  goto end;
3665  }
3666  ssn.client.next_seq = 12;
3667  ssn.client.last_ack = 10;
3668 
3669  /* close the GAP and see if we properly reassemble and update base_seq */
3670  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3671  printf("failed to add segment 4: ");
3672  goto end;
3673  }
3674  ssn.client.next_seq = 22;
3675 
3676  p->l4.hdrs.tcph->th_seq = htonl(12);
3677 
3678  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3679  FAIL_IF_NULL(seg);
3680  FAIL_IF_NOT(seg->seq == 2);
3681 
3682  ret = 1;
3683 end:
3684  FLOW_DESTROY(&f);
3685  UTHFreePacket(p);
3687  StreamTcpUTDeinit(ra_ctx);
3688  return ret;
3689 }
3690 
3691 /** \test App Layer reassembly.
3692  */
3693 static int StreamTcpReassembleInlineTest10(void)
3694 {
3695  int ret = 0;
3696  TcpReassemblyThreadCtx *ra_ctx = NULL;
3697  ThreadVars tv;
3698  TcpSession ssn;
3699  Flow *f = NULL;
3700  Packet *p = NULL;
3701 
3702  memset(&tv, 0x00, sizeof(tv));
3703 
3704  StreamTcpUTInit(&ra_ctx);
3707  StreamTcpUTSetupStream(&ssn.server, 1);
3708  ssn.server.last_ack = 2;
3709  StreamTcpUTSetupStream(&ssn.client, 1);
3710  ssn.client.last_ack = 2;
3711  ssn.data_first_seen_dir = STREAM_TOSERVER;
3712 
3713  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3714  if (f == NULL)
3715  goto end;
3716  f->protoctx = &ssn;
3717  f->proto = IPPROTO_TCP;
3718 
3719  uint8_t stream_payload1[] = "GE";
3720  uint8_t stream_payload2[] = "T /";
3721  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3722 
3723  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3724  if (p == NULL) {
3725  printf("couldn't get a packet: ");
3726  goto end;
3727  }
3728  p->l4.hdrs.tcph->th_seq = htonl(7);
3729  p->flow = f;
3731 
3732  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3733  printf("failed to add segment 1: ");
3734  goto end;
3735  }
3736  ssn.client.next_seq = 4;
3737 
3738  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3739  if (r < 0) {
3740  printf("StreamTcpReassembleAppLayer failed: ");
3741  goto end;
3742  }
3743 
3744  /* ssn.server.ra_app_base_seq should be isn here. */
3745  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3746  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3747  goto end;
3748  }
3749 
3750  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3751  printf("failed to add segment 2: ");
3752  goto end;
3753  }
3754  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3755  printf("failed to add segment 3: ");
3756  goto end;
3757  }
3758  ssn.client.next_seq = 19;
3759 
3760  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3761  if (r < 0) {
3762  printf("StreamTcpReassembleAppLayer failed: ");
3763  goto end;
3764  }
3765 
3766  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3767 
3768  ret = 1;
3769 end:
3770  UTHFreePacket(p);
3772  StreamTcpUTDeinit(ra_ctx);
3773  UTHFreeFlow(f);
3774  return ret;
3775 }
3776 
3777 /** \test test insert with overlap
3778  */
3779 static int StreamTcpReassembleInsertTest01(void)
3780 {
3781  TcpReassemblyThreadCtx *ra_ctx = NULL;
3782  ThreadVars tv;
3783  TcpSession ssn;
3784  Flow f;
3785 
3786  memset(&tv, 0x00, sizeof(tv));
3787 
3788  StreamTcpUTInit(&ra_ctx);
3790  StreamTcpUTSetupStream(&ssn.client, 1);
3792  FLOW_INITIALIZE(&f);
3793 
3794  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3795  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3796  FAIL_IF(p == NULL);
3797  p->l4.hdrs.tcph->th_seq = htonl(12);
3798  p->flow = &f;
3799 
3800  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3801  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3802  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3803  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3804  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3805  ssn.client.next_seq = 21;
3806 
3807  FLOW_DESTROY(&f);
3808  UTHFreePacket(p);
3810  StreamTcpUTDeinit(ra_ctx);
3811  PASS;
3812 }
3813 
3814 /** \test test insert with overlaps
3815  */
3816 static int StreamTcpReassembleInsertTest02(void)
3817 {
3818  int ret = 0;
3819  TcpReassemblyThreadCtx *ra_ctx = NULL;
3820  ThreadVars tv;
3821  TcpSession ssn;
3822 
3823  memset(&tv, 0x00, sizeof(tv));
3824 
3825  StreamTcpUTInit(&ra_ctx);
3827  StreamTcpUTSetupStream(&ssn.client, 1);
3828 
3829  int i;
3830  for (i = 2; i < 10; i++) {
3831  int len;
3832  len = i % 2;
3833  if (len == 0)
3834  len = 1;
3835  int seq;
3836  seq = i * 10;
3837  if (seq < 2)
3838  seq = 2;
3839 
3840  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3841  printf("failed to add segment 1: ");
3842  goto end;
3843  }
3844  }
3845  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3846  printf("failed to add segment 2: ");
3847  goto end;
3848  }
3849 
3850  ret = 1;
3851 end:
3853  StreamTcpUTDeinit(ra_ctx);
3854  return ret;
3855 }
3856 
3857 /** \test test insert with overlaps
3858  */
3859 static int StreamTcpReassembleInsertTest03(void)
3860 {
3861  int ret = 0;
3862  TcpReassemblyThreadCtx *ra_ctx = NULL;
3863  ThreadVars tv;
3864  TcpSession ssn;
3865 
3866  memset(&tv, 0x00, sizeof(tv));
3867 
3868  StreamTcpUTInit(&ra_ctx);
3870  StreamTcpUTSetupStream(&ssn.client, 1);
3871 
3872  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3873  printf("failed to add segment 2: ");
3874  goto end;
3875  }
3876 
3877  int i;
3878  for (i = 2; i < 10; i++) {
3879  int len;
3880  len = i % 2;
3881  if (len == 0)
3882  len = 1;
3883  int seq;
3884  seq = i * 10;
3885  if (seq < 2)
3886  seq = 2;
3887 
3888  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3889  printf("failed to add segment 2: ");
3890  goto end;
3891  }
3892  }
3893  ret = 1;
3894 end:
3896  StreamTcpUTDeinit(ra_ctx);
3897  return ret;
3898 }
3899 
3901 #endif /* UNITTESTS */
3902 
3903 /** \brief The Function Register the Unit tests to test the reassembly engine
3904  * for various OS policies.
3905  */
3906 
3908 {
3909 #ifdef UNITTESTS
3910  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3911  StreamTcpReassembleTest25);
3912  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3913  StreamTcpReassembleTest26);
3914  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3915  StreamTcpReassembleTest27);
3916  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3917  StreamTcpReassembleTest28);
3918  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3919  StreamTcpReassembleTest29);
3920  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3921  StreamTcpReassembleTest33);
3922  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3923  StreamTcpReassembleTest34);
3924  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3925  StreamTcpReassembleTest39);
3926  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3927  StreamTcpReassembleTest40);
3928  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3929  StreamTcpReassembleTest44);
3930  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3931  StreamTcpReassembleTest45);
3932  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3933  StreamTcpReassembleTest46);
3934  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3935  StreamTcpReassembleTest47);
3936 
3937  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3938  StreamTcpReassembleInlineTest01);
3939  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3940  StreamTcpReassembleInlineTest02);
3941  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3942  StreamTcpReassembleInlineTest03);
3943  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3944  StreamTcpReassembleInlineTest04);
3945  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3946  StreamTcpReassembleInlineTest08);
3947  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3948  StreamTcpReassembleInlineTest09);
3949 
3950  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3951  StreamTcpReassembleInlineTest10);
3952 
3953  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3954  StreamTcpReassembleInsertTest01);
3955  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3956  StreamTcpReassembleInsertTest02);
3957  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3958  StreamTcpReassembleInsertTest03);
3959 
3963  StreamTcpReassembleRawRegisterTests();
3964 #endif /* UNITTESTS */
3965 }
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
Definition: stream-tcp.h:143
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:270
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:40
TCP_GET_RAW_SEQ
#define TCP_GET_RAW_SEQ(tcph)
Definition: decode-tcp.h:80
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:58
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:56
UTHCheckGapAtPosition
int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
Definition: stream-tcp-reassemble.c:2269
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:506
TcpStream_
Definition: stream-tcp-private.h:106
len
uint8_t len
Definition: app-layer-dnp3.h:2
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:136
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:186
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:113
TcpReassemblyThreadCtx_::counter_tcp_reas_eps
ExceptionPolicyCounters counter_tcp_reas_eps
Definition: stream-tcp-reassemble.h:69
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:2125
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:166
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:381
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:97
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1909
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:97
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:91
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:94
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
TCP_STREAM_URGENT_OOB
@ TCP_STREAM_URGENT_OOB
Definition: stream-tcp.h:49
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:69
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
stream-tcp.h
StreamTcpInlineMode
bool StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:7099
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:297
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:136
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:2198
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:609
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1010
StreamTcpThread_
Definition: stream-tcp.h:92
Flow_::proto
uint8_t proto
Definition: flow.h:378
Packet_::payload
uint8_t * payload
Definition: decode.h:588
t_pcapcnt
thread_local uint64_t t_pcapcnt
Definition: stream-tcp-reassemble.c:80
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:2178
action-globals.h
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:62
Packet_::flags
uint32_t flags
Definition: decode.h:527
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:193
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:110
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:75
Flow_
Flow data structure.
Definition: flow.h:356
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:77
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:75
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:2161
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
TCPHdr_::th_win
uint16_t th_win
Definition: decode-tcp.h:156
TcpSegmentPcapHdrStorage
struct TcpSegmentPcapHdrStorage_ TcpSegmentPcapHdrStorage
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
StringParseUint16
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:337
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:156
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1398
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:40
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:89
PKT_DROP_REASON_STREAM_REASSEMBLY
@ PKT_DROP_REASON_STREAM_REASSEMBLY
Definition: decode.h:379
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1813
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir app_update_dir)
handle TCP data for the app-layer.
Definition: app-layer.c:710
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:233
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:155
MIN
#define MIN(x, y)
Definition: suricata-common.h:400
SCConfGetBool
int SCConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:497
TcpSession_::urg_offset_ts
uint16_t urg_offset_ts
Definition: stream-tcp-private.h:291
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:75
UTHSetTCPHdr
void UTHSetTCPHdr(Packet *p, TCPHdr *tcph)
Definition: util-unittest-helper.c:136
StreamTcpThreadCacheGetSegment
TcpSegment * StreamTcpThreadCacheGetSegment(void)
Definition: stream-tcp-cache.c:179
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:52
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:155
TCP_STREAM_URGENT_GAP
@ TCP_STREAM_URGENT_GAP
Definition: stream-tcp.h:50
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1830
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:515
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:219
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:39
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:260
Flow_::protoctx
void * protoctx
Definition: flow.h:441
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:710
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:100
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:589
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:223
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:615
util-unittest.h
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
AppLayerProfilingStore
#define AppLayerProfilingStore(app_tctx, p)
Definition: app-layer.h:128
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2260
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:523
PKT_DROP_REASON_STREAM_URG
@ PKT_DROP_REASON_STREAM_URG
Definition: decode.h:380
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:426
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:170
STREAMTCP_FLAG_LOSSY_BE_LIBERAL
#define STREAMTCP_FLAG_LOSSY_BE_LIBERAL
Definition: stream-tcp-private.h:194
TCPHdr_::th_ack
uint32_t th_ack
Definition: decode-tcp.h:153
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:294
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:289
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:278
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:115
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:478
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:557
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:488
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:482
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:38
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:295
StreamTcpReassembleDepthReached
bool StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:625
TCPHdr_::th_flags
uint8_t th_flags
Definition: decode-tcp.h:155
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:463
StreamTcpReassembleRealloc
void * StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
Definition: stream-tcp-reassemble.c:229
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:238
UTHCheckDataAtPosition
int UTHCheckDataAtPosition(TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
Definition: stream-tcp-reassemble.c:2292
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:104
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:279
STREAMTCP_FLAG_ASYNC
#define STREAMTCP_FLAG_ASYNC
Definition: stream-tcp-private.h:182
TCPHdr_::th_seq
uint32_t th_seq
Definition: decode-tcp.h:152
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1237
TcpStreamCnf_::urgent_policy
enum TcpStreamUrgentHandling urgent_policy
Definition: stream-tcp.h:83
STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
@ STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
Definition: decode-events.h:303
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1244
util-exception-policy.h
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
TcpSession_::urg_offset_tc
uint16_t urg_offset_tc
Definition: stream-tcp-private.h:292
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:96
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:131
ExceptionPolicyCounters_::eps_id
uint16_t eps_id[EXCEPTION_POLICY_MAX]
Definition: util-exception-policy-types.h:53
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:5542
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:190
StreamNeedsReassembly
uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:977
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1512
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:285
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:137
TCPHdr_::th_urp
uint16_t th_urp
Definition: decode-tcp.h:158
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:105
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2212
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:886
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:112
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:164
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:309
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:341
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:351
SCReturn
#define SCReturn
Definition: util-debug.h:273
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:303
Packet_
Definition: decode.h:484
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:234
stream-tcp-private.h
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:40
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:277
TcpReassemblyThreadCtx_::counter_tcp_urgent_oob
uint16_t counter_tcp_urgent_oob
Definition: stream-tcp-reassemble.h:88
conf.h
Packet_::l4
struct PacketL4 l4
Definition: decode.h:584
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:296
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:301
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1853
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:117
STREAMTCP_STREAM_FLAG_HAS_GAP
#define STREAMTCP_STREAM_FLAG_HAS_GAP
Definition: stream-tcp-private.h:217
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:596
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:287
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:722
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(void)
Creates a new app layer thread context.
Definition: app-layer.c:1106
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:398
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:234
TcpStreamCnf_::reassembly_memcap_policy
enum ExceptionPolicy reassembly_memcap_policy
Definition: stream-tcp.h:81
StreamTcpThreadCacheCleanup
void StreamTcpThreadCacheCleanup(void)
Definition: stream-tcp-cache.c:134
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:257
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:539
TH_URG
#define TH_URG
Definition: decode-tcp.h:39
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:77
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:128
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:104
VALIDATE
#define VALIDATE(e)
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:487
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:57
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a TCP packet data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:746
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:64
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
TcpSegmentPcapHdrStorage_
Definition: stream-tcp-private.h:65
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1830
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1720
STREAM_REASSEMBLY_DEPTH_REACHED
@ STREAM_REASSEMBLY_DEPTH_REACHED
Definition: decode-events.h:299
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
app-layer-frames.h
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:180
Packet_::flow
struct Flow_ * flow
Definition: decode.h:529
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:260
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:859
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
SCNtohs
#define SCNtohs(x)
Definition: suricata-common.h:423
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:447
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:259
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:634
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:124
packet.h
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:36
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:94
TcpStreamCnf_::urgent_oob_limit_policy
enum TcpStreamUrgentHandling urgent_oob_limit_policy
Definition: stream-tcp.h:84
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2241
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:135
TcpSegmentPcapHdrStorage_::alloclen
uint32_t alloclen
Definition: stream-tcp-private.h:68
ACTION_DROP
#define ACTION_DROP
Definition: action-globals.h:30
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:518
StreamTcpThreadCacheReturnSegment
void StreamTcpThreadCacheReturnSegment(TcpSegment *seg)
Definition: stream-tcp-cache.c:53
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:236
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:297
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Definition: stream-tcp-reassemble.c:2005
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:161
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TCP_STREAM_URGENT_DROP
@ TCP_STREAM_URGENT_DROP
Definition: stream-tcp.h:48
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:78
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:114
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:232
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1558
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
PacketL4::L4Hdrs::tcph
TCPHdr * tcph
Definition: decode.h:452
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:296
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:181
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:451
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:467
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayerGetCtxThread().
Definition: app-layer.c:1127
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:257
Flow_::flags
uint32_t flags
Definition: flow.h:421
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:1022
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:71
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:53
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:150
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:307
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2255
stream-tcp-cache.h
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:158
PacketL4::hdrs
union PacketL4::L4Hdrs hdrs
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:117
MISSED_ADD_PAYLOAD
#define MISSED_ADD_PAYLOAD(seq, seg, seglen)
Definition: stream-tcp-reassemble.c:2264
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:61
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:69
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:583
StreamDataRightEdge
uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:417
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:173
PacketDrop
void PacketDrop(Packet *p, const uint8_t action, enum PacketDropReason r)
issue drop action
Definition: packet.c:33
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:219
suricata.h
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap(void)
Return memcap value.
Definition: stream-tcp-reassemble.c:200
Address_::family
char family
Definition: decode.h:113
Packet_::dst
Address dst
Definition: decode.h:489
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:7112
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:258
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool
uint16_t counter_tcp_segment_from_pool
Definition: stream-tcp-reassemble.h:72
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
TCP_STREAM_URGENT_INLINE
@ TCP_STREAM_URGENT_INLINE
Definition: stream-tcp.h:46
likely
#define likely(expr)
Definition: util-optimize.h:32
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3907
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:102
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
segment_thread_pool
PoolThread * segment_thread_pool
Definition: stream-tcp-reassemble.c:82
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:195
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TCPSEG_PKT_HDR_DEFAULT_SIZE
#define TCPSEG_PKT_HDR_DEFAULT_SIZE
Definition: stream-tcp-private.h:58
TcpSession_
Definition: stream-tcp-private.h:283
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:288
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:185
flow.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:237
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:452
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
StreamReassembleLog
int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, uint64_t *progress_out, const bool eof)
Definition: stream-tcp-reassemble.c:1974
ExceptionPolicy
ExceptionPolicy
Definition: util-exception-policy-types.h:25
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:112
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:225
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:148
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
SCConfNode_
Definition: conf.h:32
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
SCConfNode_::val
char * val
Definition: conf.h:34
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamReassembleForFrame
int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof)
Definition: stream-tcp-reassemble.c:1897
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:67
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1684
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:77
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:119
TCPHdr_
Definition: decode-tcp.h:149
Packet_::src
Address src
Definition: decode.h:488
TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache
uint16_t counter_tcp_segment_from_cache
Definition: stream-tcp-reassemble.h:71
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:70
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:232
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:146
app-layer.h