suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2024 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "packet.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 #include "action-globals.h"
37 
38 #include "flow-util.h"
39 
40 #include "threadvars.h"
41 #include "tm-threads.h"
42 
43 #include "util-pool.h"
44 #include "util-unittest.h"
45 #include "util-print.h"
46 #include "util-host-os-info.h"
47 #include "util-unittest-helper.h"
48 #include "util-byte.h"
49 #include "util-device-private.h"
50 
51 #include "stream-tcp.h"
52 #include "stream-tcp-private.h"
53 #include "stream-tcp-cache.h"
54 #include "stream-tcp-reassemble.h"
55 #include "stream-tcp-inline.h"
56 #include "stream-tcp-list.h"
57 #include "stream-tcp-util.h"
58 
59 #include "stream.h"
60 
61 #include "util-debug.h"
62 #include "app-layer-protos.h"
63 #include "app-layer.h"
64 #include "app-layer-events.h"
65 #include "app-layer-parser.h"
66 #include "app-layer-frames.h"
67 
68 #include "detect-engine-state.h"
69 
70 #include "util-profiling.h"
71 #include "util-validate.h"
72 #include "util-exception-policy.h"
73 
74 #ifdef DEBUG
75 static SCMutex segment_pool_memuse_mutex;
76 static uint64_t segment_pool_memuse = 0;
77 static uint64_t segment_pool_memcnt = 0;
78 #endif
79 
80 thread_local uint64_t t_pcapcnt = UINT64_MAX;
81 
83 /* init only, protect initializing and growing pool */
84 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
85 
86 /* Memory use counter */
87 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
88 
89 static int g_tcp_session_dump_enabled = 0;
90 
91 inline bool IsTcpSessionDumpingEnabled(void)
92 {
93  return g_tcp_session_dump_enabled == 1;
94 }
95 
97 {
98  g_tcp_session_dump_enabled = 1;
99 }
100 
101 /* prototypes */
103 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
104 
106 {
107  SC_ATOMIC_INIT(ra_memuse);
108 }
109 
110 /**
111  * \brief Function to Increment the memory usage counter for the TCP reassembly
112  * segments
113  *
114  * \param size Size of the TCP segment and its payload length memory allocated
115  */
116 static void StreamTcpReassembleIncrMemuse(uint64_t size)
117 {
118  (void) SC_ATOMIC_ADD(ra_memuse, size);
119  SCLogDebug("REASSEMBLY %" PRIu64 ", incr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
120  size);
121 }
122 
123 /**
124  * \brief Function to Decrease the memory usage counter for the TCP reassembly
125  * segments
126  *
127  * \param size Size of the TCP segment and its payload length memory allocated
128  */
129 static void StreamTcpReassembleDecrMemuse(uint64_t size)
130 {
131 #ifdef UNITTESTS
132  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
133  if (RunmodeIsUnittests()) {
134  BUG_ON(presize > UINT_MAX);
135  }
136 #endif
137 
138  (void) SC_ATOMIC_SUB(ra_memuse, size);
139 
140 #ifdef UNITTESTS
141  if (RunmodeIsUnittests()) {
142  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
143  BUG_ON(postsize > presize);
144  }
145 #endif
146  SCLogDebug("REASSEMBLY %" PRIu64 ", decr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
147  size);
148 }
149 
151 {
152  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
153  return smemuse;
154 }
155 
156 /**
157  * \brief Function to Check the reassembly memory usage counter against the
158  * allowed max memory usage for TCP segments.
159  *
160  * \param size Size of the TCP segment and its payload length memory allocated
161  * \retval 1 if in bounds
162  * \retval 0 if not in bounds
163  */
165 {
166 #ifdef DEBUG
167  if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX &&
168  g_eps_stream_reassembly_memcap == t_pcapcnt))) {
169  SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt);
170  return 0;
171  }
172 #endif
173  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
174  if (memcapcopy == 0 ||
175  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
176  return 1;
177  return 0;
178 }
179 
180 /**
181  * \brief Update memcap value
182  *
183  * \param size new memcap value
184  */
186 {
187  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
188  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
189  return 1;
190  }
191 
192  return 0;
193 }
194 
195 /**
196  * \brief Return memcap value
197  *
198  * \return memcap memcap value
199  */
201 {
202  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
203  return memcapcopy;
204 }
205 
206 /* memory functions for the streaming buffer API */
207 
208 /*
209  void *(*Calloc)(size_t n, size_t size);
210 */
211 static void *ReassembleCalloc(size_t n, size_t size)
212 {
213  if (StreamTcpReassembleCheckMemcap(n * size) == 0) {
215  return NULL;
216  }
217  void *ptr = SCCalloc(n, size);
218  if (ptr == NULL) {
220  return NULL;
221  }
222  StreamTcpReassembleIncrMemuse(n * size);
223  return ptr;
224 }
225 
226 /*
227  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
228 */
229 void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
230 {
231  if (size > orig_size) {
232  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) {
233  SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap));
235  return NULL;
236  }
237  }
238  void *nptr = SCRealloc(optr, size);
239  if (nptr == NULL) {
240  SCLogDebug("realloc fail");
242  return NULL;
243  }
244  if (size > orig_size) {
245  StreamTcpReassembleIncrMemuse(size - orig_size);
246  } else {
247  StreamTcpReassembleDecrMemuse(orig_size - size);
248  }
249  return nptr;
250 }
251 
252 /*
253  void (*Free)(void *ptr, size_t size);
254 */
255 static void ReassembleFree(void *ptr, size_t size)
256 {
257  SCFree(ptr);
258  StreamTcpReassembleDecrMemuse(size);
259 }
260 
261 /** \brief alloc a tcp segment pool entry */
262 static void *TcpSegmentPoolAlloc(void)
263 {
264  SCLogDebug("segment alloc");
265  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
266  return NULL;
267  }
268 
269  TcpSegment *seg = NULL;
270 
271  seg = SCMalloc(sizeof (TcpSegment));
272  if (unlikely(seg == NULL))
273  return NULL;
274 
276  uint32_t memuse =
277  sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
278  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
279  SCFree(seg);
280  return NULL;
281  }
282 
284  if (seg->pcap_hdr_storage == NULL) {
285  SCLogError("Unable to allocate memory for "
286  "TcpSegmentPcapHdrStorage");
287  SCFree(seg);
288  return NULL;
289  } else {
290  seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
291  seg->pcap_hdr_storage->pkt_hdr =
292  SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
293  if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
294  SCLogError("Unable to allocate memory for "
295  "packet header data within "
296  "TcpSegmentPcapHdrStorage");
297  SCFree(seg->pcap_hdr_storage);
298  SCFree(seg);
299  return NULL;
300  }
301  }
302 
303  StreamTcpReassembleIncrMemuse(memuse);
304  } else {
305  seg->pcap_hdr_storage = NULL;
306  }
307 
308  return seg;
309 }
310 
311 static int TcpSegmentPoolInit(void *data, void *initdata)
312 {
313  TcpSegment *seg = (TcpSegment *) data;
314  TcpSegmentPcapHdrStorage *pcap_hdr;
315 
316  pcap_hdr = seg->pcap_hdr_storage;
317 
318  /* do this before the can bail, so TcpSegmentPoolCleanup
319  * won't have uninitialized memory to consider. */
320  memset(seg, 0, sizeof (TcpSegment));
321 
323  uint32_t memuse =
324  sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
325  seg->pcap_hdr_storage = pcap_hdr;
326  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
327  return 0;
328  }
329  StreamTcpReassembleIncrMemuse(memuse);
330  } else {
331  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
332  return 0;
333  }
334  }
335 
336 #ifdef DEBUG
337  SCMutexLock(&segment_pool_memuse_mutex);
338  segment_pool_memuse += sizeof(TcpSegment);
339  segment_pool_memcnt++;
340  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
341  SCMutexUnlock(&segment_pool_memuse_mutex);
342 #endif
343 
344  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
345  return 1;
346 }
347 
348 /** \brief clean up a tcp segment pool entry */
349 static void TcpSegmentPoolCleanup(void *ptr)
350 {
351  if (ptr == NULL)
352  return;
353 
354  TcpSegment *seg = (TcpSegment *)ptr;
355  if (seg && seg->pcap_hdr_storage) {
356  if (seg->pcap_hdr_storage->pkt_hdr) {
358  StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen);
359  }
360  SCFree(seg->pcap_hdr_storage);
361  seg->pcap_hdr_storage = NULL;
362  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage));
363  }
364 
365  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
366 
367 #ifdef DEBUG
368  SCMutexLock(&segment_pool_memuse_mutex);
369  segment_pool_memuse -= sizeof(TcpSegment);
370  segment_pool_memcnt--;
371  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
372  SCMutexUnlock(&segment_pool_memuse_mutex);
373 #endif
374 }
375 
376 /**
377  * \brief Function to return the segment back to the pool.
378  *
379  * \param seg Segment which will be returned back to the pool.
380  */
382 {
383  if (seg == NULL)
384  return;
385 
386  if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
387  seg->pcap_hdr_storage->pktlen = 0;
388  }
389 
391 }
392 
393 /**
394  * \brief return all segments in this stream into the pool(s)
395  *
396  * \param stream the stream to cleanup
397  */
399 {
400  TcpSegment *seg = NULL, *safe = NULL;
401  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
402  {
403  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
405  }
406 }
407 
408 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
409 {
410  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
411  return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq);
412  }
413  return STREAM_BASE_OFFSET(stream);
414 }
415 
416 // may contain gaps
417 uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
418 {
419  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
420  if (!eof && !StreamTcpInlineMode()) {
421  right_edge = MIN(GetAbsLastAck(stream), right_edge);
422  }
423  return right_edge;
424 }
425 
426 uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
427 {
428  uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
429  if (!eof && !StreamTcpInlineMode()) {
430  right_edge = MIN(GetAbsLastAck(stream), right_edge);
431  }
432  return right_edge;
433 }
434 
435 #ifdef UNITTESTS
436 /** \internal
437  * \brief check if segments falls before stream 'offset' */
438 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
439 {
440  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
441  return 1;
442  return 0;
443 }
444 #endif
445 
446 /** \param f locked flow */
448 {
449  if (f->protoctx == NULL)
450  return;
451 
452  TcpSession *ssn = (TcpSession *)f->protoctx;
456  if (f->alparser) {
459  }
460 }
461 
462 /** \param f locked flow */
464 {
465  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
466  return 0;
467 
468  TcpSession *ssn = (TcpSession *)f->protoctx;
470 }
471 
472 static int StreamTcpReassemblyConfig(bool quiet)
473 {
474  uint32_t segment_prealloc = 2048;
475  SCConfNode *seg = SCConfGetNode("stream.reassembly.segment-prealloc");
476  if (seg) {
477  uint32_t prealloc = 0;
478  if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) {
479  SCLogError("segment-prealloc of "
480  "%s is invalid",
481  seg->val);
482  return -1;
483  }
484  segment_prealloc = prealloc;
485  }
486  if (!quiet)
487  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
488  stream_config.prealloc_segments = segment_prealloc;
489 
490  int overlap_diff_data = 0;
491  (void)SCConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
492  if (overlap_diff_data) {
494  }
495  if (StreamTcpInlineMode()) {
497  }
498 
499  uint16_t max_regions = 8;
500  SCConfNode *mr = SCConfGetNode("stream.reassembly.max-regions");
501  if (mr) {
502  uint16_t max_r = 0;
503  if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) {
504  SCLogError("max-regions %s is invalid", mr->val);
505  return -1;
506  }
507  max_regions = max_r;
508  }
509  if (!quiet)
510  SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions);
511 
512  stream_config.prealloc_segments = segment_prealloc;
514  stream_config.sbcnf.max_regions = max_regions;
516  stream_config.sbcnf.Calloc = ReassembleCalloc;
518  stream_config.sbcnf.Free = ReassembleFree;
519 
520  return 0;
521 }
522 
523 int StreamTcpReassembleInit(bool quiet)
524 {
525  /* init the memcap/use tracker */
527 
528  if (StreamTcpReassemblyConfig(quiet) < 0)
529  return -1;
530 
531 #ifdef DEBUG
532  SCMutexInit(&segment_pool_memuse_mutex, NULL);
533 #endif
534  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
536  return 0;
537 }
538 
539 void StreamTcpReassembleFree(bool quiet)
540 {
541  SCMutexLock(&segment_thread_pool_mutex);
542  if (segment_thread_pool != NULL) {
544  segment_thread_pool = NULL;
545  }
546  SCMutexUnlock(&segment_thread_pool_mutex);
547  SCMutexDestroy(&segment_thread_pool_mutex);
548 
549 #ifdef DEBUG
550  if (segment_pool_memuse > 0)
551  SCLogDebug("segment_pool_memuse %" PRIu64 " segment_pool_memcnt %" PRIu64 "",
552  segment_pool_memuse, segment_pool_memcnt);
553  SCMutexDestroy(&segment_pool_memuse_mutex);
554 #endif
555 }
556 
558 {
559  SCEnter();
561  if (unlikely(ra_ctx == NULL))
562  return NULL;
563 
564  ra_ctx->app_tctx = AppLayerGetCtxThread();
565 
566  SCMutexLock(&segment_thread_pool_mutex);
567  if (segment_thread_pool == NULL) {
568  segment_thread_pool = PoolThreadInit(1, /* thread */
569  0, /* unlimited */
571  sizeof(TcpSegment),
572  TcpSegmentPoolAlloc,
573  TcpSegmentPoolInit, NULL,
574  TcpSegmentPoolCleanup, NULL);
575  ra_ctx->segment_thread_pool_id = 0;
576  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
578  ra_ctx->segment_thread_pool_id);
579  } else {
580  /* grow segment_thread_pool until we have an element for our thread id */
582  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
584  ra_ctx->segment_thread_pool_id);
585  }
586  SCMutexUnlock(&segment_thread_pool_mutex);
587  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
588  SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
590  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
591  }
592 
593  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
594 }
595 
597 {
598  SCEnter();
600 
601  if (ra_ctx) {
603  SCFree(ra_ctx);
604  }
605  SCReturn;
606 }
607 
608 static void StreamTcpReassembleExceptionPolicyStatsIncr(
609  ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, enum ExceptionPolicy policy)
610 {
611  uint16_t id = ra_ctx->counter_tcp_reas_eps.eps_id[policy];
612  if (likely(tv && id > 0)) {
613  StatsIncr(tv, id);
614  }
615 }
616 
617 /**
618  * \brief check if stream in pkt direction has depth reached
619  *
620  * \param p packet with *LOCKED* flow
621  *
622  * \retval true stream has depth reached
623  * \retval false stream does not have depth reached
624  */
626 {
627  if (p->flow != NULL && p->flow->protoctx != NULL) {
628  TcpSession *ssn = p->flow->protoctx;
629  TcpStream *stream;
630  if (p->flowflags & FLOW_PKT_TOSERVER) {
631  stream = &ssn->client;
632  } else {
633  stream = &ssn->server;
634  }
635 
636  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? true : false;
637  }
638 
639  return false;
640 }
641 
642 /**
643  * \internal
644  * \brief Function to check the reassembly depth value against the
645  * allowed max depth of the stream reassembly for TCP streams.
646  *
647  * \param stream stream direction
648  * \param seq sequence number where "size" starts
649  * \param size size of the segment that is added
650  *
651  * \retval size Part of the size that fits in the depth, 0 if none
652  */
653 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
654  uint32_t seq, uint32_t size)
655 {
656  SCEnter();
657 
658  /* if the configured depth value is 0, it means there is no limit on
659  reassembly depth. Otherwise carry on my boy ;) */
660  if (ssn->reassembly_depth == 0) {
661  SCReturnUInt(size);
662  }
663 
664  /* if the final flag is set, we're not accepting anymore */
666  SCReturnUInt(0);
667  }
668 
669  uint64_t seg_depth;
670  if (SEQ_GT(stream->base_seq, seq)) {
671  if (SEQ_LEQ(seq+size, stream->base_seq)) {
672  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
673  stream->base_seq, seq, seq+size);
674  SCReturnUInt(0);
675  }
676 
677  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
678  } else {
679  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
680  }
681 
682  /* if the base_seq has moved past the depth window we stop
683  * checking and just reject the rest of the packets including
684  * retransmissions. Saves us the hassle of dealing with sequence
685  * wraps as well */
686  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
687  stream->base_seq, seg_depth,
688  ssn->reassembly_depth);
689 
690  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
691  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
693  SCReturnUInt(0);
694  }
695  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
696  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
697 #if 0
698  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
699  (stream->base_seq_offset + stream->base_seq + size),
700  (stream->isn + ssn->reassembly_depth));
701 #endif
702  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
703  /* packet (partly?) fits the depth window */
704 
705  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
706  /* complete fit */
707  SCReturnUInt(size);
708  } else {
710  /* partial fit, return only what fits */
711  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
712  DEBUG_VALIDATE_BUG_ON(part > size);
713  if (part > size)
714  part = size;
715  SCReturnUInt(part);
716  }
717  }
718 
719  SCReturnUInt(0);
720 }
721 
723 {
724  if (RB_EMPTY(&stream->sb.sbb_tree)) {
725  if (stream->sb.region.stream_offset != 0)
726  return 0;
727 
728  return stream->sb.region.buf_offset;
729  }
730 
731  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
732  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
733  return stream->sb.sbb_size;
734 }
735 
736 /**
737  * \brief Insert a TCP packet data into the stream reassembly engine.
738  *
739  * \retval 0 good segment, as far as we checked.
740  * \retval -1 insert failure due to memcap
741  *
742  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
743  * or it wasn't added because of reassembly depth.
744  *
745  */
747  TcpSession *ssn, TcpStream *stream, Packet *p)
748 {
749  SCEnter();
750 
751  if (ssn->data_first_seen_dir == 0) {
752  if (PKT_IS_TOSERVER(p)) {
753  ssn->data_first_seen_dir = STREAM_TOSERVER;
754  } else {
755  ssn->data_first_seen_dir = STREAM_TOCLIENT;
756  }
757  }
758 
759  /* If the OS policy is not set then set the OS policy for this stream */
760  if (stream->os_policy == 0) {
761  StreamTcpSetOSPolicy(stream, p);
762  }
763 
766  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
767  SCReturnInt(0);
768  }
769 
770  uint16_t *urg_offset;
771  if (PKT_IS_TOSERVER(p)) {
772  urg_offset = &ssn->urg_offset_ts;
773  } else {
774  urg_offset = &ssn->urg_offset_tc;
775  }
776 
777  const TCPHdr *tcph = PacketGetTCP(p);
778  /* segment sequence number, offset by previously accepted
779  * URG OOB data. */
780  uint32_t seg_seq = TCP_GET_RAW_SEQ(tcph) - (*urg_offset);
781  uint8_t urg_data = 0;
782 
783  /* if stream_config.urgent_policy == TCP_STREAM_URGENT_DROP, we won't get here */
784  if (tcph->th_flags & TH_URG) {
785  const uint16_t urg_ptr = SCNtohs(tcph->th_urp);
786  if (urg_ptr > 0 && urg_ptr <= p->payload_len &&
789  /* track up to 64k out of band URG bytes. Fall back to inline
790  * when that budget is exceeded. */
791  if ((*urg_offset) < UINT16_MAX) {
793  (*urg_offset)++;
794 
795  if ((*urg_offset) == UINT16_MAX) {
797  }
798  } else {
799  /* OOB limit DROP is handled here */
802  SCReturnInt(0);
803  }
804  }
805  urg_data = 1; /* only treat last 1 byte as out of band. */
808  }
809 
810  /* depending on hitting the OOB limit, update urg_data or not */
812  (*urg_offset) == UINT16_MAX &&
814  urg_data = 0;
815  } else {
816  if (urg_ptr == 1 && p->payload_len == 1) {
817  SCLogDebug("no non-URG data");
818  SCReturnInt(0);
819  }
820  }
821  }
822  }
823 
824  const uint16_t payload_len = p->payload_len - urg_data;
825 
826  /* If we have reached the defined depth for either of the stream, then stop
827  reassembling the TCP session */
828  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, seg_seq, payload_len);
829  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
830 
833  /* increment stream depth counter */
836  }
837  if (size == 0) {
838  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
839  SCReturnInt(0);
840  }
841 
843  if (size > payload_len)
844  size = payload_len;
845 
846  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
847  if (seg == NULL) {
848  SCLogDebug("segment_pool is empty");
851  SCReturnInt(-1);
852  }
853 
854  DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
855  TCP_SEG_LEN(seg) = (uint16_t)size;
856  /* set SEQUENCE number, adjusted to any URG pointer offset */
857  seg->seq = seg_seq;
858 
859  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
860  if (TCP_HAS_TFO(p) && p->payload_len && (tcph->th_flags & TH_SYN))
861  seg->seq += 1;
862 
863  /* proto detection skipped, but now we do get data. Set event. */
864  if (RB_EMPTY(&stream->seg_tree) &&
866 
869  }
870 
871  int r = StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, p->payload, payload_len);
872  if (r < 0) {
873  if (r == -SC_ENOMEM) {
875  }
876  SCLogDebug("StreamTcpReassembleInsertSegment failed");
877  SCReturnInt(-1);
878  }
879  SCReturnInt(0);
880 }
881 
882 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
883  Packet *p)
884 {
885  uint8_t flag = 0;
886 
888  flag |= STREAM_START;
889  }
890 
891  if (ssn->state == TCP_CLOSED) {
892  flag |= STREAM_EOF;
893  }
894 
895  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
896  flag |= STREAM_MIDSTREAM;
897  }
898 
899  if (p->flags & PKT_PSEUDO_STREAM_END) {
900  flag |= STREAM_EOF;
901  }
902 
903  if (&ssn->client == stream) {
904  flag |= STREAM_TOSERVER;
905  } else {
906  flag |= STREAM_TOCLIENT;
907  }
909  flag |= STREAM_DEPTH;
910  }
911  return flag;
912 }
913 
914 /**
915  * \brief Check the minimum size limits for reassembly.
916  *
917  * \retval false don't reassemble yet
918  * \retval true do reassemble
919  */
920 static bool StreamTcpReassembleRawCheckLimit(
921  const TcpSession *ssn, const TcpStream *stream, const Packet *p)
922 {
923  SCEnter();
924 
925  /* if any of these flags is set we always inspect immediately */
926 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
927  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
928  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
929  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
930 
933  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
934  "is set, so not expecting any new data segments");
935  }
937  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
938  }
940  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
941  "so no new segments will be considered");
942  }
943  SCReturnBool(true);
944  }
945 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
946 
947  /* some states mean we reassemble no matter how much data we have */
948  if (ssn->state > TCP_TIME_WAIT)
949  SCReturnBool(true);
950 
951  if (p->flags & PKT_PSEUDO_STREAM_END)
952  SCReturnBool(true);
953 
954  const uint64_t last_ack_abs = GetAbsLastAck(stream);
955  int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream);
956  int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size
958 
959  /* check if we have enough data to do raw reassembly */
960  if (chunk_size <= diff) {
961  SCReturnBool(true);
962  } else {
963  SCLogDebug("%s min chunk len not yet reached: "
964  "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < "
965  "%" PRIi64,
966  PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq,
967  (stream->last_ack - stream->base_seq), chunk_size);
968  SCReturnBool(false);
969  }
970 
971  SCReturnBool(false);
972 }
973 
974 /**
975  * \brief see what if any work the TCP session still needs
976  */
977 uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
978 {
979  const TcpStream *stream = NULL;
980 #ifdef DEBUG
981  const char *dirstr = NULL;
982 #endif
983  if (direction == STREAM_TOSERVER) {
984  stream = &ssn->client;
985 #ifdef DEBUG
986  dirstr = "client";
987 #endif
988  } else {
989  stream = &ssn->server;
990 #ifdef DEBUG
991  dirstr = "server";
992 #endif
993  }
994  int use_app = 1;
995  int use_raw = 1;
996 
998  // app is dead
999  use_app = 0;
1000  }
1001 
1002  if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
1003  // raw is dead
1004  use_raw = 0;
1005  }
1006  if (use_raw) {
1007  const uint64_t right_edge =
1008  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1009  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1010  " (use: %s). Stream right edge: %" PRIu64,
1011  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1012  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1013  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
1014  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1016  }
1017  }
1018  if (use_app) {
1019  const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
1020  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1021  " (use: %s). Stream right edge: %" PRIu64,
1022  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1023  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1024  if (right_edge > STREAM_APP_PROGRESS(stream)) {
1025  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1027  }
1028  }
1029 
1030  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
1032 }
1033 
1034 #ifdef DEBUG
1035 static uint64_t GetStreamSize(TcpStream *stream)
1036 {
1037  if (stream) {
1038  uint64_t size = 0;
1039  uint32_t cnt = 0;
1040  uint64_t last_ack_abs = GetAbsLastAck(stream);
1041  uint64_t last_re = 0;
1042 
1043  SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset);
1044 
1045  TcpSegment *seg;
1046  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
1047  const uint64_t seg_abs =
1048  STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq);
1049  if (last_re != 0 && last_re < seg_abs) {
1050  const char *gacked = NULL;
1051  if (last_ack_abs >= seg_abs) {
1052  gacked = "fully ack'd";
1053  } else if (last_ack_abs > last_re) {
1054  gacked = "partly ack'd";
1055  } else {
1056  gacked = "not yet ack'd";
1057  }
1058  SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked);
1059  }
1060 
1061  const char *acked = NULL;
1062  if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) {
1063  acked = "fully ack'd";
1064  } else if (last_ack_abs > seg_abs) {
1065  acked = "partly ack'd";
1066  } else {
1067  acked = "not yet ack'd";
1068  }
1069 
1070  SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64
1071  ") ack:%s",
1072  cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg),
1073  seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked);
1074  last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg);
1075  cnt++;
1076  size += (uint64_t)TCP_SEG_LEN(seg);
1077  }
1078 
1079  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
1080  return size;
1081  }
1082  return (uint64_t)0;
1083 }
1084 
1085 static void GetSessionSize(TcpSession *ssn, Packet *p)
1086 {
1087  uint64_t size = 0;
1088  if (ssn) {
1089  size = GetStreamSize(&ssn->client);
1090  size += GetStreamSize(&ssn->server);
1091 
1092  //if (size > 900000)
1093  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1094  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1095  }
1096 }
1097 #endif
1098 
1099 static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk)
1100 {
1101  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
1102  /* only if the stream has been ack'd, consider a gap for sure
1103  * otherwise there may still be a chance of pkts coming in */
1104  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
1105  GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len)) {
1106  return true;
1107  }
1108  return false;
1109 }
1110 
1111 /** \internal
1112  *
1113  * Get buffer, or first part of the buffer if data gaps exist.
1114  *
1115  * \brief get stream data from offset
1116  * \param offset stream offset
1117  * \param check_for_gap check if there is a gap ahead. Optional as it is only
1118  * needed for app-layer incomplete support.
1119  * \retval bool pkt loss ahead */
1120 static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1121  uint64_t offset, const bool check_for_gap)
1122 {
1123  const uint8_t *mydata;
1124  uint32_t mydata_len;
1125  bool gap_ahead = false;
1126 
1127  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1128  SCLogDebug("getting one blob");
1129 
1130  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1131 
1132  *data = mydata;
1133  *data_len = mydata_len;
1134  } else {
1135  SCLogDebug("block mode");
1136  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1137  StreamingBufferBlock *blk = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1138  if (blk == NULL) {
1139  *data = NULL;
1140  *data_len = 0;
1141  return false;
1142  }
1143  SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len);
1144 
1145  /* block at expected offset */
1146  if (blk->offset == offset) {
1147  SCLogDebug("blk at offset");
1148 
1149  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
1150  DEBUG_VALIDATE_BUG_ON(blk->len != *data_len);
1151 
1152  gap_ahead = check_for_gap && GapAhead(stream, blk);
1153 
1154  /* block past out offset */
1155  } else if (blk->offset > offset) {
1156  SCLogDebug("gap, want data at offset %"PRIu64", "
1157  "got data at %"PRIu64". GAP of size %"PRIu64,
1158  offset, blk->offset, blk->offset - offset);
1159  *data = NULL;
1160  *data_len = (uint32_t)(blk->offset - offset);
1161 
1162  /* block starts before offset, but ends after */
1163  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
1164  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
1165  offset, blk->offset, blk->len);
1166  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
1167  SCLogDebug("data %p, data_len %u", *data, *data_len);
1168 
1169  gap_ahead = check_for_gap && GapAhead(stream, blk);
1170 
1171  } else {
1172  *data = NULL;
1173  *data_len = 0;
1174  }
1175  }
1176  return gap_ahead;
1177 }
1178 
1179 /** \internal
1180  * \brief check to see if we should declare a GAP
1181  * Call this when the app layer didn't get data at the requested
1182  * offset.
1183  */
1184 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
1185 {
1186  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1187  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1188  const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded;
1189 
1190  SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs);
1191  SCLogDebug("next_seq %u", stream->next_seq);
1192 
1193  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1194  * has been ack'd. This looks like a GAP. */
1195  if (last_ack_abs > app_progress) {
1196  /* however, we can accept ACKs a bit too liberally. If last_ack
1197  * is beyond next_seq, we only consider it a gap now if we do
1198  * already have data beyond the gap. */
1199  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1200  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1201  SCLogDebug("packet %" PRIu64 ": no GAP. "
1202  "next_seq %u < last_ack %u, but no data in list",
1203  p->pcap_cnt, stream->next_seq, stream->last_ack);
1204  return false;
1205  }
1206  const uint64_t next_seq_abs =
1207  STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1208  const StreamingBufferBlock *blk = stream->sb.head;
1209  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1210  /* ack'd data after the gap */
1211  SCLogDebug("packet %" PRIu64 ": GAP. "
1212  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1213  p->pcap_cnt, stream->next_seq, stream->last_ack);
1214  return true;
1215  }
1216  }
1217 
1218  SCLogDebug("packet %" PRIu64 ": GAP! "
1219  "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", "
1220  "but we have no data.",
1221  p->pcap_cnt, last_ack_abs, app_progress);
1222  return true;
1223  }
1224  SCLogDebug("packet %"PRIu64": no GAP. "
1225  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1226  p->pcap_cnt, last_ack_abs, app_progress);
1227  return false;
1228 }
1229 
1230 static inline uint32_t AdjustToAcked(const Packet *p,
1231  const TcpSession *ssn, const TcpStream *stream,
1232  const uint64_t app_progress, const uint32_t data_len)
1233 {
1234  uint32_t adjusted = data_len;
1235 
1236  /* get window of data that is acked */
1237  if (!StreamTcpInlineMode()) {
1238  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1239  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1240  (ssn->state == TCP_CLOSED &&
1241  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1242  (p->flags & PKT_PSEUDO_STREAM_END))) {
1243  // fall through, we use all available data
1244  } else {
1245  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1246  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1247 
1248  /* see if the buffer contains unack'd data as well */
1249  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1250  adjusted = (uint32_t)(last_ack_abs - app_progress);
1251  DEBUG_VALIDATE_BUG_ON(adjusted > data_len);
1252  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1253  "data is considered", adjusted);
1254  }
1255  }
1256  }
1257  return adjusted;
1258 }
1259 
1260 /** \internal
1261  * \brief get stream buffer and update the app-layer
1262  * \param stream pointer to pointer as app-layer can switch flow dir
1263  * \retval 0 success
1264  */
1265 static int ReassembleUpdateAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn,
1266  TcpStream **stream, Packet *p, enum StreamUpdateDir app_update_dir)
1267 {
1268  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1269 
1270  SCLogDebug("app progress %"PRIu64, app_progress);
1271 #ifdef DEBUG
1272  uint64_t last_ack_abs = GetAbsLastAck(*stream);
1273  SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs,
1274  (*stream)->base_seq);
1275 #endif
1276  const uint8_t *mydata;
1277  uint32_t mydata_len;
1278  bool last_was_gap = false;
1279 
1280  while (1) {
1281  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1282  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1283  bool gap_ahead =
1284  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead);
1285  SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len);
1286  if (last_was_gap && mydata_len == 0) {
1287  break;
1288  }
1289  last_was_gap = false;
1290 
1291  /* make sure to only deal with ACK'd data */
1292  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1293  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1294  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1295  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1296 
1297  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len,
1298  StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, app_update_dir);
1299  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1300 
1302  (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP;
1305 
1306  /* AppLayerHandleTCPData has likely updated progress. */
1307  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1308  app_progress = STREAM_APP_PROGRESS(*stream);
1309 
1310  /* a GAP also consumes 'data required'. TODO perhaps we can use
1311  * this to skip post GAP data until the start of a next record. */
1312  if ((*stream)->data_required > 0) {
1313  if ((*stream)->data_required > mydata_len) {
1314  (*stream)->data_required -= mydata_len;
1315  } else {
1316  (*stream)->data_required = 0;
1317  }
1318  }
1319  if (r < 0)
1320  return 0;
1321  if (no_progress_update)
1322  break;
1323  last_was_gap = true;
1324  continue;
1325 
1326  } else if (flags & STREAM_DEPTH) {
1327  SCLogDebug("DEPTH");
1328  // we're just called once with this flag, so make sure we pass it on
1329  if (mydata == NULL && mydata_len > 0) {
1330  mydata_len = 0;
1331  }
1332  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1333  SCLogDebug("GAP?1");
1334  /* Possibly a gap, but no new data. */
1335  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1336  SCReturnInt(0);
1337 
1338  mydata = NULL;
1339  mydata_len = 0;
1340  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1341  break;
1342  }
1343  DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1344 
1345  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1346  *stream, &(*stream)->sb, mydata_len, app_progress);
1347 
1348  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1349  SCLogDebug("GAP?2");
1350  if (mydata_len < (*stream)->data_required) {
1351  SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead));
1352  if (gap_ahead) {
1353  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1354  (*stream)->data_required, mydata_len);
1355  (*stream)->app_progress_rel += mydata_len;
1356  (*stream)->data_required -= mydata_len;
1357  // TODO send incomplete data to app-layer with special flag
1358  // indicating its all there is for this rec?
1359  } else {
1360  SCReturnInt(0);
1361  }
1362  app_progress = STREAM_APP_PROGRESS(*stream);
1363  continue;
1364  }
1365  }
1366  (*stream)->data_required = 0;
1367 
1368  SCLogDebug("parser");
1369  /* update the app-layer */
1370  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata,
1371  mydata_len, flags, app_update_dir);
1372  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1373  AppLayerFrameDump(p->flow);
1374  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1375  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1376  break;
1377  app_progress = new_app_progress;
1378  if (flags & STREAM_DEPTH)
1379  break;
1380  }
1381 
1382  SCReturnInt(0);
1383 }
1384 
1385 /**
1386  * \brief Update the stream reassembly upon receiving a packet.
1387  *
1388  * For IDS mode, the stream is in the opposite direction of the packet,
1389  * as the ACK-packet is ACK'ing the stream.
1390  *
1391  * One of the utilities call by this function AppLayerHandleTCPData(),
1392  * has a feature where it will call this very same function for the
1393  * stream opposing the stream it is called with. This shouldn't cause
1394  * any issues, since processing of each stream is independent of the
1395  * other stream.
1396  */
1398  TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
1399 {
1400  SCEnter();
1401 
1402  /* this function can be directly called by app layer protocol
1403  * detection. */
1406  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1407  SCReturnInt(0);
1408  }
1409 
1410 #ifdef DEBUG
1411  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1412  GetSessionSize(ssn, p);
1413 #endif
1414  /* if no segments are in the list or all are already processed,
1415  * and state is beyond established, we send an empty msg */
1416  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1417  {
1418  /* send an empty EOF msg if we have no segments but TCP state
1419  * is beyond ESTABLISHED */
1420  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1421  SCLogDebug("sending empty eof message");
1422  /* send EOF to app layer */
1423  uint8_t stream_flags = StreamGetAppLayerFlags(ssn, stream, p);
1424  DEBUG_VALIDATE_BUG_ON((stream_flags & STREAM_EOF) == 0);
1426  tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, stream_flags, app_update_dir);
1427  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1428 
1429  SCReturnInt(0);
1430  }
1431  }
1432 
1433  /* with all that out of the way, lets update the app-layer */
1434  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, app_update_dir);
1435 }
1436 
1437 /** \internal
1438  * \brief get stream data from offset
1439  * \param offset stream offset */
1440 static int GetRawBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1441  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1442 {
1443  const uint8_t *mydata;
1444  uint32_t mydata_len;
1445  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1446  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1447 
1448  /* No gaps in the stream, data must exist in the streaming buffer array */
1449  uint64_t roffset = offset;
1450  if (offset) {
1451  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1452  } else {
1453  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1454  }
1455 
1456  *data = mydata;
1457  *data_len = mydata_len;
1458  *data_offset = roffset;
1459  } else {
1460  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1461  *iter == NULL ? "starting" : "continuing", offset);
1462  if (*iter == NULL) {
1463  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1464  *iter = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1465  SCLogDebug("*iter %p", *iter);
1466  }
1467  if (*iter == NULL) {
1468  SCLogDebug("no data");
1469  *data = NULL;
1470  *data_len = 0;
1471  *data_offset = 0;
1472  return 0;
1473  }
1474  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1475 
1476  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1477  SCLogDebug("mydata %p", mydata);
1478 
1479  if ((*iter)->offset < offset) {
1480  uint64_t delta = offset - (*iter)->offset;
1481  if (delta < mydata_len) {
1482  *data = mydata + delta;
1483  *data_len = (uint32_t)(mydata_len - delta);
1484  *data_offset = offset;
1485  } else {
1486  SCLogDebug("no data (yet)");
1487  *data = NULL;
1488  *data_len = 0;
1489  *data_offset = 0;
1490  }
1491 
1492  } else {
1493  *data = mydata;
1494  *data_len = mydata_len;
1495  *data_offset = (*iter)->offset;
1496  }
1497 
1498  *iter = SBB_RB_NEXT(*iter);
1499  SCLogDebug("*iter %p", *iter);
1500  }
1501  return 0;
1502 }
1503 
1504 /** \brief does the stream engine have data to inspect?
1505  *
1506  * Returns true if there is data to inspect. In IDS case this is
1507  * about ACK'd data in the packet's direction.
1508  *
1509  * In the IPS case this is about the packet itself.
1510  */
1512 {
1513  TcpStream *stream;
1514  if (PKT_IS_TOSERVER(p)) {
1515  stream = &ssn->client;
1516  } else {
1517  stream = &ssn->server;
1518  }
1519 
1520  if (RB_EMPTY(&stream->seg_tree)) {
1521  return false;
1522  }
1523 
1526  return false;
1527 
1528  if (!StreamTcpInlineMode()) {
1529  const uint64_t segs_re_abs =
1530  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1531  if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) {
1532  return false;
1533  }
1534  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1535  return true;
1536  }
1537  } else {
1538  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1539  return true;
1540  }
1541  }
1542  return false;
1543 }
1544 
1545 /** \brief update stream engine after detection
1546  *
1547  * Tasked with progressing the 'progress' for Raw reassembly.
1548  * 2 main scenario's:
1549  * 1. progress is != 0, so we use this
1550  * 2. progress is 0, meaning the detect engine didn't touch
1551  * raw at all. In this case we need to look into progressing
1552  * raw anyway.
1553  *
1554  * Additionally, this function is tasked with disabling raw
1555  * reassembly if the app-layer requested to disable it.
1556  */
1557 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
1558 {
1559  TcpStream *stream;
1560  if (PKT_IS_TOSERVER(p)) {
1561  stream = &ssn->client;
1562  } else {
1563  stream = &ssn->server;
1564  }
1565 
1566  if (progress > STREAM_RAW_PROGRESS(stream)) {
1567  DEBUG_VALIDATE_BUG_ON(progress - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1568  uint32_t slide = (uint32_t)(progress - STREAM_RAW_PROGRESS(stream));
1569  stream->raw_progress_rel += slide;
1571 
1572  } else if (progress == 0) {
1573  uint64_t target;
1574  if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) {
1575  target = STREAM_APP_PROGRESS(stream);
1576  } else {
1577  target = GetAbsLastAck(stream);
1578  }
1579  if (target > STREAM_RAW_PROGRESS(stream)) {
1580  DEBUG_VALIDATE_BUG_ON(target - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1581  uint32_t slide = (uint32_t)(target - STREAM_RAW_PROGRESS(stream));
1582  stream->raw_progress_rel += slide;
1583  }
1585 
1586  } else {
1587  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1588  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1589  STREAM_RAW_PROGRESS(stream), stream->window);
1590  }
1591 
1592  /* if we were told to accept no more raw data, we can mark raw as
1593  * disabled now. */
1596  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1597  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1598  }
1599 
1600  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1601 }
1602 
1603 /** \internal
1604  * \brief get a buffer around the current packet and run the callback on it
1605  *
1606  * The inline/IPS scanning method takes the current payload and wraps it in
1607  * data from other segments.
1608  *
1609  * How much data is inspected is controlled by the available data, chunk_size
1610  * and the payload size of the packet.
1611  *
1612  * Large packets: if payload size is close to the chunk_size, where close is
1613  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1614  * used: payload_len + 33% of the chunk_size.
1615  * If the payload size if equal to or bigger than the chunk_size, we use
1616  * payload len + 33% of the chunk size.
1617  */
1618 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1619  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1620 {
1621  SCEnter();
1622  int r = 0;
1623 
1624  TcpStream *stream;
1625  if (PKT_IS_TOSERVER(p)) {
1626  stream = &ssn->client;
1627  } else {
1628  stream = &ssn->server;
1629  }
1630 
1631  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1633  {
1634  *progress_out = STREAM_RAW_PROGRESS(stream);
1635  return 0;
1636  }
1637 
1638  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1641  if ((chunk_size <= p->payload_len) || (((chunk_size / 3) * 2) < p->payload_len)) {
1642  chunk_size = p->payload_len + (chunk_size / 3);
1643  SCLogDebug(
1644  "packet payload len %u, so chunk_size adjusted to %u", p->payload_len, chunk_size);
1645  }
1646 
1647  const TCPHdr *tcph = PacketGetTCP(p);
1648  uint64_t packet_leftedge_abs =
1649  STREAM_BASE_OFFSET(stream) + (TCP_GET_RAW_SEQ(tcph) - stream->base_seq);
1650  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1651  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1652  packet_leftedge_abs, packet_rightedge_abs);
1653 
1654  const uint8_t *mydata = NULL;
1655  uint32_t mydata_len = 0;
1656  uint64_t mydata_offset = 0;
1657  /* simply return progress from the block we inspected. */
1658  bool return_progress = false;
1659 
1660  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1661  /* continues block */
1662  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1663  return_progress = true;
1664 
1665  } else {
1666  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1667  /* find our block */
1668  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1669  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1670  if (sbb) {
1671  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1672  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1673  mydata_offset = sbb->offset;
1674  }
1675  }
1676 
1677  /* this can only happen if the segment insert of our current 'p' failed */
1678  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1679  if ((mydata == NULL || mydata_len == 0) || /* no data */
1680  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1681  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1682  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1683  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1684  {
1685  /* no data, or data is incomplete or wrong: use packet data */
1686  mydata = p->payload;
1687  mydata_len = p->payload_len;
1688  mydata_offset = packet_leftedge_abs;
1689  //mydata_rightedge_abs = packet_rightedge_abs;
1690  } else {
1691  /* adjust buffer to match chunk_size */
1692  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1693  if (mydata_len > chunk_size) {
1694  uint32_t excess = mydata_len - chunk_size;
1695  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1696 
1697  if (mydata_rightedge_abs == packet_rightedge_abs) {
1698  mydata += excess;
1699  mydata_len -= excess;
1700  mydata_offset += excess;
1701  SCLogDebug("cutting front of the buffer with %u", excess);
1702  } else if (mydata_offset == packet_leftedge_abs) {
1703  mydata_len -= excess;
1704  SCLogDebug("cutting tail of the buffer with %u", excess);
1705  } else {
1706  DEBUG_VALIDATE_BUG_ON(mydata_offset > packet_leftedge_abs);
1707  uint32_t abs_before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1708  DEBUG_VALIDATE_BUG_ON(packet_rightedge_abs > mydata_rightedge_abs);
1709  uint32_t abs_after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1710  uint32_t before = abs_before;
1711  uint32_t after = abs_after;
1712  SCLogDebug("before %u after %u", before, after);
1713 
1714  if (after >= (chunk_size - p->payload_len) / 2) {
1715  // more trailing data than we need
1716 
1717  if (before >= (chunk_size - p->payload_len) / 2) {
1718  // also more heading data, divide evenly
1719  before = after = (chunk_size - p->payload_len) / 2;
1720  } else {
1721  // heading data is less than requested, give the
1722  // rest to the trailing data
1723  after = (chunk_size - p->payload_len) - before;
1724  }
1725  } else {
1726  // less trailing data than requested
1727 
1728  if (before >= (chunk_size - p->payload_len) / 2) {
1729  before = (chunk_size - p->payload_len) - after;
1730  } else {
1731  // both smaller than their requested size
1732  }
1733  }
1734 
1735  /* adjust the buffer */
1736  DEBUG_VALIDATE_BUG_ON(before > abs_before);
1737  uint32_t skip = abs_before - before;
1738  DEBUG_VALIDATE_BUG_ON(after > abs_after);
1739  uint32_t cut = abs_after - after;
1740  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1741  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1742  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1743  mydata += skip;
1744  mydata_len -= (skip + cut);
1745  mydata_offset += skip;
1746  }
1747  }
1748  }
1749 
1750  /* run the callback */
1751  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1752  DEBUG_VALIDATE_BUG_ON(r < 0);
1753 
1754  if (return_progress) {
1755  *progress_out = (mydata_offset + mydata_len);
1756  } else {
1757  /* several blocks of data, so we need to be a bit more careful:
1758  * - if last_ack is beyond last progress, move progress forward to last_ack
1759  * - if our block matches or starts before last ack, return right edge of
1760  * our block.
1761  */
1762  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1763  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1764 
1765  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1766  if (mydata_offset > last_ack_abs) {
1767  /* gap between us and last ack, set progress to last ack */
1768  *progress_out = last_ack_abs;
1769  } else {
1770  *progress_out = (mydata_offset + mydata_len);
1771  }
1772  } else {
1773  *progress_out = STREAM_RAW_PROGRESS(stream);
1774  }
1775  }
1776  return r;
1777 }
1778 
1779 /** \brief access 'raw' reassembly data.
1780  *
1781  * Access data as tracked by 'raw' tracker. Data is made available to
1782  * callback that is passed to this function.
1783  *
1784  * In the case of IDS the callback may be run multiple times if data
1785  * contains gaps. It will then be run for each block of data that is
1786  * continuous.
1787  *
1788  * The callback should give on of 2 return values:
1789  * - 0 ok
1790  * - 1 done
1791  * The value 1 will break the loop if there is a block list that is
1792  * inspected.
1793  *
1794  * This function will return the 'progress' value that has been
1795  * consumed until now.
1796  *
1797  * \param ssn tcp session
1798  * \param stream tcp stream
1799  * \param Callback the function pointer to the callback function
1800  * \param cb_data callback data
1801  * \param[in] progress_in progress to work from
1802  * \param[in] re right edge of data to consider
1803  * \param[out] progress_out absolute progress value of the data this
1804  * call handled.
1805  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1806  */
1807 static int StreamReassembleRawDo(const TcpSession *ssn, const TcpStream *stream,
1808  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1809  const uint64_t re, uint64_t *progress_out, bool eof)
1810 {
1811  SCEnter();
1812  int r = 0;
1813 
1814  StreamingBufferBlock *iter = NULL;
1815  uint64_t progress = progress_in;
1816 
1817  /* loop through available buffers. On no packet loss we'll have a single
1818  * iteration. On missing data we'll walk the blocks */
1819  while (1) {
1820  const uint8_t *mydata;
1821  uint32_t mydata_len;
1822  uint64_t mydata_offset = 0;
1823 
1824  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1825  if (mydata_len == 0) {
1826  SCLogDebug("no data");
1827  break;
1828  }
1829  //PrintRawDataFp(stdout, mydata, mydata_len);
1830 
1831  SCLogDebug("raw progress %"PRIu64, progress);
1832  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1833  stream, &stream->sb, mydata_len, progress);
1834 
1835  if (eof) {
1836  // inspect all remaining data, ack'd or not
1837  } else {
1838  if (re < progress) {
1839  SCLogDebug("nothing to do");
1840  goto end;
1841  }
1842 
1843  SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress);
1844  SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len,
1845  re);
1846 
1847  /* see if the buffer contains unack'd data as well */
1848  if (progress + mydata_len > re) {
1849 #ifdef DEBUG_VALIDATION
1850  uint32_t check = mydata_len;
1851 #endif
1852  mydata_len = (uint32_t)(re - progress);
1853  DEBUG_VALIDATE_BUG_ON(check < mydata_len);
1854  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1855  "data is considered", mydata_len);
1856  }
1857  }
1858  if (mydata_len == 0)
1859  break;
1860 
1861  SCLogDebug("data %p len %u", mydata, mydata_len);
1862 
1863  /* we have data. */
1864  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1865  DEBUG_VALIDATE_BUG_ON(r < 0);
1866 
1867  if (mydata_offset == progress) {
1868  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1869  progress, mydata_len, progress_in + mydata_len);
1870 
1871  progress += mydata_len;
1872  SCLogDebug("raw progress now %"PRIu64, progress);
1873 
1874  /* data is beyond the progress we'd like, and before last ack. Gap. */
1875  } else if (mydata_offset > progress && mydata_offset < re) {
1876  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1877  SCLogDebug("re %" PRIu64, re);
1878  progress = mydata_offset;
1879  SCLogDebug("raw progress now %"PRIu64, progress);
1880 
1881  } else {
1882  SCLogDebug("not increasing progress, data gap => mydata_offset "
1883  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1884  }
1885 
1886  if (iter == NULL || r == 1)
1887  break;
1888  }
1889 end:
1890  *progress_out = progress;
1891  return r;
1892 }
1893 
1895  void *cb_data, const uint64_t offset, const bool eof)
1896 {
1897  /* take app progress as the right edge of used data. */
1898  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1899  SCLogDebug("app_progress %" PRIu64, app_progress);
1900 
1901  uint64_t unused = 0;
1902  return StreamReassembleRawDo(
1903  ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof);
1904 }
1905 
1907  StreamReassembleRawFunc Callback, void *cb_data,
1908  uint64_t *progress_out, bool respect_inspect_depth)
1909 {
1910  /* handle inline separately as the logic is very different */
1911  if (StreamTcpInlineMode()) {
1912  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1913  }
1914 
1915  TcpStream *stream;
1916  if (PKT_IS_TOSERVER(p)) {
1917  stream = &ssn->client;
1918  } else {
1919  stream = &ssn->server;
1920  }
1921 
1923  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1924  {
1925  *progress_out = STREAM_RAW_PROGRESS(stream);
1926  return 0;
1927  }
1928 
1929  uint64_t progress = STREAM_RAW_PROGRESS(stream);
1930  /* if the app layer triggered a flush, and we're supposed to
1931  * use a minimal inspect depth, we actually take the app progress
1932  * as that is the right edge of the data. Then we take the window
1933  * of 'min_inspect_depth' before that. */
1934 
1935  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s "
1936  "stream->min_inspect_depth %u",
1937  respect_inspect_depth ? "true" : "false",
1938  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1939  stream->min_inspect_depth);
1940 
1941  if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) &&
1942  stream->min_inspect_depth) {
1943  progress = STREAM_APP_PROGRESS(stream);
1944  if (stream->min_inspect_depth >= progress) {
1945  progress = 0;
1946  } else {
1947  progress -= stream->min_inspect_depth;
1948  }
1949 
1950  SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream),
1951  STREAM_RAW_PROGRESS(stream));
1952 
1953  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1954  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress "
1955  "%" PRIu64,
1956  progress);
1957  }
1958 
1959  SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth,
1960  stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW"
1961  : "(no trigger)");
1962 
1963  /* absolute right edge of ack'd data */
1964  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1965  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1966 
1967  return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs,
1968  progress_out, (p->flags & PKT_PSEUDO_STREAM_END));
1969 }
1970 
1971 int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream,
1972  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1973  uint64_t *progress_out, const bool eof)
1974 {
1975  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1976  return 0;
1977 
1978  /* absolute right edge of ack'd data */
1979  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1980  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1981 
1982  return StreamReassembleRawDo(
1983  ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof);
1984 }
1985 
1986 /** \internal
1987  * \brief update app layer based on received ACK
1988  *
1989  * \retval r 0 on success, -1 on error
1990  */
1991 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1992  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1993 {
1994  SCEnter();
1995 
1996  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1997  SCReturnInt(-1);
1998 
1999  SCReturnInt(0);
2000 }
2001 
2003  TcpSession *ssn, TcpStream *stream, Packet *p)
2004 {
2005  SCEnter();
2006 
2007  DEBUG_VALIDATE_BUG_ON(!PacketIsTCP(p));
2008  const TCPHdr *tcph = PacketGetTCP(p);
2009 
2010  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
2011  ssn, stream, p, p->payload_len);
2012 
2013  /* default IDS: update opposing side (triggered by ACK) */
2015  /* inline and stream end and flow timeout packets trigger same dir handling */
2016  if (StreamTcpInlineMode()) {
2017  dir = UPDATE_DIR_PACKET;
2018  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
2019  dir = UPDATE_DIR_PACKET;
2020  } else if (tcph->th_flags & TH_RST) { // accepted rst
2021  dir = UPDATE_DIR_PACKET;
2022  } else if ((tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
2023  if (tcph->th_flags & TH_ACK) {
2024  dir = UPDATE_DIR_BOTH;
2025  } else {
2026  dir = UPDATE_DIR_PACKET;
2027  }
2028  } else if (ssn->state == TCP_CLOSED) {
2029  dir = UPDATE_DIR_BOTH;
2030  } else if ((ssn->flags & STREAMTCP_FLAG_ASYNC) != 0) {
2031  dir = UPDATE_DIR_PACKET;
2032  SCLogDebug("%" PRIu64 ": ASYNC: UPDATE_DIR_PACKET", p->pcap_cnt);
2033  }
2034 
2035  /* handle ack received */
2036  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
2037  /* we need to update the opposing stream in
2038  * StreamTcpReassembleHandleSegmentUpdateACK */
2039  TcpStream *opposing_stream = NULL;
2040  if (stream == &ssn->client) {
2041  opposing_stream = &ssn->server;
2042  } else {
2043  opposing_stream = &ssn->client;
2044  }
2045 
2046  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2047 
2048  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
2049  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
2050  SCReturnInt(-1);
2051  }
2052 
2053  /* StreamTcpReassembleHandleSegmentUpdateACK
2054  * may swap content of ssn->server and ssn->client structures.
2055  * We have to continue with initial content of the stream in such case */
2056  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2057  if (reversed_before_ack_handling != reversed_after_ack_handling) {
2058  SCLogDebug("TCP streams were swapped");
2059  stream = opposing_stream;
2060  }
2061  }
2062  /* if this segment contains data, insert it */
2063  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) &&
2064  (tcph->th_flags & TH_RST) == 0) {
2065  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
2066 
2067  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
2068  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
2069  /* failure can only be because of memcap hit, so see if this should lead to a drop */
2072  StreamTcpReassembleExceptionPolicyStatsIncr(
2074  SCReturnInt(-1);
2075  }
2076 
2077  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
2078  p->flags |= PKT_STREAM_ADD;
2079  } else {
2080  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
2081  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
2082  ssn, stream, p->payload_len,
2083  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
2084  }
2085 
2086  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
2087  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
2088  * was *just* set. In this case we trigger the AppLayer Truncate
2089  * logic, to inform the applayer no more data in this direction is
2090  * to be expected. */
2091  if ((stream->flags &
2094  {
2095  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
2096  if (dir != UPDATE_DIR_PACKET) {
2097  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
2098  "can trigger Truncate");
2099  dir = UPDATE_DIR_PACKET;
2100  }
2101  }
2102 
2103  /* in stream inline mode even if we have no data we call the reassembly
2104  * functions to handle EOF */
2105  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
2106  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
2107  StreamTcpInlineMode()?"true":"false",
2108  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
2109  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
2110  SCReturnInt(-1);
2111  }
2112  }
2113 
2114  SCReturnInt(0);
2115 }
2116 
2117 /**
2118  * \brief get a segment from the pool
2119  *
2120  * \retval seg Segment from the pool or NULL
2121  */
2123 {
2125  if (seg) {
2127  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2128  return seg;
2129  }
2130 
2131  seg = (TcpSegment *)PoolThreadGetById(
2132  segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id);
2133  SCLogDebug("seg we return is %p", seg);
2134  if (seg == NULL) {
2135  /* Increment the counter to show that we are not able to serve the
2136  segment request due to memcap limit */
2138  } else {
2139  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2141  }
2142 
2143  return seg;
2144 }
2145 
2146 /**
2147  * \brief Trigger RAW stream inspection
2148  *
2149  * Used by AppLayerTriggerRawStreamInspection to trigger RAW stream
2150  * inspection from the applayer, for example upon completion of a
2151  * HTTP request.
2152  *
2153  * It sets a flag in the stream so that the next Raw call will return
2154  * the data.
2155  *
2156  * \param ssn TcpSession
2157  */
2159 {
2160 #ifdef DEBUG
2161  BUG_ON(ssn == NULL);
2162 #endif
2163 
2164  if (ssn != NULL) {
2165  if (direction == STREAM_TOSERVER) {
2167  } else {
2169  }
2170 
2171  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
2172  }
2173 }
2174 
2175 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
2176 {
2177 #ifdef DEBUG
2178  BUG_ON(ssn == NULL);
2179 #endif
2180 
2181  if (ssn != NULL) {
2182  if (direction == STREAM_TOSERVER) {
2183  ssn->client.min_inspect_depth = depth;
2184  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
2185  } else {
2186  ssn->server.min_inspect_depth = depth;
2187  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
2188  }
2189  }
2190 }
2191 
2192 #ifdef UNITTESTS
2193 /** unit tests and it's support functions below */
2194 
2195 #define SET_ISN(stream, setseq) \
2196  (stream)->isn = (setseq); \
2197  (stream)->base_seq = (setseq) + 1
2198 
2199 /** \brief The Function to create the packet with given payload, which is used
2200  * to test the reassembly of the engine.
2201  *
2202  * \param payload The variable used to store the payload contents of the
2203  * current packet.
2204  * \param value The value which current payload will have for this packet
2205  * \param payload_len The length of the filed payload for current packet.
2206  * \param len Length of the payload array
2207  */
2208 
2209 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2210  uint8_t payload_len, uint8_t len)
2211 {
2212  uint8_t i;
2213  for (i = 0; i < payload_len; i++)
2214  payload[i] = value;
2215  for (; i < len; i++)
2216  payload = NULL;
2217 }
2218 
2219 /** \brief The Function Checks the reassembled stream contents against predefined
2220  * stream contents according to OS policy used.
2221  *
2222  * \param stream_policy Predefined value of stream for different OS policies
2223  * \param stream Reassembled stream returned from the reassembly functions
2224  */
2225 
2226 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2227 {
2228  if (StreamingBufferCompareRawData(&stream->sb,
2229  data, data_len) == 0)
2230  {
2231  SCReturnInt(0);
2232  }
2233  SCLogInfo("OK");
2234  PrintRawDataFp(stdout, data, data_len);
2235  return 1;
2236 }
2237 
2238 #define MISSED_START(isn) \
2239  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2240  TcpSession ssn; \
2241  ThreadVars tv; \
2242  memset(&tv, 0, sizeof(tv)); \
2243  \
2244  StreamTcpUTInit(&ra_ctx); \
2245  \
2246  StreamTcpUTSetupSession(&ssn); \
2247  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2248  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2249  \
2250  TcpStream *stream = &ssn.client;
2251 
2252 #define MISSED_END \
2253  StreamTcpUTClearSession(&ssn); \
2254  StreamTcpUTDeinit(ra_ctx); \
2255  PASS
2256 
2257 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2258  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2259  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2260 
2261 #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \
2262  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen));
2263 
2264 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len);
2265 
2266 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
2267 {
2268  int cnt = 0;
2269  uint64_t last_re = 0;
2270  StreamingBufferBlock *sbb = NULL;
2271  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2272  {
2273  if (sbb->offset != last_re) {
2274  // gap before us
2275  if (cnt == pos && last_re == offset && len == sbb->offset - last_re) {
2276  return 1;
2277  }
2278  cnt++;
2279  }
2280  last_re = sbb->offset + sbb->len;
2281  cnt++;
2282  }
2283  return 0;
2284 }
2285 
2287  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len);
2288 
2290  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
2291 {
2292  int cnt = 0;
2293  uint64_t last_re = 0;
2294  StreamingBufferBlock *sbb = NULL;
2295  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2296  {
2297  if (sbb->offset != last_re) {
2298  // gap before us
2299  cnt++;
2300  }
2301 
2302  if (cnt == pos && sbb->offset == offset) {
2303  const uint8_t *buf = NULL;
2304  uint32_t buf_len = 0;
2305  StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len);
2306 
2307  if (len == buf_len) {
2308  return (memcmp(data, buf, len) == 0);
2309  }
2310  }
2311 
2312  last_re = sbb->offset + sbb->len;
2313  cnt++;
2314  }
2315  return 0;
2316 }
2317 
2318 /**
2319  * \test Test the handling of packets missed by both IDS and the end host.
2320  * The packet is missed in the starting of the stream.
2321  *
2322  * \retval On success it returns 1 and on failure 0.
2323  */
2324 
2325 static int StreamTcpReassembleTest25 (void)
2326 {
2327  MISSED_START(6);
2328  MISSED_ADD_PAYLOAD(10, "BB", 2);
2329  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2330  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BB", 2) == 1);
2331  MISSED_ADD_PAYLOAD(12, "CC", 2);
2332  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2333  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BBCC", 4) == 1);
2334  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2335  MISSED_END;
2336  PASS;
2337 }
2338 
2339 /**
2340  * \test Test the handling of packets missed by both IDS and the end host.
2341  * The packet is missed in the middle of the stream.
2342  *
2343  * \retval On success it returns 1 and on failure 0.
2344  */
2345 
2346 static int StreamTcpReassembleTest26 (void)
2347 {
2348  MISSED_START(9);
2349  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2350  MISSED_ADD_PAYLOAD(15, "CC", 2);
2351  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2352  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2353  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2354  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2355  MISSED_END;
2356 }
2357 
2358 /**
2359  * \test Test the handling of packets missed by both IDS and the end host.
2360  * The packet is missed in the end of the stream.
2361  *
2362  * \retval On success it returns 1 and on failure 0.
2363  */
2364 
2365 static int StreamTcpReassembleTest27 (void)
2366 {
2367  MISSED_START(9);
2368  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2369  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2370  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2371  MISSED_END;
2372 }
2373 
2374 /**
2375  * \test Test the handling of packets missed by IDS, but the end host has
2376  * received it and send the acknowledgment of it. The packet is missed
2377  * in the starting of the stream.
2378  *
2379  * \retval On success it returns 1 and on failure 0.
2380  */
2381 
2382 static int StreamTcpReassembleTest28 (void)
2383 {
2384  MISSED_START(6);
2385  MISSED_ADD_PAYLOAD(10, "AAA", 3);
2386  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2387  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAA", 3) == 1);
2388  MISSED_ADD_PAYLOAD(13, "BB", 2);
2389  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2390  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABB", 5) == 1);
2391  ssn.state = TCP_TIME_WAIT;
2392  MISSED_ADD_PAYLOAD(15, "CC", 2);
2393  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2394  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABBCC", 7) == 1);
2395  MISSED_END;
2396 }
2397 
2398 /**
2399  * \test Test the handling of packets missed by IDS, but the end host has
2400  * received it and send the acknowledgment of it. The packet is missed
2401  * in the middle of the stream.
2402  *
2403  * \retval On success it returns 1 and on failure 0.
2404  */
2405 
2406 static int StreamTcpReassembleTest29 (void)
2407 {
2408  MISSED_START(9);
2409  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2410  ssn.state = TCP_TIME_WAIT;
2411  MISSED_ADD_PAYLOAD(15, "CC", 2);
2412  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2413  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2414  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2415  MISSED_END;
2416 }
2417 
2418 static int StreamTcpReassembleTest33(void)
2419 {
2420  TcpSession ssn;
2421  Packet *p = PacketGetFromAlloc();
2422  FAIL_IF(unlikely(p == NULL));
2423  Flow f;
2424  TCPHdr tcph;
2425  TcpReassemblyThreadCtx *ra_ctx = NULL;
2427  uint8_t packet[1460] = "";
2428 
2429  StreamTcpUTInit(&ra_ctx);
2431 
2432  memset(&f, 0, sizeof (Flow));
2433  memset(&tcph, 0, sizeof (TCPHdr));
2434  ThreadVars tv;
2435  memset(&tv, 0, sizeof (ThreadVars));
2436  FLOW_INITIALIZE(&f);
2437  f.protoctx = &ssn;
2438  f.proto = IPPROTO_TCP;
2439  p->src.family = AF_INET;
2440  p->dst.family = AF_INET;
2441  p->proto = IPPROTO_TCP;
2442  p->flow = &f;
2443  tcph.th_win = 5480;
2444  tcph.th_flags = TH_PUSH | TH_ACK;
2445  UTHSetTCPHdr(p, &tcph);
2447  p->payload = packet;
2448 
2449  tcph.th_seq = htonl(10);
2450  tcph.th_ack = htonl(31);
2451  p->payload_len = 10;
2452 
2453  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2454 
2455  tcph.th_seq = htonl(20);
2456  tcph.th_ack = htonl(31);
2457  p->payload_len = 10;
2458 
2459  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2460 
2461  tcph.th_seq = htonl(40);
2462  tcph.th_ack = htonl(31);
2463  p->payload_len = 10;
2464 
2465  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2466 
2467  tcph.th_seq = htonl(5);
2468  tcph.th_ack = htonl(31);
2469  p->payload_len = 30;
2470 
2471  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2472 
2474  StreamTcpUTDeinit(ra_ctx);
2475  SCFree(p);
2476  PASS;
2477 }
2478 
2479 static int StreamTcpReassembleTest34(void)
2480 {
2481  TcpSession ssn;
2482  Packet *p = PacketGetFromAlloc();
2483  FAIL_IF(unlikely(p == NULL));
2484  Flow f;
2485  TCPHdr tcph;
2486  TcpReassemblyThreadCtx *ra_ctx = NULL;
2488  uint8_t packet[1460] = "";
2489 
2490  StreamTcpUTInit(&ra_ctx);
2492  memset(&f, 0, sizeof (Flow));
2493  memset(&tcph, 0, sizeof (TCPHdr));
2494  ThreadVars tv;
2495  memset(&tv, 0, sizeof (ThreadVars));
2496  FLOW_INITIALIZE(&f);
2497  f.protoctx = &ssn;
2498  f.proto = IPPROTO_TCP;
2499  p->src.family = AF_INET;
2500  p->dst.family = AF_INET;
2501  p->proto = IPPROTO_TCP;
2502  p->flow = &f;
2503  tcph.th_win = 5480;
2504  tcph.th_flags = TH_PUSH | TH_ACK;
2505  UTHSetTCPHdr(p, &tcph);
2507  p->payload = packet;
2508  SET_ISN(&ssn.client, 857961230);
2509 
2510  tcph.th_seq = htonl(857961230);
2511  tcph.th_ack = htonl(31);
2512  p->payload_len = 304;
2513 
2514  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2515 
2516  tcph.th_seq = htonl(857961534);
2517  tcph.th_ack = htonl(31);
2518  p->payload_len = 1460;
2519 
2520  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2521 
2522  tcph.th_seq = htonl(857963582);
2523  tcph.th_ack = htonl(31);
2524  p->payload_len = 1460;
2525 
2526  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2527 
2528  tcph.th_seq = htonl(857960946);
2529  tcph.th_ack = htonl(31);
2530  p->payload_len = 1460;
2531 
2532  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2533 
2535  StreamTcpUTDeinit(ra_ctx);
2536  SCFree(p);
2537  PASS;
2538 }
2539 
2540 /**
2541  * \test Test to make sure that we don't return the segments until the app
2542  * layer proto has been detected and after that remove the processed
2543  * segments.
2544  *
2545  * \retval On success it returns 1 and on failure 0.
2546  */
2547 
2548 static int StreamTcpReassembleTest39 (void)
2549 {
2550  Packet *p = PacketGetFromAlloc();
2551  FAIL_IF(unlikely(p == NULL));
2552  Flow f;
2553  ThreadVars tv;
2554  StreamTcpThread stt;
2555  TCPHdr tcph;
2556  PacketQueueNoLock pq;
2557  memset(&pq,0,sizeof(PacketQueueNoLock));
2558  memset (&f, 0, sizeof(Flow));
2559  memset(&tv, 0, sizeof (ThreadVars));
2560  memset(&stt, 0, sizeof (stt));
2561  memset(&tcph, 0, sizeof (TCPHdr));
2562 
2563  FLOW_INITIALIZE(&f);
2564  f.flags = FLOW_IPV4;
2565  f.proto = IPPROTO_TCP;
2566  p->flow = &f;
2567  UTHSetTCPHdr(p, &tcph);
2568 
2569  StreamTcpUTInit(&stt.ra_ctx);
2570 
2571  /* handshake */
2572  tcph.th_win = htons(5480);
2573  tcph.th_flags = TH_SYN;
2575  p->payload_len = 0;
2576  p->payload = NULL;
2577  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2578 
2579  TcpSession *ssn = (TcpSession *)f.protoctx;
2580  FAIL_IF_NULL(ssn);
2581 
2588  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2589  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2590  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2591  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2592  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2593  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2594  FAIL_IF(ssn->data_first_seen_dir != 0);
2595 
2596  /* handshake */
2597  tcph.th_ack = htonl(1);
2598  tcph.th_flags = TH_SYN | TH_ACK;
2600  p->payload_len = 0;
2601  p->payload = NULL;
2602  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2609  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2610  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2611  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2612  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2613  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2614  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2615  FAIL_IF(ssn->data_first_seen_dir != 0);
2616 
2617  /* handshake */
2618  tcph.th_ack = htonl(1);
2619  tcph.th_seq = htonl(1);
2620  tcph.th_flags = TH_ACK;
2622  p->payload_len = 0;
2623  p->payload = NULL;
2624  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2631  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2632  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2633  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2634  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2635  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2636  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2637  FAIL_IF(ssn->data_first_seen_dir != 0);
2638 
2639  /* partial request */
2640  uint8_t request1[] = { 0x47, 0x45, };
2641  tcph.th_ack = htonl(1);
2642  tcph.th_seq = htonl(1);
2643  tcph.th_flags = TH_PUSH | TH_ACK;
2645  p->payload_len = sizeof(request1);
2646  p->payload = request1;
2647  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2654  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2655  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2656  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2657  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2658  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2659  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2660  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2661  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2662 
2663  /* response ack against partial request */
2664  tcph.th_ack = htonl(3);
2665  tcph.th_seq = htonl(1);
2666  tcph.th_flags = TH_ACK;
2668  p->payload_len = 0;
2669  p->payload = NULL;
2670  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2677  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2678  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2679  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2680  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2681  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2682  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2683  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2684  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2685 
2686  /* complete partial request */
2687  uint8_t request2[] = {
2688  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2689  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2690  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2691  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2692  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2693  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2694  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2695  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2696  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2697  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2698  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2699  tcph.th_ack = htonl(1);
2700  tcph.th_seq = htonl(3);
2701  tcph.th_flags = TH_PUSH | TH_ACK;
2703  p->payload_len = sizeof(request2);
2704  p->payload = request2;
2705  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2712  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2713  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2714  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2715  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2716  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2717  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2718  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2719  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2720  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2721 
2722  /* response - request ack */
2723  uint8_t response[] = {
2724  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2725  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2726  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2727  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2728  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2729  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2730  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2731  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2732  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2733  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2734  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2735  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2736  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2737  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2738  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2739  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2740  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2741  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2742  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2743  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2744  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2745  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2746  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2747  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2748  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2749  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2750  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2751  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2752  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2753  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2754  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2755  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2756  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2757  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2758  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2759  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2760  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2761  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2762  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2763  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2764  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2765  tcph.th_ack = htonl(88);
2766  tcph.th_seq = htonl(1);
2767  tcph.th_flags = TH_PUSH | TH_ACK;
2769  p->payload_len = sizeof(response);
2770  p->payload = response;
2771 
2772  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2779  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2780  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2781  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2782  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2784  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2785  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2786  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2787  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2788 
2789  /* response ack from request */
2790  tcph.th_ack = htonl(328);
2791  tcph.th_seq = htonl(88);
2792  tcph.th_flags = TH_ACK;
2794  p->payload_len = 0;
2795  p->payload = NULL;
2796  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2803  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2804  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2805  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2806  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2808  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2809  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2810  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2811  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2812  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2813 
2814  /* response - acking */
2815  tcph.th_ack = htonl(88);
2816  tcph.th_seq = htonl(328);
2817  tcph.th_flags = TH_PUSH | TH_ACK;
2819  p->payload_len = 0;
2820  p->payload = NULL;
2821  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2828  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2829  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2830  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2831  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2833  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2834  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2835  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2836  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2837  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2838 
2839  /* response ack from request */
2840  tcph.th_ack = htonl(328);
2841  tcph.th_seq = htonl(88);
2842  tcph.th_flags = TH_ACK;
2844  p->payload_len = 0;
2845  p->payload = NULL;
2846  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2853  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2854  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2855  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2856  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2858  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2859  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2860  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2861  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2862 
2863  /* response - acking the request again*/
2864  tcph.th_ack = htonl(88);
2865  tcph.th_seq = htonl(328);
2866  tcph.th_flags = TH_PUSH | TH_ACK;
2868  p->payload_len = 0;
2869  p->payload = NULL;
2870  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2877  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2878  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2879  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2880  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2882  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2883  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2884  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2885  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2886 
2887  /*** New Request ***/
2888 
2889  /* partial request */
2890  tcph.th_ack = htonl(328);
2891  tcph.th_seq = htonl(88);
2892  tcph.th_flags = TH_PUSH | TH_ACK;
2894  p->payload_len = sizeof(request1);
2895  p->payload = request1;
2896  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2903  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2904  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2905  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2906  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2908  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2909  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2910  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2911  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2912  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2913 
2914  /* response ack against partial request */
2915  tcph.th_ack = htonl(90);
2916  tcph.th_seq = htonl(328);
2917  tcph.th_flags = TH_ACK;
2919  p->payload_len = 0;
2920  p->payload = NULL;
2921  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2922  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2929  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2930  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2931  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2932  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2934  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2935  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2936  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2937  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2938  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2939 
2940  /* complete request */
2941  tcph.th_ack = htonl(328);
2942  tcph.th_seq = htonl(90);
2943  tcph.th_flags = TH_PUSH | TH_ACK;
2945  p->payload_len = sizeof(request2);
2946  p->payload = request2;
2947  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2954  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2955  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2956  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2957  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2959  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2960  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2961  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2962  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2963  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2964  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2965 
2966  /* response ack against second partial request */
2967  tcph.th_ack = htonl(175);
2968  tcph.th_seq = htonl(328);
2969  tcph.th_flags = TH_ACK;
2971  p->payload_len = 0;
2972  p->payload = NULL;
2973 
2974  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2981  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2982  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2983  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2984  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2986  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2987  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2988  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2989  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2990  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2991  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2992 
2993  /* response acking a request */
2994  tcph.th_ack = htonl(175);
2995  tcph.th_seq = htonl(328);
2996  tcph.th_flags = TH_ACK;
2998  p->payload_len = 0;
2999  p->payload = NULL;
3000  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3006 
3007  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3008  StreamTcpPruneSession(&f, STREAM_TOCLIENT);
3009 
3010  /* request acking a response */
3011  tcph.th_ack = htonl(328);
3012  tcph.th_seq = htonl(175);
3013  tcph.th_flags = TH_ACK;
3015  p->payload_len = 0;
3016  p->payload = NULL;
3017  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3018 
3019  StreamTcpSessionClear(ssn);
3021  SCFree(p);
3022  PASS;
3023 }
3024 
3025 /**
3026  * \test Test to make sure that we sent all the segments from the initial
3027  * segments to app layer until we have detected the app layer proto.
3028  *
3029  * \retval On success it returns 1 and on failure 0.
3030  */
3031 
3032 static int StreamTcpReassembleTest40 (void)
3033 {
3034  Packet *p = PacketGetFromAlloc();
3035  FAIL_IF_NULL(p);
3036  Flow *f = NULL;
3037  TCPHdr tcph;
3038  TcpSession ssn;
3039  memset(&tcph, 0, sizeof (TCPHdr));
3040  ThreadVars tv;
3041  memset(&tv, 0, sizeof (ThreadVars));
3042 
3043  StreamTcpInitConfig(true);
3045 
3047  FAIL_IF_NULL(ra_ctx);
3048 
3049  uint8_t httpbuf1[] = "P";
3050  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3051  uint8_t httpbuf3[] = "O";
3052  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3053  uint8_t httpbuf4[] = "S";
3054  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3055  uint8_t httpbuf5[] = "T \r\n";
3056  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3057 
3058  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3059  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3060 
3061  SET_ISN(&ssn.server, 9);
3062  ssn.server.last_ack = 10;
3063  SET_ISN(&ssn.client, 9);
3064  ssn.client.isn = 9;
3065 
3066  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3067  FAIL_IF_NULL(f);
3068  f->protoctx = &ssn;
3069  f->proto = IPPROTO_TCP;
3070  p->flow = f;
3071 
3072  tcph.th_win = htons(5480);
3073  tcph.th_seq = htonl(10);
3074  tcph.th_ack = htonl(10);
3075  tcph.th_flags = TH_ACK|TH_PUSH;
3076  UTHSetTCPHdr(p, &tcph);
3078  p->payload = httpbuf1;
3079  p->payload_len = httplen1;
3080  ssn.state = TCP_ESTABLISHED;
3081  TcpStream *s = &ssn.client;
3082  SCLogDebug("1 -- start");
3083  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3084 
3086  p->payload = httpbuf2;
3087  p->payload_len = httplen2;
3088  tcph.th_seq = htonl(10);
3089  tcph.th_ack = htonl(11);
3090  s = &ssn.server;
3091  ssn.server.last_ack = 11;
3092  SCLogDebug("2 -- start");
3093  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3094 
3096  p->payload = httpbuf3;
3097  p->payload_len = httplen3;
3098  tcph.th_seq = htonl(11);
3099  tcph.th_ack = htonl(55);
3100  s = &ssn.client;
3101  ssn.client.last_ack = 55;
3102  SCLogDebug("3 -- start");
3103  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3104 
3106  p->payload = httpbuf2;
3107  p->payload_len = httplen2;
3108  tcph.th_seq = htonl(55);
3109  tcph.th_ack = htonl(12);
3110  s = &ssn.server;
3111  ssn.server.last_ack = 12;
3112  SCLogDebug("4 -- start");
3113  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3114 
3115  /* check is have the segment in the list and flagged or not */
3116  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3117  FAIL_IF_NULL(seg);
3118  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
3119 
3121  p->payload = httpbuf4;
3122  p->payload_len = httplen4;
3123  tcph.th_seq = htonl(12);
3124  tcph.th_ack = htonl(100);
3125  s = &ssn.client;
3126  ssn.client.last_ack = 100;
3127  SCLogDebug("5 -- start");
3128  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3129 
3131  p->payload = httpbuf2;
3132  p->payload_len = httplen2;
3133  tcph.th_seq = htonl(100);
3134  tcph.th_ack = htonl(13);
3135  s = &ssn.server;
3136  ssn.server.last_ack = 13;
3137  SCLogDebug("6 -- start");
3138  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3139 
3141  p->payload = httpbuf5;
3142  p->payload_len = httplen5;
3143  tcph.th_seq = htonl(13);
3144  tcph.th_ack = htonl(145);
3145  s = &ssn.client;
3146  ssn.client.last_ack = 145;
3147  SCLogDebug("7 -- start");
3148  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3149 
3151  p->payload = httpbuf2;
3152  p->payload_len = httplen2;
3153  tcph.th_seq = htonl(145);
3154  tcph.th_ack = htonl(16);
3155  s = &ssn.server;
3156  ssn.server.last_ack = 16;
3157  SCLogDebug("8 -- start");
3158  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3159  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3160 
3163  StreamTcpFreeConfig(true);
3164  SCFree(p);
3165  UTHFreeFlow(f);
3166  PASS;
3167 }
3168 
3169 /** \test Test the memcap incrementing/decrementing and memcap check */
3170 static int StreamTcpReassembleTest44(void)
3171 {
3172  StreamTcpInitConfig(true);
3173  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
3174  StreamTcpReassembleIncrMemuse(500);
3175  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
3176  StreamTcpReassembleDecrMemuse(500);
3177  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
3179  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
3180  StreamTcpFreeConfig(true);
3181  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
3182  PASS;
3183 }
3184 
3185 /**
3186  * \test Test to make sure that reassembly_depth is enforced.
3187  *
3188  * \retval On success it returns 1 and on failure 0.
3189  */
3190 
3191 static int StreamTcpReassembleTest45 (void)
3192 {
3193  TcpReassemblyThreadCtx *ra_ctx = NULL;
3194  TcpSession ssn;
3195  ThreadVars tv;
3196  memset(&tv, 0, sizeof(tv));
3197  uint8_t payload[100] = {0};
3198  uint16_t payload_size = 100;
3199 
3200  StreamTcpUTInit(&ra_ctx);
3202 
3204  ssn.reassembly_depth = 100;
3205  StreamTcpUTSetupStream(&ssn.server, 100);
3206  StreamTcpUTSetupStream(&ssn.client, 100);
3207 
3208  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3209  FAIL_IF(r != 0);
3211 
3212  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3213  FAIL_IF(r != 0);
3215 
3219  StreamTcpUTDeinit(ra_ctx);
3220  PASS;
3221 }
3222 
3223 /**
3224  * \test Test the unlimited config value of reassembly depth.
3225  *
3226  * \retval On success it returns 1 and on failure 0.
3227  */
3228 
3229 static int StreamTcpReassembleTest46 (void)
3230 {
3231  int result = 0;
3232  TcpReassemblyThreadCtx *ra_ctx = NULL;
3233  TcpSession ssn;
3234  ThreadVars tv;
3235  memset(&tv, 0, sizeof(tv));
3236  uint8_t payload[100] = {0};
3237  uint16_t payload_size = 100;
3238 
3239  StreamTcpUTInit(&ra_ctx);
3241 
3243  StreamTcpUTSetupStream(&ssn.server, 100);
3244  StreamTcpUTSetupStream(&ssn.client, 100);
3245 
3246  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3247  if (r != 0)
3248  goto end;
3250  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3251  goto end;
3252  }
3253 
3254  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3255  if (r != 0)
3256  goto end;
3258  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3259  goto end;
3260  }
3261 
3262  result = 1;
3263 end:
3267  StreamTcpUTDeinit(ra_ctx);
3268  return result;
3269 }
3270 
3271 /**
3272  * \test Test to make sure we detect the sequence wrap around and continue
3273  * stream reassembly properly.
3274  *
3275  * \retval On success it returns 1 and on failure 0.
3276  */
3277 
3278 static int StreamTcpReassembleTest47 (void)
3279 {
3280  Packet *p = PacketGetFromAlloc();
3281  FAIL_IF(unlikely(p == NULL));
3282  Flow *f = NULL;
3283  TCPHdr tcph;
3284  TcpSession ssn;
3285  ThreadVars tv;
3286  memset(&tcph, 0, sizeof (TCPHdr));
3287  UTHSetTCPHdr(p, &tcph);
3288  memset(&tv, 0, sizeof (ThreadVars));
3289  StreamTcpInitConfig(true);
3292 
3293  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3294  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3295 
3296  SET_ISN(&ssn.server, 572799781UL);
3297  ssn.server.last_ack = 572799782UL;
3298 
3299  SET_ISN(&ssn.client, 4294967289UL);
3300  ssn.client.last_ack = 21;
3301 
3302  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3303  FAIL_IF(f == NULL);
3304  f->protoctx = &ssn;
3305  f->proto = IPPROTO_TCP;
3306  p->flow = f;
3307 
3308  tcph.th_win = htons(5480);
3309  ssn.state = TCP_ESTABLISHED;
3310  TcpStream *s = NULL;
3311  uint8_t cnt = 0;
3312 
3313  for (cnt=0; cnt < httplen1; cnt++) {
3314  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3315  tcph.th_ack = htonl(572799782UL);
3316  tcph.th_flags = TH_ACK | TH_PUSH;
3318  p->payload = &httpbuf1[cnt];
3319  p->payload_len = 1;
3320  s = &ssn.client;
3321 
3322  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3323 
3325  p->payload = NULL;
3326  p->payload_len = 0;
3327  tcph.th_seq = htonl(572799782UL);
3328  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3329  tcph.th_flags = TH_ACK;
3330  s = &ssn.server;
3331 
3332  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3333  }
3334 
3335  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3336 
3339  StreamTcpFreeConfig(true);
3340  SCFree(p);
3341  UTHFreeFlow(f);
3342  PASS;
3343 }
3344 
3345 /** \test 3 in order segments in inline reassembly */
3346 static int StreamTcpReassembleInlineTest01(void)
3347 {
3348  int ret = 0;
3349  TcpReassemblyThreadCtx *ra_ctx = NULL;
3350  ThreadVars tv;
3351  TcpSession ssn;
3352  Flow f;
3353 
3354  memset(&tv, 0x00, sizeof(tv));
3355 
3356  StreamTcpUTInit(&ra_ctx);
3359  StreamTcpUTSetupStream(&ssn.client, 1);
3360  FLOW_INITIALIZE(&f);
3361 
3362  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3363  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3364  if (p == NULL) {
3365  printf("couldn't get a packet: ");
3366  goto end;
3367  }
3368  p->l4.hdrs.tcph->th_seq = htonl(12);
3369  p->flow = &f;
3370 
3371  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3372  printf("failed to add segment 1: ");
3373  goto end;
3374  }
3375  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3376  printf("failed to add segment 2: ");
3377  goto end;
3378  }
3379  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3380  printf("failed to add segment 3: ");
3381  goto end;
3382  }
3383  ssn.client.next_seq = 17;
3384  ret = 1;
3385 end:
3386  FLOW_DESTROY(&f);
3387  UTHFreePacket(p);
3389  StreamTcpUTDeinit(ra_ctx);
3390  return ret;
3391 }
3392 
3393 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3394  * test the sliding window reassembly.
3395  */
3396 static int StreamTcpReassembleInlineTest02(void)
3397 {
3398  int ret = 0;
3399  TcpReassemblyThreadCtx *ra_ctx = NULL;
3400  ThreadVars tv;
3401  TcpSession ssn;
3402  Flow f;
3403 
3404  memset(&tv, 0x00, sizeof(tv));
3405 
3406  StreamTcpUTInit(&ra_ctx);
3409  StreamTcpUTSetupStream(&ssn.client, 1);
3410  FLOW_INITIALIZE(&f);
3411 
3412  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3413  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3414  if (p == NULL) {
3415  printf("couldn't get a packet: ");
3416  goto end;
3417  }
3418  p->l4.hdrs.tcph->th_seq = htonl(12);
3419  p->flow = &f;
3420 
3421  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3422  printf("failed to add segment 1: ");
3423  goto end;
3424  }
3425  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3426  printf("failed to add segment 2: ");
3427  goto end;
3428  }
3429  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3430  printf("failed to add segment 3: ");
3431  goto end;
3432  }
3433  ssn.client.next_seq = 17;
3434  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3435  printf("failed to add segment 4: ");
3436  goto end;
3437  }
3438  ssn.client.next_seq = 22;
3439  ret = 1;
3440 end:
3441  FLOW_DESTROY(&f);
3442  UTHFreePacket(p);
3444  StreamTcpUTDeinit(ra_ctx);
3445  return ret;
3446 }
3447 
3448 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3449  * test the sliding window reassembly with a small window size so that we
3450  * cutting off at the start (left edge)
3451  */
3452 static int StreamTcpReassembleInlineTest03(void)
3453 {
3454  int ret = 0;
3455  TcpReassemblyThreadCtx *ra_ctx = NULL;
3456  ThreadVars tv;
3457  TcpSession ssn;
3458  Flow f;
3459 
3460  memset(&tv, 0x00, sizeof(tv));
3461 
3462  StreamTcpUTInit(&ra_ctx);
3465  StreamTcpUTSetupStream(&ssn.client, 1);
3466  FLOW_INITIALIZE(&f);
3467 
3469 
3470  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3471  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3472  if (p == NULL) {
3473  printf("couldn't get a packet: ");
3474  goto end;
3475  }
3476  p->l4.hdrs.tcph->th_seq = htonl(12);
3477  p->flow = &f;
3479 
3480  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3481  printf("failed to add segment 1: ");
3482  goto end;
3483  }
3484  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3485  printf("failed to add segment 2: ");
3486  goto end;
3487  }
3488  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3489  printf("failed to add segment 3: ");
3490  goto end;
3491  }
3492  ssn.client.next_seq = 17;
3493  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3494  printf("failed to add segment 4: ");
3495  goto end;
3496  }
3497  ssn.client.next_seq = 22;
3498 
3499  p->l4.hdrs.tcph->th_seq = htonl(17);
3500  ret = 1;
3501 end:
3502  FLOW_DESTROY(&f);
3503  UTHFreePacket(p);
3505  StreamTcpUTDeinit(ra_ctx);
3506  return ret;
3507 }
3508 
3509 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3510  * test the sliding window reassembly with a small window size so that we
3511  * cutting off at the start (left edge) with small packet overlap.
3512  */
3513 static int StreamTcpReassembleInlineTest04(void)
3514 {
3515  int ret = 0;
3516  TcpReassemblyThreadCtx *ra_ctx = NULL;
3517  ThreadVars tv;
3518  TcpSession ssn;
3519  Flow f;
3520 
3521  memset(&tv, 0x00, sizeof(tv));
3522 
3523  StreamTcpUTInit(&ra_ctx);
3526  StreamTcpUTSetupStream(&ssn.client, 1);
3527  FLOW_INITIALIZE(&f);
3528 
3530 
3531  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3532  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3533  if (p == NULL) {
3534  printf("couldn't get a packet: ");
3535  goto end;
3536  }
3537  p->l4.hdrs.tcph->th_seq = htonl(12);
3538  p->flow = &f;
3540 
3541  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3542  printf("failed to add segment 1: ");
3543  goto end;
3544  }
3545  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3546  printf("failed to add segment 2: ");
3547  goto end;
3548  }
3549  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3550  printf("failed to add segment 3: ");
3551  goto end;
3552  }
3553  ssn.client.next_seq = 17;
3554  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3555  printf("failed to add segment 4: ");
3556  goto end;
3557  }
3558  ssn.client.next_seq = 22;
3559 
3560  p->l4.hdrs.tcph->th_seq = htonl(17);
3561  ret = 1;
3562 end:
3563  FLOW_DESTROY(&f);
3564  UTHFreePacket(p);
3566  StreamTcpUTDeinit(ra_ctx);
3567  return ret;
3568 }
3569 
3570 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3571  * test the sliding window reassembly with a small window size so that we
3572  * cutting off at the start (left edge). Test if the first segment is
3573  * removed from the list.
3574  */
3575 static int StreamTcpReassembleInlineTest08(void)
3576 {
3577  TcpReassemblyThreadCtx *ra_ctx = NULL;
3578  ThreadVars tv;
3579  memset(&tv, 0x00, sizeof(tv));
3580  TcpSession ssn;
3581  Flow f;
3582  StreamTcpUTInit(&ra_ctx);
3585  StreamTcpUTSetupStream(&ssn.client, 1);
3586  FLOW_INITIALIZE(&f);
3587 
3589  f.protoctx = &ssn;
3590 
3591  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3592  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3593  FAIL_IF(p == NULL);
3594  p->l4.hdrs.tcph->th_seq = htonl(12);
3595  p->flow = &f;
3597 
3598  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3599  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3600  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3601  ssn.client.next_seq = 17;
3602  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3603  ssn.client.next_seq = 22;
3604  p->l4.hdrs.tcph->th_seq = htonl(17);
3605  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3606 
3607  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3608  FAIL_IF_NULL(seg);
3609  FAIL_IF_NOT(seg->seq == 2);
3610 
3611  FLOW_DESTROY(&f);
3612  UTHFreePacket(p);
3614  StreamTcpUTDeinit(ra_ctx);
3615  PASS;
3616 }
3617 
3618 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3619  * test the sliding window reassembly with a small window size so that we
3620  * cutting off at the start (left edge). Test if the first segment is
3621  * removed from the list.
3622  */
3623 static int StreamTcpReassembleInlineTest09(void)
3624 {
3625  int ret = 0;
3626  TcpReassemblyThreadCtx *ra_ctx = NULL;
3627  ThreadVars tv;
3628  TcpSession ssn;
3629  Flow f;
3630 
3631  memset(&tv, 0x00, sizeof(tv));
3632 
3633  StreamTcpUTInit(&ra_ctx);
3636  StreamTcpUTSetupStream(&ssn.client, 1);
3637  FLOW_INITIALIZE(&f);
3638 
3640 
3641  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3642  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3643  if (p == NULL) {
3644  printf("couldn't get a packet: ");
3645  goto end;
3646  }
3647  p->l4.hdrs.tcph->th_seq = htonl(17);
3648  p->flow = &f;
3650 
3651  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3652  printf("failed to add segment 1: ");
3653  goto end;
3654  }
3655  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3656  printf("failed to add segment 2: ");
3657  goto end;
3658  }
3659  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3660  printf("failed to add segment 3: ");
3661  goto end;
3662  }
3663  ssn.client.next_seq = 12;
3664  ssn.client.last_ack = 10;
3665 
3666  /* close the GAP and see if we properly reassemble and update base_seq */
3667  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3668  printf("failed to add segment 4: ");
3669  goto end;
3670  }
3671  ssn.client.next_seq = 22;
3672 
3673  p->l4.hdrs.tcph->th_seq = htonl(12);
3674 
3675  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3676  FAIL_IF_NULL(seg);
3677  FAIL_IF_NOT(seg->seq == 2);
3678 
3679  ret = 1;
3680 end:
3681  FLOW_DESTROY(&f);
3682  UTHFreePacket(p);
3684  StreamTcpUTDeinit(ra_ctx);
3685  return ret;
3686 }
3687 
3688 /** \test App Layer reassembly.
3689  */
3690 static int StreamTcpReassembleInlineTest10(void)
3691 {
3692  int ret = 0;
3693  TcpReassemblyThreadCtx *ra_ctx = NULL;
3694  ThreadVars tv;
3695  TcpSession ssn;
3696  Flow *f = NULL;
3697  Packet *p = NULL;
3698 
3699  memset(&tv, 0x00, sizeof(tv));
3700 
3701  StreamTcpUTInit(&ra_ctx);
3704  StreamTcpUTSetupStream(&ssn.server, 1);
3705  ssn.server.last_ack = 2;
3706  StreamTcpUTSetupStream(&ssn.client, 1);
3707  ssn.client.last_ack = 2;
3708  ssn.data_first_seen_dir = STREAM_TOSERVER;
3709 
3710  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3711  if (f == NULL)
3712  goto end;
3713  f->protoctx = &ssn;
3714  f->proto = IPPROTO_TCP;
3715 
3716  uint8_t stream_payload1[] = "GE";
3717  uint8_t stream_payload2[] = "T /";
3718  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3719 
3720  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3721  if (p == NULL) {
3722  printf("couldn't get a packet: ");
3723  goto end;
3724  }
3725  p->l4.hdrs.tcph->th_seq = htonl(7);
3726  p->flow = f;
3728 
3729  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3730  printf("failed to add segment 1: ");
3731  goto end;
3732  }
3733  ssn.client.next_seq = 4;
3734 
3735  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3736  if (r < 0) {
3737  printf("StreamTcpReassembleAppLayer failed: ");
3738  goto end;
3739  }
3740 
3741  /* ssn.server.ra_app_base_seq should be isn here. */
3742  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3743  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3744  goto end;
3745  }
3746 
3747  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3748  printf("failed to add segment 2: ");
3749  goto end;
3750  }
3751  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3752  printf("failed to add segment 3: ");
3753  goto end;
3754  }
3755  ssn.client.next_seq = 19;
3756 
3757  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3758  if (r < 0) {
3759  printf("StreamTcpReassembleAppLayer failed: ");
3760  goto end;
3761  }
3762 
3763  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3764 
3765  ret = 1;
3766 end:
3767  UTHFreePacket(p);
3769  StreamTcpUTDeinit(ra_ctx);
3770  UTHFreeFlow(f);
3771  return ret;
3772 }
3773 
3774 /** \test test insert with overlap
3775  */
3776 static int StreamTcpReassembleInsertTest01(void)
3777 {
3778  TcpReassemblyThreadCtx *ra_ctx = NULL;
3779  ThreadVars tv;
3780  TcpSession ssn;
3781  Flow f;
3782 
3783  memset(&tv, 0x00, sizeof(tv));
3784 
3785  StreamTcpUTInit(&ra_ctx);
3787  StreamTcpUTSetupStream(&ssn.client, 1);
3789  FLOW_INITIALIZE(&f);
3790 
3791  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3792  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3793  FAIL_IF(p == NULL);
3794  p->l4.hdrs.tcph->th_seq = htonl(12);
3795  p->flow = &f;
3796 
3797  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3798  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3799  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3800  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3801  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3802  ssn.client.next_seq = 21;
3803 
3804  FLOW_DESTROY(&f);
3805  UTHFreePacket(p);
3807  StreamTcpUTDeinit(ra_ctx);
3808  PASS;
3809 }
3810 
3811 /** \test test insert with overlaps
3812  */
3813 static int StreamTcpReassembleInsertTest02(void)
3814 {
3815  int ret = 0;
3816  TcpReassemblyThreadCtx *ra_ctx = NULL;
3817  ThreadVars tv;
3818  TcpSession ssn;
3819 
3820  memset(&tv, 0x00, sizeof(tv));
3821 
3822  StreamTcpUTInit(&ra_ctx);
3824  StreamTcpUTSetupStream(&ssn.client, 1);
3825 
3826  int i;
3827  for (i = 2; i < 10; i++) {
3828  int len;
3829  len = i % 2;
3830  if (len == 0)
3831  len = 1;
3832  int seq;
3833  seq = i * 10;
3834  if (seq < 2)
3835  seq = 2;
3836 
3837  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3838  printf("failed to add segment 1: ");
3839  goto end;
3840  }
3841  }
3842  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3843  printf("failed to add segment 2: ");
3844  goto end;
3845  }
3846 
3847  ret = 1;
3848 end:
3850  StreamTcpUTDeinit(ra_ctx);
3851  return ret;
3852 }
3853 
3854 /** \test test insert with overlaps
3855  */
3856 static int StreamTcpReassembleInsertTest03(void)
3857 {
3858  int ret = 0;
3859  TcpReassemblyThreadCtx *ra_ctx = NULL;
3860  ThreadVars tv;
3861  TcpSession ssn;
3862 
3863  memset(&tv, 0x00, sizeof(tv));
3864 
3865  StreamTcpUTInit(&ra_ctx);
3867  StreamTcpUTSetupStream(&ssn.client, 1);
3868 
3869  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3870  printf("failed to add segment 2: ");
3871  goto end;
3872  }
3873 
3874  int i;
3875  for (i = 2; i < 10; i++) {
3876  int len;
3877  len = i % 2;
3878  if (len == 0)
3879  len = 1;
3880  int seq;
3881  seq = i * 10;
3882  if (seq < 2)
3883  seq = 2;
3884 
3885  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3886  printf("failed to add segment 2: ");
3887  goto end;
3888  }
3889  }
3890  ret = 1;
3891 end:
3893  StreamTcpUTDeinit(ra_ctx);
3894  return ret;
3895 }
3896 
3898 #endif /* UNITTESTS */
3899 
3900 /** \brief The Function Register the Unit tests to test the reassembly engine
3901  * for various OS policies.
3902  */
3903 
3905 {
3906 #ifdef UNITTESTS
3907  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3908  StreamTcpReassembleTest25);
3909  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3910  StreamTcpReassembleTest26);
3911  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3912  StreamTcpReassembleTest27);
3913  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3914  StreamTcpReassembleTest28);
3915  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3916  StreamTcpReassembleTest29);
3917  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3918  StreamTcpReassembleTest33);
3919  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3920  StreamTcpReassembleTest34);
3921  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3922  StreamTcpReassembleTest39);
3923  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3924  StreamTcpReassembleTest40);
3925  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3926  StreamTcpReassembleTest44);
3927  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3928  StreamTcpReassembleTest45);
3929  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3930  StreamTcpReassembleTest46);
3931  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3932  StreamTcpReassembleTest47);
3933 
3934  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3935  StreamTcpReassembleInlineTest01);
3936  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3937  StreamTcpReassembleInlineTest02);
3938  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3939  StreamTcpReassembleInlineTest03);
3940  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3941  StreamTcpReassembleInlineTest04);
3942  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3943  StreamTcpReassembleInlineTest08);
3944  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3945  StreamTcpReassembleInlineTest09);
3946 
3947  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3948  StreamTcpReassembleInlineTest10);
3949 
3950  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3951  StreamTcpReassembleInsertTest01);
3952  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3953  StreamTcpReassembleInsertTest02);
3954  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3955  StreamTcpReassembleInsertTest03);
3956 
3960  StreamTcpReassembleRawRegisterTests();
3961 #endif /* UNITTESTS */
3962 }
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
Definition: stream-tcp.h:143
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:270
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:40
TCP_GET_RAW_SEQ
#define TCP_GET_RAW_SEQ(tcph)
Definition: decode-tcp.h:80
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:44
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:58
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:56
util-device-private.h
UTHCheckGapAtPosition
int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
Definition: stream-tcp-reassemble.c:2266
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:523
TcpStream_
Definition: stream-tcp-private.h:106
len
uint8_t len
Definition: app-layer-dnp3.h:2
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:136
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:186
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:113
TcpReassemblyThreadCtx_::counter_tcp_reas_eps
ExceptionPolicyCounters counter_tcp_reas_eps
Definition: stream-tcp-reassemble.h:69
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:2122
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:166
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:381
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:98
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1906
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:97
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:91
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:94
stream-tcp-inline.h
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:190
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
TCP_STREAM_URGENT_OOB
@ TCP_STREAM_URGENT_OOB
Definition: stream-tcp.h:49
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:69
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
stream-tcp.h
StreamTcpInlineMode
bool StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:7099
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:136
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:2195
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:275
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:626
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1010
StreamTcpThread_
Definition: stream-tcp.h:92
Flow_::proto
uint8_t proto
Definition: flow.h:378
Packet_::payload
uint8_t * payload
Definition: decode.h:605
t_pcapcnt
thread_local uint64_t t_pcapcnt
Definition: stream-tcp-reassemble.c:80
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:2175
action-globals.h
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:62
Packet_::flags
uint32_t flags
Definition: decode.h:544
SCAppLayerParserStateSetFlag
void SCAppLayerParserStateSetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1821
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:110
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:75
Flow_
Flow data structure.
Definition: flow.h:356
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:77
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:75
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
TCPHdr_::th_win
uint16_t th_win
Definition: decode-tcp.h:156
TcpSegmentPcapHdrStorage
struct TcpSegmentPcapHdrStorage_ TcpSegmentPcapHdrStorage
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
StringParseUint16
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:337
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:156
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1397
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:40
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:89
PKT_DROP_REASON_STREAM_REASSEMBLY
@ PKT_DROP_REASON_STREAM_REASSEMBLY
Definition: decode.h:394
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1813
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir app_update_dir)
handle TCP data for the app-layer.
Definition: app-layer.c:711
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:233
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:155
MIN
#define MIN(x, y)
Definition: suricata-common.h:408
SCConfGetBool
int SCConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:497
TcpSession_::urg_offset_ts
uint16_t urg_offset_ts
Definition: stream-tcp-private.h:291
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:75
UTHSetTCPHdr
void UTHSetTCPHdr(Packet *p, TCPHdr *tcph)
Definition: util-unittest-helper.c:136
StreamTcpThreadCacheGetSegment
TcpSegment * StreamTcpThreadCacheGetSegment(void)
Definition: stream-tcp-cache.c:179
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:299
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:52
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:156
STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
@ STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
Definition: decode-events.h:306
TCP_STREAM_URGENT_GAP
@ TCP_STREAM_URGENT_GAP
Definition: stream-tcp.h:50
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1830
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:532
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:219
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:53
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:260
Flow_::protoctx
void * protoctx
Definition: flow.h:441
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:710
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:100
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:606
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:223
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:632
util-unittest.h
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
AppLayerProfilingStore
#define AppLayerProfilingStore(app_tctx, p)
Definition: app-layer.h:128
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2257
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:523
PKT_DROP_REASON_STREAM_URG
@ PKT_DROP_REASON_STREAM_URG
Definition: decode.h:395
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:426
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:170
STREAMTCP_FLAG_LOSSY_BE_LIBERAL
#define STREAMTCP_FLAG_LOSSY_BE_LIBERAL
Definition: stream-tcp-private.h:194
TCPHdr_::th_ack
uint32_t th_ack
Definition: decode-tcp.h:153
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:294
StreamTcpReassembleTriggerRawInspection
void StreamTcpReassembleTriggerRawInspection(TcpSession *ssn, int direction)
Trigger RAW stream inspection.
Definition: stream-tcp-reassemble.c:2158
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:295
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:278
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:115
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:478
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:557
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:488
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:488
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:38
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:295
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:193
StreamTcpReassembleDepthReached
bool StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:625
TCPHdr_::th_flags
uint8_t th_flags
Definition: decode-tcp.h:155
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:463
StreamTcpReassembleRealloc
void * StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
Definition: stream-tcp-reassemble.c:229
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:238
UTHCheckDataAtPosition
int UTHCheckDataAtPosition(TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
Definition: stream-tcp-reassemble.c:2289
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:104
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:279
STREAMTCP_FLAG_ASYNC
#define STREAMTCP_FLAG_ASYNC
Definition: stream-tcp-private.h:182
TCPHdr_::th_seq
uint32_t th_seq
Definition: decode-tcp.h:152
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1260
TcpStreamCnf_::urgent_policy
enum TcpStreamUrgentHandling urgent_policy
Definition: stream-tcp.h:83
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1268
util-exception-policy.h
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
TcpSession_::urg_offset_tc
uint16_t urg_offset_tc
Definition: stream-tcp-private.h:292
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:96
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:535
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:131
ExceptionPolicyCounters_::eps_id
uint16_t eps_id[EXCEPTION_POLICY_MAX]
Definition: util-exception-policy-types.h:53
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
SCEnter
#define SCEnter(...)
Definition: util-debug.h:277
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:5542
StreamNeedsReassembly
uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:977
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1511
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:285
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:137
TCPHdr_::th_urp
uint16_t th_urp
Definition: decode-tcp.h:158
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:105
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2209
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:886
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:112
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:164
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:317
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:341
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:351
SCReturn
#define SCReturn
Definition: util-debug.h:279
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:303
Packet_
Definition: decode.h:501
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:234
stream-tcp-private.h
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:54
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:283
TcpReassemblyThreadCtx_::counter_tcp_urgent_oob
uint16_t counter_tcp_urgent_oob
Definition: stream-tcp-reassemble.h:88
conf.h
Packet_::l4
struct PacketL4 l4
Definition: decode.h:601
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:301
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1853
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:117
STREAMTCP_STREAM_FLAG_HAS_GAP
#define STREAMTCP_STREAM_FLAG_HAS_GAP
Definition: stream-tcp-private.h:217
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:596
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:293
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:722
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(void)
Creates a new app layer thread context.
Definition: app-layer.c:1108
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:398
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:234
TcpStreamCnf_::reassembly_memcap_policy
enum ExceptionPolicy reassembly_memcap_policy
Definition: stream-tcp.h:81
StreamTcpThreadCacheCleanup
void StreamTcpThreadCacheCleanup(void)
Definition: stream-tcp-cache.c:134
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:270
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:539
TH_URG
#define TH_URG
Definition: decode-tcp.h:39
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:225
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:77
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:128
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:104
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
VALIDATE
#define VALIDATE(e)
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:493
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:57
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a TCP packet data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:746
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:64
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
TcpSegmentPcapHdrStorage_
Definition: stream-tcp-private.h:65
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1720
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
app-layer-frames.h
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:180
Packet_::flow
struct Flow_ * flow
Definition: decode.h:546
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:260
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:300
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:859
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
SCNtohs
#define SCNtohs(x)
Definition: suricata-common.h:431
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:447
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:259
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:634
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:124
packet.h
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:36
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:94
TcpStreamCnf_::urgent_oob_limit_policy
enum TcpStreamUrgentHandling urgent_oob_limit_policy
Definition: stream-tcp.h:84
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2238
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:135
TcpSegmentPcapHdrStorage_::alloclen
uint32_t alloclen
Definition: stream-tcp-private.h:68
ACTION_DROP
#define ACTION_DROP
Definition: action-globals.h:30
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:535
StreamTcpThreadCacheReturnSegment
void StreamTcpThreadCacheReturnSegment(TcpSegment *seg)
Definition: stream-tcp-cache.c:53
STREAM_REASSEMBLY_DEPTH_REACHED
@ STREAM_REASSEMBLY_DEPTH_REACHED
Definition: decode-events.h:302
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:236
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:297
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Definition: stream-tcp-reassemble.c:2002
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:161
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TCP_STREAM_URGENT_DROP
@ TCP_STREAM_URGENT_DROP
Definition: stream-tcp.h:48
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:78
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:114
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:258
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1557
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
PacketL4::L4Hdrs::tcph
TCPHdr * tcph
Definition: decode.h:469
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:296
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:181
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:267
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:451
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:473
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayerGetCtxThread().
Definition: app-layer.c:1129
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:257
Flow_::flags
uint32_t flags
Definition: flow.h:421
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:1022
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:71
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:53
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:150
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:307
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2252
stream-tcp-cache.h
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:158
PacketL4::hdrs
union PacketL4::L4Hdrs hdrs
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:117
MISSED_ADD_PAYLOAD
#define MISSED_ADD_PAYLOAD(seq, seg, seglen)
Definition: stream-tcp-reassemble.c:2261
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:61
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:69
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:589
StreamDataRightEdge
uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:417
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:174
PacketDrop
void PacketDrop(Packet *p, const uint8_t action, enum PacketDropReason r)
issue drop action
Definition: packet.c:33
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:219
suricata.h
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap(void)
Return memcap value.
Definition: stream-tcp-reassemble.c:200
Address_::family
char family
Definition: decode.h:113
Packet_::dst
Address dst
Definition: decode.h:506
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:7112
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:258
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool
uint16_t counter_tcp_segment_from_pool
Definition: stream-tcp-reassemble.h:72
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
TCP_STREAM_URGENT_INLINE
@ TCP_STREAM_URGENT_INLINE
Definition: stream-tcp.h:46
likely
#define likely(expr)
Definition: util-optimize.h:32
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3904
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:102
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
segment_thread_pool
PoolThread * segment_thread_pool
Definition: stream-tcp-reassemble.c:82
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:196
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TCPSEG_PKT_HDR_DEFAULT_SIZE
#define TCPSEG_PKT_HDR_DEFAULT_SIZE
Definition: stream-tcp-private.h:58
TcpSession_
Definition: stream-tcp-private.h:283
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:288
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:185
flow.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:243
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:452
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
StreamReassembleLog
int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, uint64_t *progress_out, const bool eof)
Definition: stream-tcp-reassemble.c:1971
ExceptionPolicy
ExceptionPolicy
Definition: util-exception-policy-types.h:25
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:112
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:225
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:149
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:281
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
SCConfNode_
Definition: conf.h:37
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
SCConfNode_::val
char * val
Definition: conf.h:39
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamReassembleForFrame
int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof)
Definition: stream-tcp-reassemble.c:1894
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:67
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1684
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:77
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:119
TCPHdr_
Definition: decode-tcp.h:149
Packet_::src
Address src
Definition: decode.h:505
TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache
uint16_t counter_tcp_segment_from_cache
Definition: stream-tcp-reassemble.h:71
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:70
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:232
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:146
app-layer.h