suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2024 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "packet.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 #include "action-globals.h"
37 
38 #include "flow-util.h"
39 
40 #include "threadvars.h"
41 #include "tm-threads.h"
42 
43 #include "util-pool.h"
44 #include "util-unittest.h"
45 #include "util-print.h"
46 #include "util-host-os-info.h"
47 #include "util-unittest-helper.h"
48 #include "util-byte.h"
49 #include "util-device-private.h"
50 
51 #include "stream-tcp.h"
52 #include "stream-tcp-private.h"
53 #include "stream-tcp-cache.h"
54 #include "stream-tcp-reassemble.h"
55 #include "stream-tcp-inline.h"
56 #include "stream-tcp-list.h"
57 #include "stream-tcp-util.h"
58 
59 #include "stream.h"
60 
61 #include "util-debug.h"
62 #include "app-layer-protos.h"
63 #include "app-layer.h"
64 #include "app-layer-events.h"
65 #include "app-layer-parser.h"
66 #include "app-layer-frames.h"
67 
68 #include "detect-engine-state.h"
69 
70 #include "util-profiling.h"
71 #include "util-validate.h"
72 #include "util-exception-policy.h"
73 
74 #ifdef DEBUG
75 static SCMutex segment_pool_memuse_mutex;
76 static uint64_t segment_pool_memuse = 0;
77 static uint64_t segment_pool_memcnt = 0;
78 #endif
79 
80 thread_local uint64_t t_pcapcnt = UINT64_MAX;
81 
83 /* init only, protect initializing and growing pool */
84 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
85 
86 /* Memory use counter */
87 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
88 
89 static int g_tcp_session_dump_enabled = 0;
90 
91 inline bool IsTcpSessionDumpingEnabled(void)
92 {
93  return g_tcp_session_dump_enabled == 1;
94 }
95 
97 {
98  g_tcp_session_dump_enabled = 1;
99 }
100 
101 /* prototypes */
103 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
104 
106 {
107  SC_ATOMIC_INIT(ra_memuse);
108 }
109 
110 /**
111  * \brief Function to Increment the memory usage counter for the TCP reassembly
112  * segments
113  *
114  * \param size Size of the TCP segment and its payload length memory allocated
115  */
116 static void StreamTcpReassembleIncrMemuse(uint64_t size)
117 {
118  (void) SC_ATOMIC_ADD(ra_memuse, size);
119  SCLogDebug("REASSEMBLY %" PRIu64 ", incr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
120  size);
121 }
122 
123 /**
124  * \brief Function to Decrease the memory usage counter for the TCP reassembly
125  * segments
126  *
127  * \param size Size of the TCP segment and its payload length memory allocated
128  */
129 static void StreamTcpReassembleDecrMemuse(uint64_t size)
130 {
131 #ifdef UNITTESTS
132  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
133  if (RunmodeIsUnittests()) {
134  BUG_ON(presize > UINT_MAX);
135  }
136 #endif
137 
138  (void) SC_ATOMIC_SUB(ra_memuse, size);
139 
140 #ifdef UNITTESTS
141  if (RunmodeIsUnittests()) {
142  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
143  BUG_ON(postsize > presize);
144  }
145 #endif
146  SCLogDebug("REASSEMBLY %" PRIu64 ", decr %" PRIu64, StreamTcpReassembleMemuseGlobalCounter(),
147  size);
148 }
149 
151 {
152  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
153  return smemuse;
154 }
155 
156 /**
157  * \brief Function to Check the reassembly memory usage counter against the
158  * allowed max memory usage for TCP segments.
159  *
160  * \param size Size of the TCP segment and its payload length memory allocated
161  * \retval 1 if in bounds
162  * \retval 0 if not in bounds
163  */
165 {
166 #ifdef DEBUG
167  if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX &&
168  g_eps_stream_reassembly_memcap == t_pcapcnt))) {
169  SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt);
170  return 0;
171  }
172 #endif
173  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
174  if (memcapcopy == 0 ||
175  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
176  return 1;
177  return 0;
178 }
179 
180 /**
181  * \brief Update memcap value
182  *
183  * \param size new memcap value
184  */
186 {
187  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
188  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
189  return 1;
190  }
191 
192  return 0;
193 }
194 
195 /**
196  * \brief Return memcap value
197  *
198  * \return memcap memcap value
199  */
201 {
202  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
203  return memcapcopy;
204 }
205 
206 /* memory functions for the streaming buffer API */
207 
208 /*
209  void *(*Calloc)(size_t n, size_t size);
210 */
211 static void *ReassembleCalloc(size_t n, size_t size)
212 {
213  if (StreamTcpReassembleCheckMemcap(n * size) == 0) {
215  return NULL;
216  }
217  void *ptr = SCCalloc(n, size);
218  if (ptr == NULL) {
220  return NULL;
221  }
222  StreamTcpReassembleIncrMemuse(n * size);
223  return ptr;
224 }
225 
226 /*
227  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
228 */
229 void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
230 {
231  if (size > orig_size) {
232  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) {
233  SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap));
235  return NULL;
236  }
237  }
238  void *nptr = SCRealloc(optr, size);
239  if (nptr == NULL) {
240  SCLogDebug("realloc fail");
242  return NULL;
243  }
244  if (size > orig_size) {
245  StreamTcpReassembleIncrMemuse(size - orig_size);
246  } else {
247  StreamTcpReassembleDecrMemuse(orig_size - size);
248  }
249  return nptr;
250 }
251 
252 /*
253  void (*Free)(void *ptr, size_t size);
254 */
255 static void ReassembleFree(void *ptr, size_t size)
256 {
257  SCFree(ptr);
258  StreamTcpReassembleDecrMemuse(size);
259 }
260 
261 /** \brief alloc a tcp segment pool entry */
262 static void *TcpSegmentPoolAlloc(void)
263 {
264  SCLogDebug("segment alloc");
265  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
266  return NULL;
267  }
268 
269  TcpSegment *seg = NULL;
270 
271  seg = SCMalloc(sizeof (TcpSegment));
272  if (unlikely(seg == NULL))
273  return NULL;
274 
276  uint32_t memuse =
277  sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
278  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
279  SCFree(seg);
280  return NULL;
281  }
282 
284  if (seg->pcap_hdr_storage == NULL) {
285  SCLogError("Unable to allocate memory for "
286  "TcpSegmentPcapHdrStorage");
287  SCFree(seg);
288  return NULL;
289  } else {
290  seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
291  seg->pcap_hdr_storage->pkt_hdr =
292  SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
293  if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
294  SCLogError("Unable to allocate memory for "
295  "packet header data within "
296  "TcpSegmentPcapHdrStorage");
297  SCFree(seg->pcap_hdr_storage);
298  SCFree(seg);
299  return NULL;
300  }
301  }
302 
303  StreamTcpReassembleIncrMemuse(memuse);
304  } else {
305  seg->pcap_hdr_storage = NULL;
306  }
307 
308  return seg;
309 }
310 
311 static int TcpSegmentPoolInit(void *data, void *initdata)
312 {
313  TcpSegment *seg = (TcpSegment *) data;
314  TcpSegmentPcapHdrStorage *pcap_hdr;
315 
316  pcap_hdr = seg->pcap_hdr_storage;
317 
318  /* do this before the can bail, so TcpSegmentPoolCleanup
319  * won't have uninitialized memory to consider. */
320  memset(seg, 0, sizeof (TcpSegment));
321 
323  uint32_t memuse =
324  sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
325  seg->pcap_hdr_storage = pcap_hdr;
326  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
327  return 0;
328  }
329  StreamTcpReassembleIncrMemuse(memuse);
330  } else {
331  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
332  return 0;
333  }
334  }
335 
336 #ifdef DEBUG
337  SCMutexLock(&segment_pool_memuse_mutex);
338  segment_pool_memuse += sizeof(TcpSegment);
339  segment_pool_memcnt++;
340  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
341  SCMutexUnlock(&segment_pool_memuse_mutex);
342 #endif
343 
344  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
345  return 1;
346 }
347 
348 /** \brief clean up a tcp segment pool entry */
349 static void TcpSegmentPoolCleanup(void *ptr)
350 {
351  if (ptr == NULL)
352  return;
353 
354  TcpSegment *seg = (TcpSegment *)ptr;
355  if (seg && seg->pcap_hdr_storage) {
356  if (seg->pcap_hdr_storage->pkt_hdr) {
358  StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen);
359  }
360  SCFree(seg->pcap_hdr_storage);
361  seg->pcap_hdr_storage = NULL;
362  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage));
363  }
364 
365  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
366 
367 #ifdef DEBUG
368  SCMutexLock(&segment_pool_memuse_mutex);
369  segment_pool_memuse -= sizeof(TcpSegment);
370  segment_pool_memcnt--;
371  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
372  SCMutexUnlock(&segment_pool_memuse_mutex);
373 #endif
374 }
375 
376 /**
377  * \brief Function to return the segment back to the pool.
378  *
379  * \param seg Segment which will be returned back to the pool.
380  */
382 {
383  if (seg == NULL)
384  return;
385 
386  if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
387  seg->pcap_hdr_storage->pktlen = 0;
388  }
389 
391 }
392 
393 /**
394  * \brief return all segments in this stream into the pool(s)
395  *
396  * \param stream the stream to cleanup
397  */
399 {
400  TcpSegment *seg = NULL, *safe = NULL;
401  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
402  {
403  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
405  }
406 }
407 
408 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
409 {
410  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
411  return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq);
412  }
413  return STREAM_BASE_OFFSET(stream);
414 }
415 
416 // may contain gaps
417 uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
418 {
419  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
420  if (!eof && !StreamTcpInlineMode()) {
421  right_edge = MIN(GetAbsLastAck(stream), right_edge);
422  }
423  return right_edge;
424 }
425 
426 uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
427 {
428  uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
429  if (!eof && !StreamTcpInlineMode()) {
430  right_edge = MIN(GetAbsLastAck(stream), right_edge);
431  }
432  return right_edge;
433 }
434 
435 #ifdef UNITTESTS
436 /** \internal
437  * \brief check if segments falls before stream 'offset' */
438 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
439 {
440  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
441  return 1;
442  return 0;
443 }
444 #endif
445 
446 /** \param f locked flow */
448 {
449  if (f->protoctx == NULL)
450  return;
451 
452  TcpSession *ssn = (TcpSession *)f->protoctx;
456  if (f->alparser) {
459  }
460 }
461 
462 /** \param f locked flow */
464 {
465  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
466  return 0;
467 
468  TcpSession *ssn = (TcpSession *)f->protoctx;
470 }
471 
472 static int StreamTcpReassemblyConfig(bool quiet)
473 {
474  uint32_t segment_prealloc = 2048;
475  SCConfNode *seg = SCConfGetNode("stream.reassembly.segment-prealloc");
476  if (seg) {
477  uint32_t prealloc = 0;
478  if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) {
479  SCLogError("segment-prealloc of "
480  "%s is invalid",
481  seg->val);
482  return -1;
483  }
484  segment_prealloc = prealloc;
485  }
486  if (!quiet)
487  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
488  stream_config.prealloc_segments = segment_prealloc;
489 
490  int overlap_diff_data = 0;
491  (void)SCConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
492  if (overlap_diff_data) {
494  }
495  if (StreamTcpInlineMode()) {
497  }
498 
499  uint16_t max_regions = 8;
500  SCConfNode *mr = SCConfGetNode("stream.reassembly.max-regions");
501  if (mr) {
502  uint16_t max_r = 0;
503  if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) {
504  SCLogError("max-regions %s is invalid", mr->val);
505  return -1;
506  }
507  max_regions = max_r;
508  }
509  if (!quiet)
510  SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions);
511 
512  stream_config.prealloc_segments = segment_prealloc;
514  stream_config.sbcnf.max_regions = max_regions;
516  stream_config.sbcnf.Calloc = ReassembleCalloc;
518  stream_config.sbcnf.Free = ReassembleFree;
519 
520  return 0;
521 }
522 
523 int StreamTcpReassembleInit(bool quiet)
524 {
525  /* init the memcap/use tracker */
527 
528  if (StreamTcpReassemblyConfig(quiet) < 0)
529  return -1;
530 
531 #ifdef DEBUG
532  SCMutexInit(&segment_pool_memuse_mutex, NULL);
533 #endif
534  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
536  return 0;
537 }
538 
539 void StreamTcpReassembleFree(bool quiet)
540 {
541  SCMutexLock(&segment_thread_pool_mutex);
542  if (segment_thread_pool != NULL) {
544  segment_thread_pool = NULL;
545  }
546  SCMutexUnlock(&segment_thread_pool_mutex);
547  SCMutexDestroy(&segment_thread_pool_mutex);
548 
549 #ifdef DEBUG
550  if (segment_pool_memuse > 0)
551  SCLogDebug("segment_pool_memuse %" PRIu64 " segment_pool_memcnt %" PRIu64 "",
552  segment_pool_memuse, segment_pool_memcnt);
553  SCMutexDestroy(&segment_pool_memuse_mutex);
554 #endif
555 }
556 
558 {
559  SCEnter();
561  if (unlikely(ra_ctx == NULL))
562  return NULL;
563 
564  ra_ctx->app_tctx = AppLayerGetCtxThread();
565 
566  SCMutexLock(&segment_thread_pool_mutex);
567  if (segment_thread_pool == NULL) {
568  segment_thread_pool = PoolThreadInit(1, /* thread */
569  0, /* unlimited */
571  sizeof(TcpSegment),
572  TcpSegmentPoolAlloc,
573  TcpSegmentPoolInit, NULL,
574  TcpSegmentPoolCleanup, NULL);
575  ra_ctx->segment_thread_pool_id = 0;
576  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
578  ra_ctx->segment_thread_pool_id);
579  } else {
580  /* grow segment_thread_pool until we have an element for our thread id */
582  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
584  ra_ctx->segment_thread_pool_id);
585  }
586  SCMutexUnlock(&segment_thread_pool_mutex);
587  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
588  SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
590  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
591  }
592 
593  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
594 }
595 
597 {
598  SCEnter();
600 
601  if (ra_ctx) {
603  SCFree(ra_ctx);
604  }
605  SCReturn;
606 }
607 
608 static void StreamTcpReassembleExceptionPolicyStatsIncr(
609  ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, enum ExceptionPolicy policy)
610 {
611  uint16_t id = ra_ctx->counter_tcp_reas_eps.eps_id[policy];
612  if (likely(tv && id > 0)) {
613  StatsIncr(tv, id);
614  }
615 }
616 
617 /**
618  * \brief check if stream in pkt direction has depth reached
619  *
620  * \param p packet with *LOCKED* flow
621  *
622  * \retval true stream has depth reached
623  * \retval false stream does not have depth reached
624  */
626 {
627  if (p->flow != NULL && p->flow->protoctx != NULL) {
628  TcpSession *ssn = p->flow->protoctx;
629  TcpStream *stream;
630  if (p->flowflags & FLOW_PKT_TOSERVER) {
631  stream = &ssn->client;
632  } else {
633  stream = &ssn->server;
634  }
635 
636  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? true : false;
637  }
638 
639  return false;
640 }
641 
642 /**
643  * \internal
644  * \brief Function to check the reassembly depth value against the
645  * allowed max depth of the stream reassembly for TCP streams.
646  *
647  * \param stream stream direction
648  * \param seq sequence number where "size" starts
649  * \param size size of the segment that is added
650  *
651  * \retval size Part of the size that fits in the depth, 0 if none
652  */
653 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
654  uint32_t seq, uint32_t size)
655 {
656  SCEnter();
657 
658  /* if the configured depth value is 0, it means there is no limit on
659  reassembly depth. Otherwise carry on my boy ;) */
660  if (ssn->reassembly_depth == 0) {
661  SCReturnUInt(size);
662  }
663 
664  /* if the final flag is set, we're not accepting anymore */
666  SCReturnUInt(0);
667  }
668 
669  uint64_t seg_depth;
670  if (SEQ_GT(stream->base_seq, seq)) {
671  if (SEQ_LEQ(seq+size, stream->base_seq)) {
672  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
673  stream->base_seq, seq, seq+size);
674  SCReturnUInt(0);
675  }
676 
677  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
678  } else {
679  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
680  }
681 
682  /* if the base_seq has moved past the depth window we stop
683  * checking and just reject the rest of the packets including
684  * retransmissions. Saves us the hassle of dealing with sequence
685  * wraps as well */
686  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
687  stream->base_seq, seg_depth,
688  ssn->reassembly_depth);
689 
690  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
691  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
693  SCReturnUInt(0);
694  }
695  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
696  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
697 #if 0
698  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
699  (stream->base_seq_offset + stream->base_seq + size),
700  (stream->isn + ssn->reassembly_depth));
701 #endif
702  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
703  /* packet (partly?) fits the depth window */
704 
705  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
706  /* complete fit */
707  SCReturnUInt(size);
708  } else {
710  /* partial fit, return only what fits */
711  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
712  DEBUG_VALIDATE_BUG_ON(part > size);
713  if (part > size)
714  part = size;
715  SCReturnUInt(part);
716  }
717  }
718 
719  SCReturnUInt(0);
720 }
721 
723 {
724  if (RB_EMPTY(&stream->sb.sbb_tree)) {
725  if (stream->sb.region.stream_offset != 0)
726  return 0;
727 
728  return stream->sb.region.buf_offset;
729  }
730 
731  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
732  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
733  return stream->sb.sbb_size;
734 }
735 
736 /**
737  * \brief Insert a TCP packet data into the stream reassembly engine.
738  *
739  * \retval 0 good segment, as far as we checked.
740  * \retval -1 insert failure due to memcap
741  *
742  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
743  * or it wasn't added because of reassembly depth.
744  *
745  */
747  TcpSession *ssn, TcpStream *stream, Packet *p)
748 {
749  SCEnter();
750 
751  if (ssn->data_first_seen_dir == 0) {
752  if (PKT_IS_TOSERVER(p)) {
753  ssn->data_first_seen_dir = STREAM_TOSERVER;
754  } else {
755  ssn->data_first_seen_dir = STREAM_TOCLIENT;
756  }
757  }
758 
759  /* If the OS policy is not set then set the OS policy for this stream */
760  if (stream->os_policy == 0) {
761  StreamTcpSetOSPolicy(stream, p);
762  }
763 
766  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
767  SCReturnInt(0);
768  }
769 
770  uint16_t *urg_offset;
771  if (PKT_IS_TOSERVER(p)) {
772  urg_offset = &ssn->urg_offset_ts;
773  } else {
774  urg_offset = &ssn->urg_offset_tc;
775  }
776 
777  const TCPHdr *tcph = PacketGetTCP(p);
778  /* segment sequence number, offset by previously accepted
779  * URG OOB data. */
780  uint32_t seg_seq = TCP_GET_RAW_SEQ(tcph) - (*urg_offset);
781  uint8_t urg_data = 0;
782 
783  /* if stream_config.urgent_policy == TCP_STREAM_URGENT_DROP, we won't get here */
784  if (tcph->th_flags & TH_URG) {
785  const uint16_t urg_ptr = SCNtohs(tcph->th_urp);
786  if (urg_ptr > 0 && urg_ptr <= p->payload_len &&
789  /* track up to 64k out of band URG bytes. Fall back to inline
790  * when that budget is exceeded. */
791  if ((*urg_offset) < UINT16_MAX) {
793  (*urg_offset)++;
794 
795  if ((*urg_offset) == UINT16_MAX) {
797  }
798  } else {
799  /* OOB limit DROP is handled here */
802  SCReturnInt(0);
803  }
804  }
805  urg_data = 1; /* only treat last 1 byte as out of band. */
808  }
809 
810  /* depending on hitting the OOB limit, update urg_data or not */
812  (*urg_offset) == UINT16_MAX &&
814  urg_data = 0;
815  } else {
816  if (urg_ptr == 1 && p->payload_len == 1) {
817  SCLogDebug("no non-URG data");
818  SCReturnInt(0);
819  }
820  }
821  }
822  }
823 
824  const uint16_t payload_len = p->payload_len - urg_data;
825 
826  /* If we have reached the defined depth for either of the stream, then stop
827  reassembling the TCP session */
828  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, seg_seq, payload_len);
829  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
830 
833  /* increment stream depth counter */
836  }
837  if (size == 0) {
838  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
839  SCReturnInt(0);
840  }
841 
843  if (size > payload_len)
844  size = payload_len;
845 
846  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
847  if (seg == NULL) {
848  SCLogDebug("segment_pool is empty");
851  SCReturnInt(-1);
852  }
853 
854  DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
855  TCP_SEG_LEN(seg) = (uint16_t)size;
856  /* set SEQUENCE number, adjusted to any URG pointer offset */
857  seg->seq = seg_seq;
858 
859  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
860  if (TCP_HAS_TFO(p) && p->payload_len && (tcph->th_flags & TH_SYN))
861  seg->seq += 1;
862 
863  /* proto detection skipped, but now we do get data. Set event. */
864  if (RB_EMPTY(&stream->seg_tree) &&
866 
868  }
869 
870  int r = StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, p->payload, payload_len);
871  if (r < 0) {
872  if (r == -SC_ENOMEM) {
874  }
875  SCLogDebug("StreamTcpReassembleInsertSegment failed");
876  SCReturnInt(-1);
877  }
878  SCReturnInt(0);
879 }
880 
881 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
882  Packet *p)
883 {
884  uint8_t flag = 0;
885 
887  flag |= STREAM_START;
888  }
889 
890  if (ssn->state == TCP_CLOSED) {
891  flag |= STREAM_EOF;
892  }
893 
894  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
895  flag |= STREAM_MIDSTREAM;
896  }
897 
898  if (p->flags & PKT_PSEUDO_STREAM_END) {
899  flag |= STREAM_EOF;
900  }
901 
902  if (&ssn->client == stream) {
903  flag |= STREAM_TOSERVER;
904  } else {
905  flag |= STREAM_TOCLIENT;
906  }
908  flag |= STREAM_DEPTH;
909  }
910  return flag;
911 }
912 
913 /**
914  * \brief Check the minimum size limits for reassembly.
915  *
916  * \retval false don't reassemble yet
917  * \retval true do reassemble
918  */
919 static bool StreamTcpReassembleRawCheckLimit(
920  const TcpSession *ssn, const TcpStream *stream, const Packet *p)
921 {
922  SCEnter();
923 
924  /* if any of these flags is set we always inspect immediately */
925 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
926  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
927  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
928  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
929 
932  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
933  "is set, so not expecting any new data segments");
934  }
936  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
937  }
939  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
940  "so no new segments will be considered");
941  }
942  SCReturnBool(true);
943  }
944 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
945 
946  /* some states mean we reassemble no matter how much data we have */
947  if (ssn->state > TCP_TIME_WAIT)
948  SCReturnBool(true);
949 
950  if (p->flags & PKT_PSEUDO_STREAM_END)
951  SCReturnBool(true);
952 
953  const uint64_t last_ack_abs = GetAbsLastAck(stream);
954  int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream);
955  int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size
957 
958  /* check if we have enough data to do raw reassembly */
959  if (chunk_size <= diff) {
960  SCReturnBool(true);
961  } else {
962  SCLogDebug("%s min chunk len not yet reached: "
963  "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < "
964  "%" PRIi64,
965  PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq,
966  (stream->last_ack - stream->base_seq), chunk_size);
967  SCReturnBool(false);
968  }
969 
970  SCReturnBool(false);
971 }
972 
973 /**
974  * \brief see what if any work the TCP session still needs
975  */
976 uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
977 {
978  const TcpStream *stream = NULL;
979 #ifdef DEBUG
980  const char *dirstr = NULL;
981 #endif
982  if (direction == STREAM_TOSERVER) {
983  stream = &ssn->client;
984 #ifdef DEBUG
985  dirstr = "client";
986 #endif
987  } else {
988  stream = &ssn->server;
989 #ifdef DEBUG
990  dirstr = "server";
991 #endif
992  }
993  int use_app = 1;
994  int use_raw = 1;
995 
997  // app is dead
998  use_app = 0;
999  }
1000 
1001  if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
1002  // raw is dead
1003  use_raw = 0;
1004  }
1005  if (use_raw) {
1006  const uint64_t right_edge =
1007  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1008  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1009  " (use: %s). Stream right edge: %" PRIu64,
1010  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1011  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1012  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
1013  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1015  }
1016  }
1017  if (use_app) {
1018  const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
1019  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
1020  " (use: %s). Stream right edge: %" PRIu64,
1021  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
1022  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
1023  if (right_edge > STREAM_APP_PROGRESS(stream)) {
1024  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
1026  }
1027  }
1028 
1029  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
1031 }
1032 
1033 #ifdef DEBUG
1034 static uint64_t GetStreamSize(TcpStream *stream)
1035 {
1036  if (stream) {
1037  uint64_t size = 0;
1038  uint32_t cnt = 0;
1039  uint64_t last_ack_abs = GetAbsLastAck(stream);
1040  uint64_t last_re = 0;
1041 
1042  SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset);
1043 
1044  TcpSegment *seg;
1045  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
1046  const uint64_t seg_abs =
1047  STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq);
1048  if (last_re != 0 && last_re < seg_abs) {
1049  const char *gacked = NULL;
1050  if (last_ack_abs >= seg_abs) {
1051  gacked = "fully ack'd";
1052  } else if (last_ack_abs > last_re) {
1053  gacked = "partly ack'd";
1054  } else {
1055  gacked = "not yet ack'd";
1056  }
1057  SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked);
1058  }
1059 
1060  const char *acked = NULL;
1061  if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) {
1062  acked = "fully ack'd";
1063  } else if (last_ack_abs > seg_abs) {
1064  acked = "partly ack'd";
1065  } else {
1066  acked = "not yet ack'd";
1067  }
1068 
1069  SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64
1070  ") ack:%s",
1071  cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg),
1072  seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked);
1073  last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg);
1074  cnt++;
1075  size += (uint64_t)TCP_SEG_LEN(seg);
1076  }
1077 
1078  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
1079  return size;
1080  }
1081  return (uint64_t)0;
1082 }
1083 
1084 static void GetSessionSize(TcpSession *ssn, Packet *p)
1085 {
1086  uint64_t size = 0;
1087  if (ssn) {
1088  size = GetStreamSize(&ssn->client);
1089  size += GetStreamSize(&ssn->server);
1090 
1091  //if (size > 900000)
1092  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1093  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1094  }
1095 }
1096 #endif
1097 
1098 static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk)
1099 {
1100  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
1101  /* only if the stream has been ack'd, consider a gap for sure
1102  * otherwise there may still be a chance of pkts coming in */
1103  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
1104  GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len)) {
1105  return true;
1106  }
1107  return false;
1108 }
1109 
1110 /** \internal
1111  *
1112  * Get buffer, or first part of the buffer if data gaps exist.
1113  *
1114  * \brief get stream data from offset
1115  * \param offset stream offset
1116  * \param check_for_gap check if there is a gap ahead. Optional as it is only
1117  * needed for app-layer incomplete support.
1118  * \retval bool pkt loss ahead */
1119 static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1120  uint64_t offset, const bool check_for_gap)
1121 {
1122  const uint8_t *mydata;
1123  uint32_t mydata_len;
1124  bool gap_ahead = false;
1125 
1126  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1127  SCLogDebug("getting one blob");
1128 
1129  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1130 
1131  *data = mydata;
1132  *data_len = mydata_len;
1133  } else {
1134  SCLogDebug("block mode");
1135  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1136  StreamingBufferBlock *blk = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1137  if (blk == NULL) {
1138  *data = NULL;
1139  *data_len = 0;
1140  return false;
1141  }
1142  SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len);
1143 
1144  /* block at expected offset */
1145  if (blk->offset == offset) {
1146  SCLogDebug("blk at offset");
1147 
1148  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
1149  DEBUG_VALIDATE_BUG_ON(blk->len != *data_len);
1150 
1151  gap_ahead = check_for_gap && GapAhead(stream, blk);
1152 
1153  /* block past out offset */
1154  } else if (blk->offset > offset) {
1155  SCLogDebug("gap, want data at offset %"PRIu64", "
1156  "got data at %"PRIu64". GAP of size %"PRIu64,
1157  offset, blk->offset, blk->offset - offset);
1158  *data = NULL;
1159  *data_len = (uint32_t)(blk->offset - offset);
1160 
1161  /* block starts before offset, but ends after */
1162  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
1163  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
1164  offset, blk->offset, blk->len);
1165  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
1166  SCLogDebug("data %p, data_len %u", *data, *data_len);
1167 
1168  gap_ahead = check_for_gap && GapAhead(stream, blk);
1169 
1170  } else {
1171  *data = NULL;
1172  *data_len = 0;
1173  }
1174  }
1175  return gap_ahead;
1176 }
1177 
1178 /** \internal
1179  * \brief check to see if we should declare a GAP
1180  * Call this when the app layer didn't get data at the requested
1181  * offset.
1182  */
1183 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
1184 {
1185  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1186  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1187  const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded;
1188 
1189  SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs);
1190  SCLogDebug("next_seq %u", stream->next_seq);
1191 
1192  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1193  * has been ack'd. This looks like a GAP. */
1194  if (last_ack_abs > app_progress) {
1195  /* however, we can accept ACKs a bit too liberally. If last_ack
1196  * is beyond next_seq, we only consider it a gap now if we do
1197  * already have data beyond the gap. */
1198  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1199  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1200  SCLogDebug("packet %" PRIu64 ": no GAP. "
1201  "next_seq %u < last_ack %u, but no data in list",
1202  p->pcap_cnt, stream->next_seq, stream->last_ack);
1203  return false;
1204  }
1205  const uint64_t next_seq_abs =
1206  STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1207  const StreamingBufferBlock *blk = stream->sb.head;
1208  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1209  /* ack'd data after the gap */
1210  SCLogDebug("packet %" PRIu64 ": GAP. "
1211  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1212  p->pcap_cnt, stream->next_seq, stream->last_ack);
1213  return true;
1214  }
1215  }
1216 
1217  SCLogDebug("packet %" PRIu64 ": GAP! "
1218  "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", "
1219  "but we have no data.",
1220  p->pcap_cnt, last_ack_abs, app_progress);
1221  return true;
1222  }
1223  SCLogDebug("packet %"PRIu64": no GAP. "
1224  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1225  p->pcap_cnt, last_ack_abs, app_progress);
1226  return false;
1227 }
1228 
1229 static inline uint32_t AdjustToAcked(const Packet *p,
1230  const TcpSession *ssn, const TcpStream *stream,
1231  const uint64_t app_progress, const uint32_t data_len)
1232 {
1233  uint32_t adjusted = data_len;
1234 
1235  /* get window of data that is acked */
1236  if (!StreamTcpInlineMode()) {
1237  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1238  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1239  (ssn->state == TCP_CLOSED &&
1240  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1241  (p->flags & PKT_PSEUDO_STREAM_END))) {
1242  // fall through, we use all available data
1243  } else {
1244  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1245  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1246 
1247  /* see if the buffer contains unack'd data as well */
1248  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1249  adjusted = (uint32_t)(last_ack_abs - app_progress);
1250  DEBUG_VALIDATE_BUG_ON(adjusted > data_len);
1251  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1252  "data is considered", adjusted);
1253  }
1254  }
1255  }
1256  return adjusted;
1257 }
1258 
1259 /** \internal
1260  * \brief get stream buffer and update the app-layer
1261  * \param stream pointer to pointer as app-layer can switch flow dir
1262  * \retval 0 success
1263  */
1264 static int ReassembleUpdateAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn,
1265  TcpStream **stream, Packet *p, enum StreamUpdateDir app_update_dir)
1266 {
1267  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1268 
1269  SCLogDebug("app progress %"PRIu64, app_progress);
1270 #ifdef DEBUG
1271  uint64_t last_ack_abs = GetAbsLastAck(*stream);
1272  SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs,
1273  (*stream)->base_seq);
1274 #endif
1275  const uint8_t *mydata;
1276  uint32_t mydata_len;
1277  bool last_was_gap = false;
1278 
1279  while (1) {
1280  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1281  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1282  bool gap_ahead =
1283  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead);
1284  SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len);
1285  if (last_was_gap && mydata_len == 0) {
1286  break;
1287  }
1288  last_was_gap = false;
1289 
1290  /* make sure to only deal with ACK'd data */
1291  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1292  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1293  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1294  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1295 
1296  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len,
1297  StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, app_update_dir);
1298  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1299 
1301  (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP;
1304 
1305  /* AppLayerHandleTCPData has likely updated progress. */
1306  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1307  app_progress = STREAM_APP_PROGRESS(*stream);
1308 
1309  /* a GAP also consumes 'data required'. TODO perhaps we can use
1310  * this to skip post GAP data until the start of a next record. */
1311  if ((*stream)->data_required > 0) {
1312  if ((*stream)->data_required > mydata_len) {
1313  (*stream)->data_required -= mydata_len;
1314  } else {
1315  (*stream)->data_required = 0;
1316  }
1317  }
1318  if (r < 0)
1319  return 0;
1320  if (no_progress_update)
1321  break;
1322  last_was_gap = true;
1323  continue;
1324 
1325  } else if (flags & STREAM_DEPTH) {
1326  SCLogDebug("DEPTH");
1327  // we're just called once with this flag, so make sure we pass it on
1328  if (mydata == NULL && mydata_len > 0) {
1329  mydata_len = 0;
1330  }
1331  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1332  SCLogDebug("GAP?1");
1333  /* Possibly a gap, but no new data. */
1334  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1335  SCReturnInt(0);
1336 
1337  mydata = NULL;
1338  mydata_len = 0;
1339  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1340  break;
1341  }
1342  DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1343 
1344  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1345  *stream, &(*stream)->sb, mydata_len, app_progress);
1346 
1347  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1348  SCLogDebug("GAP?2");
1349  if (mydata_len < (*stream)->data_required) {
1350  SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead));
1351  if (gap_ahead) {
1352  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1353  (*stream)->data_required, mydata_len);
1354  (*stream)->app_progress_rel += mydata_len;
1355  (*stream)->data_required -= mydata_len;
1356  // TODO send incomplete data to app-layer with special flag
1357  // indicating its all there is for this rec?
1358  } else {
1359  SCReturnInt(0);
1360  }
1361  app_progress = STREAM_APP_PROGRESS(*stream);
1362  continue;
1363  }
1364  }
1365  (*stream)->data_required = 0;
1366 
1367  SCLogDebug("parser");
1368  /* update the app-layer */
1369  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata,
1370  mydata_len, flags, app_update_dir);
1371  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1372  AppLayerFrameDump(p->flow);
1373  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1374  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1375  break;
1376  app_progress = new_app_progress;
1377  if (flags & STREAM_DEPTH)
1378  break;
1379  }
1380 
1381  SCReturnInt(0);
1382 }
1383 
1384 /**
1385  * \brief Update the stream reassembly upon receiving a packet.
1386  *
1387  * For IDS mode, the stream is in the opposite direction of the packet,
1388  * as the ACK-packet is ACK'ing the stream.
1389  *
1390  * One of the utilities call by this function AppLayerHandleTCPData(),
1391  * has a feature where it will call this very same function for the
1392  * stream opposing the stream it is called with. This shouldn't cause
1393  * any issues, since processing of each stream is independent of the
1394  * other stream.
1395  */
1397  TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
1398 {
1399  SCEnter();
1400 
1401  /* this function can be directly called by app layer protocol
1402  * detection. */
1405  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1406  SCReturnInt(0);
1407  }
1408 
1409 #ifdef DEBUG
1410  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1411  GetSessionSize(ssn, p);
1412 #endif
1413  /* if no segments are in the list or all are already processed,
1414  * and state is beyond established, we send an empty msg */
1415  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1416  {
1417  /* send an empty EOF msg if we have no segments but TCP state
1418  * is beyond ESTABLISHED */
1419  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1420  SCLogDebug("sending empty eof message");
1421  /* send EOF to app layer */
1422  uint8_t stream_flags = StreamGetAppLayerFlags(ssn, stream, p);
1424  tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, stream_flags, app_update_dir);
1425  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1426 
1427  SCReturnInt(0);
1428  }
1429  }
1430 
1431  /* with all that out of the way, lets update the app-layer */
1432  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, app_update_dir);
1433 }
1434 
1435 /** \internal
1436  * \brief get stream data from offset
1437  * \param offset stream offset */
1438 static int GetRawBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1439  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1440 {
1441  const uint8_t *mydata;
1442  uint32_t mydata_len;
1443  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1444  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1445 
1446  /* No gaps in the stream, data must exist in the streaming buffer array */
1447  uint64_t roffset = offset;
1448  if (offset) {
1449  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1450  } else {
1451  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1452  }
1453 
1454  *data = mydata;
1455  *data_len = mydata_len;
1456  *data_offset = roffset;
1457  } else {
1458  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1459  *iter == NULL ? "starting" : "continuing", offset);
1460  if (*iter == NULL) {
1461  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1462  *iter = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1463  SCLogDebug("*iter %p", *iter);
1464  }
1465  if (*iter == NULL) {
1466  SCLogDebug("no data");
1467  *data = NULL;
1468  *data_len = 0;
1469  *data_offset = 0;
1470  return 0;
1471  }
1472  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1473 
1474  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1475  SCLogDebug("mydata %p", mydata);
1476 
1477  if ((*iter)->offset < offset) {
1478  uint64_t delta = offset - (*iter)->offset;
1479  if (delta < mydata_len) {
1480  *data = mydata + delta;
1481  *data_len = (uint32_t)(mydata_len - delta);
1482  *data_offset = offset;
1483  } else {
1484  SCLogDebug("no data (yet)");
1485  *data = NULL;
1486  *data_len = 0;
1487  *data_offset = 0;
1488  }
1489 
1490  } else {
1491  *data = mydata;
1492  *data_len = mydata_len;
1493  *data_offset = (*iter)->offset;
1494  }
1495 
1496  *iter = SBB_RB_NEXT(*iter);
1497  SCLogDebug("*iter %p", *iter);
1498  }
1499  return 0;
1500 }
1501 
1502 /** \brief does the stream engine have data to inspect?
1503  *
1504  * Returns true if there is data to inspect. In IDS case this is
1505  * about ACK'd data in the packet's direction.
1506  *
1507  * In the IPS case this is about the packet itself.
1508  */
1510 {
1511  TcpStream *stream;
1512  if (PKT_IS_TOSERVER(p)) {
1513  stream = &ssn->client;
1514  } else {
1515  stream = &ssn->server;
1516  }
1517 
1518  if (RB_EMPTY(&stream->seg_tree)) {
1519  return false;
1520  }
1521 
1524  return false;
1525 
1526  if (!StreamTcpInlineMode()) {
1527  const uint64_t segs_re_abs =
1528  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1529  if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) {
1530  return false;
1531  }
1532  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1533  return true;
1534  }
1535  } else {
1536  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1537  return true;
1538  }
1539  }
1540  return false;
1541 }
1542 
1543 /** \brief update stream engine after detection
1544  *
1545  * Tasked with progressing the 'progress' for Raw reassembly.
1546  * 2 main scenario's:
1547  * 1. progress is != 0, so we use this
1548  * 2. progress is 0, meaning the detect engine didn't touch
1549  * raw at all. In this case we need to look into progressing
1550  * raw anyway.
1551  *
1552  * Additionally, this function is tasked with disabling raw
1553  * reassembly if the app-layer requested to disable it.
1554  */
1555 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
1556 {
1557  TcpStream *stream;
1558  if (PKT_IS_TOSERVER(p)) {
1559  stream = &ssn->client;
1560  } else {
1561  stream = &ssn->server;
1562  }
1563 
1564  if (progress > STREAM_RAW_PROGRESS(stream)) {
1565  DEBUG_VALIDATE_BUG_ON(progress - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1566  uint32_t slide = (uint32_t)(progress - STREAM_RAW_PROGRESS(stream));
1567  stream->raw_progress_rel += slide;
1569 
1570  } else if (progress == 0) {
1571  uint64_t target;
1572  if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) {
1573  target = STREAM_APP_PROGRESS(stream);
1574  } else {
1575  target = GetAbsLastAck(stream);
1576  }
1577  if (target > STREAM_RAW_PROGRESS(stream)) {
1578  DEBUG_VALIDATE_BUG_ON(target - STREAM_RAW_PROGRESS(stream) > UINT32_MAX);
1579  uint32_t slide = (uint32_t)(target - STREAM_RAW_PROGRESS(stream));
1580  stream->raw_progress_rel += slide;
1581  }
1583 
1584  } else {
1585  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1586  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1587  STREAM_RAW_PROGRESS(stream), stream->window);
1588  }
1589 
1590  /* if we were told to accept no more raw data, we can mark raw as
1591  * disabled now. */
1594  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1595  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1596  }
1597 
1598  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1599 }
1600 
1601 /** \internal
1602  * \brief get a buffer around the current packet and run the callback on it
1603  *
1604  * The inline/IPS scanning method takes the current payload and wraps it in
1605  * data from other segments.
1606  *
1607  * How much data is inspected is controlled by the available data, chunk_size
1608  * and the payload size of the packet.
1609  *
1610  * Large packets: if payload size is close to the chunk_size, where close is
1611  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1612  * used: payload_len + 33% of the chunk_size.
1613  * If the payload size if equal to or bigger than the chunk_size, we use
1614  * payload len + 33% of the chunk size.
1615  */
1616 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1617  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1618 {
1619  SCEnter();
1620  int r = 0;
1621 
1622  TcpStream *stream;
1623  if (PKT_IS_TOSERVER(p)) {
1624  stream = &ssn->client;
1625  } else {
1626  stream = &ssn->server;
1627  }
1628 
1629  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1631  {
1632  *progress_out = STREAM_RAW_PROGRESS(stream);
1633  return 0;
1634  }
1635 
1636  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1639  if ((chunk_size <= p->payload_len) || (((chunk_size / 3) * 2) < p->payload_len)) {
1640  chunk_size = p->payload_len + (chunk_size / 3);
1641  SCLogDebug(
1642  "packet payload len %u, so chunk_size adjusted to %u", p->payload_len, chunk_size);
1643  }
1644 
1645  const TCPHdr *tcph = PacketGetTCP(p);
1646  uint64_t packet_leftedge_abs =
1647  STREAM_BASE_OFFSET(stream) + (TCP_GET_RAW_SEQ(tcph) - stream->base_seq);
1648  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1649  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1650  packet_leftedge_abs, packet_rightedge_abs);
1651 
1652  const uint8_t *mydata = NULL;
1653  uint32_t mydata_len = 0;
1654  uint64_t mydata_offset = 0;
1655  /* simply return progress from the block we inspected. */
1656  bool return_progress = false;
1657 
1658  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1659  /* continues block */
1660  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1661  return_progress = true;
1662 
1663  } else {
1664  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1665  /* find our block */
1666  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1667  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1668  if (sbb) {
1669  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1670  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1671  mydata_offset = sbb->offset;
1672  }
1673  }
1674 
1675  /* this can only happen if the segment insert of our current 'p' failed */
1676  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1677  if ((mydata == NULL || mydata_len == 0) || /* no data */
1678  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1679  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1680  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1681  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1682  {
1683  /* no data, or data is incomplete or wrong: use packet data */
1684  mydata = p->payload;
1685  mydata_len = p->payload_len;
1686  mydata_offset = packet_leftedge_abs;
1687  //mydata_rightedge_abs = packet_rightedge_abs;
1688  } else {
1689  /* adjust buffer to match chunk_size */
1690  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1691  if (mydata_len > chunk_size) {
1692  uint32_t excess = mydata_len - chunk_size;
1693  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1694 
1695  if (mydata_rightedge_abs == packet_rightedge_abs) {
1696  mydata += excess;
1697  mydata_len -= excess;
1698  mydata_offset += excess;
1699  SCLogDebug("cutting front of the buffer with %u", excess);
1700  } else if (mydata_offset == packet_leftedge_abs) {
1701  mydata_len -= excess;
1702  SCLogDebug("cutting tail of the buffer with %u", excess);
1703  } else {
1704  DEBUG_VALIDATE_BUG_ON(mydata_offset > packet_leftedge_abs);
1705  uint32_t abs_before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1706  DEBUG_VALIDATE_BUG_ON(packet_rightedge_abs > mydata_rightedge_abs);
1707  uint32_t abs_after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1708  uint32_t before = abs_before;
1709  uint32_t after = abs_after;
1710  SCLogDebug("before %u after %u", before, after);
1711 
1712  if (after >= (chunk_size - p->payload_len) / 2) {
1713  // more trailing data than we need
1714 
1715  if (before >= (chunk_size - p->payload_len) / 2) {
1716  // also more heading data, divide evenly
1717  before = after = (chunk_size - p->payload_len) / 2;
1718  } else {
1719  // heading data is less than requested, give the
1720  // rest to the trailing data
1721  after = (chunk_size - p->payload_len) - before;
1722  }
1723  } else {
1724  // less trailing data than requested
1725 
1726  if (before >= (chunk_size - p->payload_len) / 2) {
1727  before = (chunk_size - p->payload_len) - after;
1728  } else {
1729  // both smaller than their requested size
1730  }
1731  }
1732 
1733  /* adjust the buffer */
1734  DEBUG_VALIDATE_BUG_ON(before > abs_before);
1735  uint32_t skip = abs_before - before;
1736  DEBUG_VALIDATE_BUG_ON(after > abs_after);
1737  uint32_t cut = abs_after - after;
1738  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1739  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1740  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1741  mydata += skip;
1742  mydata_len -= (skip + cut);
1743  mydata_offset += skip;
1744  }
1745  }
1746  }
1747 
1748  /* run the callback */
1749  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1750  DEBUG_VALIDATE_BUG_ON(r < 0);
1751 
1752  if (return_progress) {
1753  *progress_out = (mydata_offset + mydata_len);
1754  } else {
1755  /* several blocks of data, so we need to be a bit more careful:
1756  * - if last_ack is beyond last progress, move progress forward to last_ack
1757  * - if our block matches or starts before last ack, return right edge of
1758  * our block.
1759  */
1760  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1761  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1762 
1763  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1764  if (mydata_offset > last_ack_abs) {
1765  /* gap between us and last ack, set progress to last ack */
1766  *progress_out = last_ack_abs;
1767  } else {
1768  *progress_out = (mydata_offset + mydata_len);
1769  }
1770  } else {
1771  *progress_out = STREAM_RAW_PROGRESS(stream);
1772  }
1773  }
1774  return r;
1775 }
1776 
1777 /** \brief access 'raw' reassembly data.
1778  *
1779  * Access data as tracked by 'raw' tracker. Data is made available to
1780  * callback that is passed to this function.
1781  *
1782  * In the case of IDS the callback may be run multiple times if data
1783  * contains gaps. It will then be run for each block of data that is
1784  * continuous.
1785  *
1786  * The callback should give on of 2 return values:
1787  * - 0 ok
1788  * - 1 done
1789  * The value 1 will break the loop if there is a block list that is
1790  * inspected.
1791  *
1792  * This function will return the 'progress' value that has been
1793  * consumed until now.
1794  *
1795  * \param ssn tcp session
1796  * \param stream tcp stream
1797  * \param Callback the function pointer to the callback function
1798  * \param cb_data callback data
1799  * \param[in] progress_in progress to work from
1800  * \param[in] re right edge of data to consider
1801  * \param[out] progress_out absolute progress value of the data this
1802  * call handled.
1803  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1804  */
1805 static int StreamReassembleRawDo(const TcpSession *ssn, const TcpStream *stream,
1806  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1807  const uint64_t re, uint64_t *progress_out, bool eof)
1808 {
1809  SCEnter();
1810  int r = 0;
1811 
1812  StreamingBufferBlock *iter = NULL;
1813  uint64_t progress = progress_in;
1814 
1815  /* loop through available buffers. On no packet loss we'll have a single
1816  * iteration. On missing data we'll walk the blocks */
1817  while (1) {
1818  const uint8_t *mydata;
1819  uint32_t mydata_len;
1820  uint64_t mydata_offset = 0;
1821 
1822  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1823  if (mydata_len == 0) {
1824  SCLogDebug("no data");
1825  break;
1826  }
1827  //PrintRawDataFp(stdout, mydata, mydata_len);
1828 
1829  SCLogDebug("raw progress %"PRIu64, progress);
1830  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1831  stream, &stream->sb, mydata_len, progress);
1832 
1833  if (eof) {
1834  // inspect all remaining data, ack'd or not
1835  } else {
1836  if (re < progress) {
1837  SCLogDebug("nothing to do");
1838  goto end;
1839  }
1840 
1841  SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress);
1842  SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len,
1843  re);
1844 
1845  /* see if the buffer contains unack'd data as well */
1846  if (progress + mydata_len > re) {
1847 #ifdef DEBUG_VALIDATION
1848  uint32_t check = mydata_len;
1849 #endif
1850  mydata_len = (uint32_t)(re - progress);
1851  DEBUG_VALIDATE_BUG_ON(check < mydata_len);
1852  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1853  "data is considered", mydata_len);
1854  }
1855  }
1856  if (mydata_len == 0)
1857  break;
1858 
1859  SCLogDebug("data %p len %u", mydata, mydata_len);
1860 
1861  /* we have data. */
1862  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1863  DEBUG_VALIDATE_BUG_ON(r < 0);
1864 
1865  if (mydata_offset == progress) {
1866  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1867  progress, mydata_len, progress_in + mydata_len);
1868 
1869  progress += mydata_len;
1870  SCLogDebug("raw progress now %"PRIu64, progress);
1871 
1872  /* data is beyond the progress we'd like, and before last ack. Gap. */
1873  } else if (mydata_offset > progress && mydata_offset < re) {
1874  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1875  SCLogDebug("re %" PRIu64, re);
1876  progress = mydata_offset;
1877  SCLogDebug("raw progress now %"PRIu64, progress);
1878 
1879  } else {
1880  SCLogDebug("not increasing progress, data gap => mydata_offset "
1881  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1882  }
1883 
1884  if (iter == NULL || r == 1)
1885  break;
1886  }
1887 end:
1888  *progress_out = progress;
1889  return r;
1890 }
1891 
1893  void *cb_data, const uint64_t offset, const bool eof)
1894 {
1895  /* take app progress as the right edge of used data. */
1896  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1897  SCLogDebug("app_progress %" PRIu64, app_progress);
1898 
1899  uint64_t unused = 0;
1900  return StreamReassembleRawDo(
1901  ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof);
1902 }
1903 
1905  StreamReassembleRawFunc Callback, void *cb_data,
1906  uint64_t *progress_out, bool respect_inspect_depth)
1907 {
1908  /* handle inline separately as the logic is very different */
1909  if (StreamTcpInlineMode()) {
1910  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1911  }
1912 
1913  TcpStream *stream;
1914  if (PKT_IS_TOSERVER(p)) {
1915  stream = &ssn->client;
1916  } else {
1917  stream = &ssn->server;
1918  }
1919 
1921  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1922  {
1923  *progress_out = STREAM_RAW_PROGRESS(stream);
1924  return 0;
1925  }
1926 
1927  uint64_t progress = STREAM_RAW_PROGRESS(stream);
1928  /* if the app layer triggered a flush, and we're supposed to
1929  * use a minimal inspect depth, we actually take the app progress
1930  * as that is the right edge of the data. Then we take the window
1931  * of 'min_inspect_depth' before that. */
1932 
1933  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s "
1934  "stream->min_inspect_depth %u",
1935  respect_inspect_depth ? "true" : "false",
1936  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1937  stream->min_inspect_depth);
1938 
1939  if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) &&
1940  stream->min_inspect_depth) {
1941  progress = STREAM_APP_PROGRESS(stream);
1942  if (stream->min_inspect_depth >= progress) {
1943  progress = 0;
1944  } else {
1945  progress -= stream->min_inspect_depth;
1946  }
1947 
1948  SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream),
1949  STREAM_RAW_PROGRESS(stream));
1950 
1951  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1952  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress "
1953  "%" PRIu64,
1954  progress);
1955  }
1956 
1957  SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth,
1958  stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW"
1959  : "(no trigger)");
1960 
1961  /* absolute right edge of ack'd data */
1962  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1963  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1964 
1965  return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs,
1966  progress_out, (p->flags & PKT_PSEUDO_STREAM_END));
1967 }
1968 
1969 int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream,
1970  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1971  uint64_t *progress_out, const bool eof)
1972 {
1973  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1974  return 0;
1975 
1976  /* absolute right edge of ack'd data */
1977  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1978  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1979 
1980  return StreamReassembleRawDo(
1981  ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof);
1982 }
1983 
1984 /** \internal
1985  * \brief update app layer based on received ACK
1986  *
1987  * \retval r 0 on success, -1 on error
1988  */
1989 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1990  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1991 {
1992  SCEnter();
1993 
1994  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1995  SCReturnInt(-1);
1996 
1997  SCReturnInt(0);
1998 }
1999 
2001  TcpSession *ssn, TcpStream *stream, Packet *p)
2002 {
2003  SCEnter();
2004 
2005  DEBUG_VALIDATE_BUG_ON(!PacketIsTCP(p));
2006  const TCPHdr *tcph = PacketGetTCP(p);
2007 
2008  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
2009  ssn, stream, p, p->payload_len);
2010 
2011  /* default IDS: update opposing side (triggered by ACK) */
2013  /* inline and stream end and flow timeout packets trigger same dir handling */
2014  if (StreamTcpInlineMode()) {
2015  dir = UPDATE_DIR_PACKET;
2016  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
2017  dir = UPDATE_DIR_PACKET;
2018  } else if (tcph->th_flags & TH_RST) { // accepted rst
2019  dir = UPDATE_DIR_PACKET;
2020  } else if ((tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
2021  if (tcph->th_flags & TH_ACK) {
2022  dir = UPDATE_DIR_BOTH;
2023  } else {
2024  dir = UPDATE_DIR_PACKET;
2025  }
2026  } else if (ssn->state == TCP_CLOSED) {
2027  dir = UPDATE_DIR_BOTH;
2028  } else if ((ssn->flags & STREAMTCP_FLAG_ASYNC) != 0) {
2029  dir = UPDATE_DIR_PACKET;
2030  SCLogDebug("%" PRIu64 ": ASYNC: UPDATE_DIR_PACKET", p->pcap_cnt);
2031  }
2032 
2033  /* handle ack received */
2034  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
2035  /* we need to update the opposing stream in
2036  * StreamTcpReassembleHandleSegmentUpdateACK */
2037  TcpStream *opposing_stream = NULL;
2038  if (stream == &ssn->client) {
2039  opposing_stream = &ssn->server;
2040  } else {
2041  opposing_stream = &ssn->client;
2042  }
2043 
2044  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2045 
2046  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
2047  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
2048  SCReturnInt(-1);
2049  }
2050 
2051  /* StreamTcpReassembleHandleSegmentUpdateACK
2052  * may swap content of ssn->server and ssn->client structures.
2053  * We have to continue with initial content of the stream in such case */
2054  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2055  if (reversed_before_ack_handling != reversed_after_ack_handling) {
2056  SCLogDebug("TCP streams were swapped");
2057  stream = opposing_stream;
2058  }
2059  }
2060  /* if this segment contains data, insert it */
2061  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) &&
2062  (tcph->th_flags & TH_RST) == 0) {
2063  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
2064 
2065  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
2066  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
2067  /* failure can only be because of memcap hit, so see if this should lead to a drop */
2070  StreamTcpReassembleExceptionPolicyStatsIncr(
2072  SCReturnInt(-1);
2073  }
2074 
2075  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
2076  p->flags |= PKT_STREAM_ADD;
2077  } else {
2078  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
2079  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
2080  ssn, stream, p->payload_len,
2081  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
2082  }
2083 
2084  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
2085  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
2086  * was *just* set. In this case we trigger the AppLayer Truncate
2087  * logic, to inform the applayer no more data in this direction is
2088  * to be expected. */
2089  if ((stream->flags &
2092  {
2093  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
2094  if (dir != UPDATE_DIR_PACKET) {
2095  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
2096  "can trigger Truncate");
2097  dir = UPDATE_DIR_PACKET;
2098  }
2099  }
2100 
2101  /* in stream inline mode even if we have no data we call the reassembly
2102  * functions to handle EOF */
2103  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
2104  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
2105  StreamTcpInlineMode()?"true":"false",
2106  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
2107  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
2108  SCReturnInt(-1);
2109  }
2110  }
2111 
2112  SCReturnInt(0);
2113 }
2114 
2115 /**
2116  * \brief get a segment from the pool
2117  *
2118  * \retval seg Segment from the pool or NULL
2119  */
2121 {
2123  if (seg) {
2125  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2126  return seg;
2127  }
2128 
2129  seg = (TcpSegment *)PoolThreadGetById(
2130  segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id);
2131  SCLogDebug("seg we return is %p", seg);
2132  if (seg == NULL) {
2133  /* Increment the counter to show that we are not able to serve the
2134  segment request due to memcap limit */
2136  } else {
2137  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2139  }
2140 
2141  return seg;
2142 }
2143 
2144 /**
2145  * \brief Trigger RAW stream inspection
2146  *
2147  * Used by AppLayerTriggerRawStreamInspection to trigger RAW stream
2148  * inspection from the applayer, for example upon completion of a
2149  * HTTP request.
2150  *
2151  * It sets a flag in the stream so that the next Raw call will return
2152  * the data.
2153  *
2154  * \param ssn TcpSession
2155  */
2157 {
2158 #ifdef DEBUG
2159  BUG_ON(ssn == NULL);
2160 #endif
2161 
2162  if (ssn != NULL) {
2163  if (direction == STREAM_TOSERVER) {
2165  } else {
2167  }
2168 
2169  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
2170  }
2171 }
2172 
2173 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
2174 {
2175 #ifdef DEBUG
2176  BUG_ON(ssn == NULL);
2177 #endif
2178 
2179  if (ssn != NULL) {
2180  if (direction == STREAM_TOSERVER) {
2181  ssn->client.min_inspect_depth = depth;
2182  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
2183  } else {
2184  ssn->server.min_inspect_depth = depth;
2185  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
2186  }
2187  }
2188 }
2189 
2190 #ifdef UNITTESTS
2191 /** unit tests and it's support functions below */
2192 
2193 #define SET_ISN(stream, setseq) \
2194  (stream)->isn = (setseq); \
2195  (stream)->base_seq = (setseq) + 1
2196 
2197 /** \brief The Function to create the packet with given payload, which is used
2198  * to test the reassembly of the engine.
2199  *
2200  * \param payload The variable used to store the payload contents of the
2201  * current packet.
2202  * \param value The value which current payload will have for this packet
2203  * \param payload_len The length of the filed payload for current packet.
2204  * \param len Length of the payload array
2205  */
2206 
2207 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2208  uint8_t payload_len, uint8_t len)
2209 {
2210  uint8_t i;
2211  for (i = 0; i < payload_len; i++)
2212  payload[i] = value;
2213  for (; i < len; i++)
2214  payload = NULL;
2215 }
2216 
2217 /** \brief The Function Checks the reassembled stream contents against predefined
2218  * stream contents according to OS policy used.
2219  *
2220  * \param stream_policy Predefined value of stream for different OS policies
2221  * \param stream Reassembled stream returned from the reassembly functions
2222  */
2223 
2224 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2225 {
2226  if (StreamingBufferCompareRawData(&stream->sb,
2227  data, data_len) == 0)
2228  {
2229  SCReturnInt(0);
2230  }
2231  SCLogInfo("OK");
2232  PrintRawDataFp(stdout, data, data_len);
2233  return 1;
2234 }
2235 
2236 #define MISSED_START(isn) \
2237  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2238  TcpSession ssn; \
2239  ThreadVars tv; \
2240  memset(&tv, 0, sizeof(tv)); \
2241  \
2242  StreamTcpUTInit(&ra_ctx); \
2243  \
2244  StreamTcpUTSetupSession(&ssn); \
2245  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2246  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2247  \
2248  TcpStream *stream = &ssn.client;
2249 
2250 #define MISSED_END \
2251  StreamTcpUTClearSession(&ssn); \
2252  StreamTcpUTDeinit(ra_ctx); \
2253  PASS
2254 
2255 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2256  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2257  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2258 
2259 #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \
2260  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen));
2261 
2262 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len);
2263 
2264 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
2265 {
2266  int cnt = 0;
2267  uint64_t last_re = 0;
2268  StreamingBufferBlock *sbb = NULL;
2269  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2270  {
2271  if (sbb->offset != last_re) {
2272  // gap before us
2273  if (cnt == pos && last_re == offset && len == sbb->offset - last_re) {
2274  return 1;
2275  }
2276  cnt++;
2277  }
2278  last_re = sbb->offset + sbb->len;
2279  cnt++;
2280  }
2281  return 0;
2282 }
2283 
2285  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len);
2286 
2288  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
2289 {
2290  int cnt = 0;
2291  uint64_t last_re = 0;
2292  StreamingBufferBlock *sbb = NULL;
2293  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2294  {
2295  if (sbb->offset != last_re) {
2296  // gap before us
2297  cnt++;
2298  }
2299 
2300  if (cnt == pos && sbb->offset == offset) {
2301  const uint8_t *buf = NULL;
2302  uint32_t buf_len = 0;
2303  StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len);
2304 
2305  if (len == buf_len) {
2306  return (memcmp(data, buf, len) == 0);
2307  }
2308  }
2309 
2310  last_re = sbb->offset + sbb->len;
2311  cnt++;
2312  }
2313  return 0;
2314 }
2315 
2316 /**
2317  * \test Test the handling of packets missed by both IDS and the end host.
2318  * The packet is missed in the starting of the stream.
2319  *
2320  * \retval On success it returns 1 and on failure 0.
2321  */
2322 
2323 static int StreamTcpReassembleTest25 (void)
2324 {
2325  MISSED_START(6);
2326  MISSED_ADD_PAYLOAD(10, "BB", 2);
2327  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2328  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BB", 2) == 1);
2329  MISSED_ADD_PAYLOAD(12, "CC", 2);
2330  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2331  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BBCC", 4) == 1);
2332  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2333  MISSED_END;
2334  PASS;
2335 }
2336 
2337 /**
2338  * \test Test the handling of packets missed by both IDS and the end host.
2339  * The packet is missed in the middle of the stream.
2340  *
2341  * \retval On success it returns 1 and on failure 0.
2342  */
2343 
2344 static int StreamTcpReassembleTest26 (void)
2345 {
2346  MISSED_START(9);
2347  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2348  MISSED_ADD_PAYLOAD(15, "CC", 2);
2349  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2350  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2351  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2352  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2353  MISSED_END;
2354 }
2355 
2356 /**
2357  * \test Test the handling of packets missed by both IDS and the end host.
2358  * The packet is missed in the end of the stream.
2359  *
2360  * \retval On success it returns 1 and on failure 0.
2361  */
2362 
2363 static int StreamTcpReassembleTest27 (void)
2364 {
2365  MISSED_START(9);
2366  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2367  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2368  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2369  MISSED_END;
2370 }
2371 
2372 /**
2373  * \test Test the handling of packets missed by IDS, but the end host has
2374  * received it and send the acknowledgment of it. The packet is missed
2375  * in the starting of the stream.
2376  *
2377  * \retval On success it returns 1 and on failure 0.
2378  */
2379 
2380 static int StreamTcpReassembleTest28 (void)
2381 {
2382  MISSED_START(6);
2383  MISSED_ADD_PAYLOAD(10, "AAA", 3);
2384  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2385  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAA", 3) == 1);
2386  MISSED_ADD_PAYLOAD(13, "BB", 2);
2387  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2388  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABB", 5) == 1);
2389  ssn.state = TCP_TIME_WAIT;
2390  MISSED_ADD_PAYLOAD(15, "CC", 2);
2391  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2392  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABBCC", 7) == 1);
2393  MISSED_END;
2394 }
2395 
2396 /**
2397  * \test Test the handling of packets missed by IDS, but the end host has
2398  * received it and send the acknowledgment of it. The packet is missed
2399  * in the middle of the stream.
2400  *
2401  * \retval On success it returns 1 and on failure 0.
2402  */
2403 
2404 static int StreamTcpReassembleTest29 (void)
2405 {
2406  MISSED_START(9);
2407  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2408  ssn.state = TCP_TIME_WAIT;
2409  MISSED_ADD_PAYLOAD(15, "CC", 2);
2410  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2411  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2412  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2413  MISSED_END;
2414 }
2415 
2416 static int StreamTcpReassembleTest33(void)
2417 {
2418  TcpSession ssn;
2419  Packet *p = PacketGetFromAlloc();
2420  FAIL_IF(unlikely(p == NULL));
2421  Flow f;
2422  TCPHdr tcph;
2423  TcpReassemblyThreadCtx *ra_ctx = NULL;
2425  uint8_t packet[1460] = "";
2426 
2427  StreamTcpUTInit(&ra_ctx);
2429 
2430  memset(&f, 0, sizeof (Flow));
2431  memset(&tcph, 0, sizeof (TCPHdr));
2432  ThreadVars tv;
2433  memset(&tv, 0, sizeof (ThreadVars));
2434  FLOW_INITIALIZE(&f);
2435  f.protoctx = &ssn;
2436  f.proto = IPPROTO_TCP;
2437  p->src.family = AF_INET;
2438  p->dst.family = AF_INET;
2439  p->proto = IPPROTO_TCP;
2440  p->flow = &f;
2441  tcph.th_win = 5480;
2442  tcph.th_flags = TH_PUSH | TH_ACK;
2443  UTHSetTCPHdr(p, &tcph);
2445  p->payload = packet;
2446 
2447  tcph.th_seq = htonl(10);
2448  tcph.th_ack = htonl(31);
2449  p->payload_len = 10;
2450 
2451  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2452 
2453  tcph.th_seq = htonl(20);
2454  tcph.th_ack = htonl(31);
2455  p->payload_len = 10;
2456 
2457  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2458 
2459  tcph.th_seq = htonl(40);
2460  tcph.th_ack = htonl(31);
2461  p->payload_len = 10;
2462 
2463  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2464 
2465  tcph.th_seq = htonl(5);
2466  tcph.th_ack = htonl(31);
2467  p->payload_len = 30;
2468 
2469  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2470 
2472  FLOW_DESTROY(&f);
2473  PacketFree(p);
2474  StreamTcpUTDeinit(ra_ctx);
2475  PASS;
2476 }
2477 
2478 static int StreamTcpReassembleTest34(void)
2479 {
2480  TcpSession ssn;
2481  Packet *p = PacketGetFromAlloc();
2482  FAIL_IF(unlikely(p == NULL));
2483  Flow f;
2484  TCPHdr tcph;
2485  TcpReassemblyThreadCtx *ra_ctx = NULL;
2487  uint8_t packet[1460] = "";
2488 
2489  StreamTcpUTInit(&ra_ctx);
2491  memset(&f, 0, sizeof (Flow));
2492  memset(&tcph, 0, sizeof (TCPHdr));
2493  ThreadVars tv;
2494  memset(&tv, 0, sizeof (ThreadVars));
2495  FLOW_INITIALIZE(&f);
2496  f.protoctx = &ssn;
2497  f.proto = IPPROTO_TCP;
2498  p->src.family = AF_INET;
2499  p->dst.family = AF_INET;
2500  p->proto = IPPROTO_TCP;
2501  p->flow = &f;
2502  tcph.th_win = 5480;
2503  tcph.th_flags = TH_PUSH | TH_ACK;
2504  UTHSetTCPHdr(p, &tcph);
2506  p->payload = packet;
2507  SET_ISN(&ssn.client, 857961230);
2508 
2509  tcph.th_seq = htonl(857961230);
2510  tcph.th_ack = htonl(31);
2511  p->payload_len = 304;
2512 
2513  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2514 
2515  tcph.th_seq = htonl(857961534);
2516  tcph.th_ack = htonl(31);
2517  p->payload_len = 1460;
2518 
2519  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2520 
2521  tcph.th_seq = htonl(857963582);
2522  tcph.th_ack = htonl(31);
2523  p->payload_len = 1460;
2524 
2525  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2526 
2527  tcph.th_seq = htonl(857960946);
2528  tcph.th_ack = htonl(31);
2529  p->payload_len = 1460;
2530 
2531  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2532 
2534  FLOW_DESTROY(&f);
2535  StreamTcpUTDeinit(ra_ctx);
2536  PacketFree(p);
2537  PASS;
2538 }
2539 
2540 /**
2541  * \test Test to make sure that we don't return the segments until the app
2542  * layer proto has been detected and after that remove the processed
2543  * segments.
2544  *
2545  * \retval On success it returns 1 and on failure 0.
2546  */
2547 
2548 static int StreamTcpReassembleTest39 (void)
2549 {
2550  Packet *p = PacketGetFromAlloc();
2551  FAIL_IF(unlikely(p == NULL));
2552  Flow f;
2553  ThreadVars tv;
2554  StreamTcpThread stt;
2555  TCPHdr tcph;
2556  PacketQueueNoLock pq;
2557  memset(&pq,0,sizeof(PacketQueueNoLock));
2558  memset (&f, 0, sizeof(Flow));
2559  memset(&tv, 0, sizeof (ThreadVars));
2560  memset(&stt, 0, sizeof (stt));
2561  memset(&tcph, 0, sizeof (TCPHdr));
2562 
2563  FLOW_INITIALIZE(&f);
2564  f.flags = FLOW_IPV4;
2565  f.proto = IPPROTO_TCP;
2566  p->flow = &f;
2567  UTHSetTCPHdr(p, &tcph);
2568 
2569  StreamTcpUTInit(&stt.ra_ctx);
2570 
2571  /* handshake */
2572  tcph.th_win = htons(5480);
2573  tcph.th_flags = TH_SYN;
2575  p->payload_len = 0;
2576  p->payload = NULL;
2577  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2578 
2579  TcpSession *ssn = (TcpSession *)f.protoctx;
2580  FAIL_IF_NULL(ssn);
2581 
2588  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2589  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2590  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2591  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2592  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2593  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2594  FAIL_IF(ssn->data_first_seen_dir != 0);
2595 
2596  /* handshake */
2597  tcph.th_ack = htonl(1);
2598  tcph.th_flags = TH_SYN | TH_ACK;
2600  p->payload_len = 0;
2601  p->payload = NULL;
2602  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2609  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2610  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2611  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2612  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2613  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2614  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2615  FAIL_IF(ssn->data_first_seen_dir != 0);
2616 
2617  /* handshake */
2618  tcph.th_ack = htonl(1);
2619  tcph.th_seq = htonl(1);
2620  tcph.th_flags = TH_ACK;
2622  p->payload_len = 0;
2623  p->payload = NULL;
2624  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2631  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2632  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2633  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2634  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2635  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2636  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2637  FAIL_IF(ssn->data_first_seen_dir != 0);
2638 
2639  /* partial request */
2640  uint8_t request1[] = { 0x47, 0x45, };
2641  tcph.th_ack = htonl(1);
2642  tcph.th_seq = htonl(1);
2643  tcph.th_flags = TH_PUSH | TH_ACK;
2645  p->payload_len = sizeof(request1);
2646  p->payload = request1;
2647  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2654  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2655  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2656  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2657  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2658  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2659  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2660  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2661  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2662 
2663  /* response ack against partial request */
2664  tcph.th_ack = htonl(3);
2665  tcph.th_seq = htonl(1);
2666  tcph.th_flags = TH_ACK;
2668  p->payload_len = 0;
2669  p->payload = NULL;
2670  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2677  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2678  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2679  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2680  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2681  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2682  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2683  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2684  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2685 
2686  /* complete partial request */
2687  uint8_t request2[] = {
2688  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2689  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2690  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2691  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2692  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2693  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2694  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2695  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2696  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2697  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2698  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2699  tcph.th_ack = htonl(1);
2700  tcph.th_seq = htonl(3);
2701  tcph.th_flags = TH_PUSH | TH_ACK;
2703  p->payload_len = sizeof(request2);
2704  p->payload = request2;
2705  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2712  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2713  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2714  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2715  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2716  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2717  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2718  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2719  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2720  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2721 
2722  /* response - request ack */
2723  uint8_t response[] = {
2724  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2725  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2726  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2727  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2728  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2729  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2730  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2731  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2732  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2733  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2734  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2735  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2736  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2737  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2738  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2739  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2740  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2741  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2742  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2743  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2744  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2745  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2746  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2747  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2748  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2749  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2750  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2751  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2752  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2753  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2754  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2755  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2756  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2757  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2758  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2759  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2760  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2761  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2762  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2763  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2764  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2765  tcph.th_ack = htonl(88);
2766  tcph.th_seq = htonl(1);
2767  tcph.th_flags = TH_PUSH | TH_ACK;
2769  p->payload_len = sizeof(response);
2770  p->payload = response;
2771 
2772  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2779  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2780  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2781  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2782  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2784  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2785  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2786  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2787  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2788 
2789  /* response ack from request */
2790  tcph.th_ack = htonl(328);
2791  tcph.th_seq = htonl(88);
2792  tcph.th_flags = TH_ACK;
2794  p->payload_len = 0;
2795  p->payload = NULL;
2796  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2803  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2804  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2805  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2806  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2808  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2809  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2810  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2811  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2812  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2813 
2814  /* response - acking */
2815  tcph.th_ack = htonl(88);
2816  tcph.th_seq = htonl(328);
2817  tcph.th_flags = TH_PUSH | TH_ACK;
2819  p->payload_len = 0;
2820  p->payload = NULL;
2821  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2828  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2829  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2830  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2831  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2833  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2834  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2835  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2836  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2837  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2838 
2839  /* response ack from request */
2840  tcph.th_ack = htonl(328);
2841  tcph.th_seq = htonl(88);
2842  tcph.th_flags = TH_ACK;
2844  p->payload_len = 0;
2845  p->payload = NULL;
2846  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2853  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2854  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2855  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2856  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2858  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2859  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2860  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2861  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2862 
2863  /* response - acking the request again*/
2864  tcph.th_ack = htonl(88);
2865  tcph.th_seq = htonl(328);
2866  tcph.th_flags = TH_PUSH | TH_ACK;
2868  p->payload_len = 0;
2869  p->payload = NULL;
2870  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2877  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2878  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2879  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2880  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2882  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2883  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2884  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2885  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2886 
2887  /*** New Request ***/
2888 
2889  /* partial request */
2890  tcph.th_ack = htonl(328);
2891  tcph.th_seq = htonl(88);
2892  tcph.th_flags = TH_PUSH | TH_ACK;
2894  p->payload_len = sizeof(request1);
2895  p->payload = request1;
2896  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2903  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2904  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2905  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2906  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2908  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2909  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2910  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2911  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2912  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2913 
2914  /* response ack against partial request */
2915  tcph.th_ack = htonl(90);
2916  tcph.th_seq = htonl(328);
2917  tcph.th_flags = TH_ACK;
2919  p->payload_len = 0;
2920  p->payload = NULL;
2921  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2922  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2929  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2930  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2931  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2932  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2934  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2935  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2936  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2937  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2938  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2939 
2940  /* complete request */
2941  tcph.th_ack = htonl(328);
2942  tcph.th_seq = htonl(90);
2943  tcph.th_flags = TH_PUSH | TH_ACK;
2945  p->payload_len = sizeof(request2);
2946  p->payload = request2;
2947  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2954  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2955  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2956  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2957  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2959  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2960  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2961  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2962  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2963  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2964  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2965 
2966  /* response ack against second partial request */
2967  tcph.th_ack = htonl(175);
2968  tcph.th_seq = htonl(328);
2969  tcph.th_flags = TH_ACK;
2971  p->payload_len = 0;
2972  p->payload = NULL;
2973 
2974  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2981  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2982  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2983  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2984  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2986  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2987  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2988  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2989  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2990  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2991  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2992 
2993  /* response acking a request */
2994  tcph.th_ack = htonl(175);
2995  tcph.th_seq = htonl(328);
2996  tcph.th_flags = TH_ACK;
2998  p->payload_len = 0;
2999  p->payload = NULL;
3000  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3006 
3007  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3008  StreamTcpPruneSession(&f, STREAM_TOCLIENT);
3009 
3010  /* request acking a response */
3011  tcph.th_ack = htonl(328);
3012  tcph.th_seq = htonl(175);
3013  tcph.th_flags = TH_ACK;
3015  p->payload_len = 0;
3016  p->payload = NULL;
3017  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
3018 
3019  PacketFree(p);
3020  StreamTcpSessionClear(ssn);
3021  FLOW_DESTROY(&f);
3023  PASS;
3024 }
3025 
3026 /**
3027  * \test Test to make sure that we sent all the segments from the initial
3028  * segments to app layer until we have detected the app layer proto.
3029  *
3030  * \retval On success it returns 1 and on failure 0.
3031  */
3032 
3033 static int StreamTcpReassembleTest40 (void)
3034 {
3035  Packet *p = PacketGetFromAlloc();
3036  FAIL_IF_NULL(p);
3037  Flow *f = NULL;
3038  TCPHdr tcph;
3039  TcpSession ssn;
3040  memset(&tcph, 0, sizeof (TCPHdr));
3041  ThreadVars tv;
3042  memset(&tv, 0, sizeof (ThreadVars));
3043 
3044  StreamTcpInitConfig(true);
3046 
3048  FAIL_IF_NULL(ra_ctx);
3049 
3050  uint8_t httpbuf1[] = "P";
3051  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3052  uint8_t httpbuf3[] = "O";
3053  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3054  uint8_t httpbuf4[] = "S";
3055  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3056  uint8_t httpbuf5[] = "T \r\n";
3057  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3058 
3059  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3060  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3061 
3062  SET_ISN(&ssn.server, 9);
3063  ssn.server.last_ack = 10;
3064  SET_ISN(&ssn.client, 9);
3065  ssn.client.isn = 9;
3066 
3067  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3068  FAIL_IF_NULL(f);
3069  f->protoctx = &ssn;
3070  f->proto = IPPROTO_TCP;
3071  p->flow = f;
3072 
3073  tcph.th_win = htons(5480);
3074  tcph.th_seq = htonl(10);
3075  tcph.th_ack = htonl(10);
3076  tcph.th_flags = TH_ACK|TH_PUSH;
3077  UTHSetTCPHdr(p, &tcph);
3079  p->payload = httpbuf1;
3080  p->payload_len = httplen1;
3081  ssn.state = TCP_ESTABLISHED;
3082  TcpStream *s = &ssn.client;
3083  SCLogDebug("1 -- start");
3084  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3085 
3087  p->payload = httpbuf2;
3088  p->payload_len = httplen2;
3089  tcph.th_seq = htonl(10);
3090  tcph.th_ack = htonl(11);
3091  s = &ssn.server;
3092  ssn.server.last_ack = 11;
3093  SCLogDebug("2 -- start");
3094  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3095 
3097  p->payload = httpbuf3;
3098  p->payload_len = httplen3;
3099  tcph.th_seq = htonl(11);
3100  tcph.th_ack = htonl(55);
3101  s = &ssn.client;
3102  ssn.client.last_ack = 55;
3103  SCLogDebug("3 -- start");
3104  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3105 
3107  p->payload = httpbuf2;
3108  p->payload_len = httplen2;
3109  tcph.th_seq = htonl(55);
3110  tcph.th_ack = htonl(12);
3111  s = &ssn.server;
3112  ssn.server.last_ack = 12;
3113  SCLogDebug("4 -- start");
3114  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3115 
3116  /* check is have the segment in the list and flagged or not */
3117  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3118  FAIL_IF_NULL(seg);
3119  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
3120 
3122  p->payload = httpbuf4;
3123  p->payload_len = httplen4;
3124  tcph.th_seq = htonl(12);
3125  tcph.th_ack = htonl(100);
3126  s = &ssn.client;
3127  ssn.client.last_ack = 100;
3128  SCLogDebug("5 -- start");
3129  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3130 
3132  p->payload = httpbuf2;
3133  p->payload_len = httplen2;
3134  tcph.th_seq = htonl(100);
3135  tcph.th_ack = htonl(13);
3136  s = &ssn.server;
3137  ssn.server.last_ack = 13;
3138  SCLogDebug("6 -- start");
3139  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3140 
3142  p->payload = httpbuf5;
3143  p->payload_len = httplen5;
3144  tcph.th_seq = htonl(13);
3145  tcph.th_ack = htonl(145);
3146  s = &ssn.client;
3147  ssn.client.last_ack = 145;
3148  SCLogDebug("7 -- start");
3149  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3150 
3152  p->payload = httpbuf2;
3153  p->payload_len = httplen2;
3154  tcph.th_seq = htonl(145);
3155  tcph.th_ack = htonl(16);
3156  s = &ssn.server;
3157  ssn.server.last_ack = 16;
3158  SCLogDebug("8 -- start");
3159  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3160  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3161 
3163  PacketFree(p);
3164  UTHFreeFlow(f);
3166  StreamTcpFreeConfig(true);
3167  PASS;
3168 }
3169 
3170 /** \test Test the memcap incrementing/decrementing and memcap check */
3171 static int StreamTcpReassembleTest44(void)
3172 {
3173  StreamTcpInitConfig(true);
3174  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
3175  StreamTcpReassembleIncrMemuse(500);
3176  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
3177  StreamTcpReassembleDecrMemuse(500);
3178  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
3180  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
3181  StreamTcpFreeConfig(true);
3182  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
3183  PASS;
3184 }
3185 
3186 /**
3187  * \test Test to make sure that reassembly_depth is enforced.
3188  *
3189  * \retval On success it returns 1 and on failure 0.
3190  */
3191 
3192 static int StreamTcpReassembleTest45 (void)
3193 {
3194  TcpReassemblyThreadCtx *ra_ctx = NULL;
3195  TcpSession ssn;
3196  ThreadVars tv;
3197  memset(&tv, 0, sizeof(tv));
3198  uint8_t payload[100] = {0};
3199  uint16_t payload_size = 100;
3200 
3201  StreamTcpUTInit(&ra_ctx);
3203 
3205  ssn.reassembly_depth = 100;
3206  StreamTcpUTSetupStream(&ssn.server, 100);
3207  StreamTcpUTSetupStream(&ssn.client, 100);
3208 
3209  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3210  FAIL_IF(r != 0);
3212 
3213  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3214  FAIL_IF(r != 0);
3216 
3220  StreamTcpUTDeinit(ra_ctx);
3221  PASS;
3222 }
3223 
3224 /**
3225  * \test Test the unlimited config value of reassembly depth.
3226  *
3227  * \retval On success it returns 1 and on failure 0.
3228  */
3229 
3230 static int StreamTcpReassembleTest46 (void)
3231 {
3232  int result = 0;
3233  TcpReassemblyThreadCtx *ra_ctx = NULL;
3234  TcpSession ssn;
3235  ThreadVars tv;
3236  memset(&tv, 0, sizeof(tv));
3237  uint8_t payload[100] = {0};
3238  uint16_t payload_size = 100;
3239 
3240  StreamTcpUTInit(&ra_ctx);
3242 
3244  StreamTcpUTSetupStream(&ssn.server, 100);
3245  StreamTcpUTSetupStream(&ssn.client, 100);
3246 
3247  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3248  if (r != 0)
3249  goto end;
3251  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3252  goto end;
3253  }
3254 
3255  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3256  if (r != 0)
3257  goto end;
3259  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3260  goto end;
3261  }
3262 
3263  result = 1;
3264 end:
3268  StreamTcpUTDeinit(ra_ctx);
3269  return result;
3270 }
3271 
3272 /**
3273  * \test Test to make sure we detect the sequence wrap around and continue
3274  * stream reassembly properly.
3275  *
3276  * \retval On success it returns 1 and on failure 0.
3277  */
3278 
3279 static int StreamTcpReassembleTest47 (void)
3280 {
3281  Packet *p = PacketGetFromAlloc();
3282  FAIL_IF(unlikely(p == NULL));
3283  Flow *f = NULL;
3284  TCPHdr tcph;
3285  TcpSession ssn;
3286  ThreadVars tv;
3287  memset(&tcph, 0, sizeof (TCPHdr));
3288  UTHSetTCPHdr(p, &tcph);
3289  memset(&tv, 0, sizeof (ThreadVars));
3290  StreamTcpInitConfig(true);
3293 
3294  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3295  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3296 
3297  SET_ISN(&ssn.server, 572799781UL);
3298  ssn.server.last_ack = 572799782UL;
3299 
3300  SET_ISN(&ssn.client, 4294967289UL);
3301  ssn.client.last_ack = 21;
3302 
3303  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3304  FAIL_IF(f == NULL);
3305  f->protoctx = &ssn;
3306  f->proto = IPPROTO_TCP;
3307  p->flow = f;
3308 
3309  tcph.th_win = htons(5480);
3310  ssn.state = TCP_ESTABLISHED;
3311  TcpStream *s = NULL;
3312  uint8_t cnt = 0;
3313 
3314  for (cnt=0; cnt < httplen1; cnt++) {
3315  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3316  tcph.th_ack = htonl(572799782UL);
3317  tcph.th_flags = TH_ACK | TH_PUSH;
3319  p->payload = &httpbuf1[cnt];
3320  p->payload_len = 1;
3321  s = &ssn.client;
3322 
3323  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3324 
3326  p->payload = NULL;
3327  p->payload_len = 0;
3328  tcph.th_seq = htonl(572799782UL);
3329  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3330  tcph.th_flags = TH_ACK;
3331  s = &ssn.server;
3332 
3333  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3334  }
3335 
3336  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3337 
3339  PacketFree(p);
3340  UTHFreeFlow(f);
3342  StreamTcpFreeConfig(true);
3343  PASS;
3344 }
3345 
3346 /** \test 3 in order segments in inline reassembly */
3347 static int StreamTcpReassembleInlineTest01(void)
3348 {
3349  int ret = 0;
3350  TcpReassemblyThreadCtx *ra_ctx = NULL;
3351  ThreadVars tv;
3352  TcpSession ssn;
3353  Flow f;
3354 
3355  memset(&tv, 0x00, sizeof(tv));
3356 
3357  StreamTcpUTInit(&ra_ctx);
3360  StreamTcpUTSetupStream(&ssn.client, 1);
3361  FLOW_INITIALIZE(&f);
3362 
3363  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3364  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3365  if (p == NULL) {
3366  printf("couldn't get a packet: ");
3367  goto end;
3368  }
3369  p->l4.hdrs.tcph->th_seq = htonl(12);
3370  p->flow = &f;
3371 
3372  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3373  printf("failed to add segment 1: ");
3374  goto end;
3375  }
3376  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3377  printf("failed to add segment 2: ");
3378  goto end;
3379  }
3380  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3381  printf("failed to add segment 3: ");
3382  goto end;
3383  }
3384  ssn.client.next_seq = 17;
3385  ret = 1;
3386 end:
3387  FLOW_DESTROY(&f);
3388  UTHFreePacket(p);
3390  StreamTcpUTDeinit(ra_ctx);
3391  return ret;
3392 }
3393 
3394 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3395  * test the sliding window reassembly.
3396  */
3397 static int StreamTcpReassembleInlineTest02(void)
3398 {
3399  int ret = 0;
3400  TcpReassemblyThreadCtx *ra_ctx = NULL;
3401  ThreadVars tv;
3402  TcpSession ssn;
3403  Flow f;
3404 
3405  memset(&tv, 0x00, sizeof(tv));
3406 
3407  StreamTcpUTInit(&ra_ctx);
3410  StreamTcpUTSetupStream(&ssn.client, 1);
3411  FLOW_INITIALIZE(&f);
3412 
3413  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3414  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3415  if (p == NULL) {
3416  printf("couldn't get a packet: ");
3417  goto end;
3418  }
3419  p->l4.hdrs.tcph->th_seq = htonl(12);
3420  p->flow = &f;
3421 
3422  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3423  printf("failed to add segment 1: ");
3424  goto end;
3425  }
3426  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3427  printf("failed to add segment 2: ");
3428  goto end;
3429  }
3430  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3431  printf("failed to add segment 3: ");
3432  goto end;
3433  }
3434  ssn.client.next_seq = 17;
3435  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3436  printf("failed to add segment 4: ");
3437  goto end;
3438  }
3439  ssn.client.next_seq = 22;
3440  ret = 1;
3441 end:
3442  FLOW_DESTROY(&f);
3443  UTHFreePacket(p);
3445  StreamTcpUTDeinit(ra_ctx);
3446  return ret;
3447 }
3448 
3449 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3450  * test the sliding window reassembly with a small window size so that we
3451  * cutting off at the start (left edge)
3452  */
3453 static int StreamTcpReassembleInlineTest03(void)
3454 {
3455  int ret = 0;
3456  TcpReassemblyThreadCtx *ra_ctx = NULL;
3457  ThreadVars tv;
3458  TcpSession ssn;
3459  Flow f;
3460 
3461  memset(&tv, 0x00, sizeof(tv));
3462 
3463  StreamTcpUTInit(&ra_ctx);
3466  StreamTcpUTSetupStream(&ssn.client, 1);
3467  FLOW_INITIALIZE(&f);
3468 
3470 
3471  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3472  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3473  if (p == NULL) {
3474  printf("couldn't get a packet: ");
3475  goto end;
3476  }
3477  p->l4.hdrs.tcph->th_seq = htonl(12);
3478  p->flow = &f;
3480 
3481  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3482  printf("failed to add segment 1: ");
3483  goto end;
3484  }
3485  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3486  printf("failed to add segment 2: ");
3487  goto end;
3488  }
3489  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3490  printf("failed to add segment 3: ");
3491  goto end;
3492  }
3493  ssn.client.next_seq = 17;
3494  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3495  printf("failed to add segment 4: ");
3496  goto end;
3497  }
3498  ssn.client.next_seq = 22;
3499 
3500  p->l4.hdrs.tcph->th_seq = htonl(17);
3501  ret = 1;
3502 end:
3503  FLOW_DESTROY(&f);
3504  UTHFreePacket(p);
3506  StreamTcpUTDeinit(ra_ctx);
3507  return ret;
3508 }
3509 
3510 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3511  * test the sliding window reassembly with a small window size so that we
3512  * cutting off at the start (left edge) with small packet overlap.
3513  */
3514 static int StreamTcpReassembleInlineTest04(void)
3515 {
3516  int ret = 0;
3517  TcpReassemblyThreadCtx *ra_ctx = NULL;
3518  ThreadVars tv;
3519  TcpSession ssn;
3520  Flow f;
3521 
3522  memset(&tv, 0x00, sizeof(tv));
3523 
3524  StreamTcpUTInit(&ra_ctx);
3527  StreamTcpUTSetupStream(&ssn.client, 1);
3528  FLOW_INITIALIZE(&f);
3529 
3531 
3532  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3533  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3534  if (p == NULL) {
3535  printf("couldn't get a packet: ");
3536  goto end;
3537  }
3538  p->l4.hdrs.tcph->th_seq = htonl(12);
3539  p->flow = &f;
3541 
3542  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3543  printf("failed to add segment 1: ");
3544  goto end;
3545  }
3546  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3547  printf("failed to add segment 2: ");
3548  goto end;
3549  }
3550  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3551  printf("failed to add segment 3: ");
3552  goto end;
3553  }
3554  ssn.client.next_seq = 17;
3555  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3556  printf("failed to add segment 4: ");
3557  goto end;
3558  }
3559  ssn.client.next_seq = 22;
3560 
3561  p->l4.hdrs.tcph->th_seq = htonl(17);
3562  ret = 1;
3563 end:
3564  FLOW_DESTROY(&f);
3565  UTHFreePacket(p);
3567  StreamTcpUTDeinit(ra_ctx);
3568  return ret;
3569 }
3570 
3571 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3572  * test the sliding window reassembly with a small window size so that we
3573  * cutting off at the start (left edge). Test if the first segment is
3574  * removed from the list.
3575  */
3576 static int StreamTcpReassembleInlineTest08(void)
3577 {
3578  TcpReassemblyThreadCtx *ra_ctx = NULL;
3579  ThreadVars tv;
3580  memset(&tv, 0x00, sizeof(tv));
3581  TcpSession ssn;
3582  Flow f;
3583  StreamTcpUTInit(&ra_ctx);
3586  StreamTcpUTSetupStream(&ssn.client, 1);
3587  FLOW_INITIALIZE(&f);
3588 
3590  f.protoctx = &ssn;
3591 
3592  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3593  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3594  FAIL_IF(p == NULL);
3595  p->l4.hdrs.tcph->th_seq = htonl(12);
3596  p->flow = &f;
3598 
3599  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3600  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3601  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3602  ssn.client.next_seq = 17;
3603  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3604  ssn.client.next_seq = 22;
3605  p->l4.hdrs.tcph->th_seq = htonl(17);
3606  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3607 
3608  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3609  FAIL_IF_NULL(seg);
3610  FAIL_IF_NOT(seg->seq == 2);
3611 
3612  FLOW_DESTROY(&f);
3613  UTHFreePacket(p);
3615  StreamTcpUTDeinit(ra_ctx);
3616  PASS;
3617 }
3618 
3619 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3620  * test the sliding window reassembly with a small window size so that we
3621  * cutting off at the start (left edge). Test if the first segment is
3622  * removed from the list.
3623  */
3624 static int StreamTcpReassembleInlineTest09(void)
3625 {
3626  int ret = 0;
3627  TcpReassemblyThreadCtx *ra_ctx = NULL;
3628  ThreadVars tv;
3629  TcpSession ssn;
3630  Flow f;
3631 
3632  memset(&tv, 0x00, sizeof(tv));
3633 
3634  StreamTcpUTInit(&ra_ctx);
3637  StreamTcpUTSetupStream(&ssn.client, 1);
3638  FLOW_INITIALIZE(&f);
3639 
3641 
3642  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3643  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3644  if (p == NULL) {
3645  printf("couldn't get a packet: ");
3646  goto end;
3647  }
3648  p->l4.hdrs.tcph->th_seq = htonl(17);
3649  p->flow = &f;
3651 
3652  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3653  printf("failed to add segment 1: ");
3654  goto end;
3655  }
3656  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3657  printf("failed to add segment 2: ");
3658  goto end;
3659  }
3660  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3661  printf("failed to add segment 3: ");
3662  goto end;
3663  }
3664  ssn.client.next_seq = 12;
3665  ssn.client.last_ack = 10;
3666 
3667  /* close the GAP and see if we properly reassemble and update base_seq */
3668  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3669  printf("failed to add segment 4: ");
3670  goto end;
3671  }
3672  ssn.client.next_seq = 22;
3673 
3674  p->l4.hdrs.tcph->th_seq = htonl(12);
3675 
3676  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3677  FAIL_IF_NULL(seg);
3678  FAIL_IF_NOT(seg->seq == 2);
3679 
3680  ret = 1;
3681 end:
3682  FLOW_DESTROY(&f);
3683  UTHFreePacket(p);
3685  StreamTcpUTDeinit(ra_ctx);
3686  return ret;
3687 }
3688 
3689 /** \test App Layer reassembly.
3690  */
3691 static int StreamTcpReassembleInlineTest10(void)
3692 {
3693  int ret = 0;
3694  TcpReassemblyThreadCtx *ra_ctx = NULL;
3695  ThreadVars tv;
3696  TcpSession ssn;
3697  Flow *f = NULL;
3698  Packet *p = NULL;
3699 
3700  memset(&tv, 0x00, sizeof(tv));
3701 
3702  StreamTcpUTInit(&ra_ctx);
3705  StreamTcpUTSetupStream(&ssn.server, 1);
3706  ssn.server.last_ack = 2;
3707  StreamTcpUTSetupStream(&ssn.client, 1);
3708  ssn.client.last_ack = 2;
3709  ssn.data_first_seen_dir = STREAM_TOSERVER;
3710 
3711  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3712  if (f == NULL)
3713  goto end;
3714  f->protoctx = &ssn;
3715  f->proto = IPPROTO_TCP;
3716 
3717  uint8_t stream_payload1[] = "GE";
3718  uint8_t stream_payload2[] = "T /";
3719  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3720 
3721  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3722  if (p == NULL) {
3723  printf("couldn't get a packet: ");
3724  goto end;
3725  }
3726  p->l4.hdrs.tcph->th_seq = htonl(7);
3727  p->flow = f;
3729 
3730  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3731  printf("failed to add segment 1: ");
3732  goto end;
3733  }
3734  ssn.client.next_seq = 4;
3735 
3736  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3737  if (r < 0) {
3738  printf("StreamTcpReassembleAppLayer failed: ");
3739  goto end;
3740  }
3741 
3742  /* ssn.server.ra_app_base_seq should be isn here. */
3743  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3744  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3745  goto end;
3746  }
3747 
3748  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3749  printf("failed to add segment 2: ");
3750  goto end;
3751  }
3752  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3753  printf("failed to add segment 3: ");
3754  goto end;
3755  }
3756  ssn.client.next_seq = 19;
3757 
3758  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3759  if (r < 0) {
3760  printf("StreamTcpReassembleAppLayer failed: ");
3761  goto end;
3762  }
3763 
3764  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3765 
3766  ret = 1;
3767 end:
3768  UTHFreePacket(p);
3770  StreamTcpUTDeinit(ra_ctx);
3771  UTHFreeFlow(f);
3772  return ret;
3773 }
3774 
3775 /** \test test insert with overlap
3776  */
3777 static int StreamTcpReassembleInsertTest01(void)
3778 {
3779  TcpReassemblyThreadCtx *ra_ctx = NULL;
3780  ThreadVars tv;
3781  TcpSession ssn;
3782  Flow f;
3783 
3784  memset(&tv, 0x00, sizeof(tv));
3785 
3786  StreamTcpUTInit(&ra_ctx);
3788  StreamTcpUTSetupStream(&ssn.client, 1);
3790  FLOW_INITIALIZE(&f);
3791 
3792  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3793  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3794  FAIL_IF(p == NULL);
3795  p->l4.hdrs.tcph->th_seq = htonl(12);
3796  p->flow = &f;
3797 
3798  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3799  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3800  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3801  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3802  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3803  ssn.client.next_seq = 21;
3804 
3805  FLOW_DESTROY(&f);
3806  UTHFreePacket(p);
3808  StreamTcpUTDeinit(ra_ctx);
3809  PASS;
3810 }
3811 
3812 /** \test test insert with overlaps
3813  */
3814 static int StreamTcpReassembleInsertTest02(void)
3815 {
3816  int ret = 0;
3817  TcpReassemblyThreadCtx *ra_ctx = NULL;
3818  ThreadVars tv;
3819  TcpSession ssn;
3820 
3821  memset(&tv, 0x00, sizeof(tv));
3822 
3823  StreamTcpUTInit(&ra_ctx);
3825  StreamTcpUTSetupStream(&ssn.client, 1);
3826 
3827  int i;
3828  for (i = 2; i < 10; i++) {
3829  int len;
3830  len = i % 2;
3831  if (len == 0)
3832  len = 1;
3833  int seq;
3834  seq = i * 10;
3835  if (seq < 2)
3836  seq = 2;
3837 
3838  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3839  printf("failed to add segment 1: ");
3840  goto end;
3841  }
3842  }
3843  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3844  printf("failed to add segment 2: ");
3845  goto end;
3846  }
3847 
3848  ret = 1;
3849 end:
3851  StreamTcpUTDeinit(ra_ctx);
3852  return ret;
3853 }
3854 
3855 /** \test test insert with overlaps
3856  */
3857 static int StreamTcpReassembleInsertTest03(void)
3858 {
3859  int ret = 0;
3860  TcpReassemblyThreadCtx *ra_ctx = NULL;
3861  ThreadVars tv;
3862  TcpSession ssn;
3863 
3864  memset(&tv, 0x00, sizeof(tv));
3865 
3866  StreamTcpUTInit(&ra_ctx);
3868  StreamTcpUTSetupStream(&ssn.client, 1);
3869 
3870  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3871  printf("failed to add segment 2: ");
3872  goto end;
3873  }
3874 
3875  int i;
3876  for (i = 2; i < 10; i++) {
3877  int len;
3878  len = i % 2;
3879  if (len == 0)
3880  len = 1;
3881  int seq;
3882  seq = i * 10;
3883  if (seq < 2)
3884  seq = 2;
3885 
3886  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3887  printf("failed to add segment 2: ");
3888  goto end;
3889  }
3890  }
3891  ret = 1;
3892 end:
3894  StreamTcpUTDeinit(ra_ctx);
3895  return ret;
3896 }
3897 
3899 #endif /* UNITTESTS */
3900 
3901 /** \brief The Function Register the Unit tests to test the reassembly engine
3902  * for various OS policies.
3903  */
3904 
3906 {
3907 #ifdef UNITTESTS
3908  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3909  StreamTcpReassembleTest25);
3910  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3911  StreamTcpReassembleTest26);
3912  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3913  StreamTcpReassembleTest27);
3914  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3915  StreamTcpReassembleTest28);
3916  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3917  StreamTcpReassembleTest29);
3918  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3919  StreamTcpReassembleTest33);
3920  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3921  StreamTcpReassembleTest34);
3922  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3923  StreamTcpReassembleTest39);
3924  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3925  StreamTcpReassembleTest40);
3926  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3927  StreamTcpReassembleTest44);
3928  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3929  StreamTcpReassembleTest45);
3930  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3931  StreamTcpReassembleTest46);
3932  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3933  StreamTcpReassembleTest47);
3934 
3935  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3936  StreamTcpReassembleInlineTest01);
3937  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3938  StreamTcpReassembleInlineTest02);
3939  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3940  StreamTcpReassembleInlineTest03);
3941  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3942  StreamTcpReassembleInlineTest04);
3943  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3944  StreamTcpReassembleInlineTest08);
3945  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3946  StreamTcpReassembleInlineTest09);
3947 
3948  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3949  StreamTcpReassembleInlineTest10);
3950 
3951  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3952  StreamTcpReassembleInsertTest01);
3953  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3954  StreamTcpReassembleInsertTest02);
3955  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3956  StreamTcpReassembleInsertTest03);
3957 
3961  StreamTcpReassembleRawRegisterTests();
3962 #endif /* UNITTESTS */
3963 }
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
Definition: stream-tcp.h:143
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:270
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:40
TCP_GET_RAW_SEQ
#define TCP_GET_RAW_SEQ(tcph)
Definition: decode-tcp.h:80
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:44
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:58
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:56
util-device-private.h
UTHCheckGapAtPosition
int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
Definition: stream-tcp-reassemble.c:2264
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:98
Packet_::proto
uint8_t proto
Definition: decode.h:523
TcpStream_
Definition: stream-tcp-private.h:106
len
uint8_t len
Definition: app-layer-dnp3.h:2
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:136
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:186
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:113
TcpReassemblyThreadCtx_::counter_tcp_reas_eps
ExceptionPolicyCounters counter_tcp_reas_eps
Definition: stream-tcp-reassemble.h:69
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:2120
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:166
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:381
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:98
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1904
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:97
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:91
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:94
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:92
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
TCP_STREAM_URGENT_OOB
@ TCP_STREAM_URGENT_OOB
Definition: stream-tcp.h:49
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:69
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
stream-tcp.h
StreamTcpInlineMode
bool StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:7213
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:136
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:2193
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:279
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:626
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1026
StreamTcpThread_
Definition: stream-tcp.h:92
Flow_::proto
uint8_t proto
Definition: flow.h:370
Packet_::payload
uint8_t * payload
Definition: decode.h:605
t_pcapcnt
thread_local uint64_t t_pcapcnt
Definition: stream-tcp-reassemble.c:80
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:2173
action-globals.h
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:62
Packet_::flags
uint32_t flags
Definition: decode.h:544
SCAppLayerParserStateSetFlag
void SCAppLayerParserStateSetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1821
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:110
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:75
Flow_
Flow data structure.
Definition: flow.h:348
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:77
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:75
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
TCPHdr_::th_win
uint16_t th_win
Definition: decode-tcp.h:156
TcpSegmentPcapHdrStorage
struct TcpSegmentPcapHdrStorage_ TcpSegmentPcapHdrStorage
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
StringParseUint16
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:343
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:156
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1396
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:40
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:89
PKT_DROP_REASON_STREAM_REASSEMBLY
@ PKT_DROP_REASON_STREAM_REASSEMBLY
Definition: decode.h:394
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:246
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1813
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir app_update_dir)
handle TCP data for the app-layer.
Definition: app-layer.c:707
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:225
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:155
MIN
#define MIN(x, y)
Definition: suricata-common.h:408
SCConfGetBool
int SCConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:497
TcpSession_::urg_offset_ts
uint16_t urg_offset_ts
Definition: stream-tcp-private.h:291
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:75
UTHSetTCPHdr
void UTHSetTCPHdr(Packet *p, TCPHdr *tcph)
Definition: util-unittest-helper.c:136
StreamTcpThreadCacheGetSegment
TcpSegment * StreamTcpThreadCacheGetSegment(void)
Definition: stream-tcp-cache.c:179
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:122
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:303
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:49
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:156
STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
@ STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED
Definition: decode-events.h:310
TCP_STREAM_URGENT_GAP
@ TCP_STREAM_URGENT_GAP
Definition: stream-tcp.h:50
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1830
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:532
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:219
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:53
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:260
Flow_::protoctx
void * protoctx
Definition: flow.h:433
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:710
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:59
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:100
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:606
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:223
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:632
util-unittest.h
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
AppLayerProfilingStore
#define AppLayerProfilingStore(app_tctx, p)
Definition: app-layer.h:128
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2255
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
SCAppLayerDecoderEventsSetEventRaw
void SCAppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:96
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:523
PKT_DROP_REASON_STREAM_URG
@ PKT_DROP_REASON_STREAM_URG
Definition: decode.h:395
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:426
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:170
STREAMTCP_FLAG_LOSSY_BE_LIBERAL
#define STREAMTCP_FLAG_LOSSY_BE_LIBERAL
Definition: stream-tcp-private.h:194
TCPHdr_::th_ack
uint32_t th_ack
Definition: decode-tcp.h:153
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:294
StreamTcpReassembleTriggerRawInspection
void StreamTcpReassembleTriggerRawInspection(TcpSession *ssn, int direction)
Trigger RAW stream inspection.
Definition: stream-tcp-reassemble.c:2156
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:299
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:270
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:115
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:470
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:557
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:488
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:497
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:38
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:295
StreamTcpReassembleDepthReached
bool StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:625
TCPHdr_::th_flags
uint8_t th_flags
Definition: decode-tcp.h:155
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:463
StreamTcpReassembleRealloc
void * StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
Definition: stream-tcp-reassemble.c:229
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:238
UTHCheckDataAtPosition
int UTHCheckDataAtPosition(TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
Definition: stream-tcp-reassemble.c:2287
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:104
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:271
STREAMTCP_FLAG_ASYNC
#define STREAMTCP_FLAG_ASYNC
Definition: stream-tcp-private.h:182
TCPHdr_::th_seq
uint32_t th_seq
Definition: decode-tcp.h:152
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1262
TcpStreamCnf_::urgent_policy
enum TcpStreamUrgentHandling urgent_policy
Definition: stream-tcp.h:83
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:120
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1270
util-exception-policy.h
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
TcpSession_::urg_offset_tc
uint16_t urg_offset_tc
Definition: stream-tcp-private.h:292
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:96
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:539
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:131
ExceptionPolicyCounters_::eps_id
uint16_t eps_id[EXCEPTION_POLICY_MAX]
Definition: util-exception-policy-types.h:53
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
SCEnter
#define SCEnter(...)
Definition: util-debug.h:281
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:5656
StreamNeedsReassembly
uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:976
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:80
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1509
PacketFree
void PacketFree(Packet *p)
Return a malloced packet.
Definition: decode.c:219
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:285
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:137
TCPHdr_::th_urp
uint16_t th_urp
Definition: decode-tcp.h:158
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:316
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:105
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2207
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:886
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:112
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:164
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:317
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:341
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:351
SCReturn
#define SCReturn
Definition: util-debug.h:283
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:303
Packet_
Definition: decode.h:501
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:234
stream-tcp-private.h
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:54
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:287
TcpReassemblyThreadCtx_::counter_tcp_urgent_oob
uint16_t counter_tcp_urgent_oob
Definition: stream-tcp-reassemble.h:88
conf.h
Packet_::l4
struct PacketL4 l4
Definition: decode.h:601
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:301
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1853
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:117
STREAMTCP_STREAM_FLAG_HAS_GAP
#define STREAMTCP_STREAM_FLAG_HAS_GAP
Definition: stream-tcp-private.h:217
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:596
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:297
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:722
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(void)
Creates a new app layer thread context.
Definition: app-layer.c:1102
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:398
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:226
TcpStreamCnf_::reassembly_memcap_policy
enum ExceptionPolicy reassembly_memcap_policy
Definition: stream-tcp.h:81
StreamTcpThreadCacheCleanup
void StreamTcpThreadCacheCleanup(void)
Definition: stream-tcp-cache.c:134
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:270
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:539
TH_URG
#define TH_URG
Definition: decode-tcp.h:39
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:229
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:77
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:128
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:104
VALIDATE
#define VALIDATE(e)
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:502
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:57
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:192
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:72
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a TCP packet data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:746
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:64
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
TcpSegmentPcapHdrStorage_
Definition: stream-tcp-private.h:65
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1720
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
app-layer-frames.h
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:180
Packet_::flow
struct Flow_ * flow
Definition: decode.h:546
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:260
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:304
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:859
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
SCNtohs
#define SCNtohs(x)
Definition: suricata-common.h:431
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:447
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:259
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:634
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:124
packet.h
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:36
TcpStreamCnf_::urgent_oob_limit_policy
enum TcpStreamUrgentHandling urgent_oob_limit_policy
Definition: stream-tcp.h:84
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2236
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:135
TcpSegmentPcapHdrStorage_::alloclen
uint32_t alloclen
Definition: stream-tcp-private.h:68
ACTION_DROP
#define ACTION_DROP
Definition: action-globals.h:30
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:535
StreamTcpThreadCacheReturnSegment
void StreamTcpThreadCacheReturnSegment(TcpSegment *seg)
Definition: stream-tcp-cache.c:53
STREAM_REASSEMBLY_DEPTH_REACHED
@ STREAM_REASSEMBLY_DEPTH_REACHED
Definition: decode-events.h:306
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:236
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:297
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Definition: stream-tcp-reassemble.c:2000
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:161
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:114
TCP_STREAM_URGENT_DROP
@ TCP_STREAM_URGENT_DROP
Definition: stream-tcp.h:48
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:78
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:114
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:258
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1555
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
PacketL4::L4Hdrs::tcph
TCPHdr * tcph
Definition: decode.h:469
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:63
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:296
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
SCConfGetNode
SCConfNode * SCConfGetNode(const char *name)
Get a SCConfNode by name.
Definition: conf.c:181
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:271
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:443
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:473
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:137
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayerGetCtxThread().
Definition: app-layer.c:1123
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:257
Flow_::flags
uint32_t flags
Definition: flow.h:413
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:1022
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:71
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:53
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:150
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:307
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2250
stream-tcp-cache.h
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:158
PacketL4::hdrs
union PacketL4::L4Hdrs hdrs
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:117
MISSED_ADD_PAYLOAD
#define MISSED_ADD_PAYLOAD(seq, seg, seglen)
Definition: stream-tcp-reassemble.c:2259
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:61
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:69
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:589
StreamDataRightEdge
uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:417
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:174
PacketDrop
void PacketDrop(Packet *p, const uint8_t action, enum PacketDropReason r)
issue drop action
Definition: packet.c:34
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:219
suricata.h
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap(void)
Return memcap value.
Definition: stream-tcp-reassemble.c:200
Address_::family
char family
Definition: decode.h:113
Packet_::dst
Address dst
Definition: decode.h:506
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:7226
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:258
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:189
TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool
uint16_t counter_tcp_segment_from_pool
Definition: stream-tcp-reassemble.h:72
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
TCP_STREAM_URGENT_INLINE
@ TCP_STREAM_URGENT_INLINE
Definition: stream-tcp.h:46
likely
#define likely(expr)
Definition: util-optimize.h:32
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3905
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:102
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
segment_thread_pool
PoolThread * segment_thread_pool
Definition: stream-tcp-reassemble.c:82
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:197
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TCPSEG_PKT_HDR_DEFAULT_SIZE
#define TCPSEG_PKT_HDR_DEFAULT_SIZE
Definition: stream-tcp-private.h:58
TcpSession_
Definition: stream-tcp-private.h:283
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:288
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:185
flow.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:247
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:444
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:442
StreamReassembleLog
int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, uint64_t *progress_out, const bool eof)
Definition: stream-tcp-reassemble.c:1969
ExceptionPolicy
ExceptionPolicy
Definition: util-exception-policy-types.h:25
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:112
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:225
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:149
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:285
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:121
SCConfNode_
Definition: conf.h:37
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
SCConfNode_::val
char * val
Definition: conf.h:39
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamReassembleForFrame
int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof)
Definition: stream-tcp-reassemble.c:1892
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:67
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1684
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:77
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:119
TCPHdr_
Definition: decode-tcp.h:149
Packet_::src
Address src
Definition: decode.h:505
TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache
uint16_t counter_tcp_segment_from_cache
Definition: stream-tcp-reassemble.h:71
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:70
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:232
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:146
app-layer.h