suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "detect.h"
32 #include "flow.h"
33 #include "threads.h"
34 #include "conf.h"
35 
36 #include "flow-util.h"
37 
38 #include "threadvars.h"
39 #include "tm-threads.h"
40 
41 #include "util-pool.h"
42 #include "util-unittest.h"
43 #include "util-print.h"
44 #include "util-host-os-info.h"
45 #include "util-unittest-helper.h"
46 #include "util-byte.h"
47 #include "util-device.h"
48 
49 #include "stream-tcp.h"
50 #include "stream-tcp-private.h"
51 #include "stream-tcp-cache.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 #include "app-layer-parser.h"
64 #include "app-layer-frames.h"
65 
66 #include "detect-engine-state.h"
67 
68 #include "util-profiling.h"
69 #include "util-validate.h"
70 #include "util-exception-policy.h"
71 
72 #ifdef DEBUG
73 static SCMutex segment_pool_memuse_mutex;
74 static uint64_t segment_pool_memuse = 0;
75 static uint64_t segment_pool_memcnt = 0;
76 #endif
77 
78 thread_local uint64_t t_pcapcnt = UINT64_MAX;
79 
81 /* init only, protect initializing and growing pool */
82 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
83 
84 /* Memory use counter */
85 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
86 
87 static int g_tcp_session_dump_enabled = 0;
88 
89 inline bool IsTcpSessionDumpingEnabled(void)
90 {
91  return g_tcp_session_dump_enabled == 1;
92 }
93 
95 {
96  g_tcp_session_dump_enabled = 1;
97 }
98 
99 /* prototypes */
101 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
102 
104 {
105  SC_ATOMIC_INIT(ra_memuse);
106 }
107 
108 /**
109  * \brief Function to Increment the memory usage counter for the TCP reassembly
110  * segments
111  *
112  * \param size Size of the TCP segment and its payload length memory allocated
113  */
114 void StreamTcpReassembleIncrMemuse(uint64_t size)
115 {
116  (void) SC_ATOMIC_ADD(ra_memuse, size);
117  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
118  return;
119 }
120 
121 /**
122  * \brief Function to Decrease the memory usage counter for the TCP reassembly
123  * segments
124  *
125  * \param size Size of the TCP segment and its payload length memory allocated
126  */
127 void StreamTcpReassembleDecrMemuse(uint64_t size)
128 {
129 #ifdef UNITTESTS
130  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
131  if (RunmodeIsUnittests()) {
132  BUG_ON(presize > UINT_MAX);
133  }
134 #endif
135 
136  (void) SC_ATOMIC_SUB(ra_memuse, size);
137 
138 #ifdef UNITTESTS
139  if (RunmodeIsUnittests()) {
140  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
141  BUG_ON(postsize > presize);
142  }
143 #endif
144  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
145  return;
146 }
147 
149 {
150  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
151  return smemuse;
152 }
153 
154 /**
155  * \brief Function to Check the reassembly memory usage counter against the
156  * allowed max memory usage for TCP segments.
157  *
158  * \param size Size of the TCP segment and its payload length memory allocated
159  * \retval 1 if in bounds
160  * \retval 0 if not in bounds
161  */
163 {
164 #ifdef DEBUG
165  if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX &&
166  g_eps_stream_reassembly_memcap == t_pcapcnt))) {
167  SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt);
168  return 0;
169  }
170 #endif
171  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
172  if (memcapcopy == 0 ||
173  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
174  return 1;
175  return 0;
176 }
177 
178 /**
179  * \brief Update memcap value
180  *
181  * \param size new memcap value
182  */
184 {
185  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
186  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
187  return 1;
188  }
189 
190  return 0;
191 }
192 
193 /**
194  * \brief Return memcap value
195  *
196  * \return memcap memcap value
197  */
199 {
200  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
201  return memcapcopy;
202 }
203 
204 /* memory functions for the streaming buffer API */
205 
206 /*
207  void *(*Calloc)(size_t n, size_t size);
208 */
209 static void *ReassembleCalloc(size_t n, size_t size)
210 {
211  if (StreamTcpReassembleCheckMemcap(n * size) == 0) {
213  return NULL;
214  }
215  void *ptr = SCCalloc(n, size);
216  if (ptr == NULL) {
218  return NULL;
219  }
221  return ptr;
222 }
223 
224 /*
225  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
226 */
227 void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
228 {
229  if (size > orig_size) {
230  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) {
231  SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap));
233  return NULL;
234  }
235  }
236  void *nptr = SCRealloc(optr, size);
237  if (nptr == NULL) {
238  SCLogDebug("realloc fail");
240  return NULL;
241  }
242  if (size > orig_size) {
243  StreamTcpReassembleIncrMemuse(size - orig_size);
244  } else {
245  StreamTcpReassembleDecrMemuse(orig_size - size);
246  }
247  return nptr;
248 }
249 
250 /*
251  void (*Free)(void *ptr, size_t size);
252 */
253 static void ReassembleFree(void *ptr, size_t size)
254 {
255  SCFree(ptr);
257 }
258 
259 /** \brief alloc a tcp segment pool entry */
260 static void *TcpSegmentPoolAlloc(void)
261 {
262  SCLogDebug("segment alloc");
263  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
264  return NULL;
265  }
266 
267  TcpSegment *seg = NULL;
268 
269  seg = SCMalloc(sizeof (TcpSegment));
270  if (unlikely(seg == NULL))
271  return NULL;
272 
274  uint32_t memuse =
275  sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
276  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
277  SCFree(seg);
278  return NULL;
279  }
280 
282  if (seg->pcap_hdr_storage == NULL) {
283  SCLogError("Unable to allocate memory for "
284  "TcpSegmentPcapHdrStorage");
285  SCFree(seg);
286  return NULL;
287  } else {
288  seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
289  seg->pcap_hdr_storage->pkt_hdr =
290  SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
291  if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
292  SCLogError("Unable to allocate memory for "
293  "packet header data within "
294  "TcpSegmentPcapHdrStorage");
295  SCFree(seg->pcap_hdr_storage);
296  SCFree(seg);
297  return NULL;
298  }
299  }
300 
302  } else {
303  seg->pcap_hdr_storage = NULL;
304  }
305 
306  return seg;
307 }
308 
309 static int TcpSegmentPoolInit(void *data, void *initdata)
310 {
311  TcpSegment *seg = (TcpSegment *) data;
312  TcpSegmentPcapHdrStorage *pcap_hdr;
313 
314  pcap_hdr = seg->pcap_hdr_storage;
315 
316  /* do this before the can bail, so TcpSegmentPoolCleanup
317  * won't have uninitialized memory to consider. */
318  memset(seg, 0, sizeof (TcpSegment));
319 
321  uint32_t memuse =
322  sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
323  seg->pcap_hdr_storage = pcap_hdr;
324  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
325  return 0;
326  }
328  } else {
329  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
330  return 0;
331  }
332  }
333 
334 #ifdef DEBUG
335  SCMutexLock(&segment_pool_memuse_mutex);
336  segment_pool_memuse += sizeof(TcpSegment);
337  segment_pool_memcnt++;
338  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
339  SCMutexUnlock(&segment_pool_memuse_mutex);
340 #endif
341 
342  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
343  return 1;
344 }
345 
346 /** \brief clean up a tcp segment pool entry */
347 static void TcpSegmentPoolCleanup(void *ptr)
348 {
349  if (ptr == NULL)
350  return;
351 
352  TcpSegment *seg = (TcpSegment *)ptr;
353  if (seg && seg->pcap_hdr_storage) {
354  if (seg->pcap_hdr_storage->pkt_hdr) {
357  }
358  SCFree(seg->pcap_hdr_storage);
359  seg->pcap_hdr_storage = NULL;
361  }
362 
363  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
364 
365 #ifdef DEBUG
366  SCMutexLock(&segment_pool_memuse_mutex);
367  segment_pool_memuse -= sizeof(TcpSegment);
368  segment_pool_memcnt--;
369  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
370  SCMutexUnlock(&segment_pool_memuse_mutex);
371 #endif
372 }
373 
374 /**
375  * \brief Function to return the segment back to the pool.
376  *
377  * \param seg Segment which will be returned back to the pool.
378  */
380 {
381  if (seg == NULL)
382  return;
383 
384  if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
385  seg->pcap_hdr_storage->pktlen = 0;
386  }
387 
389 }
390 
391 /**
392  * \brief return all segments in this stream into the pool(s)
393  *
394  * \param stream the stream to cleanup
395  */
397 {
398  TcpSegment *seg = NULL, *safe = NULL;
399  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
400  {
401  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
403  }
404 }
405 
406 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
407 {
408  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
409  return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq);
410  } else {
411  return STREAM_BASE_OFFSET(stream);
412  }
413 }
414 
415 uint64_t StreamTcpGetAcked(const TcpStream *stream)
416 {
417  return GetAbsLastAck(stream);
418 }
419 
420 // may contain gaps
421 uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
422 {
423  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
424  if (!eof && !StreamTcpInlineMode()) {
425  right_edge = MIN(GetAbsLastAck(stream), right_edge);
426  }
427  return right_edge;
428 }
429 
430 uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
431 {
432  uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
433  if (!eof && !StreamTcpInlineMode()) {
434  right_edge = MIN(GetAbsLastAck(stream), right_edge);
435  }
436  return right_edge;
437 }
438 
439 #ifdef UNITTESTS
440 /** \internal
441  * \brief check if segments falls before stream 'offset' */
442 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
443 {
444  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
445  return 1;
446  return 0;
447 }
448 #endif
449 
450 /** \param f locked flow */
452 {
453  if (f->protoctx == NULL)
454  return;
455 
456  TcpSession *ssn = (TcpSession *)f->protoctx;
460  if (f->alparser) {
463  }
464 }
465 
466 /** \param f locked flow */
468 {
469  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
470  return 0;
471 
472  TcpSession *ssn = (TcpSession *)f->protoctx;
474 }
475 
476 static int StreamTcpReassemblyConfig(bool quiet)
477 {
478  uint32_t segment_prealloc = 2048;
479  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
480  if (seg) {
481  uint32_t prealloc = 0;
482  if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) {
483  SCLogError("segment-prealloc of "
484  "%s is invalid",
485  seg->val);
486  return -1;
487  }
488  segment_prealloc = prealloc;
489  }
490  if (!quiet)
491  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
492  stream_config.prealloc_segments = segment_prealloc;
493 
494  int overlap_diff_data = 0;
495  (void)ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
496  if (overlap_diff_data) {
498  }
499  if (StreamTcpInlineMode()) {
501  }
502 
503  uint16_t max_regions = 8;
504  ConfNode *mr = ConfGetNode("stream.reassembly.max-regions");
505  if (mr) {
506  uint16_t max_r = 0;
507  if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) {
508  SCLogError("max-regions %s is invalid", mr->val);
509  return -1;
510  }
511  max_regions = max_r;
512  }
513  if (!quiet)
514  SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions);
515 
516  stream_config.prealloc_segments = segment_prealloc;
518  stream_config.sbcnf.max_regions = max_regions;
520  stream_config.sbcnf.Calloc = ReassembleCalloc;
522  stream_config.sbcnf.Free = ReassembleFree;
523 
524  return 0;
525 }
526 
527 int StreamTcpReassembleInit(bool quiet)
528 {
529  /* init the memcap/use tracker */
531 
532  if (StreamTcpReassemblyConfig(quiet) < 0)
533  return -1;
534 
535 #ifdef DEBUG
536  SCMutexInit(&segment_pool_memuse_mutex, NULL);
537 #endif
538  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
540  return 0;
541 }
542 
543 void StreamTcpReassembleFree(bool quiet)
544 {
545  SCMutexLock(&segment_thread_pool_mutex);
546  if (segment_thread_pool != NULL) {
548  segment_thread_pool = NULL;
549  }
550  SCMutexUnlock(&segment_thread_pool_mutex);
551  SCMutexDestroy(&segment_thread_pool_mutex);
552 
553 #ifdef DEBUG
554  if (segment_pool_memuse > 0)
555  SCLogDebug("segment_pool_memuse %" PRIu64 " segment_pool_memcnt %" PRIu64 "",
556  segment_pool_memuse, segment_pool_memcnt);
557  SCMutexDestroy(&segment_pool_memuse_mutex);
558 #endif
559 }
560 
562 {
563  SCEnter();
565  if (unlikely(ra_ctx == NULL))
566  return NULL;
567 
568  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
569 
570  SCMutexLock(&segment_thread_pool_mutex);
571  if (segment_thread_pool == NULL) {
572  segment_thread_pool = PoolThreadInit(1, /* thread */
573  0, /* unlimited */
575  sizeof(TcpSegment),
576  TcpSegmentPoolAlloc,
577  TcpSegmentPoolInit, NULL,
578  TcpSegmentPoolCleanup, NULL);
579  ra_ctx->segment_thread_pool_id = 0;
580  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
582  ra_ctx->segment_thread_pool_id);
583  } else {
584  /* grow segment_thread_pool until we have an element for our thread id */
586  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
588  ra_ctx->segment_thread_pool_id);
589  }
590  SCMutexUnlock(&segment_thread_pool_mutex);
591  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
592  SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
594  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
595  }
596 
597  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
598 }
599 
601 {
602  SCEnter();
604 
605  if (ra_ctx) {
607  SCFree(ra_ctx);
608  }
609  SCReturn;
610 }
611 
612 /**
613  * \brief check if stream in pkt direction has depth reached
614  *
615  * \param p packet with *LOCKED* flow
616  *
617  * \retval 1 stream has depth reached
618  * \retval 0 stream does not have depth reached
619  */
621 {
622  if (p->flow != NULL && p->flow->protoctx != NULL) {
623  TcpSession *ssn = p->flow->protoctx;
624  TcpStream *stream;
625  if (p->flowflags & FLOW_PKT_TOSERVER) {
626  stream = &ssn->client;
627  } else {
628  stream = &ssn->server;
629  }
630 
631  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
632  }
633 
634  return 0;
635 }
636 
637 /**
638  * \internal
639  * \brief Function to Check the reassembly depth valuer against the
640  * allowed max depth of the stream reassembly for TCP streams.
641  *
642  * \param stream stream direction
643  * \param seq sequence number where "size" starts
644  * \param size size of the segment that is added
645  *
646  * \retval size Part of the size that fits in the depth, 0 if none
647  */
648 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
649  uint32_t seq, uint32_t size)
650 {
651  SCEnter();
652 
653  /* if the configured depth value is 0, it means there is no limit on
654  reassembly depth. Otherwise carry on my boy ;) */
655  if (ssn->reassembly_depth == 0) {
656  SCReturnUInt(size);
657  }
658 
659  /* if the final flag is set, we're not accepting anymore */
661  SCReturnUInt(0);
662  }
663 
664  uint64_t seg_depth;
665  if (SEQ_GT(stream->base_seq, seq)) {
666  if (SEQ_LEQ(seq+size, stream->base_seq)) {
667  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
668  stream->base_seq, seq, seq+size);
669  SCReturnUInt(0);
670  }
671 
672  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
673  } else {
674  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
675  }
676 
677  /* if the base_seq has moved passed the depth window we stop
678  * checking and just reject the rest of the packets including
679  * retransmissions. Saves us the hassle of dealing with sequence
680  * wraps as well */
681  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
682  stream->base_seq, seg_depth,
683  ssn->reassembly_depth);
684 
685  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
686  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
688  SCReturnUInt(0);
689  }
690  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
691  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
692 #if 0
693  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
694  (stream->base_seq_offset + stream->base_seq + size),
695  (stream->isn + ssn->reassembly_depth));
696 #endif
697  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
698  /* packet (partly?) fits the depth window */
699 
700  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
701  /* complete fit */
702  SCReturnUInt(size);
703  } else {
705  /* partial fit, return only what fits */
706  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
707  DEBUG_VALIDATE_BUG_ON(part > size);
708  if (part > size)
709  part = size;
710  SCReturnUInt(part);
711  }
712  }
713 
714  SCReturnUInt(0);
715 }
716 
718 {
719  if (RB_EMPTY(&stream->sb.sbb_tree)) {
720  if (stream->sb.region.stream_offset != 0)
721  return 0;
722 
723  return stream->sb.region.buf_offset;
724  } else {
725  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
726  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
727  return stream->sb.sbb_size;
728  }
729 }
730 
731 /**
732  * \brief Insert a packets TCP data into the stream reassembly engine.
733  *
734  * \retval 0 good segment, as far as we checked.
735  * \retval -1 insert failure due to memcap
736  *
737  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
738  * or it wasn't added because of reassembly depth.
739  *
740  */
742  TcpSession *ssn, TcpStream *stream, Packet *p)
743 {
744  SCEnter();
745 
746  if (ssn->data_first_seen_dir == 0) {
747  if (PKT_IS_TOSERVER(p)) {
748  ssn->data_first_seen_dir = STREAM_TOSERVER;
749  } else {
750  ssn->data_first_seen_dir = STREAM_TOCLIENT;
751  }
752  }
753 
754  /* If the OS policy is not set then set the OS policy for this stream */
755  if (stream->os_policy == 0) {
756  StreamTcpSetOSPolicy(stream, p);
757  }
758 
761  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
762  SCReturnInt(0);
763  }
764 
765  /* If we have reached the defined depth for either of the stream, then stop
766  reassembling the TCP session */
767  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
768  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
769 
772  /* increment stream depth counter */
775  }
776  if (size == 0) {
777  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
778  SCReturnInt(0);
779  }
780 
782  if (size > p->payload_len)
783  size = p->payload_len;
784 
785  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
786  if (seg == NULL) {
787  SCLogDebug("segment_pool is empty");
790  SCReturnInt(-1);
791  }
792 
793  DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
794  TCP_SEG_LEN(seg) = (uint16_t)size;
795  seg->seq = TCP_GET_SEQ(p);
796 
797  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
798  if (TCP_HAS_TFO(p) && p->payload_len && (p->tcph->th_flags & TH_SYN))
799  seg->seq += 1;
800 
801  /* proto detection skipped, but now we do get data. Set event. */
802  if (RB_EMPTY(&stream->seg_tree) &&
804 
807  }
808 
810  tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len);
811  if (r < 0) {
812  if (r == -SC_ENOMEM) {
814  }
815  SCLogDebug("StreamTcpReassembleInsertSegment failed");
816  SCReturnInt(-1);
817  }
818  SCReturnInt(0);
819 }
820 
821 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
822  Packet *p)
823 {
824  uint8_t flag = 0;
825 
827  flag |= STREAM_START;
828  }
829 
830  if (ssn->state == TCP_CLOSED) {
831  flag |= STREAM_EOF;
832  }
833 
834  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
835  flag |= STREAM_MIDSTREAM;
836  }
837 
838  if (p->flags & PKT_PSEUDO_STREAM_END) {
839  flag |= STREAM_EOF;
840  }
841 
842  if (&ssn->client == stream) {
843  flag |= STREAM_TOSERVER;
844  } else {
845  flag |= STREAM_TOCLIENT;
846  }
848  flag |= STREAM_DEPTH;
849  }
850  return flag;
851 }
852 
853 /**
854  * \brief Check the minimum size limits for reassembly.
855  *
856  * \retval 0 don't reassemble yet
857  * \retval 1 do reassemble
858  */
859 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
860  const TcpStream *stream, const Packet *p)
861 {
862  SCEnter();
863 
864  /* if any of these flags is set we always inspect immediately */
865 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
866  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
867  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
868  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
869 
872  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
873  "is set, so not expecting any new data segments");
874  }
876  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
877  }
879  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
880  "so no new segments will be considered");
881  }
882  SCReturnInt(1);
883  }
884 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
885 
886  /* some states mean we reassemble no matter how much data we have */
887  if (ssn->state > TCP_TIME_WAIT)
888  SCReturnInt(1);
889 
890  if (p->flags & PKT_PSEUDO_STREAM_END)
891  SCReturnInt(1);
892 
893  const uint64_t last_ack_abs = GetAbsLastAck(stream);
894  int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream);
895  int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size
897 
898  /* check if we have enough data to do raw reassembly */
899  if (chunk_size <= diff) {
900  SCReturnInt(1);
901  } else {
902  SCLogDebug("%s min chunk len not yet reached: "
903  "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < "
904  "%" PRIi64,
905  PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq,
906  (stream->last_ack - stream->base_seq), chunk_size);
907  SCReturnInt(0);
908  }
909 
910  SCReturnInt(0);
911 }
912 
913 /**
914  * \brief see what if any work the TCP session still needs
915  */
916 uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
917 {
918  const TcpStream *stream = NULL;
919 #ifdef DEBUG
920  const char *dirstr = NULL;
921 #endif
922  if (direction == STREAM_TOSERVER) {
923  stream = &ssn->client;
924 #ifdef DEBUG
925  dirstr = "client";
926 #endif
927  } else {
928  stream = &ssn->server;
929 #ifdef DEBUG
930  dirstr = "server";
931 #endif
932  }
933  int use_app = 1;
934  int use_raw = 1;
935 
937  // app is dead
938  use_app = 0;
939  }
940 
942  // raw is dead
943  use_raw = 0;
944  }
945  if (use_raw) {
946  const uint64_t right_edge =
947  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
948  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
949  " (use: %s). Stream right edge: %" PRIu64,
950  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
951  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
952  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
953  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
955  }
956  }
957  if (use_app) {
958  const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
959  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
960  " (use: %s). Stream right edge: %" PRIu64,
961  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
962  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
963  if (right_edge > STREAM_APP_PROGRESS(stream)) {
964  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
966  }
967  }
968 
969  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
971 }
972 
973 #ifdef DEBUG
974 static uint64_t GetStreamSize(TcpStream *stream)
975 {
976  if (stream) {
977  uint64_t size = 0;
978  uint32_t cnt = 0;
979  uint64_t last_ack_abs = GetAbsLastAck(stream);
980  uint64_t last_re = 0;
981 
982  SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset);
983 
984  TcpSegment *seg;
985  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
986  const uint64_t seg_abs =
987  STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq);
988  if (last_re != 0 && last_re < seg_abs) {
989  const char *gacked = NULL;
990  if (last_ack_abs >= seg_abs) {
991  gacked = "fully ack'd";
992  } else if (last_ack_abs > last_re) {
993  gacked = "partly ack'd";
994  } else {
995  gacked = "not yet ack'd";
996  }
997  SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked);
998  }
999 
1000  const char *acked = NULL;
1001  if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) {
1002  acked = "fully ack'd";
1003  } else if (last_ack_abs > seg_abs) {
1004  acked = "partly ack'd";
1005  } else {
1006  acked = "not yet ack'd";
1007  }
1008 
1009  SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64
1010  ") ack:%s",
1011  cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg),
1012  seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked);
1013  last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg);
1014  cnt++;
1015  size += (uint64_t)TCP_SEG_LEN(seg);
1016  }
1017 
1018  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
1019  return size;
1020  }
1021  return (uint64_t)0;
1022 }
1023 
1024 static void GetSessionSize(TcpSession *ssn, Packet *p)
1025 {
1026  uint64_t size = 0;
1027  if (ssn) {
1028  size = GetStreamSize(&ssn->client);
1029  size += GetStreamSize(&ssn->server);
1030 
1031  //if (size > 900000)
1032  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1033  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1034  }
1035 }
1036 #endif
1037 
1038 static StreamingBufferBlock *GetBlock(const StreamingBuffer *sb, const uint64_t offset)
1039 {
1040  StreamingBufferBlock *blk = sb->head;
1041  if (blk == NULL)
1042  return NULL;
1043 
1044  for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
1045  if (blk->offset >= offset)
1046  return blk;
1047  else if ((blk->offset + blk->len) > offset) {
1048  return blk;
1049  }
1050  }
1051  return NULL;
1052 }
1053 
1054 static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk)
1055 {
1056  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
1057  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
1058  GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len)) {
1059  return true;
1060  }
1061  return false;
1062 }
1063 
1064 /** \internal
1065  *
1066  * Get buffer, or first part of the buffer if data gaps exist.
1067  *
1068  * \brief get stream data from offset
1069  * \param offset stream offset
1070  * \param check_for_gap check if there is a gap ahead. Optional as it is only
1071  * needed for app-layer incomplete support.
1072  * \retval bool pkt loss ahead */
1073 static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1074  uint64_t offset, const bool check_for_gap)
1075 {
1076  const uint8_t *mydata;
1077  uint32_t mydata_len;
1078  bool gap_ahead = false;
1079 
1080  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1081  SCLogDebug("getting one blob");
1082 
1083  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1084 
1085  *data = mydata;
1086  *data_len = mydata_len;
1087  } else {
1088  SCLogDebug("block mode");
1089  StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
1090  if (blk == NULL) {
1091  *data = NULL;
1092  *data_len = 0;
1093  return false;
1094  }
1095  SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len);
1096 
1097  /* block at expected offset */
1098  if (blk->offset == offset) {
1099  SCLogDebug("blk at offset");
1100 
1101  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
1102  BUG_ON(blk->len != *data_len);
1103 
1104  gap_ahead = check_for_gap && GapAhead(stream, blk);
1105 
1106  /* block past out offset */
1107  } else if (blk->offset > offset) {
1108  SCLogDebug("gap, want data at offset %"PRIu64", "
1109  "got data at %"PRIu64". GAP of size %"PRIu64,
1110  offset, blk->offset, blk->offset - offset);
1111  *data = NULL;
1112  *data_len = blk->offset - offset;
1113 
1114  /* block starts before offset, but ends after */
1115  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
1116  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
1117  offset, blk->offset, blk->len);
1118  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
1119  SCLogDebug("data %p, data_len %u", *data, *data_len);
1120 
1121  gap_ahead = check_for_gap && GapAhead(stream, blk);
1122 
1123  } else {
1124  *data = NULL;
1125  *data_len = 0;
1126  }
1127  }
1128  return gap_ahead;
1129 }
1130 
1131 /** \internal
1132  * \brief check to see if we should declare a GAP
1133  * Call this when the app layer didn't get data at the requested
1134  * offset.
1135  */
1136 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
1137 {
1138  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1139  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1140  const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded;
1141 
1142  SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs);
1143  SCLogDebug("next_seq %u", stream->next_seq);
1144 
1145  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1146  * has been ack'd. This looks like a GAP. */
1147  if (last_ack_abs > app_progress) {
1148  /* however, we can accept ACKs a bit too liberally. If last_ack
1149  * is beyond next_seq, we only consider it a gap now if we do
1150  * already have data beyond the gap. */
1151  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1152  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1153  SCLogDebug("packet %" PRIu64 ": no GAP. "
1154  "next_seq %u < last_ack %u, but no data in list",
1155  p->pcap_cnt, stream->next_seq, stream->last_ack);
1156  return false;
1157  } else {
1158  const uint64_t next_seq_abs =
1159  STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1160  const StreamingBufferBlock *blk = stream->sb.head;
1161  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1162  /* ack'd data after the gap */
1163  SCLogDebug("packet %" PRIu64 ": GAP. "
1164  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1165  p->pcap_cnt, stream->next_seq, stream->last_ack);
1166  return true;
1167  }
1168  }
1169  }
1170 
1171  SCLogDebug("packet %" PRIu64 ": GAP! "
1172  "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", "
1173  "but we have no data.",
1174  p->pcap_cnt, last_ack_abs, app_progress);
1175  return true;
1176  }
1177  SCLogDebug("packet %"PRIu64": no GAP. "
1178  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1179  p->pcap_cnt, last_ack_abs, app_progress);
1180  return false;
1181 }
1182 
1183 static inline uint32_t AdjustToAcked(const Packet *p,
1184  const TcpSession *ssn, const TcpStream *stream,
1185  const uint64_t app_progress, const uint32_t data_len)
1186 {
1187  uint32_t adjusted = data_len;
1188 
1189  /* get window of data that is acked */
1190  if (!StreamTcpInlineMode()) {
1191  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1192  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1193  (ssn->state == TCP_CLOSED &&
1194  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1195  (p->flags & PKT_PSEUDO_STREAM_END))) {
1196  // fall through, we use all available data
1197  } else {
1198  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1199  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1200 
1201  /* see if the buffer contains unack'd data as well */
1202  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1203  uint32_t check = data_len;
1204  adjusted = last_ack_abs - app_progress;
1205  BUG_ON(adjusted > check);
1206  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1207  "data is considered", adjusted);
1208  }
1209  }
1210  }
1211  return adjusted;
1212 }
1213 
1214 /** \internal
1215  * \brief get stream buffer and update the app-layer
1216  * \param stream pointer to pointer as app-layer can switch flow dir
1217  * \retval 0 success
1218  */
1219 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1220  TcpReassemblyThreadCtx *ra_ctx,
1221  TcpSession *ssn, TcpStream **stream,
1222  Packet *p, enum StreamUpdateDir dir)
1223 {
1224  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1225 
1226  SCLogDebug("app progress %"PRIu64, app_progress);
1227 #ifdef DEBUG
1228  uint64_t last_ack_abs = GetAbsLastAck(*stream);
1229  SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs,
1230  (*stream)->base_seq);
1231 #endif
1232  const uint8_t *mydata;
1233  uint32_t mydata_len;
1234  bool last_was_gap = false;
1235 
1236  while (1) {
1237  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1238  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1239  bool gap_ahead =
1240  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead);
1241  SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len);
1242  if (last_was_gap && mydata_len == 0) {
1243  break;
1244  }
1245  last_was_gap = false;
1246 
1247  /* make sure to only deal with ACK'd data */
1248  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1249  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1250  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1251  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1252 
1253  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len,
1254  StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, dir);
1255  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1256 
1258  (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP;
1261 
1262  /* AppLayerHandleTCPData has likely updated progress. */
1263  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1264  app_progress = STREAM_APP_PROGRESS(*stream);
1265 
1266  /* a GAP also consumes 'data required'. TODO perhaps we can use
1267  * this to skip post GAP data until the start of a next record. */
1268  if ((*stream)->data_required > 0) {
1269  if ((*stream)->data_required > mydata_len) {
1270  (*stream)->data_required -= mydata_len;
1271  } else {
1272  (*stream)->data_required = 0;
1273  }
1274  }
1275  if (r < 0)
1276  return 0;
1277  if (no_progress_update)
1278  break;
1279  last_was_gap = true;
1280  continue;
1281 
1282  } else if (flags & STREAM_DEPTH) {
1283  SCLogDebug("DEPTH");
1284  // we're just called once with this flag, so make sure we pass it on
1285  if (mydata == NULL && mydata_len > 0) {
1286  mydata_len = 0;
1287  }
1288  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1289  SCLogDebug("GAP?1");
1290  /* Possibly a gap, but no new data. */
1291  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1292  SCReturnInt(0);
1293 
1294  mydata = NULL;
1295  mydata_len = 0;
1296  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1297  break;
1298  }
1299  DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1300 
1301  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1302  *stream, &(*stream)->sb, mydata_len, app_progress);
1303 
1304  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1305  SCLogDebug("GAP?2");
1306  if (mydata_len < (*stream)->data_required) {
1307  SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead));
1308  if (gap_ahead) {
1309  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1310  (*stream)->data_required, mydata_len);
1311  (*stream)->app_progress_rel += mydata_len;
1312  (*stream)->data_required -= mydata_len;
1313  // TODO send incomplete data to app-layer with special flag
1314  // indicating its all there is for this rec?
1315  } else {
1316  SCReturnInt(0);
1317  }
1318  app_progress = STREAM_APP_PROGRESS(*stream);
1319  continue;
1320  }
1321  }
1322  (*stream)->data_required = 0;
1323 
1324  SCLogDebug("parser");
1325  /* update the app-layer */
1326  (void)AppLayerHandleTCPData(
1327  tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata, mydata_len, flags, dir);
1328  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1329  AppLayerFrameDump(p->flow);
1330  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1331  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1332  break;
1333  app_progress = new_app_progress;
1334  if (flags & STREAM_DEPTH)
1335  break;
1336  }
1337 
1338  SCReturnInt(0);
1339 }
1340 
1341 /**
1342  * \brief Update the stream reassembly upon receiving a packet.
1343  *
1344  * For IDS mode, the stream is in the opposite direction of the packet,
1345  * as the ACK-packet is ACK'ing the stream.
1346  *
1347  * One of the utilities call by this function AppLayerHandleTCPData(),
1348  * has a feature where it will call this very same function for the
1349  * stream opposing the stream it is called with. This shouldn't cause
1350  * any issues, since processing of each stream is independent of the
1351  * other stream.
1352  */
1354  TcpSession *ssn, TcpStream *stream,
1355  Packet *p, enum StreamUpdateDir dir)
1356 {
1357  SCEnter();
1358 
1359  /* this function can be directly called by app layer protocol
1360  * detection. */
1363  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1364  SCReturnInt(0);
1365  }
1366 
1367 #ifdef DEBUG
1368  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1369  GetSessionSize(ssn, p);
1370 #endif
1371  /* if no segments are in the list or all are already processed,
1372  * and state is beyond established, we send an empty msg */
1373  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1374  {
1375  /* send an empty EOF msg if we have no segments but TCP state
1376  * is beyond ESTABLISHED */
1377  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1378  SCLogDebug("sending empty eof message");
1379  /* send EOF to app layer */
1380  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0,
1381  StreamGetAppLayerFlags(ssn, stream, p), dir);
1382  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1383 
1384  SCReturnInt(0);
1385  }
1386  }
1387 
1388  /* with all that out of the way, lets update the app-layer */
1389  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1390 }
1391 
1392 /** \internal
1393  * \brief get stream data from offset
1394  * \param offset stream offset */
1395 static int GetRawBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1396  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1397 {
1398  const uint8_t *mydata;
1399  uint32_t mydata_len;
1400  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1401  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1402 
1403  uint64_t roffset = offset;
1404  if (offset)
1405  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1406  else {
1407  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1408  }
1409 
1410  *data = mydata;
1411  *data_len = mydata_len;
1412  *data_offset = roffset;
1413  } else {
1414  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1415  *iter == NULL ? "starting" : "continuing", offset);
1416  if (*iter == NULL) {
1417  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1418  *iter = SBB_RB_FIND_INCLUSIVE((struct SBB *)&stream->sb.sbb_tree, &key);
1419  SCLogDebug("*iter %p", *iter);
1420  }
1421  if (*iter == NULL) {
1422  SCLogDebug("no data");
1423  *data = NULL;
1424  *data_len = 0;
1425  *data_offset = 0;
1426  return 0;
1427  }
1428  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1429 
1430  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1431  SCLogDebug("mydata %p", mydata);
1432 
1433  if ((*iter)->offset < offset) {
1434  uint64_t delta = offset - (*iter)->offset;
1435  if (delta < mydata_len) {
1436  *data = mydata + delta;
1437  *data_len = mydata_len - delta;
1438  *data_offset = offset;
1439  } else {
1440  SCLogDebug("no data (yet)");
1441  *data = NULL;
1442  *data_len = 0;
1443  *data_offset = 0;
1444  }
1445 
1446  } else {
1447  *data = mydata;
1448  *data_len = mydata_len;
1449  *data_offset = (*iter)->offset;
1450  }
1451 
1452  *iter = SBB_RB_NEXT(*iter);
1453  SCLogDebug("*iter %p", *iter);
1454  }
1455  return 0;
1456 }
1457 
1458 /** \brief does the stream engine have data to inspect?
1459  *
1460  * Returns true if there is data to inspect. In IDS case this is
1461  * about ACK'd data in the packet's direction.
1462  *
1463  * In the IPS case this is about the packet itself.
1464  */
1466 {
1467  TcpStream *stream;
1468  if (PKT_IS_TOSERVER(p)) {
1469  stream = &ssn->client;
1470  } else {
1471  stream = &ssn->server;
1472  }
1473 
1474  if (RB_EMPTY(&stream->seg_tree)) {
1475  return false;
1476  }
1477 
1480  return false;
1481 
1482  if (!StreamTcpInlineMode()) {
1483  const uint64_t segs_re_abs =
1484  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1485  if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) {
1486  return false;
1487  }
1488  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1489  return true;
1490  }
1491  } else {
1492  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1493  return true;
1494  }
1495  }
1496  return false;
1497 }
1498 
1499 /** \brief update stream engine after detection
1500  *
1501  * Tasked with progressing the 'progress' for Raw reassembly.
1502  * 2 main scenario's:
1503  * 1. progress is != 0, so we use this
1504  * 2. progress is 0, meaning the detect engine didn't touch
1505  * raw at all. In this case we need to look into progressing
1506  * raw anyway.
1507  *
1508  * Additionally, this function is tasked with disabling raw
1509  * reassembly if the app-layer requested to disable it.
1510  */
1511 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
1512 {
1513  TcpStream *stream;
1514  if (PKT_IS_TOSERVER(p)) {
1515  stream = &ssn->client;
1516  } else {
1517  stream = &ssn->server;
1518  }
1519 
1520  if (progress > STREAM_RAW_PROGRESS(stream)) {
1521  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1522  stream->raw_progress_rel += slide;
1524 
1525  } else if (progress == 0) {
1526  uint64_t target;
1527  if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) {
1528  target = STREAM_APP_PROGRESS(stream);
1529  } else {
1530  target = GetAbsLastAck(stream);
1531  }
1532  if (target > STREAM_RAW_PROGRESS(stream)) {
1533  uint32_t slide = target - STREAM_RAW_PROGRESS(stream);
1534  stream->raw_progress_rel += slide;
1535  }
1537 
1538  } else {
1539  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1540  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1541  STREAM_RAW_PROGRESS(stream), stream->window);
1542  }
1543 
1544  /* if we were told to accept no more raw data, we can mark raw as
1545  * disabled now. */
1548  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1549  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1550  }
1551 
1552  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1553 }
1554 
1555 /** \internal
1556  * \brief get a buffer around the current packet and run the callback on it
1557  *
1558  * The inline/IPS scanning method takes the current payload and wraps it in
1559  * data from other segments.
1560  *
1561  * How much data is inspected is controlled by the available data, chunk_size
1562  * and the payload size of the packet.
1563  *
1564  * Large packets: if payload size is close to the chunk_size, where close is
1565  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1566  * used: payload_len + 33% of the chunk_size.
1567  * If the payload size if equal to or bigger than the chunk_size, we use
1568  * payload len + 33% of the chunk size.
1569  */
1570 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1571  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1572 {
1573  SCEnter();
1574  int r = 0;
1575 
1576  TcpStream *stream;
1577  if (PKT_IS_TOSERVER(p)) {
1578  stream = &ssn->client;
1579  } else {
1580  stream = &ssn->server;
1581  }
1582 
1583  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1585  {
1586  *progress_out = STREAM_RAW_PROGRESS(stream);
1587  return 0;
1588  }
1589 
1590  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1593  if (chunk_size <= p->payload_len) {
1594  chunk_size = p->payload_len + (chunk_size / 3);
1595  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1596  p->payload_len, chunk_size);
1597  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1598  chunk_size = p->payload_len + ((chunk_size / 3));
1599  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1600  p->payload_len, chunk_size);
1601  }
1602 
1603  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1604  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1605  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1606  packet_leftedge_abs, packet_rightedge_abs);
1607 
1608  const uint8_t *mydata = NULL;
1609  uint32_t mydata_len = 0;
1610  uint64_t mydata_offset = 0;
1611  /* simply return progress from the block we inspected. */
1612  bool return_progress = false;
1613 
1614  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1615  /* continues block */
1616  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1617  return_progress = true;
1618 
1619  } else {
1620  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1621  /* find our block */
1622  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1623  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1624  if (sbb) {
1625  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1626  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1627  mydata_offset = sbb->offset;
1628  }
1629  }
1630 
1631  /* this can only happen if the segment insert of our current 'p' failed */
1632  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1633  if ((mydata == NULL || mydata_len == 0) || /* no data */
1634  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1635  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1636  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1637  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1638  {
1639  /* no data, or data is incomplete or wrong: use packet data */
1640  mydata = p->payload;
1641  mydata_len = p->payload_len;
1642  mydata_offset = packet_leftedge_abs;
1643  //mydata_rightedge_abs = packet_rightedge_abs;
1644  } else {
1645  /* adjust buffer to match chunk_size */
1646  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1647  if (mydata_len > chunk_size) {
1648  uint32_t excess = mydata_len - chunk_size;
1649  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1650 
1651  if (mydata_rightedge_abs == packet_rightedge_abs) {
1652  mydata += excess;
1653  mydata_len -= excess;
1654  mydata_offset += excess;
1655  SCLogDebug("cutting front of the buffer with %u", excess);
1656  } else if (mydata_offset == packet_leftedge_abs) {
1657  mydata_len -= excess;
1658  SCLogDebug("cutting tail of the buffer with %u", excess);
1659  } else {
1660  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1661  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1662  SCLogDebug("before %u after %u", before, after);
1663 
1664  if (after >= (chunk_size - p->payload_len) / 2) {
1665  // more trailing data than we need
1666 
1667  if (before >= (chunk_size - p->payload_len) / 2) {
1668  // also more heading data, divide evenly
1669  before = after = (chunk_size - p->payload_len) / 2;
1670  } else {
1671  // heading data is less than requested, give the
1672  // rest to the trailing data
1673  after = (chunk_size - p->payload_len) - before;
1674  }
1675  } else {
1676  // less trailing data than requested
1677 
1678  if (before >= (chunk_size - p->payload_len) / 2) {
1679  before = (chunk_size - p->payload_len) - after;
1680  } else {
1681  // both smaller than their requested size
1682  }
1683  }
1684 
1685  /* adjust the buffer */
1686  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1687  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1688  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1689  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1690  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1691  mydata += skip;
1692  mydata_len -= (skip + cut);
1693  mydata_offset += skip;
1694  }
1695  }
1696  }
1697 
1698  /* run the callback */
1699  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1700  BUG_ON(r < 0);
1701 
1702  if (return_progress) {
1703  *progress_out = (mydata_offset + mydata_len);
1704  } else {
1705  /* several blocks of data, so we need to be a bit more careful:
1706  * - if last_ack is beyond last progress, move progress forward to last_ack
1707  * - if our block matches or starts before last ack, return right edge of
1708  * our block.
1709  */
1710  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1711  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1712 
1713  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1714  if (mydata_offset > last_ack_abs) {
1715  /* gap between us and last ack, set progress to last ack */
1716  *progress_out = last_ack_abs;
1717  } else {
1718  *progress_out = (mydata_offset + mydata_len);
1719  }
1720  } else {
1721  *progress_out = STREAM_RAW_PROGRESS(stream);
1722  }
1723  }
1724  return r;
1725 }
1726 
1727 /** \brief access 'raw' reassembly data.
1728  *
1729  * Access data as tracked by 'raw' tracker. Data is made available to
1730  * callback that is passed to this function.
1731  *
1732  * In the case of IDS the callback may be run multiple times if data
1733  * contains gaps. It will then be run for each block of data that is
1734  * continuous.
1735  *
1736  * The callback should give on of 2 return values:
1737  * - 0 ok
1738  * - 1 done
1739  * The value 1 will break the loop if there is a block list that is
1740  * inspected.
1741  *
1742  * This function will return the 'progress' value that has been
1743  * consumed until now.
1744  *
1745  * \param ssn tcp session
1746  * \param stream tcp stream
1747  * \param Callback the function pointer to the callback function
1748  * \param cb_data callback data
1749  * \param[in] progress_in progress to work from
1750  * \param[in] re right edge of data to consider
1751  * \param[out] progress_out absolute progress value of the data this
1752  * call handled.
1753  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1754  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1755  *
1756  * `respect_inspect_depth` is used to avoid useless inspection of too
1757  * much data.
1758  */
1759 static int StreamReassembleRawDo(const TcpSession *ssn, const TcpStream *stream,
1760  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1761  const uint64_t re, uint64_t *progress_out, bool eof, bool respect_inspect_depth)
1762 {
1763  SCEnter();
1764  int r = 0;
1765 
1766  StreamingBufferBlock *iter = NULL;
1767  uint64_t progress = progress_in;
1768 
1769  /* loop through available buffers. On no packet loss we'll have a single
1770  * iteration. On missing data we'll walk the blocks */
1771  while (1) {
1772  const uint8_t *mydata;
1773  uint32_t mydata_len;
1774  uint64_t mydata_offset = 0;
1775 
1776  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1777  if (mydata_len == 0) {
1778  SCLogDebug("no data");
1779  break;
1780  }
1781  //PrintRawDataFp(stdout, mydata, mydata_len);
1782 
1783  SCLogDebug("raw progress %"PRIu64, progress);
1784  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1785  stream, &stream->sb, mydata_len, progress);
1786 
1787  if (eof) {
1788  // inspect all remaining data, ack'd or not
1789  } else {
1790  if (re < progress) {
1791  SCLogDebug("nothing to do");
1792  goto end;
1793  }
1794 
1795  SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress);
1796  SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len,
1797  re);
1798 
1799  /* see if the buffer contains unack'd data as well */
1800  if (progress + mydata_len > re) {
1801  uint32_t check = mydata_len;
1802  mydata_len = re - progress;
1803  BUG_ON(check < mydata_len);
1804  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1805  "data is considered", mydata_len);
1806  }
1807  }
1808  if (mydata_len == 0)
1809  break;
1810 
1811  SCLogDebug("data %p len %u", mydata, mydata_len);
1812 
1813  /* we have data. */
1814  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1815  BUG_ON(r < 0);
1816 
1817  if (mydata_offset == progress) {
1818  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1819  progress, mydata_len, progress_in + mydata_len);
1820 
1821  progress += mydata_len;
1822  SCLogDebug("raw progress now %"PRIu64, progress);
1823 
1824  /* data is beyond the progress we'd like, and before last ack. Gap. */
1825  } else if (mydata_offset > progress && mydata_offset < re) {
1826  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1827  SCLogDebug("re %" PRIu64, re);
1828  progress = mydata_offset;
1829  SCLogDebug("raw progress now %"PRIu64, progress);
1830 
1831  } else {
1832  SCLogDebug("not increasing progress, data gap => mydata_offset "
1833  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1834  }
1835 
1836  if (iter == NULL || r == 1)
1837  break;
1838  }
1839 end:
1840  *progress_out = progress;
1841  return r;
1842 }
1843 
1845  void *cb_data, const uint64_t offset, const bool eof)
1846 {
1847  /* take app progress as the right edge of used data. */
1848  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1849  SCLogDebug("app_progress %" PRIu64, app_progress);
1850 
1851  uint64_t unused = 0;
1852  return StreamReassembleRawDo(
1853  ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof, false);
1854 }
1855 
1857  StreamReassembleRawFunc Callback, void *cb_data,
1858  uint64_t *progress_out, bool respect_inspect_depth)
1859 {
1860  /* handle inline separately as the logic is very different */
1861  if (StreamTcpInlineMode()) {
1862  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1863  }
1864 
1865  TcpStream *stream;
1866  if (PKT_IS_TOSERVER(p)) {
1867  stream = &ssn->client;
1868  } else {
1869  stream = &ssn->server;
1870  }
1871 
1873  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1874  {
1875  *progress_out = STREAM_RAW_PROGRESS(stream);
1876  return 0;
1877  }
1878 
1879  uint64_t progress = STREAM_RAW_PROGRESS(stream);
1880  /* if the app layer triggered a flush, and we're supposed to
1881  * use a minimal inspect depth, we actually take the app progress
1882  * as that is the right edge of the data. Then we take the window
1883  * of 'min_inspect_depth' before that. */
1884 
1885  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s "
1886  "stream->min_inspect_depth %u",
1887  respect_inspect_depth ? "true" : "false",
1888  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1889  stream->min_inspect_depth);
1890 
1891  if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) &&
1892  stream->min_inspect_depth) {
1893  progress = STREAM_APP_PROGRESS(stream);
1894  if (stream->min_inspect_depth >= progress) {
1895  progress = 0;
1896  } else {
1897  progress -= stream->min_inspect_depth;
1898  }
1899 
1900  SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream),
1901  STREAM_RAW_PROGRESS(stream));
1902 
1903  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1904  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress "
1905  "%" PRIu64,
1906  progress);
1907  }
1908 
1909  SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth,
1910  stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW"
1911  : "(no trigger)");
1912 
1913  /* absolute right edge of ack'd data */
1914  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1915  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1916 
1917  return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs,
1918  progress_out, (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1919 }
1920 
1921 int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream,
1922  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1923  uint64_t *progress_out, const bool eof)
1924 {
1925  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1926  return 0;
1927 
1928  /* absolute right edge of ack'd data */
1929  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1930  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1931 
1932  return StreamReassembleRawDo(
1933  ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof, false);
1934 }
1935 
1936 /** \internal
1937  * \brief update app layer based on received ACK
1938  *
1939  * \retval r 0 on success, -1 on error
1940  */
1941 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1942  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1943 {
1944  SCEnter();
1945 
1946  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1947  SCReturnInt(-1);
1948 
1949  SCReturnInt(0);
1950 }
1951 
1953  TcpSession *ssn, TcpStream *stream, Packet *p)
1954 {
1955  SCEnter();
1956 
1957  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1958 
1959  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1960  ssn, stream, p, p->payload_len);
1961 
1962  /* default IDS: update opposing side (triggered by ACK) */
1964  /* inline and stream end and flow timeout packets trigger same dir handling */
1965  if (StreamTcpInlineMode()) {
1966  dir = UPDATE_DIR_PACKET;
1967  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1968  dir = UPDATE_DIR_PACKET;
1969  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1970  dir = UPDATE_DIR_PACKET;
1971  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1972  if (p->tcph->th_flags & TH_ACK) {
1973  dir = UPDATE_DIR_BOTH;
1974  } else {
1975  dir = UPDATE_DIR_PACKET;
1976  }
1977  } else if (ssn->state == TCP_CLOSED) {
1978  dir = UPDATE_DIR_BOTH;
1979  }
1980 
1981  /* handle ack received */
1982  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
1983  /* we need to update the opposing stream in
1984  * StreamTcpReassembleHandleSegmentUpdateACK */
1985  TcpStream *opposing_stream = NULL;
1986  if (stream == &ssn->client) {
1987  opposing_stream = &ssn->server;
1988  } else {
1989  opposing_stream = &ssn->client;
1990  }
1991 
1992  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
1993 
1994  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
1995  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1996  SCReturnInt(-1);
1997  }
1998 
1999  /* StreamTcpReassembleHandleSegmentUpdateACK
2000  * may swap content of ssn->server and ssn->client structures.
2001  * We have to continue with initial content of the stream in such case */
2002  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2003  if (reversed_before_ack_handling != reversed_after_ack_handling) {
2004  SCLogDebug("TCP streams were swapped");
2005  stream = opposing_stream;
2006  }
2007  }
2008  /* if this segment contains data, insert it */
2009  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) &&
2010  (p->tcph->th_flags & TH_RST) == 0) {
2011  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
2012 
2013  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
2014  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
2015  /* failure can only be because of memcap hit, so see if this should lead to a drop */
2018  SCReturnInt(-1);
2019  }
2020 
2021  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
2022  p->flags |= PKT_STREAM_ADD;
2023  } else {
2024  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
2025  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
2026  ssn, stream, p->payload_len,
2027  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
2028  }
2029 
2030  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
2031  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
2032  * was *just* set. In this case we trigger the AppLayer Truncate
2033  * logic, to inform the applayer no more data in this direction is
2034  * to be expected. */
2035  if ((stream->flags &
2038  {
2039  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
2040  if (dir != UPDATE_DIR_PACKET) {
2041  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
2042  "can trigger Truncate");
2043  dir = UPDATE_DIR_PACKET;
2044  }
2045  }
2046 
2047  /* in stream inline mode even if we have no data we call the reassembly
2048  * functions to handle EOF */
2049  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
2050  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
2051  StreamTcpInlineMode()?"true":"false",
2052  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
2053  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
2054  SCReturnInt(-1);
2055  }
2056  }
2057 
2058  SCReturnInt(0);
2059 }
2060 
2061 /**
2062  * \brief get a segment from the pool
2063  *
2064  * \retval seg Segment from the pool or NULL
2065  */
2067 {
2069  if (seg) {
2071  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2072  return seg;
2073  }
2074 
2075  seg = (TcpSegment *)PoolThreadGetById(
2076  segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id);
2077  SCLogDebug("seg we return is %p", seg);
2078  if (seg == NULL) {
2079  /* Increment the counter to show that we are not able to serve the
2080  segment request due to memcap limit */
2082  } else {
2083  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2085  }
2086 
2087  return seg;
2088 }
2089 
2090 /**
2091  * \brief Trigger RAW stream reassembly
2092  *
2093  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
2094  * reassembly from the applayer, for example upon completion of a
2095  * HTTP request.
2096  *
2097  * It sets a flag in the stream so that the next Raw call will return
2098  * the data.
2099  *
2100  * \param ssn TcpSession
2101  */
2103 {
2104 #ifdef DEBUG
2105  BUG_ON(ssn == NULL);
2106 #endif
2107 
2108  if (ssn != NULL) {
2109  if (direction == STREAM_TOSERVER) {
2111  } else {
2113  }
2114 
2115  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
2116  }
2117 }
2118 
2119 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
2120 {
2121 #ifdef DEBUG
2122  BUG_ON(ssn == NULL);
2123 #endif
2124 
2125  if (ssn != NULL) {
2126  if (direction == STREAM_TOSERVER) {
2127  ssn->client.min_inspect_depth = depth;
2128  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
2129  } else {
2130  ssn->server.min_inspect_depth = depth;
2131  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
2132  }
2133  }
2134 }
2135 
2136 #ifdef UNITTESTS
2137 /** unit tests and it's support functions below */
2138 
2139 #define SET_ISN(stream, setseq) \
2140  (stream)->isn = (setseq); \
2141  (stream)->base_seq = (setseq) + 1
2142 
2143 /** \brief The Function to create the packet with given payload, which is used
2144  * to test the reassembly of the engine.
2145  *
2146  * \param payload The variable used to store the payload contents of the
2147  * current packet.
2148  * \param value The value which current payload will have for this packet
2149  * \param payload_len The length of the filed payload for current packet.
2150  * \param len Length of the payload array
2151  */
2152 
2153 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2154  uint8_t payload_len, uint8_t len)
2155 {
2156  uint8_t i;
2157  for (i = 0; i < payload_len; i++)
2158  payload[i] = value;
2159  for (; i < len; i++)
2160  payload = NULL;
2161 }
2162 
2163 /** \brief The Function Checks the reassembled stream contents against predefined
2164  * stream contents according to OS policy used.
2165  *
2166  * \param stream_policy Predefined value of stream for different OS policies
2167  * \param stream Reassembled stream returned from the reassembly functions
2168  */
2169 
2170 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
2171 {
2172  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
2173  {
2174  //PrintRawDataFp(stdout, stream_policy, sp_size);
2175  return 0;
2176  }
2177  return 1;
2178 }
2179 
2180 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2181 {
2182  if (StreamingBufferCompareRawData(&stream->sb,
2183  data, data_len) == 0)
2184  {
2185  SCReturnInt(0);
2186  }
2187  SCLogInfo("OK");
2188  PrintRawDataFp(stdout, data, data_len);
2189  return 1;
2190 }
2191 
2192 #define MISSED_START(isn) \
2193  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2194  TcpSession ssn; \
2195  ThreadVars tv; \
2196  memset(&tv, 0, sizeof(tv)); \
2197  \
2198  StreamTcpUTInit(&ra_ctx); \
2199  \
2200  StreamTcpUTSetupSession(&ssn); \
2201  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2202  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2203  \
2204  TcpStream *stream = &ssn.client;
2205 
2206 #define MISSED_END \
2207  StreamTcpUTClearSession(&ssn); \
2208  StreamTcpUTDeinit(ra_ctx); \
2209  PASS
2210 
2211 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2212  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2213  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2214 
2215 #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \
2216  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen));
2217 
2218 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len);
2219 
2220 int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
2221 {
2222  int cnt = 0;
2223  uint64_t last_re = 0;
2224  StreamingBufferBlock *sbb = NULL;
2225  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2226  {
2227  if (sbb->offset != last_re) {
2228  // gap before us
2229  if (cnt == pos && last_re == offset && len == sbb->offset - last_re) {
2230  return 1;
2231  }
2232  cnt++;
2233  }
2234  last_re = sbb->offset + sbb->len;
2235  cnt++;
2236  }
2237  return 0;
2238 }
2239 
2241  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len);
2242 
2244  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
2245 {
2246  int cnt = 0;
2247  uint64_t last_re = 0;
2248  StreamingBufferBlock *sbb = NULL;
2249  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2250  {
2251  if (sbb->offset != last_re) {
2252  // gap before us
2253  cnt++;
2254  }
2255 
2256  if (cnt == pos && sbb->offset == offset) {
2257  const uint8_t *buf = NULL;
2258  uint32_t buf_len = 0;
2259  StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len);
2260 
2261  if (len == buf_len) {
2262  return (memcmp(data, buf, len) == 0);
2263  }
2264  }
2265 
2266  last_re = sbb->offset + sbb->len;
2267  cnt++;
2268  }
2269  return 0;
2270 }
2271 
2272 /**
2273  * \test Test the handling of packets missed by both IDS and the end host.
2274  * The packet is missed in the starting of the stream.
2275  *
2276  * \retval On success it returns 1 and on failure 0.
2277  */
2278 
2279 static int StreamTcpReassembleTest25 (void)
2280 {
2281  MISSED_START(6);
2282  MISSED_ADD_PAYLOAD(10, "BB", 2);
2283  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2284  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BB", 2) == 1);
2285  MISSED_ADD_PAYLOAD(12, "CC", 2);
2286  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2287  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "BBCC", 4) == 1);
2288  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2289  MISSED_END;
2290  PASS;
2291 }
2292 
2293 /**
2294  * \test Test the handling of packets missed by both IDS and the end host.
2295  * The packet is missed in the middle of the stream.
2296  *
2297  * \retval On success it returns 1 and on failure 0.
2298  */
2299 
2300 static int StreamTcpReassembleTest26 (void)
2301 {
2302  MISSED_START(9);
2303  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2304  MISSED_ADD_PAYLOAD(15, "CC", 2);
2305  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2306  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2307  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2308  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2309  MISSED_END;
2310 }
2311 
2312 /**
2313  * \test Test the handling of packets missed by both IDS and the end host.
2314  * The packet is missed in the end of the stream.
2315  *
2316  * \retval On success it returns 1 and on failure 0.
2317  */
2318 
2319 static int StreamTcpReassembleTest27 (void)
2320 {
2321  MISSED_START(9);
2322  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2323  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2324  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2325  MISSED_END;
2326 }
2327 
2328 /**
2329  * \test Test the handling of packets missed by IDS, but the end host has
2330  * received it and send the acknowledgment of it. The packet is missed
2331  * in the starting of the stream.
2332  *
2333  * \retval On success it returns 1 and on failure 0.
2334  */
2335 
2336 static int StreamTcpReassembleTest28 (void)
2337 {
2338  MISSED_START(6);
2339  MISSED_ADD_PAYLOAD(10, "AAA", 3);
2340  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2341  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAA", 3) == 1);
2342  MISSED_ADD_PAYLOAD(13, "BB", 2);
2343  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2344  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABB", 5) == 1);
2345  ssn.state = TCP_TIME_WAIT;
2346  MISSED_ADD_PAYLOAD(15, "CC", 2);
2347  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 0, 0, 3) == 1);
2348  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 1, 3, "AAABBCC", 7) == 1);
2349  MISSED_END;
2350 }
2351 
2352 /**
2353  * \test Test the handling of packets missed by IDS, but the end host has
2354  * received it and send the acknowledgment of it. The packet is missed
2355  * in the middle of the stream.
2356  *
2357  * \retval On success it returns 1 and on failure 0.
2358  */
2359 
2360 static int StreamTcpReassembleTest29 (void)
2361 {
2362  MISSED_START(9);
2363  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2364  ssn.state = TCP_TIME_WAIT;
2365  MISSED_ADD_PAYLOAD(15, "CC", 2);
2366  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 0, 0, "AAA", 3) == 1);
2367  FAIL_IF_NOT(UTHCheckGapAtPosition(stream, 1, 3, 2) == 1);
2368  FAIL_IF_NOT(UTHCheckDataAtPosition(stream, 2, 5, "CC", 2) == 1);
2369  MISSED_END;
2370 }
2371 
2372 static int StreamTcpReassembleTest33(void)
2373 {
2374  TcpSession ssn;
2375  Packet *p = PacketGetFromAlloc();
2376  FAIL_IF(unlikely(p == NULL));
2377  Flow f;
2378  TCPHdr tcph;
2379  TcpReassemblyThreadCtx *ra_ctx = NULL;
2381  uint8_t packet[1460] = "";
2382 
2383  StreamTcpUTInit(&ra_ctx);
2385 
2386  memset(&f, 0, sizeof (Flow));
2387  memset(&tcph, 0, sizeof (TCPHdr));
2388  ThreadVars tv;
2389  memset(&tv, 0, sizeof (ThreadVars));
2390  FLOW_INITIALIZE(&f);
2391  f.protoctx = &ssn;
2392  f.proto = IPPROTO_TCP;
2393  p->src.family = AF_INET;
2394  p->dst.family = AF_INET;
2395  p->proto = IPPROTO_TCP;
2396  p->flow = &f;
2397  tcph.th_win = 5480;
2398  tcph.th_flags = TH_PUSH | TH_ACK;
2399  p->tcph = &tcph;
2401  p->payload = packet;
2402 
2403  p->tcph->th_seq = htonl(10);
2404  p->tcph->th_ack = htonl(31);
2405  p->payload_len = 10;
2406 
2407  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2408 
2409  p->tcph->th_seq = htonl(20);
2410  p->tcph->th_ack = htonl(31);
2411  p->payload_len = 10;
2412 
2413  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2414 
2415  p->tcph->th_seq = htonl(40);
2416  p->tcph->th_ack = htonl(31);
2417  p->payload_len = 10;
2418 
2419  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2420 
2421  p->tcph->th_seq = htonl(5);
2422  p->tcph->th_ack = htonl(31);
2423  p->payload_len = 30;
2424 
2425  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2426 
2428  StreamTcpUTDeinit(ra_ctx);
2429  SCFree(p);
2430  PASS;
2431 }
2432 
2433 static int StreamTcpReassembleTest34(void)
2434 {
2435  TcpSession ssn;
2436  Packet *p = PacketGetFromAlloc();
2437  FAIL_IF(unlikely(p == NULL));
2438  Flow f;
2439  TCPHdr tcph;
2440  TcpReassemblyThreadCtx *ra_ctx = NULL;
2442  uint8_t packet[1460] = "";
2443 
2444  StreamTcpUTInit(&ra_ctx);
2446  memset(&f, 0, sizeof (Flow));
2447  memset(&tcph, 0, sizeof (TCPHdr));
2448  ThreadVars tv;
2449  memset(&tv, 0, sizeof (ThreadVars));
2450  FLOW_INITIALIZE(&f);
2451  f.protoctx = &ssn;
2452  f.proto = IPPROTO_TCP;
2453  p->src.family = AF_INET;
2454  p->dst.family = AF_INET;
2455  p->proto = IPPROTO_TCP;
2456  p->flow = &f;
2457  tcph.th_win = 5480;
2458  tcph.th_flags = TH_PUSH | TH_ACK;
2459  p->tcph = &tcph;
2461  p->payload = packet;
2462  SET_ISN(&ssn.client, 857961230);
2463 
2464  p->tcph->th_seq = htonl(857961230);
2465  p->tcph->th_ack = htonl(31);
2466  p->payload_len = 304;
2467 
2468  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2469 
2470  p->tcph->th_seq = htonl(857961534);
2471  p->tcph->th_ack = htonl(31);
2472  p->payload_len = 1460;
2473 
2474  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2475 
2476  p->tcph->th_seq = htonl(857963582);
2477  p->tcph->th_ack = htonl(31);
2478  p->payload_len = 1460;
2479 
2480  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2481 
2482  p->tcph->th_seq = htonl(857960946);
2483  p->tcph->th_ack = htonl(31);
2484  p->payload_len = 1460;
2485 
2486  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2487 
2489  StreamTcpUTDeinit(ra_ctx);
2490  SCFree(p);
2491  PASS;
2492 }
2493 
2494 /**
2495  * \test Test to make sure that we don't return the segments until the app
2496  * layer proto has been detected and after that remove the processed
2497  * segments.
2498  *
2499  * \retval On success it returns 1 and on failure 0.
2500  */
2501 
2502 static int StreamTcpReassembleTest39 (void)
2503 {
2504  Packet *p = PacketGetFromAlloc();
2505  FAIL_IF(unlikely(p == NULL));
2506  Flow f;
2507  ThreadVars tv;
2508  StreamTcpThread stt;
2509  TCPHdr tcph;
2510  PacketQueueNoLock pq;
2511  memset(&pq,0,sizeof(PacketQueueNoLock));
2512  memset (&f, 0, sizeof(Flow));
2513  memset(&tv, 0, sizeof (ThreadVars));
2514  memset(&stt, 0, sizeof (stt));
2515  memset(&tcph, 0, sizeof (TCPHdr));
2516 
2517  FLOW_INITIALIZE(&f);
2518  f.flags = FLOW_IPV4;
2519  f.proto = IPPROTO_TCP;
2520  p->flow = &f;
2521  p->tcph = &tcph;
2522 
2523  StreamTcpUTInit(&stt.ra_ctx);
2524 
2525  /* handshake */
2526  tcph.th_win = htons(5480);
2527  tcph.th_flags = TH_SYN;
2529  p->payload_len = 0;
2530  p->payload = NULL;
2531  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2532 
2533  TcpSession *ssn = (TcpSession *)f.protoctx;
2534  FAIL_IF_NULL(ssn);
2535 
2542  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2543  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2544  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2545  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2546  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2547  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2548  FAIL_IF(ssn->data_first_seen_dir != 0);
2549 
2550  /* handshake */
2551  p->tcph->th_ack = htonl(1);
2552  p->tcph->th_flags = TH_SYN | TH_ACK;
2554  p->payload_len = 0;
2555  p->payload = NULL;
2556  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2563  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2564  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2565  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2566  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2567  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2568  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2569  FAIL_IF(ssn->data_first_seen_dir != 0);
2570 
2571  /* handshake */
2572  p->tcph->th_ack = htonl(1);
2573  p->tcph->th_seq = htonl(1);
2574  p->tcph->th_flags = TH_ACK;
2576  p->payload_len = 0;
2577  p->payload = NULL;
2578  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2585  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2586  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2587  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2588  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2589  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2590  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2591  FAIL_IF(ssn->data_first_seen_dir != 0);
2592 
2593  /* partial request */
2594  uint8_t request1[] = { 0x47, 0x45, };
2595  p->tcph->th_ack = htonl(1);
2596  p->tcph->th_seq = htonl(1);
2597  p->tcph->th_flags = TH_PUSH | TH_ACK;
2599  p->payload_len = sizeof(request1);
2600  p->payload = request1;
2601  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2608  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2609  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2610  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2611  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2612  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2613  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2614  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2615  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2616 
2617  /* response ack against partial request */
2618  p->tcph->th_ack = htonl(3);
2619  p->tcph->th_seq = htonl(1);
2620  p->tcph->th_flags = TH_ACK;
2622  p->payload_len = 0;
2623  p->payload = NULL;
2624  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2631  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2632  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2633  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2634  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2635  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2636  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2637  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2638  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2639 
2640  /* complete partial request */
2641  uint8_t request2[] = {
2642  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2643  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2644  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2645  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2646  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2647  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2648  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2649  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2650  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2651  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2652  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2653  p->tcph->th_ack = htonl(1);
2654  p->tcph->th_seq = htonl(3);
2655  p->tcph->th_flags = TH_PUSH | TH_ACK;
2657  p->payload_len = sizeof(request2);
2658  p->payload = request2;
2659  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2666  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2667  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2668  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2669  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2670  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2671  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2672  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2673  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2674  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2675 
2676  /* response - request ack */
2677  uint8_t response[] = {
2678  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2679  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2680  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2681  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2682  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2683  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2684  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2685  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2686  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2687  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2688  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2689  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2690  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2691  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2692  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2693  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2694  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2695  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2696  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2697  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2698  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2699  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2700  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2701  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2702  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2703  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2704  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2705  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2706  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2707  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2708  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2709  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2710  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2711  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2712  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2713  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2714  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2715  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2716  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2717  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2718  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2719  p->tcph->th_ack = htonl(88);
2720  p->tcph->th_seq = htonl(1);
2721  p->tcph->th_flags = TH_PUSH | TH_ACK;
2723  p->payload_len = sizeof(response);
2724  p->payload = response;
2725 
2726  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2733  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2734  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2735  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2736  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2738  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2739  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2740  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2741  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2742 
2743  /* response ack from request */
2744  p->tcph->th_ack = htonl(328);
2745  p->tcph->th_seq = htonl(88);
2746  p->tcph->th_flags = TH_ACK;
2748  p->payload_len = 0;
2749  p->payload = NULL;
2750  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2757  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2758  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2759  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2760  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2762  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2763  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2764  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2765  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2766  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2767 
2768  /* response - acking */
2769  p->tcph->th_ack = htonl(88);
2770  p->tcph->th_seq = htonl(328);
2771  p->tcph->th_flags = TH_PUSH | TH_ACK;
2773  p->payload_len = 0;
2774  p->payload = NULL;
2775  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2782  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2783  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2784  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2785  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2787  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2788  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2789  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2790  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2791  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2792 
2793  /* response ack from request */
2794  p->tcph->th_ack = htonl(328);
2795  p->tcph->th_seq = htonl(88);
2796  p->tcph->th_flags = TH_ACK;
2798  p->payload_len = 0;
2799  p->payload = NULL;
2800  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2807  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2808  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2809  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2810  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2812  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2813  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2814  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2815  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2816 
2817  /* response - acking the request again*/
2818  p->tcph->th_ack = htonl(88);
2819  p->tcph->th_seq = htonl(328);
2820  p->tcph->th_flags = TH_PUSH | TH_ACK;
2822  p->payload_len = 0;
2823  p->payload = NULL;
2824  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2831  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2832  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2833  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2834  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2836  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2837  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2838  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2839  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2840 
2841  /*** New Request ***/
2842 
2843  /* partial request */
2844  p->tcph->th_ack = htonl(328);
2845  p->tcph->th_seq = htonl(88);
2846  p->tcph->th_flags = TH_PUSH | TH_ACK;
2848  p->payload_len = sizeof(request1);
2849  p->payload = request1;
2850  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2857  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2858  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2859  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2860  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2862  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2863  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2864  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2865  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2866  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2867 
2868  /* response ack against partial request */
2869  p->tcph->th_ack = htonl(90);
2870  p->tcph->th_seq = htonl(328);
2871  p->tcph->th_flags = TH_ACK;
2873  p->payload_len = 0;
2874  p->payload = NULL;
2875  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2876  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2883  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2884  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2885  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2886  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2888  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2889  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2890  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2891  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2892  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2893 
2894  /* complete request */
2895  p->tcph->th_ack = htonl(328);
2896  p->tcph->th_seq = htonl(90);
2897  p->tcph->th_flags = TH_PUSH | TH_ACK;
2899  p->payload_len = sizeof(request2);
2900  p->payload = request2;
2901  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2908  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2909  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2910  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2911  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2913  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2914  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2915  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2916  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2917  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2918  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2919 
2920  /* response ack against second partial request */
2921  p->tcph->th_ack = htonl(175);
2922  p->tcph->th_seq = htonl(328);
2923  p->tcph->th_flags = TH_ACK;
2925  p->payload_len = 0;
2926  p->payload = NULL;
2927 
2928  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2935  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2936  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2937  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2938  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2940  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2941  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2942  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2943  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2944  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2945  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2946 
2947  /* response acking a request */
2948  p->tcph->th_ack = htonl(175);
2949  p->tcph->th_seq = htonl(328);
2950  p->tcph->th_flags = TH_ACK;
2952  p->payload_len = 0;
2953  p->payload = NULL;
2954  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2960 
2961  StreamTcpPruneSession(&f, STREAM_TOSERVER);
2962  StreamTcpPruneSession(&f, STREAM_TOCLIENT);
2963 
2964  /* request acking a response */
2965  p->tcph->th_ack = htonl(328);
2966  p->tcph->th_seq = htonl(175);
2967  p->tcph->th_flags = TH_ACK;
2969  p->payload_len = 0;
2970  p->payload = NULL;
2971  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2972 
2973  StreamTcpSessionClear(ssn);
2975  SCFree(p);
2976  PASS;
2977 }
2978 
2979 /**
2980  * \test Test to make sure that we sent all the segments from the initial
2981  * segments to app layer until we have detected the app layer proto.
2982  *
2983  * \retval On success it returns 1 and on failure 0.
2984  */
2985 
2986 static int StreamTcpReassembleTest40 (void)
2987 {
2988  Packet *p = PacketGetFromAlloc();
2989  FAIL_IF_NULL(p);
2990  Flow *f = NULL;
2991  TCPHdr tcph;
2992  TcpSession ssn;
2993  memset(&tcph, 0, sizeof (TCPHdr));
2994  ThreadVars tv;
2995  memset(&tv, 0, sizeof (ThreadVars));
2996 
2997  StreamTcpInitConfig(true);
2999 
3001  FAIL_IF_NULL(ra_ctx);
3002 
3003  uint8_t httpbuf1[] = "P";
3004  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3005  uint8_t httpbuf3[] = "O";
3006  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3007  uint8_t httpbuf4[] = "S";
3008  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3009  uint8_t httpbuf5[] = "T \r\n";
3010  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3011 
3012  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3013  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3014 
3015  SET_ISN(&ssn.server, 9);
3016  ssn.server.last_ack = 10;
3017  SET_ISN(&ssn.client, 9);
3018  ssn.client.isn = 9;
3019 
3020  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3021  FAIL_IF_NULL(f);
3022  f->protoctx = &ssn;
3023  f->proto = IPPROTO_TCP;
3024  p->flow = f;
3025 
3026  tcph.th_win = htons(5480);
3027  tcph.th_seq = htonl(10);
3028  tcph.th_ack = htonl(10);
3029  tcph.th_flags = TH_ACK|TH_PUSH;
3030  p->tcph = &tcph;
3032  p->payload = httpbuf1;
3033  p->payload_len = httplen1;
3034  ssn.state = TCP_ESTABLISHED;
3035  TcpStream *s = &ssn.client;
3036  SCLogDebug("1 -- start");
3037  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3038 
3040  p->payload = httpbuf2;
3041  p->payload_len = httplen2;
3042  tcph.th_seq = htonl(10);
3043  tcph.th_ack = htonl(11);
3044  s = &ssn.server;
3045  ssn.server.last_ack = 11;
3046  SCLogDebug("2 -- start");
3047  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3048 
3050  p->payload = httpbuf3;
3051  p->payload_len = httplen3;
3052  tcph.th_seq = htonl(11);
3053  tcph.th_ack = htonl(55);
3054  s = &ssn.client;
3055  ssn.client.last_ack = 55;
3056  SCLogDebug("3 -- start");
3057  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3058 
3060  p->payload = httpbuf2;
3061  p->payload_len = httplen2;
3062  tcph.th_seq = htonl(55);
3063  tcph.th_ack = htonl(12);
3064  s = &ssn.server;
3065  ssn.server.last_ack = 12;
3066  SCLogDebug("4 -- start");
3067  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3068 
3069  /* check is have the segment in the list and flagged or not */
3070  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3071  FAIL_IF_NULL(seg);
3072  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
3073 
3075  p->payload = httpbuf4;
3076  p->payload_len = httplen4;
3077  tcph.th_seq = htonl(12);
3078  tcph.th_ack = htonl(100);
3079  s = &ssn.client;
3080  ssn.client.last_ack = 100;
3081  SCLogDebug("5 -- start");
3082  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3083 
3085  p->payload = httpbuf2;
3086  p->payload_len = httplen2;
3087  tcph.th_seq = htonl(100);
3088  tcph.th_ack = htonl(13);
3089  s = &ssn.server;
3090  ssn.server.last_ack = 13;
3091  SCLogDebug("6 -- start");
3092  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3093 
3095  p->payload = httpbuf5;
3096  p->payload_len = httplen5;
3097  tcph.th_seq = htonl(13);
3098  tcph.th_ack = htonl(145);
3099  s = &ssn.client;
3100  ssn.client.last_ack = 145;
3101  SCLogDebug("7 -- start");
3102  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3103 
3105  p->payload = httpbuf2;
3106  p->payload_len = httplen2;
3107  tcph.th_seq = htonl(145);
3108  tcph.th_ack = htonl(16);
3109  s = &ssn.server;
3110  ssn.server.last_ack = 16;
3111  SCLogDebug("8 -- start");
3112  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3113  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3114 
3117  StreamTcpFreeConfig(true);
3118  SCFree(p);
3119  UTHFreeFlow(f);
3120  PASS;
3121 }
3122 
3123 /** \test Test the memcap incrementing/decrementing and memcap check */
3124 static int StreamTcpReassembleTest44(void)
3125 {
3126  StreamTcpInitConfig(true);
3127  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
3129  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
3131  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
3133  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
3134  StreamTcpFreeConfig(true);
3135  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
3136  PASS;
3137 }
3138 
3139 /**
3140  * \test Test to make sure that reassembly_depth is enforced.
3141  *
3142  * \retval On success it returns 1 and on failure 0.
3143  */
3144 
3145 static int StreamTcpReassembleTest45 (void)
3146 {
3147  TcpReassemblyThreadCtx *ra_ctx = NULL;
3148  TcpSession ssn;
3149  ThreadVars tv;
3150  memset(&tv, 0, sizeof(tv));
3151  uint8_t payload[100] = {0};
3152  uint16_t payload_size = 100;
3153 
3154  StreamTcpUTInit(&ra_ctx);
3156 
3158  ssn.reassembly_depth = 100;
3159  StreamTcpUTSetupStream(&ssn.server, 100);
3160  StreamTcpUTSetupStream(&ssn.client, 100);
3161 
3162  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3163  FAIL_IF(r != 0);
3165 
3166  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3167  FAIL_IF(r != 0);
3169 
3173  StreamTcpUTDeinit(ra_ctx);
3174  PASS;
3175 }
3176 
3177 /**
3178  * \test Test the unlimited config value of reassembly depth.
3179  *
3180  * \retval On success it returns 1 and on failure 0.
3181  */
3182 
3183 static int StreamTcpReassembleTest46 (void)
3184 {
3185  int result = 0;
3186  TcpReassemblyThreadCtx *ra_ctx = NULL;
3187  TcpSession ssn;
3188  ThreadVars tv;
3189  memset(&tv, 0, sizeof(tv));
3190  uint8_t payload[100] = {0};
3191  uint16_t payload_size = 100;
3192 
3193  StreamTcpUTInit(&ra_ctx);
3195 
3197  StreamTcpUTSetupStream(&ssn.server, 100);
3198  StreamTcpUTSetupStream(&ssn.client, 100);
3199 
3200  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3201  if (r != 0)
3202  goto end;
3204  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3205  goto end;
3206  }
3207 
3208  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3209  if (r != 0)
3210  goto end;
3212  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3213  goto end;
3214  }
3215 
3216  result = 1;
3217 end:
3221  StreamTcpUTDeinit(ra_ctx);
3222  return result;
3223 }
3224 
3225 /**
3226  * \test Test to make sure we detect the sequence wrap around and continue
3227  * stream reassembly properly.
3228  *
3229  * \retval On success it returns 1 and on failure 0.
3230  */
3231 
3232 static int StreamTcpReassembleTest47 (void)
3233 {
3234  Packet *p = PacketGetFromAlloc();
3235  FAIL_IF(unlikely(p == NULL));
3236  Flow *f = NULL;
3237  TCPHdr tcph;
3238  TcpSession ssn;
3239  ThreadVars tv;
3240  memset(&tcph, 0, sizeof (TCPHdr));
3241  memset(&tv, 0, sizeof (ThreadVars));
3242  StreamTcpInitConfig(true);
3245 
3246  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3247  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3248 
3249  SET_ISN(&ssn.server, 572799781UL);
3250  ssn.server.last_ack = 572799782UL;
3251 
3252  SET_ISN(&ssn.client, 4294967289UL);
3253  ssn.client.last_ack = 21;
3254 
3255  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3256  FAIL_IF(f == NULL);
3257  f->protoctx = &ssn;
3258  f->proto = IPPROTO_TCP;
3259  p->flow = f;
3260 
3261  tcph.th_win = htons(5480);
3262  ssn.state = TCP_ESTABLISHED;
3263  TcpStream *s = NULL;
3264  uint8_t cnt = 0;
3265 
3266  for (cnt=0; cnt < httplen1; cnt++) {
3267  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3268  tcph.th_ack = htonl(572799782UL);
3269  tcph.th_flags = TH_ACK|TH_PUSH;
3270  p->tcph = &tcph;
3272  p->payload = &httpbuf1[cnt];
3273  p->payload_len = 1;
3274  s = &ssn.client;
3275 
3276  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3277 
3279  p->payload = NULL;
3280  p->payload_len = 0;
3281  tcph.th_seq = htonl(572799782UL);
3282  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3283  tcph.th_flags = TH_ACK;
3284  p->tcph = &tcph;
3285  s = &ssn.server;
3286 
3287  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3288  }
3289 
3290  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3291 
3294  StreamTcpFreeConfig(true);
3295  SCFree(p);
3296  UTHFreeFlow(f);
3297  PASS;
3298 }
3299 
3300 /** \test 3 in order segments in inline reassembly */
3301 static int StreamTcpReassembleInlineTest01(void)
3302 {
3303  int ret = 0;
3304  TcpReassemblyThreadCtx *ra_ctx = NULL;
3305  ThreadVars tv;
3306  TcpSession ssn;
3307  Flow f;
3308 
3309  memset(&tv, 0x00, sizeof(tv));
3310 
3311  StreamTcpUTInit(&ra_ctx);
3314  StreamTcpUTSetupStream(&ssn.client, 1);
3315  FLOW_INITIALIZE(&f);
3316 
3317  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3318  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3319  if (p == NULL) {
3320  printf("couldn't get a packet: ");
3321  goto end;
3322  }
3323  p->tcph->th_seq = htonl(12);
3324  p->flow = &f;
3325 
3326  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3327  printf("failed to add segment 1: ");
3328  goto end;
3329  }
3330  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3331  printf("failed to add segment 2: ");
3332  goto end;
3333  }
3334  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3335  printf("failed to add segment 3: ");
3336  goto end;
3337  }
3338  ssn.client.next_seq = 17;
3339  ret = 1;
3340 end:
3341  FLOW_DESTROY(&f);
3342  UTHFreePacket(p);
3344  StreamTcpUTDeinit(ra_ctx);
3345  return ret;
3346 }
3347 
3348 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3349  * test the sliding window reassembly.
3350  */
3351 static int StreamTcpReassembleInlineTest02(void)
3352 {
3353  int ret = 0;
3354  TcpReassemblyThreadCtx *ra_ctx = NULL;
3355  ThreadVars tv;
3356  TcpSession ssn;
3357  Flow f;
3358 
3359  memset(&tv, 0x00, sizeof(tv));
3360 
3361  StreamTcpUTInit(&ra_ctx);
3364  StreamTcpUTSetupStream(&ssn.client, 1);
3365  FLOW_INITIALIZE(&f);
3366 
3367  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3368  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3369  if (p == NULL) {
3370  printf("couldn't get a packet: ");
3371  goto end;
3372  }
3373  p->tcph->th_seq = htonl(12);
3374  p->flow = &f;
3375 
3376  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3377  printf("failed to add segment 1: ");
3378  goto end;
3379  }
3380  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3381  printf("failed to add segment 2: ");
3382  goto end;
3383  }
3384  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3385  printf("failed to add segment 3: ");
3386  goto end;
3387  }
3388  ssn.client.next_seq = 17;
3389  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3390  printf("failed to add segment 4: ");
3391  goto end;
3392  }
3393  ssn.client.next_seq = 22;
3394  ret = 1;
3395 end:
3396  FLOW_DESTROY(&f);
3397  UTHFreePacket(p);
3399  StreamTcpUTDeinit(ra_ctx);
3400  return ret;
3401 }
3402 
3403 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3404  * test the sliding window reassembly with a small window size so that we
3405  * cutting off at the start (left edge)
3406  */
3407 static int StreamTcpReassembleInlineTest03(void)
3408 {
3409  int ret = 0;
3410  TcpReassemblyThreadCtx *ra_ctx = NULL;
3411  ThreadVars tv;
3412  TcpSession ssn;
3413  Flow f;
3414 
3415  memset(&tv, 0x00, sizeof(tv));
3416 
3417  StreamTcpUTInit(&ra_ctx);
3420  StreamTcpUTSetupStream(&ssn.client, 1);
3421  FLOW_INITIALIZE(&f);
3422 
3424 
3425  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3426  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3427  if (p == NULL) {
3428  printf("couldn't get a packet: ");
3429  goto end;
3430  }
3431  p->tcph->th_seq = htonl(12);
3432  p->flow = &f;
3434 
3435  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3436  printf("failed to add segment 1: ");
3437  goto end;
3438  }
3439  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3440  printf("failed to add segment 2: ");
3441  goto end;
3442  }
3443  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3444  printf("failed to add segment 3: ");
3445  goto end;
3446  }
3447  ssn.client.next_seq = 17;
3448  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3449  printf("failed to add segment 4: ");
3450  goto end;
3451  }
3452  ssn.client.next_seq = 22;
3453 
3454  p->tcph->th_seq = htonl(17);
3455  ret = 1;
3456 end:
3457  FLOW_DESTROY(&f);
3458  UTHFreePacket(p);
3460  StreamTcpUTDeinit(ra_ctx);
3461  return ret;
3462 }
3463 
3464 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3465  * test the sliding window reassembly with a small window size so that we
3466  * cutting off at the start (left edge) with small packet overlap.
3467  */
3468 static int StreamTcpReassembleInlineTest04(void)
3469 {
3470  int ret = 0;
3471  TcpReassemblyThreadCtx *ra_ctx = NULL;
3472  ThreadVars tv;
3473  TcpSession ssn;
3474  Flow f;
3475 
3476  memset(&tv, 0x00, sizeof(tv));
3477 
3478  StreamTcpUTInit(&ra_ctx);
3481  StreamTcpUTSetupStream(&ssn.client, 1);
3482  FLOW_INITIALIZE(&f);
3483 
3485 
3486  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3487  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3488  if (p == NULL) {
3489  printf("couldn't get a packet: ");
3490  goto end;
3491  }
3492  p->tcph->th_seq = htonl(12);
3493  p->flow = &f;
3495 
3496  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3497  printf("failed to add segment 1: ");
3498  goto end;
3499  }
3500  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3501  printf("failed to add segment 2: ");
3502  goto end;
3503  }
3504  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3505  printf("failed to add segment 3: ");
3506  goto end;
3507  }
3508  ssn.client.next_seq = 17;
3509  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3510  printf("failed to add segment 4: ");
3511  goto end;
3512  }
3513  ssn.client.next_seq = 22;
3514 
3515  p->tcph->th_seq = htonl(17);
3516  ret = 1;
3517 end:
3518  FLOW_DESTROY(&f);
3519  UTHFreePacket(p);
3521  StreamTcpUTDeinit(ra_ctx);
3522  return ret;
3523 }
3524 
3525 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3526  * test the sliding window reassembly with a small window size so that we
3527  * cutting off at the start (left edge). Test if the first segment is
3528  * removed from the list.
3529  */
3530 static int StreamTcpReassembleInlineTest08(void)
3531 {
3532  TcpReassemblyThreadCtx *ra_ctx = NULL;
3533  ThreadVars tv;
3534  memset(&tv, 0x00, sizeof(tv));
3535  TcpSession ssn;
3536  Flow f;
3537  StreamTcpUTInit(&ra_ctx);
3540  StreamTcpUTSetupStream(&ssn.client, 1);
3541  FLOW_INITIALIZE(&f);
3542 
3544  f.protoctx = &ssn;
3545 
3546  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3547  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3548  FAIL_IF(p == NULL);
3549  p->tcph->th_seq = htonl(12);
3550  p->flow = &f;
3552 
3553  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3554  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3555  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3556  ssn.client.next_seq = 17;
3557  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3558  ssn.client.next_seq = 22;
3559  p->tcph->th_seq = htonl(17);
3560  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3561 
3562  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3563  FAIL_IF_NULL(seg);
3564  FAIL_IF_NOT(seg->seq == 2);
3565 
3566  FLOW_DESTROY(&f);
3567  UTHFreePacket(p);
3569  StreamTcpUTDeinit(ra_ctx);
3570  PASS;
3571 }
3572 
3573 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3574  * test the sliding window reassembly with a small window size so that we
3575  * cutting off at the start (left edge). Test if the first segment is
3576  * removed from the list.
3577  */
3578 static int StreamTcpReassembleInlineTest09(void)
3579 {
3580  int ret = 0;
3581  TcpReassemblyThreadCtx *ra_ctx = NULL;
3582  ThreadVars tv;
3583  TcpSession ssn;
3584  Flow f;
3585 
3586  memset(&tv, 0x00, sizeof(tv));
3587 
3588  StreamTcpUTInit(&ra_ctx);
3591  StreamTcpUTSetupStream(&ssn.client, 1);
3592  FLOW_INITIALIZE(&f);
3593 
3595 
3596  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3597  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3598  if (p == NULL) {
3599  printf("couldn't get a packet: ");
3600  goto end;
3601  }
3602  p->tcph->th_seq = htonl(17);
3603  p->flow = &f;
3605 
3606  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3607  printf("failed to add segment 1: ");
3608  goto end;
3609  }
3610  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3611  printf("failed to add segment 2: ");
3612  goto end;
3613  }
3614  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3615  printf("failed to add segment 3: ");
3616  goto end;
3617  }
3618  ssn.client.next_seq = 12;
3619  ssn.client.last_ack = 10;
3620 
3621  /* close the GAP and see if we properly reassemble and update base_seq */
3622  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3623  printf("failed to add segment 4: ");
3624  goto end;
3625  }
3626  ssn.client.next_seq = 22;
3627 
3628  p->tcph->th_seq = htonl(12);
3629 
3630  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3631  FAIL_IF_NULL(seg);
3632  FAIL_IF_NOT(seg->seq == 2);
3633 
3634  ret = 1;
3635 end:
3636  FLOW_DESTROY(&f);
3637  UTHFreePacket(p);
3639  StreamTcpUTDeinit(ra_ctx);
3640  return ret;
3641 }
3642 
3643 /** \test App Layer reassembly.
3644  */
3645 static int StreamTcpReassembleInlineTest10(void)
3646 {
3647  int ret = 0;
3648  TcpReassemblyThreadCtx *ra_ctx = NULL;
3649  ThreadVars tv;
3650  TcpSession ssn;
3651  Flow *f = NULL;
3652  Packet *p = NULL;
3653 
3654  memset(&tv, 0x00, sizeof(tv));
3655 
3656  StreamTcpUTInit(&ra_ctx);
3659  StreamTcpUTSetupStream(&ssn.server, 1);
3660  ssn.server.last_ack = 2;
3661  StreamTcpUTSetupStream(&ssn.client, 1);
3662  ssn.client.last_ack = 2;
3663  ssn.data_first_seen_dir = STREAM_TOSERVER;
3664 
3665  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3666  if (f == NULL)
3667  goto end;
3668  f->protoctx = &ssn;
3669  f->proto = IPPROTO_TCP;
3670 
3671  uint8_t stream_payload1[] = "GE";
3672  uint8_t stream_payload2[] = "T /";
3673  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3674 
3675  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3676  if (p == NULL) {
3677  printf("couldn't get a packet: ");
3678  goto end;
3679  }
3680  p->tcph->th_seq = htonl(7);
3681  p->flow = f;
3683 
3684  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3685  printf("failed to add segment 1: ");
3686  goto end;
3687  }
3688  ssn.client.next_seq = 4;
3689 
3690  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3691  if (r < 0) {
3692  printf("StreamTcpReassembleAppLayer failed: ");
3693  goto end;
3694  }
3695 
3696  /* ssn.server.ra_app_base_seq should be isn here. */
3697  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3698  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3699  goto end;
3700  }
3701 
3702  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3703  printf("failed to add segment 2: ");
3704  goto end;
3705  }
3706  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3707  printf("failed to add segment 3: ");
3708  goto end;
3709  }
3710  ssn.client.next_seq = 19;
3711 
3712  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3713  if (r < 0) {
3714  printf("StreamTcpReassembleAppLayer failed: ");
3715  goto end;
3716  }
3717 
3718  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3719 
3720  ret = 1;
3721 end:
3722  UTHFreePacket(p);
3724  StreamTcpUTDeinit(ra_ctx);
3725  UTHFreeFlow(f);
3726  return ret;
3727 }
3728 
3729 /** \test test insert with overlap
3730  */
3731 static int StreamTcpReassembleInsertTest01(void)
3732 {
3733  TcpReassemblyThreadCtx *ra_ctx = NULL;
3734  ThreadVars tv;
3735  TcpSession ssn;
3736  Flow f;
3737 
3738  memset(&tv, 0x00, sizeof(tv));
3739 
3740  StreamTcpUTInit(&ra_ctx);
3742  StreamTcpUTSetupStream(&ssn.client, 1);
3744  FLOW_INITIALIZE(&f);
3745 
3746  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3747  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3748  FAIL_IF(p == NULL);
3749  p->tcph->th_seq = htonl(12);
3750  p->flow = &f;
3751 
3752  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3753  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3754  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3755  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3756  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3757  ssn.client.next_seq = 21;
3758 
3759  FLOW_DESTROY(&f);
3760  UTHFreePacket(p);
3762  StreamTcpUTDeinit(ra_ctx);
3763  PASS;
3764 }
3765 
3766 /** \test test insert with overlaps
3767  */
3768 static int StreamTcpReassembleInsertTest02(void)
3769 {
3770  int ret = 0;
3771  TcpReassemblyThreadCtx *ra_ctx = NULL;
3772  ThreadVars tv;
3773  TcpSession ssn;
3774 
3775  memset(&tv, 0x00, sizeof(tv));
3776 
3777  StreamTcpUTInit(&ra_ctx);
3779  StreamTcpUTSetupStream(&ssn.client, 1);
3780 
3781  int i;
3782  for (i = 2; i < 10; i++) {
3783  int len;
3784  len = i % 2;
3785  if (len == 0)
3786  len = 1;
3787  int seq;
3788  seq = i * 10;
3789  if (seq < 2)
3790  seq = 2;
3791 
3792  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3793  printf("failed to add segment 1: ");
3794  goto end;
3795  }
3796  }
3797  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3798  printf("failed to add segment 2: ");
3799  goto end;
3800  }
3801 
3802  ret = 1;
3803 end:
3805  StreamTcpUTDeinit(ra_ctx);
3806  return ret;
3807 }
3808 
3809 /** \test test insert with overlaps
3810  */
3811 static int StreamTcpReassembleInsertTest03(void)
3812 {
3813  int ret = 0;
3814  TcpReassemblyThreadCtx *ra_ctx = NULL;
3815  ThreadVars tv;
3816  TcpSession ssn;
3817 
3818  memset(&tv, 0x00, sizeof(tv));
3819 
3820  StreamTcpUTInit(&ra_ctx);
3822  StreamTcpUTSetupStream(&ssn.client, 1);
3823 
3824  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3825  printf("failed to add segment 2: ");
3826  goto end;
3827  }
3828 
3829  int i;
3830  for (i = 2; i < 10; i++) {
3831  int len;
3832  len = i % 2;
3833  if (len == 0)
3834  len = 1;
3835  int seq;
3836  seq = i * 10;
3837  if (seq < 2)
3838  seq = 2;
3839 
3840  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3841  printf("failed to add segment 2: ");
3842  goto end;
3843  }
3844  }
3845  ret = 1;
3846 end:
3848  StreamTcpUTDeinit(ra_ctx);
3849  return ret;
3850 }
3851 
3853 #endif /* UNITTESTS */
3854 
3855 /** \brief The Function Register the Unit tests to test the reassembly engine
3856  * for various OS policies.
3857  */
3858 
3860 {
3861 #ifdef UNITTESTS
3862  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3863  StreamTcpReassembleTest25);
3864  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3865  StreamTcpReassembleTest26);
3866  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3867  StreamTcpReassembleTest27);
3868  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3869  StreamTcpReassembleTest28);
3870  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3871  StreamTcpReassembleTest29);
3872  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3873  StreamTcpReassembleTest33);
3874  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3875  StreamTcpReassembleTest34);
3876  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3877  StreamTcpReassembleTest39);
3878  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3879  StreamTcpReassembleTest40);
3880  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3881  StreamTcpReassembleTest44);
3882  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3883  StreamTcpReassembleTest45);
3884  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3885  StreamTcpReassembleTest46);
3886  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3887  StreamTcpReassembleTest47);
3888 
3889  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3890  StreamTcpReassembleInlineTest01);
3891  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3892  StreamTcpReassembleInlineTest02);
3893  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3894  StreamTcpReassembleInlineTest03);
3895  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3896  StreamTcpReassembleInlineTest04);
3897  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3898  StreamTcpReassembleInlineTest08);
3899  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3900  StreamTcpReassembleInlineTest09);
3901 
3902  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3903  StreamTcpReassembleInlineTest10);
3904 
3905  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3906  StreamTcpReassembleInsertTest01);
3907  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3908  StreamTcpReassembleInsertTest02);
3909  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3910  StreamTcpReassembleInsertTest03);
3911 
3915  StreamTcpReassembleRawRegisterTests();
3916 #endif /* UNITTESTS */
3917 }
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
Definition: stream-tcp.h:129
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:270
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:40
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:57
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:55
UTHCheckGapAtPosition
int UTHCheckGapAtPosition(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
Definition: stream-tcp-reassemble.c:2220
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:458
TcpStream_
Definition: stream-tcp-private.h:106
len
uint8_t len
Definition: app-layer-dnp3.h:2
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:69
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:186
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:113
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:2066
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:167
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:379
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:97
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1856
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:97
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:89
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:94
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:69
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:483
stream-tcp.h
StreamTcpInlineMode
bool StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6853
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:386
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:620
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:637
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:136
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:2139
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:607
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1029
StreamTcpThread_
Definition: stream-tcp.h:79
Flow_::proto
uint8_t proto
Definition: flow.h:372
Packet_::payload
uint8_t * payload
Definition: decode.h:586
t_pcapcnt
thread_local uint64_t t_pcapcnt
Definition: stream-tcp-reassemble.c:78
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:2119
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:61
Packet_::flags
uint32_t flags
Definition: decode.h:473
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:181
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:110
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:64
Flow_
Flow data structure.
Definition: flow.h:350
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:77
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:72
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:2102
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
TcpSegmentPcapHdrStorage
struct TcpSegmentPcapHdrStorage_ TcpSegmentPcapHdrStorage
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:332
StringParseUint16
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:337
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:156
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:40
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:76
PKT_DROP_REASON_STREAM_REASSEMBLY
@ PKT_DROP_REASON_STREAM_REASSEMBLY
Definition: decode.h:404
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1754
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:222
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:155
MIN
#define MIN(x, y)
Definition: suricata-common.h:391
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:75
StreamTcpThreadCacheGetSegment
TcpSegment * StreamTcpThreadCacheGetSegment(void)
Definition: stream-tcp-cache.c:179
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:292
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:155
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1771
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:467
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:115
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:39
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:244
Flow_::protoctx
void * protoctx
Definition: flow.h:440
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:710
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:96
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:587
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:223
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:613
util-unittest.h
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2211
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:527
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:430
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:170
STREAMTCP_FLAG_LOSSY_BE_LIBERAL
#define STREAMTCP_FLAG_LOSSY_BE_LIBERAL
Definition: stream-tcp-private.h:194
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:292
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:272
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:172
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:115
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:474
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:561
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:359
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:501
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1353
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:38
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:293
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:467
StreamTcpReassembleRealloc
void * StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
Definition: stream-tcp-reassemble.c:227
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:251
UTHCheckDataAtPosition
int UTHCheckDataAtPosition(TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
Definition: stream-tcp-reassemble.c:2243
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:104
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:273
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1013
StreamTcpGetAcked
uint64_t StreamTcpGetAcked(const TcpStream *stream)
Definition: stream-tcp-reassemble.c:415
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1020
util-exception-policy.h
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:94
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:131
util-print.h
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:51
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:36
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:1005
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:5312
StreamNeedsReassembly
uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:916
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1465
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:285
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:137
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:103
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2153
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:890
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:135
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:162
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
TCP_GET_SEQ
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:114
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:341
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
SCReturn
#define SCReturn
Definition: util-debug.h:273
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:301
Packet_
Definition: decode.h:436
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:234
stream-tcp-private.h
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:40
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:277
conf.h
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:299
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1793
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:117
STREAMTCP_STREAM_FLAG_HAS_GAP
#define STREAMTCP_STREAM_FLAG_HAS_GAP
Definition: stream-tcp-private.h:217
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:600
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:287
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:717
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:396
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:223
TcpStreamCnf_::reassembly_memcap_policy
enum ExceptionPolicy reassembly_memcap_policy
Definition: stream-tcp.h:70
StreamTcpThreadCacheCleanup
void StreamTcpThreadCacheCleanup(void)
Definition: stream-tcp-cache.c:134
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:252
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:543
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:74
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:128
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:100
VALIDATE
#define VALIDATE(e)
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:506
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:56
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamingBuffer_
Definition: util-streaming-buffer.h:108
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:741
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:63
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
TcpSegmentPcapHdrStorage_
Definition: stream-tcp-private.h:65
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1796
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1662
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
app-layer-frames.h
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:180
Packet_::flow
struct Flow_ * flow
Definition: decode.h:475
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:260
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:293
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:690
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:451
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:259
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:175
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir dir)
handle TCP data for the app-layer.
Definition: app-layer.c:657
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:566
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:124
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:30
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:91
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2192
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:135
TcpSegmentPcapHdrStorage_::alloclen
uint32_t alloclen
Definition: stream-tcp-private.h:68
Packet_::app_update_direction
uint8_t app_update_direction
Definition: decode.h:470
StreamTcpThreadCacheReturnSegment
void StreamTcpThreadCacheReturnSegment(TcpSegment *seg)
Definition: stream-tcp-cache.c:53
STREAM_REASSEMBLY_DEPTH_REACHED
@ STREAM_REASSEMBLY_DEPTH_REACHED
Definition: decode-events.h:295
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:236
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:295
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Definition: stream-tcp-reassemble.c:1952
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:161
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:67
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:114
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:173
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, const uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1511
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:294
StreamTcpReassembleIncrMemuse
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:114
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:50
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:450
ConfNode_
Definition: conf.h:32
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:486
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayerGetCtxThread().
Definition: app-layer.c:1026
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:257
Flow_::flags
uint32_t flags
Definition: flow.h:420
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:810
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:71
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:53
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:148
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:305
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2206
stream-tcp-cache.h
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:158
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:102
MISSED_ADD_PAYLOAD
#define MISSED_ADD_PAYLOAD(seq, seg, seglen)
Definition: stream-tcp-reassemble.c:2215
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:60
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:58
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:542
StreamDataRightEdge
uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:421
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:173
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:219
suricata.h
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap(void)
Return memcap value.
Definition: stream-tcp-reassemble.c:198
Address_::family
char family
Definition: decode.h:116
Packet_::dst
Address dst
Definition: decode.h:441
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:6868
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:258
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:53
TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool
uint16_t counter_tcp_segment_from_pool
Definition: stream-tcp-reassemble.h:69
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:238
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3859
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:102
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
segment_thread_pool
PoolThread * segment_thread_pool
Definition: stream-tcp-reassemble.c:80
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:233
StreamTcpReassembleDecrMemuse
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:127
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TCPSEG_PKT_HDR_DEFAULT_SIZE
#define TCPSEG_PKT_HDR_DEFAULT_SIZE
Definition: stream-tcp-private.h:58
TcpSession_
Definition: stream-tcp-private.h:283
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:288
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:183
flow.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:237
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:451
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:449
StreamReassembleLog
int StreamReassembleLog(const TcpSession *ssn, const TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, uint64_t *progress_out, const bool eof)
Definition: stream-tcp-reassemble.c:1921
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:108
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:225
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:148
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamReassembleForFrame
int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof)
Definition: stream-tcp-reassemble.c:1844
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:66
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1625
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:66
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:103
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:121
Packet_::src
Address src
Definition: decode.h:440
TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache
uint16_t counter_tcp_segment_from_cache
Definition: stream-tcp-reassemble.h:68
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:70
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:232
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:146
app-layer.h
StreamTcpCheckStreamContents
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
Definition: stream-tcp-reassemble.c:2170