suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "detect.h"
32 #include "flow.h"
33 #include "threads.h"
34 #include "conf.h"
35 
36 #include "flow-util.h"
37 
38 #include "threadvars.h"
39 #include "tm-threads.h"
40 
41 #include "util-pool.h"
42 #include "util-unittest.h"
43 #include "util-print.h"
44 #include "util-host-os-info.h"
45 #include "util-unittest-helper.h"
46 #include "util-byte.h"
47 #include "util-device.h"
48 
49 #include "stream-tcp.h"
50 #include "stream-tcp-private.h"
51 #include "stream-tcp-cache.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 #include "app-layer-parser.h"
64 #include "app-layer-frames.h"
65 
66 #include "detect-engine-state.h"
67 
68 #include "util-profiling.h"
69 #include "util-validate.h"
70 #include "util-exception-policy.h"
71 
72 #ifdef DEBUG
73 static SCMutex segment_pool_memuse_mutex;
74 static uint64_t segment_pool_memuse = 0;
75 static uint64_t segment_pool_memcnt = 0;
76 #endif
77 
78 thread_local uint64_t t_pcapcnt = UINT64_MAX;
79 
81 /* init only, protect initializing and growing pool */
82 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
83 
84 /* Memory use counter */
85 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
86 
87 static int g_tcp_session_dump_enabled = 0;
88 
89 inline bool IsTcpSessionDumpingEnabled(void)
90 {
91  return g_tcp_session_dump_enabled == 1;
92 }
93 
95 {
96  g_tcp_session_dump_enabled = 1;
97 }
98 
99 /* prototypes */
101 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
102 
104 {
105  SC_ATOMIC_INIT(ra_memuse);
106 }
107 
108 /**
109  * \brief Function to Increment the memory usage counter for the TCP reassembly
110  * segments
111  *
112  * \param size Size of the TCP segment and its payload length memory allocated
113  */
114 void StreamTcpReassembleIncrMemuse(uint64_t size)
115 {
116  (void) SC_ATOMIC_ADD(ra_memuse, size);
117  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
118  return;
119 }
120 
121 /**
122  * \brief Function to Decrease the memory usage counter for the TCP reassembly
123  * segments
124  *
125  * \param size Size of the TCP segment and its payload length memory allocated
126  */
127 void StreamTcpReassembleDecrMemuse(uint64_t size)
128 {
129 #ifdef UNITTESTS
130  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
131  if (RunmodeIsUnittests()) {
132  BUG_ON(presize > UINT_MAX);
133  }
134 #endif
135 
136  (void) SC_ATOMIC_SUB(ra_memuse, size);
137 
138 #ifdef UNITTESTS
139  if (RunmodeIsUnittests()) {
140  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
141  BUG_ON(postsize > presize);
142  }
143 #endif
144  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
145  return;
146 }
147 
149 {
150  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
151  return smemuse;
152 }
153 
154 /**
155  * \brief Function to Check the reassembly memory usage counter against the
156  * allowed max memory usage for TCP segments.
157  *
158  * \param size Size of the TCP segment and its payload length memory allocated
159  * \retval 1 if in bounds
160  * \retval 0 if not in bounds
161  */
163 {
164 #ifdef DEBUG
165  if (unlikely((g_eps_stream_reassembly_memcap != UINT64_MAX &&
166  g_eps_stream_reassembly_memcap == t_pcapcnt))) {
167  SCLogNotice("simulating memcap reached condition for packet %" PRIu64, t_pcapcnt);
168  return 0;
169  }
170 #endif
171  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
172  if (memcapcopy == 0 ||
173  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
174  return 1;
175  return 0;
176 }
177 
178 /**
179  * \brief Update memcap value
180  *
181  * \param size new memcap value
182  */
184 {
185  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
186  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
187  return 1;
188  }
189 
190  return 0;
191 }
192 
193 /**
194  * \brief Return memcap value
195  *
196  * \return memcap memcap value
197  */
199 {
200  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
201  return memcapcopy;
202 }
203 
204 /* memory functions for the streaming buffer API */
205 
206 /*
207  void *(*Calloc)(size_t n, size_t size);
208 */
209 static void *ReassembleCalloc(size_t n, size_t size)
210 {
211  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
212  return NULL;
213  void *ptr = SCCalloc(n, size);
214  if (ptr == NULL)
215  return NULL;
217  return ptr;
218 }
219 
220 /*
221  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
222 */
223 void *StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
224 {
225  if (size > orig_size) {
226  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0) {
227  SCLogDebug("memcap hit at %" PRIu64, SC_ATOMIC_GET(stream_config.reassembly_memcap));
228  return NULL;
229  }
230  }
231  void *nptr = SCRealloc(optr, size);
232  if (nptr == NULL) {
233  SCLogDebug("realloc fail");
234  return NULL;
235  }
236  if (size > orig_size) {
237  StreamTcpReassembleIncrMemuse(size - orig_size);
238  } else {
239  StreamTcpReassembleDecrMemuse(orig_size - size);
240  }
241  return nptr;
242 }
243 
244 /*
245  void (*Free)(void *ptr, size_t size);
246 */
247 static void ReassembleFree(void *ptr, size_t size)
248 {
249  SCFree(ptr);
251 }
252 
253 /** \brief alloc a tcp segment pool entry */
254 static void *TcpSegmentPoolAlloc(void)
255 {
256  SCLogDebug("segment alloc");
257  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
258  return NULL;
259  }
260 
261  TcpSegment *seg = NULL;
262 
263  seg = SCMalloc(sizeof (TcpSegment));
264  if (unlikely(seg == NULL))
265  return NULL;
266 
268  uint32_t memuse =
269  sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
270  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
271  SCFree(seg);
272  return NULL;
273  }
274 
276  if (seg->pcap_hdr_storage == NULL) {
277  SCLogError("Unable to allocate memory for "
278  "TcpSegmentPcapHdrStorage");
279  SCFree(seg);
280  return NULL;
281  } else {
282  seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
283  seg->pcap_hdr_storage->pkt_hdr =
284  SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
285  if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
286  SCLogError("Unable to allocate memory for "
287  "packet header data within "
288  "TcpSegmentPcapHdrStorage");
289  SCFree(seg->pcap_hdr_storage);
290  SCFree(seg);
291  return NULL;
292  }
293  }
294 
296  } else {
297  seg->pcap_hdr_storage = NULL;
298  }
299 
300  return seg;
301 }
302 
303 static int TcpSegmentPoolInit(void *data, void *initdata)
304 {
305  TcpSegment *seg = (TcpSegment *) data;
306  TcpSegmentPcapHdrStorage *pcap_hdr;
307 
308  pcap_hdr = seg->pcap_hdr_storage;
309 
310  /* do this before the can bail, so TcpSegmentPoolCleanup
311  * won't have uninitialized memory to consider. */
312  memset(seg, 0, sizeof (TcpSegment));
313 
315  uint32_t memuse =
316  sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
317  seg->pcap_hdr_storage = pcap_hdr;
318  if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
319  return 0;
320  }
322  } else {
323  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
324  return 0;
325  }
326  }
327 
328 #ifdef DEBUG
329  SCMutexLock(&segment_pool_memuse_mutex);
330  segment_pool_memuse += sizeof(TcpSegment);
331  segment_pool_memcnt++;
332  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
333  SCMutexUnlock(&segment_pool_memuse_mutex);
334 #endif
335 
336  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
337  return 1;
338 }
339 
340 /** \brief clean up a tcp segment pool entry */
341 static void TcpSegmentPoolCleanup(void *ptr)
342 {
343  if (ptr == NULL)
344  return;
345 
346  TcpSegment *seg = (TcpSegment *)ptr;
347  if (seg && seg->pcap_hdr_storage) {
348  if (seg->pcap_hdr_storage->pkt_hdr) {
351  }
352  SCFree(seg->pcap_hdr_storage);
353  seg->pcap_hdr_storage = NULL;
355  }
356 
357  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
358 
359 #ifdef DEBUG
360  SCMutexLock(&segment_pool_memuse_mutex);
361  segment_pool_memuse -= sizeof(TcpSegment);
362  segment_pool_memcnt--;
363  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
364  SCMutexUnlock(&segment_pool_memuse_mutex);
365 #endif
366 }
367 
368 /**
369  * \brief Function to return the segment back to the pool.
370  *
371  * \param seg Segment which will be returned back to the pool.
372  */
374 {
375  if (seg == NULL)
376  return;
377 
378  if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
379  seg->pcap_hdr_storage->pktlen = 0;
380  }
381 
383 }
384 
385 /**
386  * \brief return all segments in this stream into the pool(s)
387  *
388  * \param stream the stream to cleanup
389  */
391 {
392  TcpSegment *seg = NULL, *safe = NULL;
393  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
394  {
395  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
397  }
398 }
399 
400 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
401 {
402  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
403  return STREAM_BASE_OFFSET(stream) + (stream->last_ack - stream->base_seq);
404  } else {
405  return STREAM_BASE_OFFSET(stream);
406  }
407 }
408 
409 uint64_t StreamTcpGetAcked(const TcpStream *stream)
410 {
411  return GetAbsLastAck(stream);
412 }
413 
414 // may contain gaps
415 uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
416 {
417  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
418  if (!eof && StreamTcpInlineMode() == FALSE) {
419  right_edge = MIN(GetAbsLastAck(stream), right_edge);
420  }
421  return right_edge;
422 }
423 
424 uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
425 {
426  uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
427  if (!eof && StreamTcpInlineMode() == FALSE) {
428  right_edge = MIN(GetAbsLastAck(stream), right_edge);
429  }
430  return right_edge;
431 }
432 
433 #ifdef UNITTESTS
434 /** \internal
435  * \brief check if segments falls before stream 'offset' */
436 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
437 {
438  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
439  return 1;
440  return 0;
441 }
442 #endif
443 
444 /** \param f locked flow */
446 {
447  if (f->protoctx == NULL)
448  return;
449 
450  TcpSession *ssn = (TcpSession *)f->protoctx;
454  if (f->alparser) {
457  }
458 }
459 
460 /** \param f locked flow */
462 {
463  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
464  return 0;
465 
466  TcpSession *ssn = (TcpSession *)f->protoctx;
468 }
469 
470 static int StreamTcpReassemblyConfig(bool quiet)
471 {
472  uint32_t segment_prealloc = 2048;
473  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
474  if (seg) {
475  uint32_t prealloc = 0;
476  if (StringParseUint32(&prealloc, 10, (uint16_t)strlen(seg->val), seg->val) < 0) {
477  SCLogError("segment-prealloc of "
478  "%s is invalid",
479  seg->val);
480  return -1;
481  }
482  segment_prealloc = prealloc;
483  }
484  if (!quiet)
485  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
486  stream_config.prealloc_segments = segment_prealloc;
487 
488  int overlap_diff_data = 0;
489  (void)ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
490  if (overlap_diff_data) {
492  }
493  if (StreamTcpInlineMode() == TRUE) {
495  }
496 
497  uint16_t max_regions = 8;
498  ConfNode *mr = ConfGetNode("stream.reassembly.max-regions");
499  if (mr) {
500  uint16_t max_r = 0;
501  if (StringParseUint16(&max_r, 10, (uint16_t)strlen(mr->val), mr->val) < 0) {
502  SCLogError("max-regions %s is invalid", mr->val);
503  return -1;
504  }
505  max_regions = max_r;
506  }
507  if (!quiet)
508  SCLogConfig("stream.reassembly \"max-regions\": %u", max_regions);
509 
510  stream_config.prealloc_segments = segment_prealloc;
512  stream_config.sbcnf.max_regions = max_regions;
514  stream_config.sbcnf.Calloc = ReassembleCalloc;
516  stream_config.sbcnf.Free = ReassembleFree;
517 
518  return 0;
519 }
520 
521 int StreamTcpReassembleInit(bool quiet)
522 {
523  /* init the memcap/use tracker */
525 
526  if (StreamTcpReassemblyConfig(quiet) < 0)
527  return -1;
528 
529 #ifdef DEBUG
530  SCMutexInit(&segment_pool_memuse_mutex, NULL);
531 #endif
532  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
534  return 0;
535 }
536 
537 void StreamTcpReassembleFree(bool quiet)
538 {
539  SCMutexLock(&segment_thread_pool_mutex);
540  if (segment_thread_pool != NULL) {
542  segment_thread_pool = NULL;
543  }
544  SCMutexUnlock(&segment_thread_pool_mutex);
545  SCMutexDestroy(&segment_thread_pool_mutex);
546 
547 #ifdef DEBUG
548  if (segment_pool_memuse > 0)
549  SCLogDebug("segment_pool_memuse %" PRIu64 "", segment_pool_memuse);
550  SCMutexDestroy(&segment_pool_memuse_mutex);
551 #endif
552 }
553 
555 {
556  SCEnter();
558  if (unlikely(ra_ctx == NULL))
559  return NULL;
560 
561  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
562 
563  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
564 
565  SCMutexLock(&segment_thread_pool_mutex);
566  if (segment_thread_pool == NULL) {
567  segment_thread_pool = PoolThreadInit(1, /* thread */
568  0, /* unlimited */
570  sizeof(TcpSegment),
571  TcpSegmentPoolAlloc,
572  TcpSegmentPoolInit, NULL,
573  TcpSegmentPoolCleanup, NULL);
574  ra_ctx->segment_thread_pool_id = 0;
575  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
577  ra_ctx->segment_thread_pool_id);
578  } else {
579  /* grow segment_thread_pool until we have an element for our thread id */
581  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
583  ra_ctx->segment_thread_pool_id);
584  }
585  SCMutexUnlock(&segment_thread_pool_mutex);
586  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
587  SCLogError("failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
589  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
590  }
591 
592  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
593 }
594 
596 {
597  SCEnter();
599 
600  if (ra_ctx) {
602  SCFree(ra_ctx);
603  }
604  SCReturn;
605 }
606 
607 /**
608  * \brief check if stream in pkt direction has depth reached
609  *
610  * \param p packet with *LOCKED* flow
611  *
612  * \retval 1 stream has depth reached
613  * \retval 0 stream does not have depth reached
614  */
616 {
617  if (p->flow != NULL && p->flow->protoctx != NULL) {
618  TcpSession *ssn = p->flow->protoctx;
619  TcpStream *stream;
620  if (p->flowflags & FLOW_PKT_TOSERVER) {
621  stream = &ssn->client;
622  } else {
623  stream = &ssn->server;
624  }
625 
626  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
627  }
628 
629  return 0;
630 }
631 
632 /**
633  * \internal
634  * \brief Function to Check the reassembly depth valuer against the
635  * allowed max depth of the stream reassembly for TCP streams.
636  *
637  * \param stream stream direction
638  * \param seq sequence number where "size" starts
639  * \param size size of the segment that is added
640  *
641  * \retval size Part of the size that fits in the depth, 0 if none
642  */
643 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
644  uint32_t seq, uint32_t size)
645 {
646  SCEnter();
647 
648  /* if the configured depth value is 0, it means there is no limit on
649  reassembly depth. Otherwise carry on my boy ;) */
650  if (ssn->reassembly_depth == 0) {
651  SCReturnUInt(size);
652  }
653 
654  /* if the final flag is set, we're not accepting anymore */
656  SCReturnUInt(0);
657  }
658 
659  uint64_t seg_depth;
660  if (SEQ_GT(stream->base_seq, seq)) {
661  if (SEQ_LEQ(seq+size, stream->base_seq)) {
662  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
663  stream->base_seq, seq, seq+size);
664  SCReturnUInt(0);
665  }
666 
667  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
668  } else {
669  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
670  }
671 
672  /* if the base_seq has moved passed the depth window we stop
673  * checking and just reject the rest of the packets including
674  * retransmissions. Saves us the hassle of dealing with sequence
675  * wraps as well */
676  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
677  stream->base_seq, seg_depth,
678  ssn->reassembly_depth);
679 
680  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
681  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
683  SCReturnUInt(0);
684  }
685  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
686  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
687 #if 0
688  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
689  (stream->base_seq_offset + stream->base_seq + size),
690  (stream->isn + ssn->reassembly_depth));
691 #endif
692  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
693  /* packet (partly?) fits the depth window */
694 
695  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
696  /* complete fit */
697  SCReturnUInt(size);
698  } else {
700  /* partial fit, return only what fits */
701  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
702  DEBUG_VALIDATE_BUG_ON(part > size);
703  if (part > size)
704  part = size;
705  SCReturnUInt(part);
706  }
707  }
708 
709  SCReturnUInt(0);
710 }
711 
713 {
714  if (RB_EMPTY(&stream->sb.sbb_tree)) {
715  if (stream->sb.region.stream_offset != 0)
716  return 0;
717 
718  return stream->sb.region.buf_offset;
719  } else {
720  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
721  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
722  return stream->sb.sbb_size;
723  }
724 }
725 
726 /**
727  * \brief Insert a packets TCP data into the stream reassembly engine.
728  *
729  * \retval 0 good segment, as far as we checked.
730  * \retval -1 insert failure due to memcap
731  *
732  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
733  * or it wasn't added because of reassembly depth.
734  *
735  */
737  TcpSession *ssn, TcpStream *stream, Packet *p)
738 {
739  SCEnter();
740 
741  if (ssn->data_first_seen_dir == 0) {
742  if (PKT_IS_TOSERVER(p)) {
743  ssn->data_first_seen_dir = STREAM_TOSERVER;
744  } else {
745  ssn->data_first_seen_dir = STREAM_TOCLIENT;
746  }
747  }
748 
749  /* If the OS policy is not set then set the OS policy for this stream */
750  if (stream->os_policy == 0) {
751  StreamTcpSetOSPolicy(stream, p);
752  }
753 
756  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
757  SCReturnInt(0);
758  }
759 
760  /* If we have reached the defined depth for either of the stream, then stop
761  reassembling the TCP session */
762  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
763  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
764 
767  /* increment stream depth counter */
770  }
771  if (size == 0) {
772  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
773  SCReturnInt(0);
774  }
775 
777  if (size > p->payload_len)
778  size = p->payload_len;
779 
780  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
781  if (seg == NULL) {
782  SCLogDebug("segment_pool is empty");
785  SCReturnInt(-1);
786  }
787 
788  DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
789  TCP_SEG_LEN(seg) = (uint16_t)size;
790  seg->seq = TCP_GET_SEQ(p);
791 
792  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
793  if (TCP_HAS_TFO(p) && p->payload_len && (p->tcph->th_flags & TH_SYN))
794  seg->seq += 1;
795 
796  /* proto detection skipped, but now we do get data. Set event. */
797  if (RB_EMPTY(&stream->seg_tree) &&
799 
802  }
803 
805  tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len);
806  if (r < 0) {
807  if (r == -ENOMEM) {
809  }
810  SCLogDebug("StreamTcpReassembleInsertSegment failed");
811  SCReturnInt(-1);
812  }
813  SCReturnInt(0);
814 }
815 
816 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
817  Packet *p)
818 {
819  uint8_t flag = 0;
820 
822  flag |= STREAM_START;
823  }
824 
825  if (ssn->state == TCP_CLOSED) {
826  flag |= STREAM_EOF;
827  }
828 
829  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
830  flag |= STREAM_MIDSTREAM;
831  }
832 
833  if (p->flags & PKT_PSEUDO_STREAM_END) {
834  flag |= STREAM_EOF;
835  }
836 
837  if (&ssn->client == stream) {
838  flag |= STREAM_TOSERVER;
839  } else {
840  flag |= STREAM_TOCLIENT;
841  }
843  flag |= STREAM_DEPTH;
844  }
845  return flag;
846 }
847 
848 /**
849  * \brief Check the minimum size limits for reassembly.
850  *
851  * \retval 0 don't reassemble yet
852  * \retval 1 do reassemble
853  */
854 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
855  const TcpStream *stream, const Packet *p)
856 {
857  SCEnter();
858 
859  /* if any of these flags is set we always inspect immediately */
860 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
861  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
862  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
863  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
864 
867  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
868  "is set, so not expecting any new data segments");
869  }
871  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
872  }
874  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
875  "so no new segments will be considered");
876  }
877  SCReturnInt(1);
878  }
879 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
880 
881  /* some states mean we reassemble no matter how much data we have */
882  if (ssn->state > TCP_TIME_WAIT)
883  SCReturnInt(1);
884 
885  if (p->flags & PKT_PSEUDO_STREAM_END)
886  SCReturnInt(1);
887 
888  const uint64_t last_ack_abs = GetAbsLastAck(stream);
889  int64_t diff = last_ack_abs - STREAM_RAW_PROGRESS(stream);
890  int64_t chunk_size = PKT_IS_TOSERVER(p) ? (int64_t)stream_config.reassembly_toserver_chunk_size
892 
893  /* check if we have enough data to do raw reassembly */
894  if (chunk_size <= diff) {
895  SCReturnInt(1);
896  } else {
897  SCLogDebug("%s min chunk len not yet reached: "
898  "last_ack %" PRIu32 ", ra_raw_base_seq %" PRIu32 ", %" PRIu32 " < "
899  "%" PRIi64,
900  PKT_IS_TOSERVER(p) ? "toserver" : "toclient", stream->last_ack, stream->base_seq,
901  (stream->last_ack - stream->base_seq), chunk_size);
902  SCReturnInt(0);
903  }
904 
905  SCReturnInt(0);
906 }
907 
908 /**
909  * \brief see what if any work the TCP session still needs
910  */
911 uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
912 {
913  const TcpStream *stream = NULL;
914 #ifdef DEBUG
915  const char *dirstr = NULL;
916 #endif
917  if (direction == STREAM_TOSERVER) {
918  stream = &ssn->client;
919 #ifdef DEBUG
920  dirstr = "client";
921 #endif
922  } else {
923  stream = &ssn->server;
924 #ifdef DEBUG
925  dirstr = "server";
926 #endif
927  }
928  int use_app = 1;
929  int use_raw = 1;
930 
932  // app is dead
933  use_app = 0;
934  }
935 
937  // raw is dead
938  use_raw = 0;
939  }
940  if (use_raw) {
941  const uint64_t right_edge =
942  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
943  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
944  " (use: %s). Stream right edge: %" PRIu64,
945  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
946  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
947  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
948  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
950  }
951  }
952  if (use_app) {
953  const uint64_t right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
954  SCLogDebug("%s: app %" PRIu64 " (use: %s), raw %" PRIu64
955  " (use: %s). Stream right edge: %" PRIu64,
956  dirstr, STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
957  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no", right_edge);
958  if (right_edge > STREAM_APP_PROGRESS(stream)) {
959  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
961  }
962  }
963 
964  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
966 }
967 
968 #ifdef DEBUG
969 static uint64_t GetStreamSize(TcpStream *stream)
970 {
971  if (stream) {
972  uint64_t size = 0;
973  uint32_t cnt = 0;
974  uint64_t last_ack_abs = GetAbsLastAck(stream);
975  uint64_t last_re = 0;
976 
977  SCLogDebug("stream_offset %" PRIu64, stream->sb.region.stream_offset);
978 
979  TcpSegment *seg;
980  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
981  const uint64_t seg_abs =
982  STREAM_BASE_OFFSET(stream) + (uint64_t)(seg->seq - stream->base_seq);
983  if (last_re != 0 && last_re < seg_abs) {
984  const char *gacked = NULL;
985  if (last_ack_abs >= seg_abs) {
986  gacked = "fully ack'd";
987  } else if (last_ack_abs > last_re) {
988  gacked = "partly ack'd";
989  } else {
990  gacked = "not yet ack'd";
991  }
992  SCLogDebug(" -> gap of size %" PRIu64 ", ack:%s", seg_abs - last_re, gacked);
993  }
994 
995  const char *acked = NULL;
996  if (last_ack_abs >= seg_abs + (uint64_t)TCP_SEG_LEN(seg)) {
997  acked = "fully ack'd";
998  } else if (last_ack_abs > seg_abs) {
999  acked = "partly ack'd";
1000  } else {
1001  acked = "not yet ack'd";
1002  }
1003 
1004  SCLogDebug("%u -> seg %p seq %u abs %" PRIu64 " size %u abs %" PRIu64 " (%" PRIu64
1005  ") ack:%s",
1006  cnt, seg, seg->seq, seg_abs, TCP_SEG_LEN(seg),
1007  seg_abs + (uint64_t)TCP_SEG_LEN(seg), STREAM_BASE_OFFSET(stream), acked);
1008  last_re = seg_abs + (uint64_t)TCP_SEG_LEN(seg);
1009  cnt++;
1010  size += (uint64_t)TCP_SEG_LEN(seg);
1011  }
1012 
1013  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
1014  return size;
1015  }
1016  return (uint64_t)0;
1017 }
1018 
1019 static void GetSessionSize(TcpSession *ssn, Packet *p)
1020 {
1021  uint64_t size = 0;
1022  if (ssn) {
1023  size = GetStreamSize(&ssn->client);
1024  size += GetStreamSize(&ssn->server);
1025 
1026  //if (size > 900000)
1027  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1028  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
1029  }
1030 }
1031 #endif
1032 
1033 static StreamingBufferBlock *GetBlock(const StreamingBuffer *sb, const uint64_t offset)
1034 {
1035  StreamingBufferBlock *blk = sb->head;
1036  if (blk == NULL)
1037  return NULL;
1038 
1039  for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
1040  if (blk->offset >= offset)
1041  return blk;
1042  else if ((blk->offset + blk->len) > offset) {
1043  return blk;
1044  }
1045  }
1046  return NULL;
1047 }
1048 
1049 static inline bool GapAhead(const TcpStream *stream, StreamingBufferBlock *cur_blk)
1050 {
1051  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
1052  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
1053  GetAbsLastAck(stream) > (cur_blk->offset + cur_blk->len)) {
1054  return true;
1055  }
1056  return false;
1057 }
1058 
1059 /** \internal
1060  *
1061  * Get buffer, or first part of the buffer if data gaps exist.
1062  *
1063  * \brief get stream data from offset
1064  * \param offset stream offset
1065  * \param check_for_gap check if there is a gap ahead. Optional as it is only
1066  * needed for app-layer incomplete support.
1067  * \retval bool pkt loss ahead */
1068 static bool GetAppBuffer(const TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1069  uint64_t offset, const bool check_for_gap)
1070 {
1071  const uint8_t *mydata;
1072  uint32_t mydata_len;
1073  bool gap_ahead = false;
1074 
1075  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1076  SCLogDebug("getting one blob");
1077 
1078  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1079 
1080  *data = mydata;
1081  *data_len = mydata_len;
1082  } else {
1083  SCLogDebug("block mode");
1084  StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
1085  if (blk == NULL) {
1086  *data = NULL;
1087  *data_len = 0;
1088  return false;
1089  }
1090  SCLogDebug("blk %p blk->offset %" PRIu64 ", blk->len %u", blk, blk->offset, blk->len);
1091 
1092  /* block at expected offset */
1093  if (blk->offset == offset) {
1094  SCLogDebug("blk at offset");
1095 
1096  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
1097  BUG_ON(blk->len != *data_len);
1098 
1099  gap_ahead = check_for_gap && GapAhead(stream, blk);
1100 
1101  /* block past out offset */
1102  } else if (blk->offset > offset) {
1103  SCLogDebug("gap, want data at offset %"PRIu64", "
1104  "got data at %"PRIu64". GAP of size %"PRIu64,
1105  offset, blk->offset, blk->offset - offset);
1106  *data = NULL;
1107  *data_len = blk->offset - offset;
1108 
1109  /* block starts before offset, but ends after */
1110  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
1111  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
1112  offset, blk->offset, blk->len);
1113  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
1114  SCLogDebug("data %p, data_len %u", *data, *data_len);
1115 
1116  gap_ahead = check_for_gap && GapAhead(stream, blk);
1117 
1118  } else {
1119  *data = NULL;
1120  *data_len = 0;
1121  }
1122  }
1123  return gap_ahead;
1124 }
1125 
1126 /** \internal
1127  * \brief check to see if we should declare a GAP
1128  * Call this when the app layer didn't get data at the requested
1129  * offset.
1130  */
1131 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
1132 {
1133  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1134  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1135  const uint64_t last_ack_abs = GetAbsLastAck(stream) - (uint64_t)ackadded;
1136 
1137  SCLogDebug("last_ack %u abs %" PRIu64, stream->last_ack, last_ack_abs);
1138  SCLogDebug("next_seq %u", stream->next_seq);
1139 
1140  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1141  * has been ack'd. This looks like a GAP. */
1142  if (last_ack_abs > app_progress) {
1143  /* however, we can accept ACKs a bit too liberally. If last_ack
1144  * is beyond next_seq, we only consider it a gap now if we do
1145  * already have data beyond the gap. */
1146  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1147  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1148  SCLogDebug("packet %" PRIu64 ": no GAP. "
1149  "next_seq %u < last_ack %u, but no data in list",
1150  p->pcap_cnt, stream->next_seq, stream->last_ack);
1151  return false;
1152  } else {
1153  const uint64_t next_seq_abs =
1154  STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1155  const StreamingBufferBlock *blk = stream->sb.head;
1156  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1157  /* ack'd data after the gap */
1158  SCLogDebug("packet %" PRIu64 ": GAP. "
1159  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1160  p->pcap_cnt, stream->next_seq, stream->last_ack);
1161  return true;
1162  }
1163  }
1164  }
1165 
1166  SCLogDebug("packet %" PRIu64 ": GAP! "
1167  "last_ack_abs %" PRIu64 " > app_progress %" PRIu64 ", "
1168  "but we have no data.",
1169  p->pcap_cnt, last_ack_abs, app_progress);
1170  return true;
1171  }
1172  SCLogDebug("packet %"PRIu64": no GAP. "
1173  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1174  p->pcap_cnt, last_ack_abs, app_progress);
1175  return false;
1176 }
1177 
1178 static inline uint32_t AdjustToAcked(const Packet *p,
1179  const TcpSession *ssn, const TcpStream *stream,
1180  const uint64_t app_progress, const uint32_t data_len)
1181 {
1182  uint32_t adjusted = data_len;
1183 
1184  /* get window of data that is acked */
1185  if (StreamTcpInlineMode() == FALSE) {
1186  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1187  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1188  (ssn->state == TCP_CLOSED &&
1189  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1190  (p->flags & PKT_PSEUDO_STREAM_END))) {
1191  // fall through, we use all available data
1192  } else {
1193  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1194  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1195 
1196  /* see if the buffer contains unack'd data as well */
1197  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1198  uint32_t check = data_len;
1199  adjusted = last_ack_abs - app_progress;
1200  BUG_ON(adjusted > check);
1201  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1202  "data is considered", adjusted);
1203  }
1204  }
1205  }
1206  return adjusted;
1207 }
1208 
1209 /** \internal
1210  * \brief get stream buffer and update the app-layer
1211  * \param stream pointer to pointer as app-layer can switch flow dir
1212  * \retval 0 success
1213  */
1214 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1215  TcpReassemblyThreadCtx *ra_ctx,
1216  TcpSession *ssn, TcpStream **stream,
1217  Packet *p, enum StreamUpdateDir dir)
1218 {
1219  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1220 
1221  SCLogDebug("app progress %"PRIu64, app_progress);
1222 #ifdef DEBUG
1223  uint64_t last_ack_abs = GetAbsLastAck(*stream);
1224  SCLogDebug("last_ack %u (abs %" PRIu64 "), base_seq %u", (*stream)->last_ack, last_ack_abs,
1225  (*stream)->base_seq);
1226 #endif
1227  const uint8_t *mydata;
1228  uint32_t mydata_len;
1229  bool last_was_gap = false;
1230 
1231  while (1) {
1232  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1233  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1234  bool gap_ahead =
1235  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress, check_for_gap_ahead);
1236  SCLogDebug("gap_ahead %s mydata_len %u", BOOL2STR(gap_ahead), mydata_len);
1237  if (last_was_gap && mydata_len == 0) {
1238  break;
1239  }
1240  last_was_gap = false;
1241 
1242  /* make sure to only deal with ACK'd data */
1243  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1244  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1245  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1246  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1247 
1248  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1249  NULL, mydata_len,
1250  StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP);
1251  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1252 
1254  (*stream)->flags |= STREAMTCP_STREAM_FLAG_HAS_GAP;
1257 
1258  /* AppLayerHandleTCPData has likely updated progress. */
1259  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1260  app_progress = STREAM_APP_PROGRESS(*stream);
1261 
1262  /* a GAP also consumes 'data required'. TODO perhaps we can use
1263  * this to skip post GAP data until the start of a next record. */
1264  if ((*stream)->data_required > 0) {
1265  if ((*stream)->data_required > mydata_len) {
1266  (*stream)->data_required -= mydata_len;
1267  } else {
1268  (*stream)->data_required = 0;
1269  }
1270  }
1271  if (r < 0)
1272  return 0;
1273  if (no_progress_update)
1274  break;
1275  last_was_gap = true;
1276  continue;
1277 
1278  } else if (flags & STREAM_DEPTH) {
1279  SCLogDebug("DEPTH");
1280  // we're just called once with this flag, so make sure we pass it on
1281  if (mydata == NULL && mydata_len > 0) {
1282  mydata_len = 0;
1283  }
1284  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1285  SCLogDebug("GAP?1");
1286  /* Possibly a gap, but no new data. */
1287  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1288  SCReturnInt(0);
1289 
1290  mydata = NULL;
1291  mydata_len = 0;
1292  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1293  break;
1294  }
1295  DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1296 
1297  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1298  *stream, &(*stream)->sb, mydata_len, app_progress);
1299 
1300  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1301  SCLogDebug("GAP?2");
1302  if (mydata_len < (*stream)->data_required) {
1303  SCLogDebug("GAP?3 gap_head %s", BOOL2STR(gap_ahead));
1304  if (gap_ahead) {
1305  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1306  (*stream)->data_required, mydata_len);
1307  (*stream)->app_progress_rel += mydata_len;
1308  (*stream)->data_required -= mydata_len;
1309  // TODO send incomplete data to app-layer with special flag
1310  // indicating its all there is for this rec?
1311  } else {
1312  SCReturnInt(0);
1313  }
1314  app_progress = STREAM_APP_PROGRESS(*stream);
1315  continue;
1316  }
1317  }
1318  (*stream)->data_required = 0;
1319 
1320  SCLogDebug("parser");
1321  /* update the app-layer */
1322  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1323  (uint8_t *)mydata, mydata_len, flags);
1324  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1325  AppLayerFrameDump(p->flow);
1326  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1327  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1328  break;
1329  app_progress = new_app_progress;
1330  if (flags & STREAM_DEPTH)
1331  break;
1332  }
1333 
1334  SCReturnInt(0);
1335 }
1336 
1337 /**
1338  * \brief Update the stream reassembly upon receiving a packet.
1339  *
1340  * For IDS mode, the stream is in the opposite direction of the packet,
1341  * as the ACK-packet is ACK'ing the stream.
1342  *
1343  * One of the utilities call by this function AppLayerHandleTCPData(),
1344  * has a feature where it will call this very same function for the
1345  * stream opposing the stream it is called with. This shouldn't cause
1346  * any issues, since processing of each stream is independent of the
1347  * other stream.
1348  */
1350  TcpSession *ssn, TcpStream *stream,
1351  Packet *p, enum StreamUpdateDir dir)
1352 {
1353  SCEnter();
1354 
1355  /* this function can be directly called by app layer protocol
1356  * detection. */
1359  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1360  SCReturnInt(0);
1361  }
1362 
1363 #ifdef DEBUG
1364  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1365  GetSessionSize(ssn, p);
1366 #endif
1367  /* if no segments are in the list or all are already processed,
1368  * and state is beyond established, we send an empty msg */
1369  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1370  {
1371  /* send an empty EOF msg if we have no segments but TCP state
1372  * is beyond ESTABLISHED */
1373  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1374  SCLogDebug("sending empty eof message");
1375  /* send EOF to app layer */
1376  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1377  NULL, 0,
1378  StreamGetAppLayerFlags(ssn, stream, p));
1379  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1380 
1381  SCReturnInt(0);
1382  }
1383  }
1384 
1385  /* with all that out of the way, lets update the app-layer */
1386  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1387 }
1388 
1389 /** \internal
1390  * \brief get stream data from offset
1391  * \param offset stream offset */
1392 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1393  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1394 {
1395  const uint8_t *mydata;
1396  uint32_t mydata_len;
1397  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1398  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1399 
1400  uint64_t roffset = offset;
1401  if (offset)
1402  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1403  else {
1404  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1405  }
1406 
1407  *data = mydata;
1408  *data_len = mydata_len;
1409  *data_offset = roffset;
1410  } else {
1411  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1412  *iter == NULL ? "starting" : "continuing", offset);
1413  if (*iter == NULL) {
1414  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1415  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1416  SCLogDebug("*iter %p", *iter);
1417  }
1418  if (*iter == NULL) {
1419  SCLogDebug("no data");
1420  *data = NULL;
1421  *data_len = 0;
1422  *data_offset = 0;
1423  return 0;
1424  }
1425  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1426 
1427  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1428  SCLogDebug("mydata %p", mydata);
1429 
1430  if ((*iter)->offset < offset) {
1431  uint64_t delta = offset - (*iter)->offset;
1432  if (delta < mydata_len) {
1433  *data = mydata + delta;
1434  *data_len = mydata_len - delta;
1435  *data_offset = offset;
1436  } else {
1437  SCLogDebug("no data (yet)");
1438  *data = NULL;
1439  *data_len = 0;
1440  *data_offset = 0;
1441  }
1442 
1443  } else {
1444  *data = mydata;
1445  *data_len = mydata_len;
1446  *data_offset = (*iter)->offset;
1447  }
1448 
1449  *iter = SBB_RB_NEXT(*iter);
1450  SCLogDebug("*iter %p", *iter);
1451  }
1452  return 0;
1453 }
1454 
1455 /** \brief does the stream engine have data to inspect?
1456  *
1457  * Returns true if there is data to inspect. In IDS case this is
1458  * about ACK'd data in the packet's direction.
1459  *
1460  * In the IPS case this is about the packet itself.
1461  */
1463 {
1464  TcpStream *stream;
1465  if (PKT_IS_TOSERVER(p)) {
1466  stream = &ssn->client;
1467  } else {
1468  stream = &ssn->server;
1469  }
1470 
1471  if (RB_EMPTY(&stream->seg_tree)) {
1472  return false;
1473  }
1474 
1477  return false;
1478 
1479  if (StreamTcpInlineMode() == FALSE) {
1480  const uint64_t segs_re_abs =
1481  STREAM_BASE_OFFSET(stream) + stream->segs_right_edge - stream->base_seq;
1482  if (STREAM_RAW_PROGRESS(stream) == segs_re_abs) {
1483  return false;
1484  }
1485  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1486  return true;
1487  }
1488  } else {
1489  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1490  return true;
1491  }
1492  }
1493  return false;
1494 }
1495 
1496 /** \brief update stream engine after detection
1497  *
1498  * Tasked with progressing the 'progress' for Raw reassembly.
1499  * 2 main scenario's:
1500  * 1. progress is != 0, so we use this
1501  * 2. progress is 0, meaning the detect engine didn't touch
1502  * raw at all. In this case we need to look into progressing
1503  * raw anyway.
1504  *
1505  * Additionally, this function is tasked with disabling raw
1506  * reassembly if the app-layer requested to disable it.
1507  */
1508 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1509 {
1510  TcpStream *stream;
1511  if (PKT_IS_TOSERVER(p)) {
1512  stream = &ssn->client;
1513  } else {
1514  stream = &ssn->server;
1515  }
1516 
1517  if (progress > STREAM_RAW_PROGRESS(stream)) {
1518  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1519  stream->raw_progress_rel += slide;
1521 
1522  /* if app is active and beyond raw, sync raw to app */
1523  } else if (progress == 0 && STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1525  /* if trigger raw is set we sync the 2 trackers */
1527  {
1528  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1529  stream->raw_progress_rel += slide;
1531 
1532  /* otherwise mix in the tcp window */
1533  } else {
1534  uint64_t tcp_window = stream->window;
1535  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1536  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1537  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1538  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1539  stream->raw_progress_rel += slide;
1540  }
1541  }
1542  }
1543  /* app is dead */
1544  } else if (progress == 0) {
1545  uint64_t tcp_window = stream->window;
1546  const uint64_t stream_right_edge = StreamingBufferGetConsecutiveDataRightEdge(&stream->sb);
1547  if (tcp_window < stream_right_edge) {
1548  uint64_t new_raw = stream_right_edge - tcp_window;
1549  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1550  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1551  stream->raw_progress_rel += slide;
1552  }
1553  }
1555 
1556  } else {
1557  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1558  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1559  STREAM_RAW_PROGRESS(stream), stream->window);
1560  }
1561 
1562  /* if we were told to accept no more raw data, we can mark raw as
1563  * disabled now. */
1566  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1567  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1568  }
1569 
1570  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1571 }
1572 
1573 /** \internal
1574  * \brief get a buffer around the current packet and run the callback on it
1575  *
1576  * The inline/IPS scanning method takes the current payload and wraps it in
1577  * data from other segments.
1578  *
1579  * How much data is inspected is controlled by the available data, chunk_size
1580  * and the payload size of the packet.
1581  *
1582  * Large packets: if payload size is close to the chunk_size, where close is
1583  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1584  * used: payload_len + 33% of the chunk_size.
1585  * If the payload size if equal to or bigger than the chunk_size, we use
1586  * payload len + 33% of the chunk size.
1587  */
1588 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1589  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1590 {
1591  SCEnter();
1592  int r = 0;
1593 
1594  TcpStream *stream;
1595  if (PKT_IS_TOSERVER(p)) {
1596  stream = &ssn->client;
1597  } else {
1598  stream = &ssn->server;
1599  }
1600 
1601  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1603  {
1604  *progress_out = STREAM_RAW_PROGRESS(stream);
1605  return 0;
1606  }
1607 
1608  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1611  if (chunk_size <= p->payload_len) {
1612  chunk_size = p->payload_len + (chunk_size / 3);
1613  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1614  p->payload_len, chunk_size);
1615  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1616  chunk_size = p->payload_len + ((chunk_size / 3));
1617  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1618  p->payload_len, chunk_size);
1619  }
1620 
1621  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1622  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1623  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1624  packet_leftedge_abs, packet_rightedge_abs);
1625 
1626  const uint8_t *mydata = NULL;
1627  uint32_t mydata_len = 0;
1628  uint64_t mydata_offset = 0;
1629  /* simply return progress from the block we inspected. */
1630  bool return_progress = false;
1631 
1632  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1633  /* continues block */
1634  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1635  return_progress = true;
1636 
1637  } else {
1638  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1639  /* find our block */
1640  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1641  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1642  if (sbb) {
1643  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1644  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1645  mydata_offset = sbb->offset;
1646  }
1647  }
1648 
1649  /* this can only happen if the segment insert of our current 'p' failed */
1650  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1651  if ((mydata == NULL || mydata_len == 0) || /* no data */
1652  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1653  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1654  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1655  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1656  {
1657  /* no data, or data is incomplete or wrong: use packet data */
1658  mydata = p->payload;
1659  mydata_len = p->payload_len;
1660  mydata_offset = packet_leftedge_abs;
1661  //mydata_rightedge_abs = packet_rightedge_abs;
1662  } else {
1663  /* adjust buffer to match chunk_size */
1664  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1665  if (mydata_len > chunk_size) {
1666  uint32_t excess = mydata_len - chunk_size;
1667  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1668 
1669  if (mydata_rightedge_abs == packet_rightedge_abs) {
1670  mydata += excess;
1671  mydata_len -= excess;
1672  mydata_offset += excess;
1673  SCLogDebug("cutting front of the buffer with %u", excess);
1674  } else if (mydata_offset == packet_leftedge_abs) {
1675  mydata_len -= excess;
1676  SCLogDebug("cutting tail of the buffer with %u", excess);
1677  } else {
1678  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1679  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1680  SCLogDebug("before %u after %u", before, after);
1681 
1682  if (after >= (chunk_size - p->payload_len) / 2) {
1683  // more trailing data than we need
1684 
1685  if (before >= (chunk_size - p->payload_len) / 2) {
1686  // also more heading data, divide evenly
1687  before = after = (chunk_size - p->payload_len) / 2;
1688  } else {
1689  // heading data is less than requested, give the
1690  // rest to the trailing data
1691  after = (chunk_size - p->payload_len) - before;
1692  }
1693  } else {
1694  // less trailing data than requested
1695 
1696  if (before >= (chunk_size - p->payload_len) / 2) {
1697  before = (chunk_size - p->payload_len) - after;
1698  } else {
1699  // both smaller than their requested size
1700  }
1701  }
1702 
1703  /* adjust the buffer */
1704  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1705  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1706  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1707  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1708  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1709  mydata += skip;
1710  mydata_len -= (skip + cut);
1711  mydata_offset += skip;
1712  }
1713  }
1714  }
1715 
1716  /* run the callback */
1717  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1718  BUG_ON(r < 0);
1719 
1720  if (return_progress) {
1721  *progress_out = (mydata_offset + mydata_len);
1722  } else {
1723  /* several blocks of data, so we need to be a bit more careful:
1724  * - if last_ack is beyond last progress, move progress forward to last_ack
1725  * - if our block matches or starts before last ack, return right edge of
1726  * our block.
1727  */
1728  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1729  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1730 
1731  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1732  if (mydata_offset > last_ack_abs) {
1733  /* gap between us and last ack, set progress to last ack */
1734  *progress_out = last_ack_abs;
1735  } else {
1736  *progress_out = (mydata_offset + mydata_len);
1737  }
1738  } else {
1739  *progress_out = STREAM_RAW_PROGRESS(stream);
1740  }
1741  }
1742  return r;
1743 }
1744 
1745 /** \brief access 'raw' reassembly data.
1746  *
1747  * Access data as tracked by 'raw' tracker. Data is made available to
1748  * callback that is passed to this function.
1749  *
1750  * In the case of IDS the callback may be run multiple times if data
1751  * contains gaps. It will then be run for each block of data that is
1752  * continuous.
1753  *
1754  * The callback should give on of 2 return values:
1755  * - 0 ok
1756  * - 1 done
1757  * The value 1 will break the loop if there is a block list that is
1758  * inspected.
1759  *
1760  * This function will return the 'progress' value that has been
1761  * consumed until now.
1762  *
1763  * \param ssn tcp session
1764  * \param stream tcp stream
1765  * \param Callback the function pointer to the callback function
1766  * \param cb_data callback data
1767  * \param[in] progress_in progress to work from
1768  * \param[in] re right edge of data to consider
1769  * \param[out] progress_out absolute progress value of the data this
1770  * call handled.
1771  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1772  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1773  *
1774  * `respect_inspect_depth` is used to avoid useless inspection of too
1775  * much data.
1776  */
1777 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1778  StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in,
1779  const uint64_t re, uint64_t *progress_out, bool eof, bool respect_inspect_depth)
1780 {
1781  SCEnter();
1782  int r = 0;
1783 
1784  StreamingBufferBlock *iter = NULL;
1785  uint64_t progress = progress_in;
1786 
1787  /* loop through available buffers. On no packet loss we'll have a single
1788  * iteration. On missing data we'll walk the blocks */
1789  while (1) {
1790  const uint8_t *mydata;
1791  uint32_t mydata_len;
1792  uint64_t mydata_offset = 0;
1793 
1794  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1795  if (mydata_len == 0) {
1796  SCLogDebug("no data");
1797  break;
1798  }
1799  //PrintRawDataFp(stdout, mydata, mydata_len);
1800 
1801  SCLogDebug("raw progress %"PRIu64, progress);
1802  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1803  stream, &stream->sb, mydata_len, progress);
1804 
1805  if (eof) {
1806  // inspect all remaining data, ack'd or not
1807  } else {
1808  if (re < progress) {
1809  SCLogDebug("nothing to do");
1810  goto end;
1811  }
1812 
1813  SCLogDebug("re %" PRIu64 ", raw_progress %" PRIu64, re, progress);
1814  SCLogDebug("raw_progress + mydata_len %" PRIu64 ", re %" PRIu64, progress + mydata_len,
1815  re);
1816 
1817  /* see if the buffer contains unack'd data as well */
1818  if (progress + mydata_len > re) {
1819  uint32_t check = mydata_len;
1820  mydata_len = re - progress;
1821  BUG_ON(check < mydata_len);
1822  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1823  "data is considered", mydata_len);
1824  }
1825  }
1826  if (mydata_len == 0)
1827  break;
1828 
1829  SCLogDebug("data %p len %u", mydata, mydata_len);
1830 
1831  /* we have data. */
1832  r = Callback(cb_data, mydata, mydata_len, mydata_offset);
1833  BUG_ON(r < 0);
1834 
1835  if (mydata_offset == progress) {
1836  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1837  progress, mydata_len, progress_in + mydata_len);
1838 
1839  progress += mydata_len;
1840  SCLogDebug("raw progress now %"PRIu64, progress);
1841 
1842  /* data is beyond the progress we'd like, and before last ack. Gap. */
1843  } else if (mydata_offset > progress && mydata_offset < re) {
1844  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1845  SCLogDebug("re %" PRIu64, re);
1846  progress = mydata_offset;
1847  SCLogDebug("raw progress now %"PRIu64, progress);
1848 
1849  } else {
1850  SCLogDebug("not increasing progress, data gap => mydata_offset "
1851  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1852  }
1853 
1854  if (iter == NULL || r == 1)
1855  break;
1856  }
1857 end:
1858  *progress_out = progress;
1859  return r;
1860 }
1861 
1863  void *cb_data, const uint64_t offset, const bool eof)
1864 {
1865  /* take app progress as the right edge of used data. */
1866  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1867  SCLogDebug("app_progress %" PRIu64, app_progress);
1868 
1869  uint64_t unused = 0;
1870  return StreamReassembleRawDo(
1871  ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof, false);
1872 }
1873 
1875  StreamReassembleRawFunc Callback, void *cb_data,
1876  uint64_t *progress_out, bool respect_inspect_depth)
1877 {
1878  /* handle inline separately as the logic is very different */
1879  if (StreamTcpInlineMode() == TRUE) {
1880  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1881  }
1882 
1883  TcpStream *stream;
1884  if (PKT_IS_TOSERVER(p)) {
1885  stream = &ssn->client;
1886  } else {
1887  stream = &ssn->server;
1888  }
1889 
1891  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1892  {
1893  *progress_out = STREAM_RAW_PROGRESS(stream);
1894  return 0;
1895  }
1896 
1897  uint64_t progress = STREAM_RAW_PROGRESS(stream);
1898  /* if the app layer triggered a flush, and we're supposed to
1899  * use a minimal inspect depth, we actually take the app progress
1900  * as that is the right edge of the data. Then we take the window
1901  * of 'min_inspect_depth' before that. */
1902 
1903  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s "
1904  "stream->min_inspect_depth %u",
1905  respect_inspect_depth ? "true" : "false",
1906  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1907  stream->min_inspect_depth);
1908 
1909  if (respect_inspect_depth && (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) &&
1910  stream->min_inspect_depth) {
1911  progress = STREAM_APP_PROGRESS(stream);
1912  if (stream->min_inspect_depth >= progress) {
1913  progress = 0;
1914  } else {
1915  progress -= stream->min_inspect_depth;
1916  }
1917 
1918  SCLogDebug("stream app %" PRIu64 ", raw %" PRIu64, STREAM_APP_PROGRESS(stream),
1919  STREAM_RAW_PROGRESS(stream));
1920 
1921  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1922  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress "
1923  "%" PRIu64,
1924  progress);
1925  }
1926 
1927  SCLogDebug("progress %" PRIu64 ", min inspect depth %u %s", progress, stream->min_inspect_depth,
1928  stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW"
1929  : "(no trigger)");
1930 
1931  /* absolute right edge of ack'd data */
1932  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1933  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1934 
1935  return StreamReassembleRawDo(ssn, stream, Callback, cb_data, progress, last_ack_abs,
1936  progress_out, (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1937 }
1938 
1940  StreamReassembleRawFunc Callback, void *cb_data,
1941  uint64_t progress_in,
1942  uint64_t *progress_out, bool eof)
1943 {
1944  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1945  return 0;
1946 
1947  /* absolute right edge of ack'd data */
1948  const uint64_t last_ack_abs = GetAbsLastAck(stream);
1949  SCLogDebug("last_ack_abs %" PRIu64, last_ack_abs);
1950 
1951  return StreamReassembleRawDo(
1952  ssn, stream, Callback, cb_data, progress_in, last_ack_abs, progress_out, eof, false);
1953 }
1954 
1955 /** \internal
1956  * \brief update app layer based on received ACK
1957  *
1958  * \retval r 0 on success, -1 on error
1959  */
1960 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1961  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1962 {
1963  SCEnter();
1964 
1965  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1966  SCReturnInt(-1);
1967 
1968  SCReturnInt(0);
1969 }
1970 
1972  TcpSession *ssn, TcpStream *stream, Packet *p)
1973 {
1974  SCEnter();
1975 
1976  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1977 
1978  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1979  ssn, stream, p, p->payload_len);
1980 
1981  /* default IDS: update opposing side (triggered by ACK) */
1983  /* inline and stream end and flow timeout packets trigger same dir handling */
1984  if (StreamTcpInlineMode()) {
1985  dir = UPDATE_DIR_PACKET;
1986  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1987  dir = UPDATE_DIR_PACKET;
1988  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1989  dir = UPDATE_DIR_PACKET;
1990  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1991  if (p->tcph->th_flags & TH_ACK) {
1992  dir = UPDATE_DIR_BOTH;
1993  } else {
1994  dir = UPDATE_DIR_PACKET;
1995  }
1996  } else if (ssn->state == TCP_CLOSED) {
1997  dir = UPDATE_DIR_BOTH;
1998  }
1999 
2000  /* handle ack received */
2001  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
2002  /* we need to update the opposing stream in
2003  * StreamTcpReassembleHandleSegmentUpdateACK */
2004  TcpStream *opposing_stream = NULL;
2005  if (stream == &ssn->client) {
2006  opposing_stream = &ssn->server;
2007  } else {
2008  opposing_stream = &ssn->client;
2009  }
2010 
2011  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2012 
2013  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
2014  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
2015  SCReturnInt(-1);
2016  }
2017 
2018  /* StreamTcpReassembleHandleSegmentUpdateACK
2019  * may swap content of ssn->server and ssn->client structures.
2020  * We have to continue with initial content of the stream in such case */
2021  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
2022  if (reversed_before_ack_handling != reversed_after_ack_handling) {
2023  SCLogDebug("TCP streams were swapped");
2024  stream = opposing_stream;
2025  }
2026  }
2027  /* if this segment contains data, insert it */
2028  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
2029  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
2030 
2031  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
2032  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
2033  /* failure can only be because of memcap hit, so see if this should lead to a drop */
2036  SCReturnInt(-1);
2037  }
2038 
2039  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
2040  p->flags |= PKT_STREAM_ADD;
2041  } else {
2042  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
2043  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
2044  ssn, stream, p->payload_len,
2045  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
2046 
2047  }
2048 
2049  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
2050  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
2051  * was *just* set. In this case we trigger the AppLayer Truncate
2052  * logic, to inform the applayer no more data in this direction is
2053  * to be expected. */
2054  if ((stream->flags &
2057  {
2058  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
2059  if (dir != UPDATE_DIR_PACKET) {
2060  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
2061  "can trigger Truncate");
2062  dir = UPDATE_DIR_PACKET;
2063  }
2064  }
2065 
2066  /* in stream inline mode even if we have no data we call the reassembly
2067  * functions to handle EOF */
2068  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
2069  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
2070  StreamTcpInlineMode()?"true":"false",
2071  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
2072  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
2073  SCReturnInt(-1);
2074  }
2075  }
2076 
2077  SCReturnInt(0);
2078 }
2079 
2080 /**
2081  * \brief get a segment from the pool
2082  *
2083  * \retval seg Segment from the pool or NULL
2084  */
2086 {
2088  if (seg) {
2090  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2091  return seg;
2092  }
2093 
2094  seg = (TcpSegment *)PoolThreadGetById(
2095  segment_thread_pool, (uint16_t)ra_ctx->segment_thread_pool_id);
2096  SCLogDebug("seg we return is %p", seg);
2097  if (seg == NULL) {
2098  /* Increment the counter to show that we are not able to serve the
2099  segment request due to memcap limit */
2101  } else {
2102  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
2104  }
2105 
2106  return seg;
2107 }
2108 
2109 /**
2110  * \brief Trigger RAW stream reassembly
2111  *
2112  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
2113  * reassembly from the applayer, for example upon completion of a
2114  * HTTP request.
2115  *
2116  * It sets a flag in the stream so that the next Raw call will return
2117  * the data.
2118  *
2119  * \param ssn TcpSession
2120  */
2122 {
2123 #ifdef DEBUG
2124  BUG_ON(ssn == NULL);
2125 #endif
2126 
2127  if (ssn != NULL) {
2128  if (direction == STREAM_TOSERVER) {
2130  } else {
2132  }
2133 
2134  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
2135  }
2136 }
2137 
2138 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
2139 {
2140 #ifdef DEBUG
2141  BUG_ON(ssn == NULL);
2142 #endif
2143 
2144  if (ssn != NULL) {
2145  if (direction == STREAM_TOSERVER) {
2146  ssn->client.min_inspect_depth = depth;
2147  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
2148  } else {
2149  ssn->server.min_inspect_depth = depth;
2150  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
2151  }
2152  }
2153 }
2154 
2155 #ifdef UNITTESTS
2156 /** unit tests and it's support functions below */
2157 
2158 #define SET_ISN(stream, setseq) \
2159  (stream)->isn = (setseq); \
2160  (stream)->base_seq = (setseq) + 1
2161 
2162 /** \brief The Function to create the packet with given payload, which is used
2163  * to test the reassembly of the engine.
2164  *
2165  * \param payload The variable used to store the payload contents of the
2166  * current packet.
2167  * \param value The value which current payload will have for this packet
2168  * \param payload_len The length of the filed payload for current packet.
2169  * \param len Length of the payload array
2170  */
2171 
2172 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2173  uint8_t payload_len, uint8_t len)
2174 {
2175  uint8_t i;
2176  for (i = 0; i < payload_len; i++)
2177  payload[i] = value;
2178  for (; i < len; i++)
2179  payload = NULL;
2180 }
2181 
2182 /** \brief The Function Checks the reassembled stream contents against predefined
2183  * stream contents according to OS policy used.
2184  *
2185  * \param stream_policy Predefined value of stream for different OS policies
2186  * \param stream Reassembled stream returned from the reassembly functions
2187  */
2188 
2189 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
2190 {
2191  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
2192  {
2193  //PrintRawDataFp(stdout, stream_policy, sp_size);
2194  return 0;
2195  }
2196  return 1;
2197 }
2198 
2199 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2200 {
2201  if (StreamingBufferCompareRawData(&stream->sb,
2202  data, data_len) == 0)
2203  {
2204  SCReturnInt(0);
2205  }
2206  SCLogInfo("OK");
2207  PrintRawDataFp(stdout, data, data_len);
2208  return 1;
2209 }
2210 
2211 #define MISSED_START(isn) \
2212  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2213  TcpSession ssn; \
2214  ThreadVars tv; \
2215  memset(&tv, 0, sizeof(tv)); \
2216  \
2217  StreamTcpUTInit(&ra_ctx); \
2218  \
2219  StreamTcpUTSetupSession(&ssn); \
2220  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2221  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2222  \
2223  TcpStream *stream = &ssn.client;
2224 
2225 #define MISSED_END \
2226  StreamTcpUTClearSession(&ssn); \
2227  StreamTcpUTDeinit(ra_ctx); \
2228  PASS
2229 
2230 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2231  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2232  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2233 
2234 #define MISSED_ADD_PAYLOAD(seq, seg, seglen) \
2235  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen));
2236 
2237 int UTHCheckGapAtPostion(TcpStream *stream, int pos, uint64_t offset, uint32_t len);
2238 
2239 int UTHCheckGapAtPostion(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
2240 {
2241  int cnt = 0;
2242  uint64_t last_re = 0;
2243  StreamingBufferBlock *sbb = NULL;
2244  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2245  {
2246  if (sbb->offset != last_re) {
2247  // gap before us
2248  if (cnt == pos && last_re == offset && len == sbb->offset - last_re) {
2249  return 1;
2250  }
2251  cnt++;
2252  }
2253  last_re = sbb->offset + sbb->len;
2254  cnt++;
2255  }
2256  return 0;
2257 }
2258 
2260  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len);
2261 
2263  TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
2264 {
2265  int cnt = 0;
2266  uint64_t last_re = 0;
2267  StreamingBufferBlock *sbb = NULL;
2268  RB_FOREACH(sbb, SBB, &stream->sb.sbb_tree)
2269  {
2270  if (sbb->offset != last_re) {
2271  // gap before us
2272  cnt++;
2273  }
2274 
2275  if (cnt == pos && sbb->offset == offset) {
2276  const uint8_t *buf = NULL;
2277  uint32_t buf_len = 0;
2278  StreamingBufferSBBGetData(&stream->sb, sbb, &buf, &buf_len);
2279 
2280  if (len == buf_len) {
2281  return (memcmp(data, buf, len) == 0);
2282  }
2283  }
2284 
2285  last_re = sbb->offset + sbb->len;
2286  cnt++;
2287  }
2288  return 0;
2289 }
2290 
2291 /**
2292  * \test Test the handling of packets missed by both IDS and the end host.
2293  * The packet is missed in the starting of the stream.
2294  *
2295  * \retval On success it returns 1 and on failure 0.
2296  */
2297 
2298 static int StreamTcpReassembleTest25 (void)
2299 {
2300  MISSED_START(6);
2301  MISSED_ADD_PAYLOAD(10, "BB", 2);
2302  FAIL_IF_NOT(UTHCheckGapAtPostion(stream, 0, 0, 3) == 1);
2303  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 1, 3, "BB", 2) == 1);
2304  MISSED_ADD_PAYLOAD(12, "CC", 2);
2305  FAIL_IF_NOT(UTHCheckGapAtPostion(stream, 0, 0, 3) == 1);
2306  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 1, 3, "BBCC", 4) == 1);
2307  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2308  MISSED_END;
2309  PASS;
2310 }
2311 
2312 /**
2313  * \test Test the handling of packets missed by both IDS and the end host.
2314  * The packet is missed in the middle of the stream.
2315  *
2316  * \retval On success it returns 1 and on failure 0.
2317  */
2318 
2319 static int StreamTcpReassembleTest26 (void)
2320 {
2321  MISSED_START(9);
2322  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2323  MISSED_ADD_PAYLOAD(15, "CC", 2);
2324  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 0, 0, "AAA", 3) == 1);
2325  FAIL_IF_NOT(UTHCheckGapAtPostion(stream, 1, 3, 2) == 1);
2326  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 2, 5, "CC", 2) == 1);
2327  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2328  MISSED_END;
2329 }
2330 
2331 /**
2332  * \test Test the handling of packets missed by both IDS and the end host.
2333  * The packet is missed in the end of the stream.
2334  *
2335  * \retval On success it returns 1 and on failure 0.
2336  */
2337 
2338 static int StreamTcpReassembleTest27 (void)
2339 {
2340  MISSED_START(9);
2341  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2342  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2343  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2344  MISSED_END;
2345 }
2346 
2347 /**
2348  * \test Test the handling of packets missed by IDS, but the end host has
2349  * received it and send the acknowledgment of it. The packet is missed
2350  * in the starting of the stream.
2351  *
2352  * \retval On success it returns 1 and on failure 0.
2353  */
2354 
2355 static int StreamTcpReassembleTest28 (void)
2356 {
2357  MISSED_START(6);
2358  MISSED_ADD_PAYLOAD(10, "AAA", 3);
2359  FAIL_IF_NOT(UTHCheckGapAtPostion(stream, 0, 0, 3) == 1);
2360  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 1, 3, "AAA", 3) == 1);
2361  MISSED_ADD_PAYLOAD(13, "BB", 2);
2362  FAIL_IF_NOT(UTHCheckGapAtPostion(stream, 0, 0, 3) == 1);
2363  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 1, 3, "AAABB", 5) == 1);
2364  ssn.state = TCP_TIME_WAIT;
2365  MISSED_ADD_PAYLOAD(15, "CC", 2);
2366  FAIL_IF_NOT(UTHCheckGapAtPostion(stream, 0, 0, 3) == 1);
2367  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 1, 3, "AAABBCC", 7) == 1);
2368  MISSED_END;
2369 }
2370 
2371 /**
2372  * \test Test the handling of packets missed by IDS, but the end host has
2373  * received it and send the acknowledgment of it. The packet is missed
2374  * in the middle of the stream.
2375  *
2376  * \retval On success it returns 1 and on failure 0.
2377  */
2378 
2379 static int StreamTcpReassembleTest29 (void)
2380 {
2381  MISSED_START(9);
2382  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2383  ssn.state = TCP_TIME_WAIT;
2384  MISSED_ADD_PAYLOAD(15, "CC", 2);
2385  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 0, 0, "AAA", 3) == 1);
2386  FAIL_IF_NOT(UTHCheckGapAtPostion(stream, 1, 3, 2) == 1);
2387  FAIL_IF_NOT(UTHCheckDataAtPostion(stream, 2, 5, "CC", 2) == 1);
2388  MISSED_END;
2389 }
2390 
2391 static int StreamTcpReassembleTest33(void)
2392 {
2393  TcpSession ssn;
2394  Packet *p = PacketGetFromAlloc();
2395  FAIL_IF(unlikely(p == NULL));
2396  Flow f;
2397  TCPHdr tcph;
2398  TcpReassemblyThreadCtx *ra_ctx = NULL;
2400  uint8_t packet[1460] = "";
2401 
2402  StreamTcpUTInit(&ra_ctx);
2404 
2405  memset(&f, 0, sizeof (Flow));
2406  memset(&tcph, 0, sizeof (TCPHdr));
2407  ThreadVars tv;
2408  memset(&tv, 0, sizeof (ThreadVars));
2409  FLOW_INITIALIZE(&f);
2410  f.protoctx = &ssn;
2411  f.proto = IPPROTO_TCP;
2412  p->src.family = AF_INET;
2413  p->dst.family = AF_INET;
2414  p->proto = IPPROTO_TCP;
2415  p->flow = &f;
2416  tcph.th_win = 5480;
2417  tcph.th_flags = TH_PUSH | TH_ACK;
2418  p->tcph = &tcph;
2420  p->payload = packet;
2421 
2422  p->tcph->th_seq = htonl(10);
2423  p->tcph->th_ack = htonl(31);
2424  p->payload_len = 10;
2425 
2426  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2427 
2428  p->tcph->th_seq = htonl(20);
2429  p->tcph->th_ack = htonl(31);
2430  p->payload_len = 10;
2431 
2432  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2433 
2434  p->tcph->th_seq = htonl(40);
2435  p->tcph->th_ack = htonl(31);
2436  p->payload_len = 10;
2437 
2438  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2439 
2440  p->tcph->th_seq = htonl(5);
2441  p->tcph->th_ack = htonl(31);
2442  p->payload_len = 30;
2443 
2444  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2445 
2447  StreamTcpUTDeinit(ra_ctx);
2448  SCFree(p);
2449  PASS;
2450 }
2451 
2452 static int StreamTcpReassembleTest34(void)
2453 {
2454  TcpSession ssn;
2455  Packet *p = PacketGetFromAlloc();
2456  FAIL_IF(unlikely(p == NULL));
2457  Flow f;
2458  TCPHdr tcph;
2459  TcpReassemblyThreadCtx *ra_ctx = NULL;
2461  uint8_t packet[1460] = "";
2462 
2463  StreamTcpUTInit(&ra_ctx);
2465  memset(&f, 0, sizeof (Flow));
2466  memset(&tcph, 0, sizeof (TCPHdr));
2467  ThreadVars tv;
2468  memset(&tv, 0, sizeof (ThreadVars));
2469  FLOW_INITIALIZE(&f);
2470  f.protoctx = &ssn;
2471  f.proto = IPPROTO_TCP;
2472  p->src.family = AF_INET;
2473  p->dst.family = AF_INET;
2474  p->proto = IPPROTO_TCP;
2475  p->flow = &f;
2476  tcph.th_win = 5480;
2477  tcph.th_flags = TH_PUSH | TH_ACK;
2478  p->tcph = &tcph;
2480  p->payload = packet;
2481  SET_ISN(&ssn.client, 857961230);
2482 
2483  p->tcph->th_seq = htonl(857961230);
2484  p->tcph->th_ack = htonl(31);
2485  p->payload_len = 304;
2486 
2487  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2488 
2489  p->tcph->th_seq = htonl(857961534);
2490  p->tcph->th_ack = htonl(31);
2491  p->payload_len = 1460;
2492 
2493  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2494 
2495  p->tcph->th_seq = htonl(857963582);
2496  p->tcph->th_ack = htonl(31);
2497  p->payload_len = 1460;
2498 
2499  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2500 
2501  p->tcph->th_seq = htonl(857960946);
2502  p->tcph->th_ack = htonl(31);
2503  p->payload_len = 1460;
2504 
2505  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, &ssn.client, p) == -1);
2506 
2508  StreamTcpUTDeinit(ra_ctx);
2509  SCFree(p);
2510  PASS;
2511 }
2512 
2513 /**
2514  * \test Test to make sure that we don't return the segments until the app
2515  * layer proto has been detected and after that remove the processed
2516  * segments.
2517  *
2518  * \retval On success it returns 1 and on failure 0.
2519  */
2520 
2521 static int StreamTcpReassembleTest39 (void)
2522 {
2523  Packet *p = PacketGetFromAlloc();
2524  FAIL_IF(unlikely(p == NULL));
2525  Flow f;
2526  ThreadVars tv;
2527  StreamTcpThread stt;
2528  TCPHdr tcph;
2529  PacketQueueNoLock pq;
2530  memset(&pq,0,sizeof(PacketQueueNoLock));
2531  memset (&f, 0, sizeof(Flow));
2532  memset(&tv, 0, sizeof (ThreadVars));
2533  memset(&stt, 0, sizeof (stt));
2534  memset(&tcph, 0, sizeof (TCPHdr));
2535 
2536  FLOW_INITIALIZE(&f);
2537  f.flags = FLOW_IPV4;
2538  f.proto = IPPROTO_TCP;
2539  p->flow = &f;
2540  p->tcph = &tcph;
2541 
2542  StreamTcpUTInit(&stt.ra_ctx);
2543 
2544  /* handshake */
2545  tcph.th_win = htons(5480);
2546  tcph.th_flags = TH_SYN;
2548  p->payload_len = 0;
2549  p->payload = NULL;
2550  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2551 
2552  TcpSession *ssn = (TcpSession *)f.protoctx;
2553  FAIL_IF_NULL(ssn);
2554 
2561  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2562  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2563  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2564  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2565  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2566  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2567  FAIL_IF(ssn->data_first_seen_dir != 0);
2568 
2569  /* handshake */
2570  p->tcph->th_ack = htonl(1);
2571  p->tcph->th_flags = TH_SYN | TH_ACK;
2573  p->payload_len = 0;
2574  p->payload = NULL;
2575  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2582  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2583  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2584  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2585  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2586  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2587  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2588  FAIL_IF(ssn->data_first_seen_dir != 0);
2589 
2590  /* handshake */
2591  p->tcph->th_ack = htonl(1);
2592  p->tcph->th_seq = htonl(1);
2593  p->tcph->th_flags = TH_ACK;
2595  p->payload_len = 0;
2596  p->payload = NULL;
2597  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2604  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2605  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2606  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2607  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2608  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2609  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2610  FAIL_IF(ssn->data_first_seen_dir != 0);
2611 
2612  /* partial request */
2613  uint8_t request1[] = { 0x47, 0x45, };
2614  p->tcph->th_ack = htonl(1);
2615  p->tcph->th_seq = htonl(1);
2616  p->tcph->th_flags = TH_PUSH | TH_ACK;
2618  p->payload_len = sizeof(request1);
2619  p->payload = request1;
2620  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2627  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2628  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2629  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2630  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2631  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2632  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2633  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2634  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2635 
2636  /* response ack against partial request */
2637  p->tcph->th_ack = htonl(3);
2638  p->tcph->th_seq = htonl(1);
2639  p->tcph->th_flags = TH_ACK;
2641  p->payload_len = 0;
2642  p->payload = NULL;
2643  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2650  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2651  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2652  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2653  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2654  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2655  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2656  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2657  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2658 
2659  /* complete partial request */
2660  uint8_t request2[] = {
2661  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2662  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2663  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2664  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2665  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2666  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2667  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2668  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2669  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2670  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2671  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2672  p->tcph->th_ack = htonl(1);
2673  p->tcph->th_seq = htonl(3);
2674  p->tcph->th_flags = TH_PUSH | TH_ACK;
2676  p->payload_len = sizeof(request2);
2677  p->payload = request2;
2678  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2685  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2686  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2687  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2688  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2689  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2690  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2691  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2692  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2693  FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2694 
2695  /* response - request ack */
2696  uint8_t response[] = {
2697  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2698  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2699  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2700  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2701  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2702  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2703  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2704  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2705  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2706  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2707  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2708  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2709  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2710  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2711  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2712  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2713  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2714  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2715  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2716  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2717  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2718  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2719  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2720  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2721  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2722  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2723  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2724  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2725  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2726  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2727  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2728  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2729  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2730  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2731  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2732  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2733  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2734  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2735  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2736  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2737  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2738  p->tcph->th_ack = htonl(88);
2739  p->tcph->th_seq = htonl(1);
2740  p->tcph->th_flags = TH_PUSH | TH_ACK;
2742  p->payload_len = sizeof(response);
2743  p->payload = response;
2744 
2745  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2752  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2753  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2754  FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2755  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2757  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2758  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2759  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2760  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2761 
2762  /* response ack from request */
2763  p->tcph->th_ack = htonl(328);
2764  p->tcph->th_seq = htonl(88);
2765  p->tcph->th_flags = TH_ACK;
2767  p->payload_len = 0;
2768  p->payload = NULL;
2769  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2776  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2777  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2778  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2779  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2781  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2782  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2783  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2784  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2785  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2786 
2787  /* response - acking */
2788  p->tcph->th_ack = htonl(88);
2789  p->tcph->th_seq = htonl(328);
2790  p->tcph->th_flags = TH_PUSH | TH_ACK;
2792  p->payload_len = 0;
2793  p->payload = NULL;
2794  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2801  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2802  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2803  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2804  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2806  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2807  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2808  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2809  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2810  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2811 
2812  /* response ack from request */
2813  p->tcph->th_ack = htonl(328);
2814  p->tcph->th_seq = htonl(88);
2815  p->tcph->th_flags = TH_ACK;
2817  p->payload_len = 0;
2818  p->payload = NULL;
2819  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2826  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2827  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2828  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2829  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2831  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2832  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2833  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2834  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2835 
2836  /* response - acking the request again*/
2837  p->tcph->th_ack = htonl(88);
2838  p->tcph->th_seq = htonl(328);
2839  p->tcph->th_flags = TH_PUSH | TH_ACK;
2841  p->payload_len = 0;
2842  p->payload = NULL;
2843  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2850  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2851  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2852  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2853  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2855  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2856  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2857  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2858  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2859 
2860  /*** New Request ***/
2861 
2862  /* partial request */
2863  p->tcph->th_ack = htonl(328);
2864  p->tcph->th_seq = htonl(88);
2865  p->tcph->th_flags = TH_PUSH | TH_ACK;
2867  p->payload_len = sizeof(request1);
2868  p->payload = request1;
2869  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2876  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2877  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2878  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2879  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2881  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2882  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2883  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2884  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2885  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2886 
2887  /* response ack against partial request */
2888  p->tcph->th_ack = htonl(90);
2889  p->tcph->th_seq = htonl(328);
2890  p->tcph->th_flags = TH_ACK;
2892  p->payload_len = 0;
2893  p->payload = NULL;
2894  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2895  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2902  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2903  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2904  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2905  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2907  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2908  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2909  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2910  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2911  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2912 
2913  /* complete request */
2914  p->tcph->th_ack = htonl(328);
2915  p->tcph->th_seq = htonl(90);
2916  p->tcph->th_flags = TH_PUSH | TH_ACK;
2918  p->payload_len = sizeof(request2);
2919  p->payload = request2;
2920  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2927  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2928  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2929  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2930  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2932  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2933  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2934  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2935  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2936  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2937  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2938 
2939  /* response ack against second partial request */
2940  p->tcph->th_ack = htonl(175);
2941  p->tcph->th_seq = htonl(328);
2942  p->tcph->th_flags = TH_ACK;
2944  p->payload_len = 0;
2945  p->payload = NULL;
2946 
2947  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2954  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2955  FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2956  FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2957  FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2959  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2960  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2961  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2962  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2963  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2964  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2965 
2966  /* response acking a request */
2967  p->tcph->th_ack = htonl(175);
2968  p->tcph->th_seq = htonl(328);
2969  p->tcph->th_flags = TH_ACK;
2971  p->payload_len = 0;
2972  p->payload = NULL;
2973  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2979 
2980  StreamTcpPruneSession(&f, STREAM_TOSERVER);
2981  StreamTcpPruneSession(&f, STREAM_TOCLIENT);
2982 
2983  /* request acking a response */
2984  p->tcph->th_ack = htonl(328);
2985  p->tcph->th_seq = htonl(175);
2986  p->tcph->th_flags = TH_ACK;
2988  p->payload_len = 0;
2989  p->payload = NULL;
2990  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2991 
2992  StreamTcpSessionClear(ssn);
2994  SCFree(p);
2995  PASS;
2996 }
2997 
2998 /**
2999  * \test Test to make sure that we sent all the segments from the initial
3000  * segments to app layer until we have detected the app layer proto.
3001  *
3002  * \retval On success it returns 1 and on failure 0.
3003  */
3004 
3005 static int StreamTcpReassembleTest40 (void)
3006 {
3007  Packet *p = PacketGetFromAlloc();
3008  FAIL_IF_NULL(p);
3009  Flow *f = NULL;
3010  TCPHdr tcph;
3011  TcpSession ssn;
3012  memset(&tcph, 0, sizeof (TCPHdr));
3013  ThreadVars tv;
3014  memset(&tv, 0, sizeof (ThreadVars));
3015 
3016  StreamTcpInitConfig(true);
3018 
3020  FAIL_IF_NULL(ra_ctx);
3021 
3022  uint8_t httpbuf1[] = "P";
3023  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3024  uint8_t httpbuf3[] = "O";
3025  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3026  uint8_t httpbuf4[] = "S";
3027  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3028  uint8_t httpbuf5[] = "T \r\n";
3029  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3030 
3031  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3032  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3033 
3034  SET_ISN(&ssn.server, 9);
3035  ssn.server.last_ack = 10;
3036  SET_ISN(&ssn.client, 9);
3037  ssn.client.isn = 9;
3038 
3039  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3040  FAIL_IF_NULL(f);
3041  f->protoctx = &ssn;
3042  f->proto = IPPROTO_TCP;
3043  p->flow = f;
3044 
3045  tcph.th_win = htons(5480);
3046  tcph.th_seq = htonl(10);
3047  tcph.th_ack = htonl(10);
3048  tcph.th_flags = TH_ACK|TH_PUSH;
3049  p->tcph = &tcph;
3051  p->payload = httpbuf1;
3052  p->payload_len = httplen1;
3053  ssn.state = TCP_ESTABLISHED;
3054  TcpStream *s = &ssn.client;
3055  SCLogDebug("1 -- start");
3056  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3057 
3059  p->payload = httpbuf2;
3060  p->payload_len = httplen2;
3061  tcph.th_seq = htonl(10);
3062  tcph.th_ack = htonl(11);
3063  s = &ssn.server;
3064  ssn.server.last_ack = 11;
3065  SCLogDebug("2 -- start");
3066  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3067 
3069  p->payload = httpbuf3;
3070  p->payload_len = httplen3;
3071  tcph.th_seq = htonl(11);
3072  tcph.th_ack = htonl(55);
3073  s = &ssn.client;
3074  ssn.client.last_ack = 55;
3075  SCLogDebug("3 -- start");
3076  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3077 
3079  p->payload = httpbuf2;
3080  p->payload_len = httplen2;
3081  tcph.th_seq = htonl(55);
3082  tcph.th_ack = htonl(12);
3083  s = &ssn.server;
3084  ssn.server.last_ack = 12;
3085  SCLogDebug("4 -- start");
3086  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3087 
3088  /* check is have the segment in the list and flagged or not */
3089  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3090  FAIL_IF_NULL(seg);
3091  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
3092 
3094  p->payload = httpbuf4;
3095  p->payload_len = httplen4;
3096  tcph.th_seq = htonl(12);
3097  tcph.th_ack = htonl(100);
3098  s = &ssn.client;
3099  ssn.client.last_ack = 100;
3100  SCLogDebug("5 -- start");
3101  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3102 
3104  p->payload = httpbuf2;
3105  p->payload_len = httplen2;
3106  tcph.th_seq = htonl(100);
3107  tcph.th_ack = htonl(13);
3108  s = &ssn.server;
3109  ssn.server.last_ack = 13;
3110  SCLogDebug("6 -- start");
3111  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3112 
3114  p->payload = httpbuf5;
3115  p->payload_len = httplen5;
3116  tcph.th_seq = htonl(13);
3117  tcph.th_ack = htonl(145);
3118  s = &ssn.client;
3119  ssn.client.last_ack = 145;
3120  SCLogDebug("7 -- start");
3121  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3122 
3124  p->payload = httpbuf2;
3125  p->payload_len = httplen2;
3126  tcph.th_seq = htonl(145);
3127  tcph.th_ack = htonl(16);
3128  s = &ssn.server;
3129  ssn.server.last_ack = 16;
3130  SCLogDebug("8 -- start");
3131  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3132  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3133 
3136  StreamTcpFreeConfig(true);
3137  SCFree(p);
3138  UTHFreeFlow(f);
3139  PASS;
3140 }
3141 
3142 /** \test Test the memcap incrementing/decrementing and memcap check */
3143 static int StreamTcpReassembleTest44(void)
3144 {
3145  StreamTcpInitConfig(true);
3146  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
3148  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
3150  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
3152  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
3153  StreamTcpFreeConfig(true);
3154  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
3155  PASS;
3156 }
3157 
3158 /**
3159  * \test Test to make sure that reassembly_depth is enforced.
3160  *
3161  * \retval On success it returns 1 and on failure 0.
3162  */
3163 
3164 static int StreamTcpReassembleTest45 (void)
3165 {
3166  TcpReassemblyThreadCtx *ra_ctx = NULL;
3167  TcpSession ssn;
3168  ThreadVars tv;
3169  memset(&tv, 0, sizeof(tv));
3170  uint8_t payload[100] = {0};
3171  uint16_t payload_size = 100;
3172 
3173  StreamTcpUTInit(&ra_ctx);
3175 
3177  ssn.reassembly_depth = 100;
3178  StreamTcpUTSetupStream(&ssn.server, 100);
3179  StreamTcpUTSetupStream(&ssn.client, 100);
3180 
3181  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3182  FAIL_IF(r != 0);
3184 
3185  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3186  FAIL_IF(r != 0);
3188 
3192  StreamTcpUTDeinit(ra_ctx);
3193  PASS;
3194 }
3195 
3196 /**
3197  * \test Test the unlimited config value of reassembly depth.
3198  *
3199  * \retval On success it returns 1 and on failure 0.
3200  */
3201 
3202 static int StreamTcpReassembleTest46 (void)
3203 {
3204  int result = 0;
3205  TcpReassemblyThreadCtx *ra_ctx = NULL;
3206  TcpSession ssn;
3207  ThreadVars tv;
3208  memset(&tv, 0, sizeof(tv));
3209  uint8_t payload[100] = {0};
3210  uint16_t payload_size = 100;
3211 
3212  StreamTcpUTInit(&ra_ctx);
3214 
3216  StreamTcpUTSetupStream(&ssn.server, 100);
3217  StreamTcpUTSetupStream(&ssn.client, 100);
3218 
3219  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3220  if (r != 0)
3221  goto end;
3223  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3224  goto end;
3225  }
3226 
3227  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3228  if (r != 0)
3229  goto end;
3231  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3232  goto end;
3233  }
3234 
3235  result = 1;
3236 end:
3240  StreamTcpUTDeinit(ra_ctx);
3241  return result;
3242 }
3243 
3244 /**
3245  * \test Test to make sure we detect the sequence wrap around and continue
3246  * stream reassembly properly.
3247  *
3248  * \retval On success it returns 1 and on failure 0.
3249  */
3250 
3251 static int StreamTcpReassembleTest47 (void)
3252 {
3253  Packet *p = PacketGetFromAlloc();
3254  FAIL_IF(unlikely(p == NULL));
3255  Flow *f = NULL;
3256  TCPHdr tcph;
3257  TcpSession ssn;
3258  ThreadVars tv;
3259  memset(&tcph, 0, sizeof (TCPHdr));
3260  memset(&tv, 0, sizeof (ThreadVars));
3261  StreamTcpInitConfig(true);
3264 
3265  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3266  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3267 
3268  SET_ISN(&ssn.server, 572799781UL);
3269  ssn.server.last_ack = 572799782UL;
3270 
3271  SET_ISN(&ssn.client, 4294967289UL);
3272  ssn.client.last_ack = 21;
3273 
3274  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3275  FAIL_IF(f == NULL);
3276  f->protoctx = &ssn;
3277  f->proto = IPPROTO_TCP;
3278  p->flow = f;
3279 
3280  tcph.th_win = htons(5480);
3281  ssn.state = TCP_ESTABLISHED;
3282  TcpStream *s = NULL;
3283  uint8_t cnt = 0;
3284 
3285  for (cnt=0; cnt < httplen1; cnt++) {
3286  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3287  tcph.th_ack = htonl(572799782UL);
3288  tcph.th_flags = TH_ACK|TH_PUSH;
3289  p->tcph = &tcph;
3291  p->payload = &httpbuf1[cnt];
3292  p->payload_len = 1;
3293  s = &ssn.client;
3294 
3295  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3296 
3298  p->payload = NULL;
3299  p->payload_len = 0;
3300  tcph.th_seq = htonl(572799782UL);
3301  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3302  tcph.th_flags = TH_ACK;
3303  p->tcph = &tcph;
3304  s = &ssn.server;
3305 
3306  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p) == -1);
3307  }
3308 
3309  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3310 
3313  StreamTcpFreeConfig(true);
3314  SCFree(p);
3315  UTHFreeFlow(f);
3316  PASS;
3317 }
3318 
3319 /** \test 3 in order segments in inline reassembly */
3320 static int StreamTcpReassembleInlineTest01(void)
3321 {
3322  int ret = 0;
3323  TcpReassemblyThreadCtx *ra_ctx = NULL;
3324  ThreadVars tv;
3325  TcpSession ssn;
3326  Flow f;
3327 
3328  memset(&tv, 0x00, sizeof(tv));
3329 
3330  StreamTcpUTInit(&ra_ctx);
3333  StreamTcpUTSetupStream(&ssn.client, 1);
3334  FLOW_INITIALIZE(&f);
3335 
3336  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3337  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3338  if (p == NULL) {
3339  printf("couldn't get a packet: ");
3340  goto end;
3341  }
3342  p->tcph->th_seq = htonl(12);
3343  p->flow = &f;
3344 
3345  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3346  printf("failed to add segment 1: ");
3347  goto end;
3348  }
3349  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3350  printf("failed to add segment 2: ");
3351  goto end;
3352  }
3353  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3354  printf("failed to add segment 3: ");
3355  goto end;
3356  }
3357  ssn.client.next_seq = 17;
3358  ret = 1;
3359 end:
3360  FLOW_DESTROY(&f);
3361  UTHFreePacket(p);
3363  StreamTcpUTDeinit(ra_ctx);
3364  return ret;
3365 }
3366 
3367 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3368  * test the sliding window reassembly.
3369  */
3370 static int StreamTcpReassembleInlineTest02(void)
3371 {
3372  int ret = 0;
3373  TcpReassemblyThreadCtx *ra_ctx = NULL;
3374  ThreadVars tv;
3375  TcpSession ssn;
3376  Flow f;
3377 
3378  memset(&tv, 0x00, sizeof(tv));
3379 
3380  StreamTcpUTInit(&ra_ctx);
3383  StreamTcpUTSetupStream(&ssn.client, 1);
3384  FLOW_INITIALIZE(&f);
3385 
3386  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3387  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3388  if (p == NULL) {
3389  printf("couldn't get a packet: ");
3390  goto end;
3391  }
3392  p->tcph->th_seq = htonl(12);
3393  p->flow = &f;
3394 
3395  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3396  printf("failed to add segment 1: ");
3397  goto end;
3398  }
3399  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3400  printf("failed to add segment 2: ");
3401  goto end;
3402  }
3403  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3404  printf("failed to add segment 3: ");
3405  goto end;
3406  }
3407  ssn.client.next_seq = 17;
3408  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3409  printf("failed to add segment 4: ");
3410  goto end;
3411  }
3412  ssn.client.next_seq = 22;
3413  ret = 1;
3414 end:
3415  FLOW_DESTROY(&f);
3416  UTHFreePacket(p);
3418  StreamTcpUTDeinit(ra_ctx);
3419  return ret;
3420 }
3421 
3422 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3423  * test the sliding window reassembly with a small window size so that we
3424  * cutting off at the start (left edge)
3425  */
3426 static int StreamTcpReassembleInlineTest03(void)
3427 {
3428  int ret = 0;
3429  TcpReassemblyThreadCtx *ra_ctx = NULL;
3430  ThreadVars tv;
3431  TcpSession ssn;
3432  Flow f;
3433 
3434  memset(&tv, 0x00, sizeof(tv));
3435 
3436  StreamTcpUTInit(&ra_ctx);
3439  StreamTcpUTSetupStream(&ssn.client, 1);
3440  FLOW_INITIALIZE(&f);
3441 
3443 
3444  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3445  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3446  if (p == NULL) {
3447  printf("couldn't get a packet: ");
3448  goto end;
3449  }
3450  p->tcph->th_seq = htonl(12);
3451  p->flow = &f;
3453 
3454  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3455  printf("failed to add segment 1: ");
3456  goto end;
3457  }
3458  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3459  printf("failed to add segment 2: ");
3460  goto end;
3461  }
3462  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3463  printf("failed to add segment 3: ");
3464  goto end;
3465  }
3466  ssn.client.next_seq = 17;
3467  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3468  printf("failed to add segment 4: ");
3469  goto end;
3470  }
3471  ssn.client.next_seq = 22;
3472 
3473  p->tcph->th_seq = htonl(17);
3474  ret = 1;
3475 end:
3476  FLOW_DESTROY(&f);
3477  UTHFreePacket(p);
3479  StreamTcpUTDeinit(ra_ctx);
3480  return ret;
3481 }
3482 
3483 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3484  * test the sliding window reassembly with a small window size so that we
3485  * cutting off at the start (left edge) with small packet overlap.
3486  */
3487 static int StreamTcpReassembleInlineTest04(void)
3488 {
3489  int ret = 0;
3490  TcpReassemblyThreadCtx *ra_ctx = NULL;
3491  ThreadVars tv;
3492  TcpSession ssn;
3493  Flow f;
3494 
3495  memset(&tv, 0x00, sizeof(tv));
3496 
3497  StreamTcpUTInit(&ra_ctx);
3500  StreamTcpUTSetupStream(&ssn.client, 1);
3501  FLOW_INITIALIZE(&f);
3502 
3504 
3505  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3506  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3507  if (p == NULL) {
3508  printf("couldn't get a packet: ");
3509  goto end;
3510  }
3511  p->tcph->th_seq = htonl(12);
3512  p->flow = &f;
3514 
3515  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3516  printf("failed to add segment 1: ");
3517  goto end;
3518  }
3519  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3520  printf("failed to add segment 2: ");
3521  goto end;
3522  }
3523  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3524  printf("failed to add segment 3: ");
3525  goto end;
3526  }
3527  ssn.client.next_seq = 17;
3528  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3529  printf("failed to add segment 4: ");
3530  goto end;
3531  }
3532  ssn.client.next_seq = 22;
3533 
3534  p->tcph->th_seq = htonl(17);
3535  ret = 1;
3536 end:
3537  FLOW_DESTROY(&f);
3538  UTHFreePacket(p);
3540  StreamTcpUTDeinit(ra_ctx);
3541  return ret;
3542 }
3543 
3544 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3545  * test the sliding window reassembly with a small window size so that we
3546  * cutting off at the start (left edge). Test if the first segment is
3547  * removed from the list.
3548  */
3549 static int StreamTcpReassembleInlineTest08(void)
3550 {
3551  TcpReassemblyThreadCtx *ra_ctx = NULL;
3552  ThreadVars tv;
3553  memset(&tv, 0x00, sizeof(tv));
3554  TcpSession ssn;
3555  Flow f;
3556  StreamTcpUTInit(&ra_ctx);
3559  StreamTcpUTSetupStream(&ssn.client, 1);
3560  FLOW_INITIALIZE(&f);
3561 
3563  f.protoctx = &ssn;
3564 
3565  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3566  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3567  FAIL_IF(p == NULL);
3568  p->tcph->th_seq = htonl(12);
3569  p->flow = &f;
3571 
3572  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3573  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3574  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3575  ssn.client.next_seq = 17;
3576  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3577  ssn.client.next_seq = 22;
3578  p->tcph->th_seq = htonl(17);
3579  StreamTcpPruneSession(&f, STREAM_TOSERVER);
3580 
3581  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3582  FAIL_IF_NULL(seg);
3583  FAIL_IF_NOT(seg->seq == 2);
3584 
3585  FLOW_DESTROY(&f);
3586  UTHFreePacket(p);
3588  StreamTcpUTDeinit(ra_ctx);
3589  PASS;
3590 }
3591 
3592 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3593  * test the sliding window reassembly with a small window size so that we
3594  * cutting off at the start (left edge). Test if the first segment is
3595  * removed from the list.
3596  */
3597 static int StreamTcpReassembleInlineTest09(void)
3598 {
3599  int ret = 0;
3600  TcpReassemblyThreadCtx *ra_ctx = NULL;
3601  ThreadVars tv;
3602  TcpSession ssn;
3603  Flow f;
3604 
3605  memset(&tv, 0x00, sizeof(tv));
3606 
3607  StreamTcpUTInit(&ra_ctx);
3610  StreamTcpUTSetupStream(&ssn.client, 1);
3611  FLOW_INITIALIZE(&f);
3612 
3614 
3615  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3616  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3617  if (p == NULL) {
3618  printf("couldn't get a packet: ");
3619  goto end;
3620  }
3621  p->tcph->th_seq = htonl(17);
3622  p->flow = &f;
3624 
3625  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3626  printf("failed to add segment 1: ");
3627  goto end;
3628  }
3629  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3630  printf("failed to add segment 2: ");
3631  goto end;
3632  }
3633  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3634  printf("failed to add segment 3: ");
3635  goto end;
3636  }
3637  ssn.client.next_seq = 12;
3638  ssn.client.last_ack = 10;
3639 
3640  /* close the GAP and see if we properly reassemble and update base_seq */
3641  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3642  printf("failed to add segment 4: ");
3643  goto end;
3644  }
3645  ssn.client.next_seq = 22;
3646 
3647  p->tcph->th_seq = htonl(12);
3648 
3649  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3650  FAIL_IF_NULL(seg);
3651  FAIL_IF_NOT(seg->seq == 2);
3652 
3653  ret = 1;
3654 end:
3655  FLOW_DESTROY(&f);
3656  UTHFreePacket(p);
3658  StreamTcpUTDeinit(ra_ctx);
3659  return ret;
3660 }
3661 
3662 /** \test App Layer reassembly.
3663  */
3664 static int StreamTcpReassembleInlineTest10(void)
3665 {
3666  int ret = 0;
3667  TcpReassemblyThreadCtx *ra_ctx = NULL;
3668  ThreadVars tv;
3669  TcpSession ssn;
3670  Flow *f = NULL;
3671  Packet *p = NULL;
3672 
3673  memset(&tv, 0x00, sizeof(tv));
3674 
3675  StreamTcpUTInit(&ra_ctx);
3678  StreamTcpUTSetupStream(&ssn.server, 1);
3679  ssn.server.last_ack = 2;
3680  StreamTcpUTSetupStream(&ssn.client, 1);
3681  ssn.client.last_ack = 2;
3682  ssn.data_first_seen_dir = STREAM_TOSERVER;
3683 
3684  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3685  if (f == NULL)
3686  goto end;
3687  f->protoctx = &ssn;
3688  f->proto = IPPROTO_TCP;
3689 
3690  uint8_t stream_payload1[] = "GE";
3691  uint8_t stream_payload2[] = "T /";
3692  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3693 
3694  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3695  if (p == NULL) {
3696  printf("couldn't get a packet: ");
3697  goto end;
3698  }
3699  p->tcph->th_seq = htonl(7);
3700  p->flow = f;
3702 
3703  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3704  printf("failed to add segment 1: ");
3705  goto end;
3706  }
3707  ssn.client.next_seq = 4;
3708 
3709  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3710  if (r < 0) {
3711  printf("StreamTcpReassembleAppLayer failed: ");
3712  goto end;
3713  }
3714 
3715  /* ssn.server.ra_app_base_seq should be isn here. */
3716  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3717  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3718  goto end;
3719  }
3720 
3721  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3722  printf("failed to add segment 2: ");
3723  goto end;
3724  }
3725  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3726  printf("failed to add segment 3: ");
3727  goto end;
3728  }
3729  ssn.client.next_seq = 19;
3730 
3731  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3732  if (r < 0) {
3733  printf("StreamTcpReassembleAppLayer failed: ");
3734  goto end;
3735  }
3736 
3737  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3738 
3739  ret = 1;
3740 end:
3741  UTHFreePacket(p);
3743  StreamTcpUTDeinit(ra_ctx);
3744  UTHFreeFlow(f);
3745  return ret;
3746 }
3747 
3748 /** \test test insert with overlap
3749  */
3750 static int StreamTcpReassembleInsertTest01(void)
3751 {
3752  TcpReassemblyThreadCtx *ra_ctx = NULL;
3753  ThreadVars tv;
3754  TcpSession ssn;
3755  Flow f;
3756 
3757  memset(&tv, 0x00, sizeof(tv));
3758 
3759  StreamTcpUTInit(&ra_ctx);
3761  StreamTcpUTSetupStream(&ssn.client, 1);
3763  FLOW_INITIALIZE(&f);
3764 
3765  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3766  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3767  FAIL_IF(p == NULL);
3768  p->tcph->th_seq = htonl(12);
3769  p->flow = &f;
3770 
3771  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3772  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3773  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3774  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3775  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3776  ssn.client.next_seq = 21;
3777 
3778  FLOW_DESTROY(&f);
3779  UTHFreePacket(p);
3781  StreamTcpUTDeinit(ra_ctx);
3782  PASS;
3783 }
3784 
3785 /** \test test insert with overlaps
3786  */
3787 static int StreamTcpReassembleInsertTest02(void)
3788 {
3789  int ret = 0;
3790  TcpReassemblyThreadCtx *ra_ctx = NULL;
3791  ThreadVars tv;
3792  TcpSession ssn;
3793 
3794  memset(&tv, 0x00, sizeof(tv));
3795 
3796  StreamTcpUTInit(&ra_ctx);
3798  StreamTcpUTSetupStream(&ssn.client, 1);
3799 
3800  int i;
3801  for (i = 2; i < 10; i++) {
3802  int len;
3803  len = i % 2;
3804  if (len == 0)
3805  len = 1;
3806  int seq;
3807  seq = i * 10;
3808  if (seq < 2)
3809  seq = 2;
3810 
3811  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3812  printf("failed to add segment 1: ");
3813  goto end;
3814  }
3815  }
3816  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3817  printf("failed to add segment 2: ");
3818  goto end;
3819  }
3820 
3821  ret = 1;
3822 end:
3824  StreamTcpUTDeinit(ra_ctx);
3825  return ret;
3826 }
3827 
3828 /** \test test insert with overlaps
3829  */
3830 static int StreamTcpReassembleInsertTest03(void)
3831 {
3832  int ret = 0;
3833  TcpReassemblyThreadCtx *ra_ctx = NULL;
3834  ThreadVars tv;
3835  TcpSession ssn;
3836 
3837  memset(&tv, 0x00, sizeof(tv));
3838 
3839  StreamTcpUTInit(&ra_ctx);
3841  StreamTcpUTSetupStream(&ssn.client, 1);
3842 
3843  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3844  printf("failed to add segment 2: ");
3845  goto end;
3846  }
3847 
3848  int i;
3849  for (i = 2; i < 10; i++) {
3850  int len;
3851  len = i % 2;
3852  if (len == 0)
3853  len = 1;
3854  int seq;
3855  seq = i * 10;
3856  if (seq < 2)
3857  seq = 2;
3858 
3859  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3860  printf("failed to add segment 2: ");
3861  goto end;
3862  }
3863  }
3864  ret = 1;
3865 end:
3867  StreamTcpUTDeinit(ra_ctx);
3868  return ret;
3869 }
3870 
3872 #endif /* UNITTESTS */
3873 
3874 /** \brief The Function Register the Unit tests to test the reassembly engine
3875  * for various OS policies.
3876  */
3877 
3879 {
3880 #ifdef UNITTESTS
3881  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3882  StreamTcpReassembleTest25);
3883  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3884  StreamTcpReassembleTest26);
3885  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3886  StreamTcpReassembleTest27);
3887  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3888  StreamTcpReassembleTest28);
3889  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3890  StreamTcpReassembleTest29);
3891  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3892  StreamTcpReassembleTest33);
3893  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3894  StreamTcpReassembleTest34);
3895  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3896  StreamTcpReassembleTest39);
3897  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3898  StreamTcpReassembleTest40);
3899  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3900  StreamTcpReassembleTest44);
3901  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3902  StreamTcpReassembleTest45);
3903  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3904  StreamTcpReassembleTest46);
3905  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3906  StreamTcpReassembleTest47);
3907 
3908  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3909  StreamTcpReassembleInlineTest01);
3910  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3911  StreamTcpReassembleInlineTest02);
3912  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3913  StreamTcpReassembleInlineTest03);
3914  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3915  StreamTcpReassembleInlineTest04);
3916  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3917  StreamTcpReassembleInlineTest08);
3918  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3919  StreamTcpReassembleInlineTest09);
3920 
3921  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3922  StreamTcpReassembleInlineTest10);
3923 
3924  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3925  StreamTcpReassembleInsertTest01);
3926  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3927  StreamTcpReassembleInsertTest02);
3928  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3929  StreamTcpReassembleInsertTest03);
3930 
3934  StreamTcpReassembleRawRegisterTests();
3935 #endif /* UNITTESTS */
3936 }
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
Definition: stream-tcp.h:139
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:269
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:56
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:54
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:450
UTHCheckGapAtPostion
int UTHCheckGapAtPostion(TcpStream *stream, int pos, uint64_t offset, uint32_t len)
Definition: stream-tcp-reassemble.c:2239
TcpStream_
Definition: stream-tcp-private.h:106
len
uint8_t len
Definition: app-layer-dnp3.h:2
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:66
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:186
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:113
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:2085
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:167
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:373
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:97
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:67
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1874
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:97
IsTcpSessionDumpingEnabled
bool IsTcpSessionDumpingEnabled(void)
Definition: stream-tcp-reassemble.c:89
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:94
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:70
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:112
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:482
stream-tcp.h
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
PKT_DROP_REASON_STREAM_MEMCAP
@ PKT_DROP_REASON_STREAM_MEMCAP
Definition: decode.h:402
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:615
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:633
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:135
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:2158
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:594
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1013
StreamTcpThread_
Definition: stream-tcp.h:79
Flow_::proto
uint8_t proto
Definition: flow.h:379
Packet_::payload
uint8_t * payload
Definition: decode.h:573
StreamTcpInlineMode
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6850
t_pcapcnt
thread_local uint64_t t_pcapcnt
Definition: stream-tcp-reassemble.c:78
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:2138
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:60
Packet_::flags
uint32_t flags
Definition: decode.h:463
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:181
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:110
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:64
Flow_
Flow data structure.
Definition: flow.h:357
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:77
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:71
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:2121
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
TcpSegmentPcapHdrStorage
struct TcpSegmentPcapHdrStorage_ TcpSegmentPcapHdrStorage
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
StringParseUint16
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:337
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:155
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:40
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:76
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:36
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1508
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1594
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:227
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:154
MIN
#define MIN(x, y)
Definition: suricata-common.h:380
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:75
StreamTcpThreadCacheGetSegment
TcpSegment * StreamTcpThreadCacheGetSegment(void)
Definition: stream-tcp-cache.c:179
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:107
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:155
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1611
PKT_APPLAYER_UPDATE
#define PKT_APPLAYER_UPDATE
Definition: decode.h:1056
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:459
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:115
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:39
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:244
Flow_::protoctx
void * protoctx
Definition: flow.h:447
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:710
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:97
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:574
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:222
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:68
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:600
util-unittest.h
TcpSegmentPcapHdrStorage_::pktlen
uint32_t pktlen
Definition: stream-tcp-private.h:67
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2230
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
STREAM_REASSEMBLY_DEPTH_REACHED
@ STREAM_REASSEMBLY_DEPTH_REACHED
Definition: decode-events.h:294
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:521
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:424
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:169
STREAMTCP_FLAG_LOSSY_BE_LIBERAL
#define STREAMTCP_FLAG_LOSSY_BE_LIBERAL
Definition: stream-tcp-private.h:193
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:291
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:277
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:115
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:481
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:554
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:359
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:521
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1349
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:40
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:292
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:461
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:644
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:183
StreamTcpReassembleRealloc
void * StreamTcpReassembleRealloc(void *optr, size_t orig_size, size_t size)
Definition: stream-tcp-reassemble.c:223
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:251
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:104
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:278
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:998
StreamTcpGetAcked
uint64_t StreamTcpGetAcked(const TcpStream *stream)
Definition: stream-tcp-reassemble.c:409
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1005
util-exception-policy.h
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:143
EnableTcpSessionDumping
void EnableTcpSessionDumping(void)
Definition: stream-tcp-reassemble.c:94
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:130
StreamReassembleLog
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
Definition: stream-tcp-reassemble.c:1939
util-print.h
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:50
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:993
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:5306
StreamNeedsReassembly
uint8_t StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:911
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1462
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:284
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:136
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:103
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2172
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:888
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:143
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:162
TRUE
#define TRUE
Definition: suricata-common.h:33
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:186
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
TCP_GET_SEQ
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:114
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:113
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
FALSE
#define FALSE
Definition: suricata-common.h:34
SCReturn
#define SCReturn
Definition: util-debug.h:273
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:64
stream.h
TcpSegment
Definition: stream-tcp-private.h:72
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:300
Packet_
Definition: decode.h:428
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:233
stream-tcp-private.h
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:40
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:277
conf.h
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:298
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1633
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:117
STREAMTCP_STREAM_FLAG_HAS_GAP
#define STREAMTCP_STREAM_FLAG_HAS_GAP
Definition: stream-tcp-private.h:216
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:595
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:287
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:712
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:88
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:390
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:228
TcpStreamCnf_::reassembly_memcap_policy
enum ExceptionPolicy reassembly_memcap_policy
Definition: stream-tcp.h:70
StreamTcpThreadCacheCleanup
void StreamTcpThreadCacheCleanup(void)
Definition: stream-tcp-cache.c:134
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:232
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:537
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:73
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:127
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:81
VALIDATE
#define VALIDATE(e)
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:526
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:55
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:111
StreamingBuffer_
Definition: util-streaming-buffer.h:109
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:736
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:62
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:161
TcpSegmentPcapHdrStorage_
Definition: stream-tcp-private.h:65
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint16_t flag)
Definition: app-layer-parser.c:1814
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1502
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
app-layer-frames.h
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:179
Packet_::flow
struct Flow_ * flow
Definition: decode.h:465
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:259
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:695
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:50
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:445
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:258
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:553
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:124
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:30
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:91
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2211
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:134
TcpSegmentPcapHdrStorage_::alloclen
uint32_t alloclen
Definition: stream-tcp-private.h:68
StreamTcpThreadCacheReturnSegment
void StreamTcpThreadCacheReturnSegment(TcpSegment *seg)
Definition: stream-tcp-cache.c:53
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:292
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:235
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:294
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Definition: stream-tcp-reassemble.c:1971
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:160
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:67
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:114
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:173
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:293
StreamTcpReassembleIncrMemuse
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:114
TcpSegmentPcapHdrStorage_::pkt_hdr
uint8_t * pkt_hdr
Definition: stream-tcp-private.h:69
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
TcpSegment::pcap_hdr_storage
TcpSegmentPcapHdrStorage * pcap_hdr_storage
Definition: stream-tcp-private.h:78
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:457
ConfNode_
Definition: conf.h:32
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:485
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:1015
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:256
Flow_::flags
uint32_t flags
Definition: flow.h:427
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:815
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:72
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:53
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:148
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:304
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2225
stream-tcp-cache.h
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:157
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:112
MISSED_ADD_PAYLOAD
#define MISSED_ADD_PAYLOAD(seq, seg, seglen)
Definition: stream-tcp-reassemble.c:2234
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:59
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:58
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:542
StreamDataRightEdge
uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:415
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:173
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:200
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:218
suricata.h
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap(void)
Return memcap value.
Definition: stream-tcp-reassemble.c:198
Address_::family
char family
Definition: decode.h:115
Packet_::dst
Address dst
Definition: decode.h:433
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:6865
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:257
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:53
TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool
uint16_t counter_tcp_segment_from_pool
Definition: stream-tcp-reassemble.h:68
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:237
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3878
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:102
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:144
segment_thread_pool
PoolThread * segment_thread_pool
Definition: stream-tcp-reassemble.c:80
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:230
StreamTcpReassembleDecrMemuse
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:127
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
TCPSEG_PKT_HDR_DEFAULT_SIZE
#define TCPSEG_PKT_HDR_DEFAULT_SIZE
Definition: stream-tcp-private.h:58
TcpSession_
Definition: stream-tcp-private.h:282
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:287
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:183
flow.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:237
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:458
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:110
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:456
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:109
UTHCheckDataAtPostion
int UTHCheckDataAtPostion(TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len)
Definition: stream-tcp-reassemble.c:2262
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:224
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:69
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:148
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:291
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:99
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:97
StreamReassembleForFrame
int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof)
Definition: stream-tcp-reassemble.c:1862
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:65
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1468
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:66
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:96
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:129
Packet_::src
Address src
Definition: decode.h:432
TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache
uint16_t counter_tcp_segment_from_cache
Definition: stream-tcp-reassemble.h:67
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:71
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:89
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:231
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:145
app-layer.h
StreamTcpCheckStreamContents
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
Definition: stream-tcp-reassemble.c:2189