suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 #include "app-layer-parser.h"
64 
65 #include "detect-engine-state.h"
66 
67 #include "util-profiling.h"
68 #include "util-validate.h"
69 
70 #ifdef DEBUG
71 static SCMutex segment_pool_memuse_mutex;
72 static uint64_t segment_pool_memuse = 0;
73 static uint64_t segment_pool_memcnt = 0;
74 #endif
75 
76 static PoolThread *segment_thread_pool = NULL;
77 /* init only, protect initializing and growing pool */
78 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
79 
80 /* Memory use counter */
81 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
82 
83 /* prototypes */
85 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346  if (f->alparser) {
349  }
350 }
351 
352 /** \param f locked flow */
354 {
355  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
356  return 0;
357 
358  TcpSession *ssn = (TcpSession *)f->protoctx;
360 }
361 
362 static int StreamTcpReassemblyConfig(char quiet)
363 {
364  uint32_t segment_prealloc = 2048;
365  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
366  if (seg) {
367  uint32_t prealloc = 0;
368  if (StringParseUint32(&prealloc, 10, strlen(seg->val), seg->val) < 0)
369  {
370  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
371  "%s is invalid", seg->val);
372  return -1;
373  }
374  segment_prealloc = prealloc;
375  }
376  if (!quiet)
377  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
378  stream_config.prealloc_segments = segment_prealloc;
379 
380  int overlap_diff_data = 0;
381  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
382  if (overlap_diff_data) {
384  }
385  if (StreamTcpInlineMode() == TRUE) {
387  }
388 
391  stream_config.sbcnf.Malloc = ReassembleMalloc;
392  stream_config.sbcnf.Calloc = ReassembleCalloc;
393  stream_config.sbcnf.Realloc = ReassembleRealloc;
394  stream_config.sbcnf.Free = ReassembleFree;
395 
396  return 0;
397 }
398 
399 int StreamTcpReassembleInit(char quiet)
400 {
401  /* init the memcap/use tracker */
403 
404  if (StreamTcpReassemblyConfig(quiet) < 0)
405  return -1;
406 
407 #ifdef DEBUG
408  SCMutexInit(&segment_pool_memuse_mutex, NULL);
409 #endif
410  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
412  return 0;
413 }
414 
415 void StreamTcpReassembleFree(char quiet)
416 {
417  SCMutexLock(&segment_thread_pool_mutex);
418  if (segment_thread_pool != NULL) {
419  PoolThreadFree(segment_thread_pool);
420  segment_thread_pool = NULL;
421  }
422  SCMutexUnlock(&segment_thread_pool_mutex);
423  SCMutexDestroy(&segment_thread_pool_mutex);
424 
425 #ifdef DEBUG
426  if (segment_pool_memuse > 0)
427  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
428  if (segment_pool_memcnt > 0)
429  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
430  SCMutexDestroy(&segment_pool_memuse_mutex);
431 #endif
432 }
433 
435 {
436  SCEnter();
438  if (unlikely(ra_ctx == NULL))
439  return NULL;
440 
441  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
442 
443  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
444 
445  SCMutexLock(&segment_thread_pool_mutex);
446  if (segment_thread_pool == NULL) {
447  segment_thread_pool = PoolThreadInit(1, /* thread */
448  0, /* unlimited */
450  sizeof(TcpSegment),
451  TcpSegmentPoolAlloc,
452  TcpSegmentPoolInit, NULL,
453  TcpSegmentPoolCleanup, NULL);
454  ra_ctx->segment_thread_pool_id = 0;
455  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
456  PoolThreadSize(segment_thread_pool),
457  ra_ctx->segment_thread_pool_id);
458  } else {
459  /* grow segment_thread_pool until we have a element for our thread id */
460  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
461  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
462  PoolThreadSize(segment_thread_pool),
463  ra_ctx->segment_thread_pool_id);
464  }
465  SCMutexUnlock(&segment_thread_pool_mutex);
466  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
467  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
469  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
470  }
471 
472  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
473 }
474 
476 {
477  SCEnter();
479  SCFree(ra_ctx);
480  SCReturn;
481 }
482 
483 /**
484  * \brief check if stream in pkt direction has depth reached
485  *
486  * \param p packet with *LOCKED* flow
487  *
488  * \retval 1 stream has depth reached
489  * \retval 0 stream does not have depth reached
490  */
492 {
493  if (p->flow != NULL && p->flow->protoctx != NULL) {
494  TcpSession *ssn = p->flow->protoctx;
495  TcpStream *stream;
496  if (p->flowflags & FLOW_PKT_TOSERVER) {
497  stream = &ssn->client;
498  } else {
499  stream = &ssn->server;
500  }
501 
502  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
503  }
504 
505  return 0;
506 }
507 
508 /**
509  * \internal
510  * \brief Function to Check the reassembly depth valuer against the
511  * allowed max depth of the stream reassembly for TCP streams.
512  *
513  * \param stream stream direction
514  * \param seq sequence number where "size" starts
515  * \param size size of the segment that is added
516  *
517  * \retval size Part of the size that fits in the depth, 0 if none
518  */
519 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
520  uint32_t seq, uint32_t size)
521 {
522  SCEnter();
523 
524  /* if the configured depth value is 0, it means there is no limit on
525  reassembly depth. Otherwise carry on my boy ;) */
526  if (ssn->reassembly_depth == 0) {
527  SCReturnUInt(size);
528  }
529 
530  /* if the final flag is set, we're not accepting anymore */
532  SCReturnUInt(0);
533  }
534 
535  uint64_t seg_depth;
536  if (SEQ_GT(stream->base_seq, seq)) {
537  if (SEQ_LEQ(seq+size, stream->base_seq)) {
538  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
539  stream->base_seq, seq, seq+size);
540  SCReturnUInt(0);
541  }
542 
543  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
544  } else {
545  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
546  }
547 
548  /* if the base_seq has moved passed the depth window we stop
549  * checking and just reject the rest of the packets including
550  * retransmissions. Saves us the hassle of dealing with sequence
551  * wraps as well */
552  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
553  stream->base_seq, seg_depth,
554  ssn->reassembly_depth);
555 
556  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
557  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
559  SCReturnUInt(0);
560  }
561  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
562  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
563 #if 0
564  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
565  (stream->base_seq_offset + stream->base_seq + size),
566  (stream->isn + ssn->reassembly_depth));
567 #endif
568  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
569  /* packet (partly?) fits the depth window */
570 
571  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
572  /* complete fit */
573  SCReturnUInt(size);
574  } else {
576  /* partial fit, return only what fits */
577  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
578  DEBUG_VALIDATE_BUG_ON(part > size);
579  if (part > size)
580  part = size;
581  SCReturnUInt(part);
582  }
583  }
584 
585  SCReturnUInt(0);
586 }
587 
588 /**
589  * \brief Insert a packets TCP data into the stream reassembly engine.
590  *
591  * \retval 0 good segment, as far as we checked.
592  * \retval -1 badness, reason to drop in inline mode
593  *
594  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
595  * or it wasn't added because of reassembly depth.
596  *
597  */
599  TcpSession *ssn, TcpStream *stream, Packet *p)
600 {
601  SCEnter();
602 
603  if (ssn->data_first_seen_dir == 0) {
604  if (PKT_IS_TOSERVER(p)) {
606  } else {
608  }
609  }
610 
611  /* If the OS policy is not set then set the OS policy for this stream */
612  if (stream->os_policy == 0) {
613  StreamTcpSetOSPolicy(stream, p);
614  }
615 
618  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
619  SCReturnInt(0);
620  }
621 
622  /* If we have reached the defined depth for either of the stream, then stop
623  reassembling the TCP session */
624  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
625  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
626 
628  /* increment stream depth counter */
630  }
631  if (size == 0) {
632  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
633  SCReturnInt(0);
634  }
635 
637  if (size > p->payload_len)
638  size = p->payload_len;
639 
640  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
641  if (seg == NULL) {
642  SCLogDebug("segment_pool is empty");
644  SCReturnInt(-1);
645  }
646 
647  TCP_SEG_LEN(seg) = size;
648  seg->seq = TCP_GET_SEQ(p);
649 
650  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
651  if (TCP_HAS_TFO(p) && p->payload_len && p->tcph->th_flags == TH_SYN)
652  seg->seq += 1;
653 
654  /* proto detection skipped, but now we do get data. Set event. */
655  if (RB_EMPTY(&stream->seg_tree) &&
657 
660  }
661 
662  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
663  SCLogDebug("StreamTcpReassembleInsertSegment failed");
664  SCReturnInt(-1);
665  }
666  SCReturnInt(0);
667 }
668 
669 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
670  Packet *p)
671 {
672  uint8_t flag = 0;
673 
675  flag |= STREAM_START;
676  }
677 
678  if (ssn->state == TCP_CLOSED) {
679  flag |= STREAM_EOF;
680  }
681 
682  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
683  flag |= STREAM_MIDSTREAM;
684  }
685 
686  if (p->flags & PKT_PSEUDO_STREAM_END) {
687  flag |= STREAM_EOF;
688  }
689 
690  if (&ssn->client == stream) {
691  flag |= STREAM_TOSERVER;
692  } else {
693  flag |= STREAM_TOCLIENT;
694  }
696  flag |= STREAM_DEPTH;
697  }
698  return flag;
699 }
700 
701 /**
702  * \brief Check the minimum size limits for reassembly.
703  *
704  * \retval 0 don't reassemble yet
705  * \retval 1 do reassemble
706  */
707 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
708  const TcpStream *stream, const Packet *p)
709 {
710  SCEnter();
711 
712  /* if any of these flags is set we always inspect immediately */
713 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
714  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
715  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
716  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
717 
720  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
721  "is set, so not expecting any new data segments");
722  }
724  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
725  }
727  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
728  "so no new segments will be considered");
729  }
730  SCReturnInt(1);
731  }
732 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
733 
734  /* some states mean we reassemble no matter how much data we have */
735  if (ssn->state > TCP_TIME_WAIT)
736  SCReturnInt(1);
737 
738  if (p->flags & PKT_PSEUDO_STREAM_END)
739  SCReturnInt(1);
740 
741  /* check if we have enough data to do raw reassembly */
742  if (PKT_IS_TOSERVER(p)) {
743  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
744  uint32_t delta = stream->last_ack - stream->base_seq;
745  /* get max absolute offset */
746  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
747 
748  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
749  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
750  SCReturnInt(1);
751  } else {
752  SCLogDebug("toserver min chunk len not yet reached: "
753  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
754  "%"PRIu32"", stream->last_ack, stream->base_seq,
755  (stream->last_ack - stream->base_seq),
757  SCReturnInt(0);
758  }
759  }
760  } else {
761  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
762  uint32_t delta = stream->last_ack - stream->base_seq;
763  /* get max absolute offset */
764  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
765 
766  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
767 
768  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
769  SCReturnInt(1);
770  } else {
771  SCLogDebug("toclient min chunk len not yet reached: "
772  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
773  "%"PRIu32"", stream->last_ack, stream->base_seq,
774  (stream->last_ack - stream->base_seq),
776  SCReturnInt(0);
777  }
778  }
779  }
780 
781  SCReturnInt(0);
782 }
783 
784 /**
785  * \brief see what if any work the TCP session still needs
786  */
787 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
788 {
789  const TcpStream *stream = NULL;
790 #ifdef DEBUG
791  const char *dirstr = NULL;
792 #endif
793  if (direction == STREAM_TOSERVER) {
794  stream = &ssn->client;
795 #ifdef DEBUG
796  dirstr = "client";
797 #endif
798  } else {
799  stream = &ssn->server;
800 #ifdef DEBUG
801  dirstr = "server";
802 #endif
803  }
804 
805  int use_app = 1;
806  int use_raw = 1;
807 
809  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
810  {
811  // app is dead
812  use_app = 0;
813  }
814 
816  // raw is dead
817  use_raw = 0;
818  }
819 
820  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
821 
822  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
823  dirstr,
824  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
825  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
826  right_edge);
827  if (use_raw) {
828  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
829  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
831  }
832  }
833  if (use_app) {
834  if (right_edge > STREAM_APP_PROGRESS(stream)) {
835  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
837  }
838  }
839 
840  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
842 }
843 
844 #ifdef DEBUG
845 static uint64_t GetStreamSize(TcpStream *stream)
846 {
847  if (stream) {
848  uint64_t size = 0;
849  uint32_t cnt = 0;
850 
851  TcpSegment *seg;
852  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
853  cnt++;
854  size += (uint64_t)TCP_SEG_LEN(seg);
855  }
856 
857  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
858  return size;
859  }
860  return (uint64_t)0;
861 }
862 
863 static void GetSessionSize(TcpSession *ssn, Packet *p)
864 {
865  uint64_t size = 0;
866  if (ssn) {
867  size = GetStreamSize(&ssn->client);
868  size += GetStreamSize(&ssn->server);
869 
870  //if (size > 900000)
871  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
872  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
873  }
874 }
875 #endif
876 
877 static StreamingBufferBlock *GetBlock(StreamingBuffer *sb, const uint64_t offset)
878 {
879  StreamingBufferBlock *blk = sb->head;
880  if (blk == NULL)
881  return NULL;
882 
883  for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
884  if (blk->offset >= offset)
885  return blk;
886  else if ((blk->offset + blk->len) > offset) {
887  return blk;
888  }
889  }
890  return NULL;
891 }
892 
893 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
894 {
895  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
896  return STREAM_BASE_OFFSET(stream) +
897  (stream->last_ack - stream->base_seq);
898  } else {
899  return STREAM_BASE_OFFSET(stream);
900  }
901 }
902 
903 static inline bool GapAhead(TcpStream *stream, StreamingBufferBlock *cur_blk)
904 {
905  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
906  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
907  GetAbsLastAck(stream) >= (cur_blk->offset + cur_blk->len)) {
908  return true;
909  }
910  return false;
911 }
912 
913 /** \internal
914  *
915  * Get buffer, or first part of the buffer if data gaps exist.
916  *
917  * \brief get stream data from offset
918  * \param offset stream offset
919  * \param check_for_gap check if there is a gap ahead. Optional as it is only
920  * needed for app-layer incomplete support.
921  * \retval bool pkt loss ahead */
922 static bool GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
923  uint64_t offset, const bool check_for_gap)
924 {
925  const uint8_t *mydata;
926  uint32_t mydata_len;
927  bool gap_ahead = false;
928 
929  if (RB_EMPTY(&stream->sb.sbb_tree)) {
930  SCLogDebug("getting one blob");
931 
932  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
933 
934  *data = mydata;
935  *data_len = mydata_len;
936  } else {
937  StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
938  if (blk == NULL) {
939  *data = NULL;
940  *data_len = 0;
941  return false;
942  }
943 
944  /* block at expected offset */
945  if (blk->offset == offset) {
946 
947  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
948 
949  gap_ahead = check_for_gap && GapAhead(stream, blk);
950 
951  /* block past out offset */
952  } else if (blk->offset > offset) {
953  SCLogDebug("gap, want data at offset %"PRIu64", "
954  "got data at %"PRIu64". GAP of size %"PRIu64,
955  offset, blk->offset, blk->offset - offset);
956  *data = NULL;
957  *data_len = blk->offset - offset;
958 
959  /* block starts before offset, but ends after */
960  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
961  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
962  offset, blk->offset, blk->len);
963  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
964  SCLogDebug("data %p, data_len %u", *data, *data_len);
965 
966  gap_ahead = check_for_gap && GapAhead(stream, blk);
967 
968  } else {
969  *data = NULL;
970  *data_len = 0;
971  }
972  }
973  return gap_ahead;
974 }
975 
976 /** \internal
977  * \brief check to see if we should declare a GAP
978  * Call this when the app layer didn't get data at the requested
979  * offset.
980  */
981 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
982 {
983  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
984  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
985 
986  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
987  /* get window of data that is acked */
988  const uint32_t delta = stream->last_ack - stream->base_seq;
989  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
990  /* get max absolute offset */
991  last_ack_abs += delta;
992 
993  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
994  last_ack_abs -= ackadded;
995 
996  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
997  SCLogDebug("next_seq %u", stream->next_seq);
998 
999  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1000  * has been ack'd. This looks like a GAP. */
1001  if (last_ack_abs > app_progress) {
1002  /* however, we can accept ACKs a bit too liberally. If last_ack
1003  * is beyond next_seq, we only consider it a gap now if we do
1004  * already have data beyond the gap. */
1005  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1006  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1007  SCLogDebug("packet %"PRIu64": no GAP. "
1008  "next_seq %u < last_ack %u, but no data in list",
1009  p->pcap_cnt, stream->next_seq, stream->last_ack);
1010  return false;
1011  } else {
1012  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1013  const StreamingBufferBlock *blk = stream->sb.head;
1014  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1015  /* ack'd data after the gap */
1016  SCLogDebug("packet %"PRIu64": GAP. "
1017  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1018  p->pcap_cnt, stream->next_seq, stream->last_ack);
1019  return true;
1020  }
1021  }
1022  }
1023 
1024  SCLogDebug("packet %"PRIu64": GAP! "
1025  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
1026  "but we have no data.",
1027  p->pcap_cnt, last_ack_abs, app_progress);
1028  return true;
1029  }
1030  }
1031  SCLogDebug("packet %"PRIu64": no GAP. "
1032  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1033  p->pcap_cnt, last_ack_abs, app_progress);
1034  return false;
1035 }
1036 
1037 static inline uint32_t AdjustToAcked(const Packet *p,
1038  const TcpSession *ssn, const TcpStream *stream,
1039  const uint64_t app_progress, const uint32_t data_len)
1040 {
1041  uint32_t adjusted = data_len;
1042 
1043  /* get window of data that is acked */
1044  if (StreamTcpInlineMode() == FALSE) {
1045  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1046  if ((ssn->state < TCP_CLOSED ||
1047  (ssn->state == TCP_CLOSED &&
1048  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1050  {
1051  // fall through, we use all available data
1052  } else {
1053  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1054  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1055  /* get window of data that is acked */
1056  uint32_t delta = stream->last_ack - stream->base_seq;
1057  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1058  /* get max absolute offset */
1059  last_ack_abs += delta;
1060  }
1061 
1062  /* see if the buffer contains unack'd data as well */
1063  if (app_progress + data_len > last_ack_abs) {
1064  uint32_t check = data_len;
1065  adjusted = last_ack_abs - app_progress;
1066  BUG_ON(adjusted > check);
1067  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1068  "data is considered", adjusted);
1069  }
1070  }
1071  }
1072  return adjusted;
1073 }
1074 
1075 /** \internal
1076  * \brief get stream buffer and update the app-layer
1077  * \param stream pointer to pointer as app-layer can switch flow dir
1078  * \retval 0 success
1079  */
1080 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1081  TcpReassemblyThreadCtx *ra_ctx,
1082  TcpSession *ssn, TcpStream **stream,
1083  Packet *p, enum StreamUpdateDir dir)
1084 {
1085  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1086 
1087  SCLogDebug("app progress %"PRIu64, app_progress);
1088  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1089 
1090  const uint8_t *mydata;
1091  uint32_t mydata_len;
1092  bool gap_ahead = false;
1093 
1094  while (1) {
1095  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1096  gap_ahead = GetAppBuffer(*stream, &mydata, &mydata_len,
1097  app_progress, check_for_gap_ahead);
1098  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1099  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1100  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1101 
1102  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1103  NULL, mydata_len,
1104  StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP);
1105  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1106 
1109 
1110  /* AppLayerHandleTCPData has likely updated progress. */
1111  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1112  app_progress = STREAM_APP_PROGRESS(*stream);
1113 
1114  /* a GAP also consumes 'data required'. TODO perhaps we can use
1115  * this to skip post GAP data until the start of a next record. */
1116  if ((*stream)->data_required > 0) {
1117  if ((*stream)->data_required > mydata_len) {
1118  (*stream)->data_required -= mydata_len;
1119  } else {
1120  (*stream)->data_required = 0;
1121  }
1122  }
1123  if (r < 0)
1124  return 0;
1125  if (no_progress_update)
1126  break;
1127  continue;
1128 
1129  } else if (mydata == NULL || mydata_len == 0) {
1130  /* Possibly a gap, but no new data. */
1131  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1132  SCReturnInt(0);
1133 
1134  mydata = NULL;
1135  mydata_len = 0;
1136  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1137  break;
1138  }
1139 
1140  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1141  *stream, &(*stream)->sb, mydata_len, app_progress);
1142 
1143  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1144  /* get window of data that is acked */
1145  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1146  if (mydata_len == 0 && (flags & STREAM_EOF) == 0)
1147  SCReturnInt(0);
1148 
1149  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1150  if (mydata_len < (*stream)->data_required) {
1151  if (gap_ahead) {
1152  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1153  (*stream)->data_required, mydata_len);
1154  (*stream)->app_progress_rel += mydata_len;
1155  (*stream)->data_required -= mydata_len;
1156  // TODO send incomplete data to app-layer with special flag
1157  // indicating its all there is for this rec?
1158  } else {
1159  SCReturnInt(0);
1160  }
1161  app_progress = STREAM_APP_PROGRESS(*stream);
1162  continue;
1163  }
1164  }
1165  (*stream)->data_required = 0;
1166 
1167  /* update the app-layer */
1168  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1169  (uint8_t *)mydata, mydata_len, flags);
1170  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1171  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1172  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1173  break;
1174  app_progress = new_app_progress;
1175  }
1176 
1177  SCReturnInt(0);
1178 }
1179 
1180 /**
1181  * \brief Update the stream reassembly upon receiving a packet.
1182  *
1183  * For IDS mode, the stream is in the opposite direction of the packet,
1184  * as the ACK-packet is ACK'ing the stream.
1185  *
1186  * One of the utilities call by this function AppLayerHandleTCPData(),
1187  * has a feature where it will call this very same function for the
1188  * stream opposing the stream it is called with. This shouldn't cause
1189  * any issues, since processing of each stream is independent of the
1190  * other stream.
1191  */
1193  TcpSession *ssn, TcpStream *stream,
1194  Packet *p, enum StreamUpdateDir dir)
1195 {
1196  SCEnter();
1197 
1198  /* this function can be directly called by app layer protocol
1199  * detection. */
1202  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1203  SCReturnInt(0);
1204  }
1205 
1206 #ifdef DEBUG
1207  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1208  GetSessionSize(ssn, p);
1209 #endif
1210  /* if no segments are in the list or all are already processed,
1211  * and state is beyond established, we send an empty msg */
1212  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1213  {
1214  /* send an empty EOF msg if we have no segments but TCP state
1215  * is beyond ESTABLISHED */
1216  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1217  SCLogDebug("sending empty eof message");
1218  /* send EOF to app layer */
1219  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1220  NULL, 0,
1221  StreamGetAppLayerFlags(ssn, stream, p));
1222  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1223 
1224  SCReturnInt(0);
1225  }
1226  }
1227 
1228  /* with all that out of the way, lets update the app-layer */
1229  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1230 }
1231 
1232 /** \internal
1233  * \brief get stream data from offset
1234  * \param offset stream offset */
1235 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1236  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1237 {
1238  const uint8_t *mydata;
1239  uint32_t mydata_len;
1240  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1241  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1242 
1243  uint64_t roffset = offset;
1244  if (offset)
1245  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1246  else {
1247  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1248  }
1249 
1250  *data = mydata;
1251  *data_len = mydata_len;
1252  *data_offset = roffset;
1253  } else {
1254  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1255  *iter == NULL ? "starting" : "continuing", offset);
1256  if (*iter == NULL) {
1257  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1258  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1259  SCLogDebug("*iter %p", *iter);
1260  }
1261  if (*iter == NULL) {
1262  SCLogDebug("no data");
1263  *data = NULL;
1264  *data_len = 0;
1265  *data_offset = 0;
1266  return 0;
1267  }
1268  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1269 
1270  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1271  SCLogDebug("mydata %p", mydata);
1272 
1273  if ((*iter)->offset < offset) {
1274  uint64_t delta = offset - (*iter)->offset;
1275  if (delta < mydata_len) {
1276  *data = mydata + delta;
1277  *data_len = mydata_len - delta;
1278  *data_offset = offset;
1279  } else {
1280  SCLogDebug("no data (yet)");
1281  *data = NULL;
1282  *data_len = 0;
1283  *data_offset = 0;
1284  }
1285 
1286  } else {
1287  *data = mydata;
1288  *data_len = mydata_len;
1289  *data_offset = (*iter)->offset;
1290  }
1291 
1292  *iter = SBB_RB_NEXT(*iter);
1293  SCLogDebug("*iter %p", *iter);
1294  }
1295  return 0;
1296 }
1297 
1298 /** \brief does the stream engine have data to inspect?
1299  *
1300  * Returns true if there is data to inspect. In IDS case this is
1301  * about ACK'd data in the packet's direction.
1302  *
1303  * In the IPS case this is about the packet itself.
1304  */
1306 {
1307  TcpStream *stream;
1308  if (PKT_IS_TOSERVER(p)) {
1309  stream = &ssn->client;
1310  } else {
1311  stream = &ssn->server;
1312  }
1313 
1314  if (RB_EMPTY(&stream->seg_tree)) {
1315  return false;
1316  }
1317 
1320  return false;
1321 
1322  if (StreamTcpInlineMode() == FALSE) {
1323  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1324  return false;
1325  }
1326  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1327  return true;
1328  }
1329  } else {
1330  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1331  return true;
1332  }
1333  }
1334  return false;
1335 }
1336 
1337 /** \brief update stream engine after detection
1338  *
1339  * Tasked with progressing the 'progress' for Raw reassembly.
1340  * 2 main scenario's:
1341  * 1. progress is != 0, so we use this
1342  * 2. progress is 0, meaning the detect engine didn't touch
1343  * raw at all. In this case we need to look into progressing
1344  * raw anyway.
1345  *
1346  * Additionally, this function is tasked with disabling raw
1347  * reassembly if the app-layer requested to disable it.
1348  */
1349 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1350 {
1351  TcpStream *stream;
1352  if (PKT_IS_TOSERVER(p)) {
1353  stream = &ssn->client;
1354  } else {
1355  stream = &ssn->server;
1356  }
1357 
1358  if (progress > STREAM_RAW_PROGRESS(stream)) {
1359  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1360  stream->raw_progress_rel += slide;
1362 
1363  /* if app is active and beyond raw, sync raw to app */
1364  } else if (progress == 0 &&
1365  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1367  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1368  {
1369  /* if trigger raw is set we sync the 2 trackers */
1371  {
1372  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1373  stream->raw_progress_rel += slide;
1375 
1376  /* otherwise mix in the tcp window */
1377  } else {
1378  uint64_t tcp_window = stream->window;
1379  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1380  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1381  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1382  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1383  stream->raw_progress_rel += slide;
1384  }
1385  }
1386  }
1387  /* app is dead */
1388  } else if (progress == 0) {
1389  uint64_t tcp_window = stream->window;
1390  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1391  if (tcp_window < stream_right_edge) {
1392  uint64_t new_raw = stream_right_edge - tcp_window;
1393  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1394  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1395  stream->raw_progress_rel += slide;
1396  }
1397  }
1399 
1400  } else {
1401  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1402  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1403  STREAM_RAW_PROGRESS(stream), stream->window);
1404  }
1405 
1406  /* if we were told to accept no more raw data, we can mark raw as
1407  * disabled now. */
1410  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1411  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1412  }
1413 
1414  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1415 }
1416 
1417 /** \internal
1418  * \brief get a buffer around the current packet and run the callback on it
1419  *
1420  * The inline/IPS scanning method takes the current payload and wraps it in
1421  * data from other segments.
1422  *
1423  * How much data is inspected is controlled by the available data, chunk_size
1424  * and the payload size of the packet.
1425  *
1426  * Large packets: if payload size is close to the chunk_size, where close is
1427  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1428  * used: payload_len + 33% of the chunk_size.
1429  * If the payload size if equal to or bigger than the chunk_size, we use
1430  * payload len + 33% of the chunk size.
1431  */
1432 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1433  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1434 {
1435  SCEnter();
1436  int r = 0;
1437 
1438  TcpStream *stream;
1439  if (PKT_IS_TOSERVER(p)) {
1440  stream = &ssn->client;
1441  } else {
1442  stream = &ssn->server;
1443  }
1444 
1445  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1447  {
1448  *progress_out = STREAM_RAW_PROGRESS(stream);
1449  return 0;
1450  }
1451 
1452  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1455  if (chunk_size <= p->payload_len) {
1456  chunk_size = p->payload_len + (chunk_size / 3);
1457  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1458  p->payload_len, chunk_size);
1459  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1460  chunk_size = p->payload_len + ((chunk_size / 3));
1461  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1462  p->payload_len, chunk_size);
1463  }
1464 
1465  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1466  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1467  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1468  packet_leftedge_abs, packet_rightedge_abs);
1469 
1470  const uint8_t *mydata = NULL;
1471  uint32_t mydata_len = 0;
1472  uint64_t mydata_offset = 0;
1473  /* simply return progress from the block we inspected. */
1474  bool return_progress = false;
1475 
1476  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1477  /* continues block */
1478  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1479  return_progress = true;
1480 
1481  } else {
1482  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1483  /* find our block */
1484  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1485  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1486  if (sbb) {
1487  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1488  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1489  mydata_offset = sbb->offset;
1490  }
1491  }
1492 
1493  /* this can only happen if the segment insert of our current 'p' failed */
1494  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1495  if ((mydata == NULL || mydata_len == 0) || /* no data */
1496  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1497  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1498  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1499  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1500  {
1501  /* no data, or data is incomplete or wrong: use packet data */
1502  mydata = p->payload;
1503  mydata_len = p->payload_len;
1504  mydata_offset = packet_leftedge_abs;
1505  //mydata_rightedge_abs = packet_rightedge_abs;
1506  } else {
1507  /* adjust buffer to match chunk_size */
1508  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1509  if (mydata_len > chunk_size) {
1510  uint32_t excess = mydata_len - chunk_size;
1511  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1512 
1513  if (mydata_rightedge_abs == packet_rightedge_abs) {
1514  mydata += excess;
1515  mydata_len -= excess;
1516  mydata_offset += excess;
1517  SCLogDebug("cutting front of the buffer with %u", excess);
1518  } else if (mydata_offset == packet_leftedge_abs) {
1519  mydata_len -= excess;
1520  SCLogDebug("cutting tail of the buffer with %u", excess);
1521  } else {
1522  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1523  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1524  SCLogDebug("before %u after %u", before, after);
1525 
1526  if (after >= (chunk_size - p->payload_len) / 2) {
1527  // more trailing data than we need
1528 
1529  if (before >= (chunk_size - p->payload_len) / 2) {
1530  // also more heading data, divide evenly
1531  before = after = (chunk_size - p->payload_len) / 2;
1532  } else {
1533  // heading data is less than requested, give the
1534  // rest to the trailing data
1535  after = (chunk_size - p->payload_len) - before;
1536  }
1537  } else {
1538  // less trailing data than requested
1539 
1540  if (before >= (chunk_size - p->payload_len) / 2) {
1541  before = (chunk_size - p->payload_len) - after;
1542  } else {
1543  // both smaller than their requested size
1544  }
1545  }
1546 
1547  /* adjust the buffer */
1548  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1549  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1550  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1551  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1552  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1553  mydata += skip;
1554  mydata_len -= (skip + cut);
1555  mydata_offset += skip;
1556  }
1557  }
1558  }
1559 
1560  /* run the callback */
1561  r = Callback(cb_data, mydata, mydata_len);
1562  BUG_ON(r < 0);
1563 
1564  if (return_progress) {
1565  *progress_out = (mydata_offset + mydata_len);
1566  } else {
1567  /* several blocks of data, so we need to be a bit more careful:
1568  * - if last_ack is beyond last progress, move progress forward to last_ack
1569  * - if our block matches or starts before last ack, return right edge of
1570  * our block.
1571  */
1572  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1573  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1574  uint32_t delta = stream->last_ack - stream->base_seq;
1575  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1576  /* get max absolute offset */
1577  last_ack_abs += delta;
1578  }
1579  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1580 
1581  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1582  if (mydata_offset > last_ack_abs) {
1583  /* gap between us and last ack, set progress to last ack */
1584  *progress_out = last_ack_abs;
1585  } else {
1586  *progress_out = (mydata_offset + mydata_len);
1587  }
1588  } else {
1589  *progress_out = STREAM_RAW_PROGRESS(stream);
1590  }
1591  }
1592  return r;
1593 }
1594 
1595 /** \brief access 'raw' reassembly data.
1596  *
1597  * Access data as tracked by 'raw' tracker. Data is made available to
1598  * callback that is passed to this function.
1599  *
1600  * In the case of IDS the callback may be run multiple times if data
1601  * contains gaps. It will then be run for each block of data that is
1602  * continuous.
1603  *
1604  * The callback should give on of 2 return values:
1605  * - 0 ok
1606  * - 1 done
1607  * The value 1 will break the loop if there is a block list that is
1608  * inspected.
1609  *
1610  * This function will return the 'progress' value that has been
1611  * consumed until now.
1612  *
1613  * \param ssn tcp session
1614  * \param stream tcp stream
1615  * \param Callback the function pointer to the callback function
1616  * \param cb_data callback data
1617  * \param[in] progress_in progress to work from
1618  * \param[out] progress_out absolute progress value of the data this
1619  * call handled.
1620  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1621  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1622  *
1623  * `respect_inspect_depth` is used to avoid useless inspection of too
1624  * much data.
1625  */
1626 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1627  StreamReassembleRawFunc Callback, void *cb_data,
1628  const uint64_t progress_in,
1629  uint64_t *progress_out, bool eof,
1630  bool respect_inspect_depth)
1631 {
1632  SCEnter();
1633  int r = 0;
1634 
1635  StreamingBufferBlock *iter = NULL;
1636  uint64_t progress = progress_in;
1637  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1638 
1639  /* if the app layer triggered a flush, and we're supposed to
1640  * use a minimal inspect depth, we actually take the app progress
1641  * as that is the right edge of the data. Then we take the window
1642  * of 'min_inspect_depth' before that. */
1643 
1644  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1645  respect_inspect_depth ? "true" : "false",
1646  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1647  stream->min_inspect_depth);
1648 
1649  if (respect_inspect_depth &&
1651  && stream->min_inspect_depth)
1652  {
1653  progress = STREAM_APP_PROGRESS(stream);
1654  if (stream->min_inspect_depth >= progress) {
1655  progress = 0;
1656  } else {
1657  progress -= stream->min_inspect_depth;
1658  }
1659 
1660  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1661 
1662  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1663  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1664  }
1665 
1666  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1667 
1668  /* get window of data that is acked */
1669  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1670  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1671  uint32_t delta = stream->last_ack - stream->base_seq;
1672  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1673  /* get max absolute offset */
1674  last_ack_abs += delta;
1675  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1676  }
1677 
1678  /* loop through available buffers. On no packet loss we'll have a single
1679  * iteration. On missing data we'll walk the blocks */
1680  while (1) {
1681  const uint8_t *mydata;
1682  uint32_t mydata_len;
1683  uint64_t mydata_offset = 0;
1684 
1685  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1686  if (mydata_len == 0) {
1687  SCLogDebug("no data");
1688  break;
1689  }
1690  //PrintRawDataFp(stdout, mydata, mydata_len);
1691 
1692  SCLogDebug("raw progress %"PRIu64, progress);
1693  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1694  stream, &stream->sb, mydata_len, progress);
1695 
1696  if (eof) {
1697  // inspect all remaining data, ack'd or not
1698  } else {
1699  if (last_ack_abs < progress) {
1700  SCLogDebug("nothing to do");
1701  goto end;
1702  }
1703 
1704  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1705  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1706 
1707  /* see if the buffer contains unack'd data as well */
1708  if (progress + mydata_len > last_ack_abs) {
1709  uint32_t check = mydata_len;
1710  mydata_len = last_ack_abs - progress;
1711  BUG_ON(check < mydata_len);
1712  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1713  "data is considered", mydata_len);
1714  }
1715 
1716  }
1717  if (mydata_len == 0)
1718  break;
1719 
1720  SCLogDebug("data %p len %u", mydata, mydata_len);
1721 
1722  /* we have data. */
1723  r = Callback(cb_data, mydata, mydata_len);
1724  BUG_ON(r < 0);
1725 
1726  if (mydata_offset == progress) {
1727  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1728  progress, mydata_len, progress_in + mydata_len);
1729 
1730  progress += mydata_len;
1731  SCLogDebug("raw progress now %"PRIu64, progress);
1732 
1733  /* data is beyond the progress we'd like, and before last ack. Gap. */
1734  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1735  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1736  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1737  progress = mydata_offset;
1738  SCLogDebug("raw progress now %"PRIu64, progress);
1739 
1740  } else {
1741  SCLogDebug("not increasing progress, data gap => mydata_offset "
1742  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1743  }
1744 
1745  if (iter == NULL || r == 1)
1746  break;
1747  }
1748 end:
1749  *progress_out = progress;
1750  return r;
1751 }
1752 
1754  StreamReassembleRawFunc Callback, void *cb_data,
1755  uint64_t *progress_out, bool respect_inspect_depth)
1756 {
1757  /* handle inline separately as the logic is very different */
1758  if (StreamTcpInlineMode() == TRUE) {
1759  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1760  }
1761 
1762  TcpStream *stream;
1763  if (PKT_IS_TOSERVER(p)) {
1764  stream = &ssn->client;
1765  } else {
1766  stream = &ssn->server;
1767  }
1768 
1770  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1771  {
1772  *progress_out = STREAM_RAW_PROGRESS(stream);
1773  return 0;
1774  }
1775 
1776  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1777  STREAM_RAW_PROGRESS(stream), progress_out,
1778  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1779 }
1780 
1782  StreamReassembleRawFunc Callback, void *cb_data,
1783  uint64_t progress_in,
1784  uint64_t *progress_out, bool eof)
1785 {
1786  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1787  return 0;
1788 
1789  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1790  progress_in, progress_out, eof, false);
1791 }
1792 
1793 /** \internal
1794  * \brief update app layer based on received ACK
1795  *
1796  * \retval r 0 on success, -1 on error
1797  */
1798 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1799  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1800 {
1801  SCEnter();
1802 
1803  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1804  SCReturnInt(-1);
1805 
1806  SCReturnInt(0);
1807 }
1808 
1810  TcpSession *ssn, TcpStream *stream,
1811  Packet *p, PacketQueueNoLock *pq)
1812 {
1813  SCEnter();
1814 
1815  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1816 
1817  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1818  ssn, stream, p, p->payload_len);
1819 
1820  /* we need to update the opposing stream in
1821  * StreamTcpReassembleHandleSegmentUpdateACK */
1822  TcpStream *opposing_stream = NULL;
1823  if (stream == &ssn->client) {
1824  opposing_stream = &ssn->server;
1825  } else {
1826  opposing_stream = &ssn->client;
1827  }
1828 
1829  /* default IDS: update opposing side (triggered by ACK) */
1831  /* inline and stream end and flow timeout packets trigger same dir handling */
1832  if (StreamTcpInlineMode()) {
1833  dir = UPDATE_DIR_PACKET;
1834  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1835  dir = UPDATE_DIR_PACKET;
1836  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1837  dir = UPDATE_DIR_PACKET;
1838  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1839  dir = UPDATE_DIR_PACKET;
1840  } else if (ssn->state == TCP_CLOSED) {
1841  dir = UPDATE_DIR_BOTH;
1842  }
1843 
1844  /* handle ack received */
1845  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1846  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1847  {
1848  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1849  SCReturnInt(-1);
1850  }
1851 
1852  /* if this segment contains data, insert it */
1853  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1854  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1855 
1856  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1857  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1858  SCReturnInt(-1);
1859  }
1860 
1861  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1862  p->flags |= PKT_STREAM_ADD;
1863  } else {
1864  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1865  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1866  ssn, stream, p->payload_len,
1867  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1868 
1869  }
1870 
1871  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1872  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1873  * was *just* set. In this case we trigger the AppLayer Truncate
1874  * logic, to inform the applayer no more data in this direction is
1875  * to be expected. */
1876  if ((stream->flags &
1879  {
1880  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1881  if (dir != UPDATE_DIR_PACKET) {
1882  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1883  "can trigger Truncate");
1884  dir = UPDATE_DIR_PACKET;
1885  }
1886  }
1887 
1888  /* in stream inline mode even if we have no data we call the reassembly
1889  * functions to handle EOF */
1890  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1891  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1892  StreamTcpInlineMode()?"true":"false",
1893  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1894  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1895  SCReturnInt(-1);
1896  }
1897  }
1898 
1899  SCReturnInt(0);
1900 }
1901 
1902 /**
1903  * \brief get a segment from the pool
1904  *
1905  * \retval seg Segment from the pool or NULL
1906  */
1908 {
1909  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1910  SCLogDebug("seg we return is %p", seg);
1911  if (seg == NULL) {
1912  /* Increment the counter to show that we are not able to serve the
1913  segment request due to memcap limit */
1915  } else {
1916  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1917  }
1918 
1919  return seg;
1920 }
1921 
1922 /**
1923  * \brief Trigger RAW stream reassembly
1924  *
1925  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1926  * reassembly from the applayer, for example upon completion of a
1927  * HTTP request.
1928  *
1929  * It sets a flag in the stream so that the next Raw call will return
1930  * the data.
1931  *
1932  * \param ssn TcpSession
1933  */
1935 {
1936 #ifdef DEBUG
1937  BUG_ON(ssn == NULL);
1938 #endif
1939 
1940  if (ssn != NULL) {
1941  if (direction == STREAM_TOSERVER) {
1943  } else {
1945  }
1946 
1947  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1948  }
1949 }
1950 
1951 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1952 {
1953 #ifdef DEBUG
1954  BUG_ON(ssn == NULL);
1955 #endif
1956 
1957  if (ssn != NULL) {
1958  if (direction == STREAM_TOSERVER) {
1959  ssn->client.min_inspect_depth = depth;
1960  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1961  } else {
1962  ssn->server.min_inspect_depth = depth;
1963  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1964  }
1965  }
1966 }
1967 
1968 #ifdef UNITTESTS
1969 /** unit tests and it's support functions below */
1970 
1971 #define SET_ISN(stream, setseq) \
1972  (stream)->isn = (setseq); \
1973  (stream)->base_seq = (setseq) + 1
1974 
1975 /** \brief The Function to create the packet with given payload, which is used
1976  * to test the reassembly of the engine.
1977  *
1978  * \param payload The variable used to store the payload contents of the
1979  * current packet.
1980  * \param value The value which current payload will have for this packet
1981  * \param payload_len The length of the filed payload for current packet.
1982  * \param len Length of the payload array
1983  */
1984 
1985 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1986  uint8_t payload_len, uint8_t len)
1987 {
1988  uint8_t i;
1989  for (i = 0; i < payload_len; i++)
1990  payload[i] = value;
1991  for (; i < len; i++)
1992  payload = NULL;
1993 }
1994 
1995 /** \brief The Function Checks the reassembled stream contents against predefined
1996  * stream contents according to OS policy used.
1997  *
1998  * \param stream_policy Predefined value of stream for different OS policies
1999  * \param stream Reassembled stream returned from the reassembly functions
2000  */
2001 
2002 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
2003 {
2004  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
2005  {
2006  //PrintRawDataFp(stdout, stream_policy, sp_size);
2007  return 0;
2008  }
2009  return 1;
2010 }
2011 
2012 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2013 {
2014  if (StreamingBufferCompareRawData(&stream->sb,
2015  data, data_len) == 0)
2016  {
2017  SCReturnInt(0);
2018  }
2019  SCLogInfo("OK");
2020  PrintRawDataFp(stdout, data, data_len);
2021  return 1;
2022 }
2023 
2024 #define MISSED_START(isn) \
2025  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2026  TcpSession ssn; \
2027  ThreadVars tv; \
2028  memset(&tv, 0, sizeof(tv)); \
2029  \
2030  StreamTcpUTInit(&ra_ctx); \
2031  \
2032  StreamTcpUTSetupSession(&ssn); \
2033  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2034  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2035  \
2036  TcpStream *stream = &ssn.client;
2037 
2038 #define MISSED_END \
2039  StreamTcpUTClearSession(&ssn); \
2040  StreamTcpUTDeinit(ra_ctx); \
2041  PASS
2042 
2043 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2044  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2045  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2046 
2047 /**
2048  * \test Test the handling of packets missed by both IDS and the end host.
2049  * The packet is missed in the starting of the stream.
2050  *
2051  * \retval On success it returns 1 and on failure 0.
2052  */
2053 
2054 static int StreamTcpReassembleTest25 (void)
2055 {
2056  MISSED_START(6);
2057  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
2058  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
2059  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2060  MISSED_END;
2061 }
2062 
2063 /**
2064  * \test Test the handling of packets missed by both IDS and the end host.
2065  * The packet is missed in the middle of the stream.
2066  *
2067  * \retval On success it returns 1 and on failure 0.
2068  */
2069 
2070 static int StreamTcpReassembleTest26 (void)
2071 {
2072  MISSED_START(9);
2073  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2074  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2075  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2076  MISSED_END;
2077 }
2078 
2079 /**
2080  * \test Test the handling of packets missed by both IDS and the end host.
2081  * The packet is missed in the end of the stream.
2082  *
2083  * \retval On success it returns 1 and on failure 0.
2084  */
2085 
2086 static int StreamTcpReassembleTest27 (void)
2087 {
2088  MISSED_START(9);
2089  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2090  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2091  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2092  MISSED_END;
2093 }
2094 
2095 /**
2096  * \test Test the handling of packets missed by IDS, but the end host has
2097  * received it and send the acknowledgment of it. The packet is missed
2098  * in the starting of the stream.
2099  *
2100  * \retval On success it returns 1 and on failure 0.
2101  */
2102 
2103 static int StreamTcpReassembleTest28 (void)
2104 {
2105  MISSED_START(6);
2106  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2107  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2108  ssn.state = TCP_TIME_WAIT;
2109  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2110  MISSED_END;
2111 }
2112 
2113 /**
2114  * \test Test the handling of packets missed by IDS, but the end host has
2115  * received it and send the acknowledgment of it. The packet is missed
2116  * in the middle of the stream.
2117  *
2118  * \retval On success it returns 1 and on failure 0.
2119  */
2120 
2121 static int StreamTcpReassembleTest29 (void)
2122 {
2123  MISSED_START(9);
2124  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2125  ssn.state = TCP_TIME_WAIT;
2126  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2127  MISSED_END;
2128 }
2129 
2130 static int StreamTcpReassembleTest33(void)
2131 {
2132  TcpSession ssn;
2133  Packet *p = PacketGetFromAlloc();
2134  FAIL_IF(unlikely(p == NULL));
2135  Flow f;
2136  TCPHdr tcph;
2137  TcpReassemblyThreadCtx *ra_ctx = NULL;
2139  uint8_t packet[1460] = "";
2140 
2141  StreamTcpUTInit(&ra_ctx);
2143 
2144  PacketQueueNoLock pq;
2145  memset(&pq,0,sizeof(PacketQueueNoLock));
2146  memset(&f, 0, sizeof (Flow));
2147  memset(&tcph, 0, sizeof (TCPHdr));
2148  ThreadVars tv;
2149  memset(&tv, 0, sizeof (ThreadVars));
2150  FLOW_INITIALIZE(&f);
2151  f.protoctx = &ssn;
2152  f.proto = IPPROTO_TCP;
2153  p->src.family = AF_INET;
2154  p->dst.family = AF_INET;
2155  p->proto = IPPROTO_TCP;
2156  p->flow = &f;
2157  tcph.th_win = 5480;
2158  tcph.th_flags = TH_PUSH | TH_ACK;
2159  p->tcph = &tcph;
2161  p->payload = packet;
2162 
2163  p->tcph->th_seq = htonl(10);
2164  p->tcph->th_ack = htonl(31);
2165  p->payload_len = 10;
2166 
2167  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2168 
2169  p->tcph->th_seq = htonl(20);
2170  p->tcph->th_ack = htonl(31);
2171  p->payload_len = 10;
2172 
2173  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2174 
2175  p->tcph->th_seq = htonl(40);
2176  p->tcph->th_ack = htonl(31);
2177  p->payload_len = 10;
2178 
2179  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2180 
2181  p->tcph->th_seq = htonl(5);
2182  p->tcph->th_ack = htonl(31);
2183  p->payload_len = 30;
2184 
2185  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2186 
2188  StreamTcpUTDeinit(ra_ctx);
2189  SCFree(p);
2190  PASS;
2191 }
2192 
2193 static int StreamTcpReassembleTest34(void)
2194 {
2195  TcpSession ssn;
2196  Packet *p = PacketGetFromAlloc();
2197  FAIL_IF(unlikely(p == NULL));
2198  Flow f;
2199  TCPHdr tcph;
2200  TcpReassemblyThreadCtx *ra_ctx = NULL;
2202  uint8_t packet[1460] = "";
2203 
2204  StreamTcpUTInit(&ra_ctx);
2206  PacketQueueNoLock pq;
2207  memset(&pq,0,sizeof(PacketQueueNoLock));
2208  memset(&f, 0, sizeof (Flow));
2209  memset(&tcph, 0, sizeof (TCPHdr));
2210  ThreadVars tv;
2211  memset(&tv, 0, sizeof (ThreadVars));
2212  FLOW_INITIALIZE(&f);
2213  f.protoctx = &ssn;
2214  f.proto = IPPROTO_TCP;
2215  p->src.family = AF_INET;
2216  p->dst.family = AF_INET;
2217  p->proto = IPPROTO_TCP;
2218  p->flow = &f;
2219  tcph.th_win = 5480;
2220  tcph.th_flags = TH_PUSH | TH_ACK;
2221  p->tcph = &tcph;
2223  p->payload = packet;
2224  SET_ISN(&ssn.client, 857961230);
2225 
2226  p->tcph->th_seq = htonl(857961230);
2227  p->tcph->th_ack = htonl(31);
2228  p->payload_len = 304;
2229 
2230  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2231 
2232  p->tcph->th_seq = htonl(857961534);
2233  p->tcph->th_ack = htonl(31);
2234  p->payload_len = 1460;
2235 
2236  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2237 
2238  p->tcph->th_seq = htonl(857963582);
2239  p->tcph->th_ack = htonl(31);
2240  p->payload_len = 1460;
2241 
2242  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2243 
2244  p->tcph->th_seq = htonl(857960946);
2245  p->tcph->th_ack = htonl(31);
2246  p->payload_len = 1460;
2247 
2248  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2249 
2251  StreamTcpUTDeinit(ra_ctx);
2252  SCFree(p);
2253  PASS;
2254 }
2255 
2256 /** \test Test the bug 76 condition */
2257 static int StreamTcpReassembleTest37(void)
2258 {
2259  TcpSession ssn;
2260  Flow f;
2261  TCPHdr tcph;
2262  TcpReassemblyThreadCtx *ra_ctx = NULL;
2263  uint8_t packet[1460] = "";
2264  PacketQueueNoLock pq;
2265  ThreadVars tv;
2266  memset(&tv, 0, sizeof (ThreadVars));
2267 
2268  Packet *p = PacketGetFromAlloc();
2269  FAIL_IF(unlikely(p == NULL));
2270 
2271  StreamTcpUTInit(&ra_ctx);
2273  memset(&pq,0,sizeof(PacketQueueNoLock));
2274  memset(&f, 0, sizeof (Flow));
2275  memset(&tcph, 0, sizeof (TCPHdr));
2276  memset(&tv, 0, sizeof (ThreadVars));
2277 
2278  FLOW_INITIALIZE(&f);
2279  f.protoctx = &ssn;
2280  f.proto = IPPROTO_TCP;
2281  p->src.family = AF_INET;
2282  p->dst.family = AF_INET;
2283  p->proto = IPPROTO_TCP;
2284  p->flow = &f;
2285  tcph.th_win = 5480;
2286  tcph.th_flags = TH_PUSH | TH_ACK;
2287  p->tcph = &tcph;
2289  p->payload = packet;
2291 
2292  p->tcph->th_seq = htonl(3061088537UL);
2293  p->tcph->th_ack = htonl(1729548549UL);
2294  p->payload_len = 1391;
2295  ssn.client.last_ack = 3061091137UL;
2296  SET_ISN(&ssn.client, 3061091309UL);
2297 
2298  /* pre base_seq, so should be rejected */
2299  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2300 
2301  p->tcph->th_seq = htonl(3061089928UL);
2302  p->tcph->th_ack = htonl(1729548549UL);
2303  p->payload_len = 1391;
2304  ssn.client.last_ack = 3061091137UL;
2305 
2306  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2307 
2308  p->tcph->th_seq = htonl(3061091319UL);
2309  p->tcph->th_ack = htonl(1729548549UL);
2310  p->payload_len = 1391;
2311  ssn.client.last_ack = 3061091137UL;
2312 
2313  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2314 
2316  StreamTcpUTDeinit(ra_ctx);
2317  SCFree(p);
2318  PASS;
2319 }
2320 
2321 /**
2322  * \test Test to make sure that we don't return the segments until the app
2323  * layer proto has been detected and after that remove the processed
2324  * segments.
2325  *
2326  * \retval On success it returns 1 and on failure 0.
2327  */
2328 
2329 static int StreamTcpReassembleTest39 (void)
2330 {
2331  Packet *p = PacketGetFromAlloc();
2332  FAIL_IF(unlikely(p == NULL));
2333  Flow f;
2334  ThreadVars tv;
2335  StreamTcpThread stt;
2336  TCPHdr tcph;
2337  PacketQueueNoLock pq;
2338  memset(&pq,0,sizeof(PacketQueueNoLock));
2339  memset (&f, 0, sizeof(Flow));
2340  memset(&tv, 0, sizeof (ThreadVars));
2341  memset(&stt, 0, sizeof (stt));
2342  memset(&tcph, 0, sizeof (TCPHdr));
2343 
2344  FLOW_INITIALIZE(&f);
2345  f.flags = FLOW_IPV4;
2346  f.proto = IPPROTO_TCP;
2347  p->flow = &f;
2348  p->tcph = &tcph;
2349 
2350  StreamTcpUTInit(&stt.ra_ctx);
2351 
2352  /* handshake */
2353  tcph.th_win = htons(5480);
2354  tcph.th_flags = TH_SYN;
2356  p->payload_len = 0;
2357  p->payload = NULL;
2358  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2359 
2360  TcpSession *ssn = (TcpSession *)f.protoctx;
2361  FAIL_IF_NULL(ssn);
2362 
2373  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2374  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2375  FAIL_IF(ssn->data_first_seen_dir != 0);
2376 
2377  /* handshake */
2378  p->tcph->th_ack = htonl(1);
2379  p->tcph->th_flags = TH_SYN | TH_ACK;
2381  p->payload_len = 0;
2382  p->payload = NULL;
2383  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2394  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2395  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2396  FAIL_IF(ssn->data_first_seen_dir != 0);
2397 
2398  /* handshake */
2399  p->tcph->th_ack = htonl(1);
2400  p->tcph->th_seq = htonl(1);
2401  p->tcph->th_flags = TH_ACK;
2403  p->payload_len = 0;
2404  p->payload = NULL;
2405  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2416  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2417  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2418  FAIL_IF(ssn->data_first_seen_dir != 0);
2419 
2420  /* partial request */
2421  uint8_t request1[] = { 0x47, 0x45, };
2422  p->tcph->th_ack = htonl(1);
2423  p->tcph->th_seq = htonl(1);
2424  p->tcph->th_flags = TH_PUSH | TH_ACK;
2426  p->payload_len = sizeof(request1);
2427  p->payload = request1;
2428  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2439  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2440  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2441  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2443 
2444  /* response ack against partial request */
2445  p->tcph->th_ack = htonl(3);
2446  p->tcph->th_seq = htonl(1);
2447  p->tcph->th_flags = TH_ACK;
2449  p->payload_len = 0;
2450  p->payload = NULL;
2451  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2462  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2463  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2464  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2466 
2467  /* complete partial request */
2468  uint8_t request2[] = {
2469  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2470  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2471  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2472  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2473  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2474  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2475  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2476  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2477  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2478  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2479  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2480  p->tcph->th_ack = htonl(1);
2481  p->tcph->th_seq = htonl(3);
2482  p->tcph->th_flags = TH_PUSH | TH_ACK;
2484  p->payload_len = sizeof(request2);
2485  p->payload = request2;
2486  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2497  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2498  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2499  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2500  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2502 
2503  /* response - request ack */
2504  uint8_t response[] = {
2505  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2506  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2507  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2508  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2509  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2510  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2511  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2512  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2513  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2514  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2515  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2516  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2517  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2518  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2519  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2520  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2521  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2522  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2523  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2524  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2525  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2526  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2527  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2528  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2529  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2530  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2531  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2532  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2533  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2534  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2535  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2536  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2537  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2538  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2539  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2540  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2541  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2542  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2543  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2544  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2545  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2546  p->tcph->th_ack = htonl(88);
2547  p->tcph->th_seq = htonl(1);
2548  p->tcph->th_flags = TH_PUSH | TH_ACK;
2550  p->payload_len = sizeof(response);
2551  p->payload = response;
2552 
2553  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2565  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2566  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2567  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2568  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2569 
2570  /* response ack from request */
2571  p->tcph->th_ack = htonl(328);
2572  p->tcph->th_seq = htonl(88);
2573  p->tcph->th_flags = TH_ACK;
2575  p->payload_len = 0;
2576  p->payload = NULL;
2577  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2589  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2590  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2591  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2592  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2593  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2594 
2595  /* response - acking */
2596  p->tcph->th_ack = htonl(88);
2597  p->tcph->th_seq = htonl(328);
2598  p->tcph->th_flags = TH_PUSH | TH_ACK;
2600  p->payload_len = 0;
2601  p->payload = NULL;
2602  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2614  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2615  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2616  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2617  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2618  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2619 
2620  /* response ack from request */
2621  p->tcph->th_ack = htonl(328);
2622  p->tcph->th_seq = htonl(88);
2623  p->tcph->th_flags = TH_ACK;
2625  p->payload_len = 0;
2626  p->payload = NULL;
2627  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2639  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2640  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2641  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2642  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2643 
2644  /* response - acking the request again*/
2645  p->tcph->th_ack = htonl(88);
2646  p->tcph->th_seq = htonl(328);
2647  p->tcph->th_flags = TH_PUSH | TH_ACK;
2649  p->payload_len = 0;
2650  p->payload = NULL;
2651  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2663  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2664  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2665  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2666  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2667 
2668  /*** New Request ***/
2669 
2670  /* partial request */
2671  p->tcph->th_ack = htonl(328);
2672  p->tcph->th_seq = htonl(88);
2673  p->tcph->th_flags = TH_PUSH | TH_ACK;
2675  p->payload_len = sizeof(request1);
2676  p->payload = request1;
2677  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2689  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2690  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2691  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2692  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2693  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2694 
2695  /* response ack against partial request */
2696  p->tcph->th_ack = htonl(90);
2697  p->tcph->th_seq = htonl(328);
2698  p->tcph->th_flags = TH_ACK;
2700  p->payload_len = 0;
2701  p->payload = NULL;
2702  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2703  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2715  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2716  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2717  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2718  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2719  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2720 
2721  /* complete request */
2722  p->tcph->th_ack = htonl(328);
2723  p->tcph->th_seq = htonl(90);
2724  p->tcph->th_flags = TH_PUSH | TH_ACK;
2726  p->payload_len = sizeof(request2);
2727  p->payload = request2;
2728  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2740  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2741  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2742  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2743  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2744  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2745  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2746 
2747  /* response ack against second partial request */
2748  p->tcph->th_ack = htonl(175);
2749  p->tcph->th_seq = htonl(328);
2750  p->tcph->th_flags = TH_ACK;
2752  p->payload_len = 0;
2753  p->payload = NULL;
2754 
2755  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2767  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2768  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2769  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2770  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2771  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2772  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2773 
2774  /* response acking a request */
2775  p->tcph->th_ack = htonl(175);
2776  p->tcph->th_seq = htonl(328);
2777  p->tcph->th_flags = TH_ACK;
2779  p->payload_len = 0;
2780  p->payload = NULL;
2781  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2787 
2790 
2791  /* request acking a response */
2792  p->tcph->th_ack = htonl(328);
2793  p->tcph->th_seq = htonl(175);
2794  p->tcph->th_flags = TH_ACK;
2796  p->payload_len = 0;
2797  p->payload = NULL;
2798  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2799 
2800  StreamTcpSessionClear(ssn);
2802  SCFree(p);
2803  PASS;
2804 }
2805 
2806 /**
2807  * \test Test to make sure that we sent all the segments from the initial
2808  * segments to app layer until we have detected the app layer proto.
2809  *
2810  * \retval On success it returns 1 and on failure 0.
2811  */
2812 
2813 static int StreamTcpReassembleTest40 (void)
2814 {
2815  Packet *p = PacketGetFromAlloc();
2816  FAIL_IF_NULL(p);
2817  Flow *f = NULL;
2818  TCPHdr tcph;
2819  TcpSession ssn;
2820  PacketQueueNoLock pq;
2821  memset(&pq,0,sizeof(PacketQueueNoLock));
2822  memset(&tcph, 0, sizeof (TCPHdr));
2823  ThreadVars tv;
2824  memset(&tv, 0, sizeof (ThreadVars));
2825 
2828 
2830  FAIL_IF_NULL(ra_ctx);
2831 
2832  uint8_t httpbuf1[] = "P";
2833  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2834  uint8_t httpbuf3[] = "O";
2835  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2836  uint8_t httpbuf4[] = "S";
2837  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2838  uint8_t httpbuf5[] = "T \r\n";
2839  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2840 
2841  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2842  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2843 
2844  SET_ISN(&ssn.server, 9);
2845  ssn.server.last_ack = 10;
2846  SET_ISN(&ssn.client, 9);
2847  ssn.client.isn = 9;
2848 
2849  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2850  FAIL_IF_NULL(f);
2851  f->protoctx = &ssn;
2852  f->proto = IPPROTO_TCP;
2853  p->flow = f;
2854 
2855  tcph.th_win = htons(5480);
2856  tcph.th_seq = htonl(10);
2857  tcph.th_ack = htonl(10);
2858  tcph.th_flags = TH_ACK|TH_PUSH;
2859  p->tcph = &tcph;
2861  p->payload = httpbuf1;
2862  p->payload_len = httplen1;
2863  ssn.state = TCP_ESTABLISHED;
2864  TcpStream *s = &ssn.client;
2865  SCLogDebug("1 -- start");
2866  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2867 
2869  p->payload = httpbuf2;
2870  p->payload_len = httplen2;
2871  tcph.th_seq = htonl(10);
2872  tcph.th_ack = htonl(11);
2873  s = &ssn.server;
2874  ssn.server.last_ack = 11;
2875  SCLogDebug("2 -- start");
2876  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2877 
2879  p->payload = httpbuf3;
2880  p->payload_len = httplen3;
2881  tcph.th_seq = htonl(11);
2882  tcph.th_ack = htonl(55);
2883  s = &ssn.client;
2884  ssn.client.last_ack = 55;
2885  SCLogDebug("3 -- start");
2886  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2887 
2889  p->payload = httpbuf2;
2890  p->payload_len = httplen2;
2891  tcph.th_seq = htonl(55);
2892  tcph.th_ack = htonl(12);
2893  s = &ssn.server;
2894  ssn.server.last_ack = 12;
2895  SCLogDebug("4 -- start");
2896  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2897 
2898  /* check is have the segment in the list and flagged or not */
2899  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2900  FAIL_IF_NULL(seg);
2901  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2902 
2904  p->payload = httpbuf4;
2905  p->payload_len = httplen4;
2906  tcph.th_seq = htonl(12);
2907  tcph.th_ack = htonl(100);
2908  s = &ssn.client;
2909  ssn.client.last_ack = 100;
2910  SCLogDebug("5 -- start");
2911  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2912 
2914  p->payload = httpbuf2;
2915  p->payload_len = httplen2;
2916  tcph.th_seq = htonl(100);
2917  tcph.th_ack = htonl(13);
2918  s = &ssn.server;
2919  ssn.server.last_ack = 13;
2920  SCLogDebug("6 -- start");
2921  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2922 
2924  p->payload = httpbuf5;
2925  p->payload_len = httplen5;
2926  tcph.th_seq = htonl(13);
2927  tcph.th_ack = htonl(145);
2928  s = &ssn.client;
2929  ssn.client.last_ack = 145;
2930  SCLogDebug("7 -- start");
2931  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2932 
2934  p->payload = httpbuf2;
2935  p->payload_len = httplen2;
2936  tcph.th_seq = htonl(145);
2937  tcph.th_ack = htonl(16);
2938  s = &ssn.server;
2939  ssn.server.last_ack = 16;
2940  SCLogDebug("8 -- start");
2941  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2942  FAIL_IF(f->alproto != ALPROTO_HTTP);
2943 
2947  SCFree(p);
2948  UTHFreeFlow(f);
2949  PASS;
2950 }
2951 
2952 /** \test Test the memcap incrementing/decrementing and memcap check */
2953 static int StreamTcpReassembleTest44(void)
2954 {
2956  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2958  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2960  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2962  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2964  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2965  PASS;
2966 }
2967 
2968 /**
2969  * \test Test to make sure that reassembly_depth is enforced.
2970  *
2971  * \retval On success it returns 1 and on failure 0.
2972  */
2973 
2974 static int StreamTcpReassembleTest45 (void)
2975 {
2976  TcpReassemblyThreadCtx *ra_ctx = NULL;
2977  TcpSession ssn;
2978  ThreadVars tv;
2979  memset(&tv, 0, sizeof(tv));
2980  uint8_t payload[100] = {0};
2981  uint16_t payload_size = 100;
2982 
2983  StreamTcpUTInit(&ra_ctx);
2985 
2987  ssn.reassembly_depth = 100;
2988  StreamTcpUTSetupStream(&ssn.server, 100);
2989  StreamTcpUTSetupStream(&ssn.client, 100);
2990 
2991  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2992  FAIL_IF(r != 0);
2994 
2995  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2996  FAIL_IF(r != 0);
2998 
3002  StreamTcpUTDeinit(ra_ctx);
3003  PASS;
3004 }
3005 
3006 /**
3007  * \test Test the unlimited config value of reassembly depth.
3008  *
3009  * \retval On success it returns 1 and on failure 0.
3010  */
3011 
3012 static int StreamTcpReassembleTest46 (void)
3013 {
3014  int result = 0;
3015  TcpReassemblyThreadCtx *ra_ctx = NULL;
3016  TcpSession ssn;
3017  ThreadVars tv;
3018  memset(&tv, 0, sizeof(tv));
3019  uint8_t payload[100] = {0};
3020  uint16_t payload_size = 100;
3021 
3022  StreamTcpUTInit(&ra_ctx);
3024 
3026  StreamTcpUTSetupStream(&ssn.server, 100);
3027  StreamTcpUTSetupStream(&ssn.client, 100);
3028 
3029  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3030  if (r != 0)
3031  goto end;
3033  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3034  goto end;
3035  }
3036 
3037  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3038  if (r != 0)
3039  goto end;
3041  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3042  goto end;
3043  }
3044 
3045  result = 1;
3046 end:
3050  StreamTcpUTDeinit(ra_ctx);
3051  return result;
3052 }
3053 
3054 /**
3055  * \test Test to make sure we detect the sequence wrap around and continue
3056  * stream reassembly properly.
3057  *
3058  * \retval On success it returns 1 and on failure 0.
3059  */
3060 
3061 static int StreamTcpReassembleTest47 (void)
3062 {
3063  Packet *p = PacketGetFromAlloc();
3064  FAIL_IF(unlikely(p == NULL));
3065  Flow *f = NULL;
3066  TCPHdr tcph;
3067  TcpSession ssn;
3068  ThreadVars tv;
3069  PacketQueueNoLock pq;
3070  memset(&pq,0,sizeof(PacketQueueNoLock));
3071  memset(&tcph, 0, sizeof (TCPHdr));
3072  memset(&tv, 0, sizeof (ThreadVars));
3076 
3077  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3078  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3079 
3080  SET_ISN(&ssn.server, 572799781UL);
3081  ssn.server.last_ack = 572799782UL;
3082 
3083  SET_ISN(&ssn.client, 4294967289UL);
3084  ssn.client.last_ack = 21;
3085 
3086  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3087  FAIL_IF(f == NULL);
3088  f->protoctx = &ssn;
3089  f->proto = IPPROTO_TCP;
3090  p->flow = f;
3091 
3092  tcph.th_win = htons(5480);
3093  ssn.state = TCP_ESTABLISHED;
3094  TcpStream *s = NULL;
3095  uint8_t cnt = 0;
3096 
3097  for (cnt=0; cnt < httplen1; cnt++) {
3098  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3099  tcph.th_ack = htonl(572799782UL);
3100  tcph.th_flags = TH_ACK|TH_PUSH;
3101  p->tcph = &tcph;
3103  p->payload = &httpbuf1[cnt];
3104  p->payload_len = 1;
3105  s = &ssn.client;
3106 
3107  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3108 
3110  p->payload = NULL;
3111  p->payload_len = 0;
3112  tcph.th_seq = htonl(572799782UL);
3113  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3114  tcph.th_flags = TH_ACK;
3115  p->tcph = &tcph;
3116  s = &ssn.server;
3117 
3118  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3119  }
3120 
3121  FAIL_IF(f->alproto != ALPROTO_HTTP);
3122 
3126  SCFree(p);
3127  UTHFreeFlow(f);
3128  PASS;
3129 }
3130 
3131 /** \test 3 in order segments in inline reassembly */
3132 static int StreamTcpReassembleInlineTest01(void)
3133 {
3134  int ret = 0;
3135  TcpReassemblyThreadCtx *ra_ctx = NULL;
3136  ThreadVars tv;
3137  TcpSession ssn;
3138  Flow f;
3139 
3140  memset(&tv, 0x00, sizeof(tv));
3141 
3142  StreamTcpUTInit(&ra_ctx);
3145  StreamTcpUTSetupStream(&ssn.client, 1);
3146  FLOW_INITIALIZE(&f);
3147 
3148  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3149  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3150  if (p == NULL) {
3151  printf("couldn't get a packet: ");
3152  goto end;
3153  }
3154  p->tcph->th_seq = htonl(12);
3155  p->flow = &f;
3156 
3157  FLOWLOCK_WRLOCK(&f);
3158  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3159  printf("failed to add segment 1: ");
3160  goto end;
3161  }
3162  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3163  printf("failed to add segment 2: ");
3164  goto end;
3165  }
3166  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3167  printf("failed to add segment 3: ");
3168  goto end;
3169  }
3170  ssn.client.next_seq = 17;
3171  ret = 1;
3172 end:
3173  FLOWLOCK_UNLOCK(&f);
3174  FLOW_DESTROY(&f);
3175  UTHFreePacket(p);
3177  StreamTcpUTDeinit(ra_ctx);
3178  return ret;
3179 }
3180 
3181 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3182  * test the sliding window reassembly.
3183  */
3184 static int StreamTcpReassembleInlineTest02(void)
3185 {
3186  int ret = 0;
3187  TcpReassemblyThreadCtx *ra_ctx = NULL;
3188  ThreadVars tv;
3189  TcpSession ssn;
3190  Flow f;
3191 
3192  memset(&tv, 0x00, sizeof(tv));
3193 
3194  StreamTcpUTInit(&ra_ctx);
3197  StreamTcpUTSetupStream(&ssn.client, 1);
3198  FLOW_INITIALIZE(&f);
3199 
3200  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3201  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3202  if (p == NULL) {
3203  printf("couldn't get a packet: ");
3204  goto end;
3205  }
3206  p->tcph->th_seq = htonl(12);
3207  p->flow = &f;
3208 
3209  FLOWLOCK_WRLOCK(&f);
3210  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3211  printf("failed to add segment 1: ");
3212  goto end;
3213  }
3214  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3215  printf("failed to add segment 2: ");
3216  goto end;
3217  }
3218  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3219  printf("failed to add segment 3: ");
3220  goto end;
3221  }
3222  ssn.client.next_seq = 17;
3223  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3224  printf("failed to add segment 4: ");
3225  goto end;
3226  }
3227  ssn.client.next_seq = 22;
3228  ret = 1;
3229 end:
3230  FLOWLOCK_UNLOCK(&f);
3231  FLOW_DESTROY(&f);
3232  UTHFreePacket(p);
3234  StreamTcpUTDeinit(ra_ctx);
3235  return ret;
3236 }
3237 
3238 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3239  * test the sliding window reassembly with a small window size so that we
3240  * cutting off at the start (left edge)
3241  */
3242 static int StreamTcpReassembleInlineTest03(void)
3243 {
3244  int ret = 0;
3245  TcpReassemblyThreadCtx *ra_ctx = NULL;
3246  ThreadVars tv;
3247  TcpSession ssn;
3248  Flow f;
3249 
3250  memset(&tv, 0x00, sizeof(tv));
3251 
3252  StreamTcpUTInit(&ra_ctx);
3255  StreamTcpUTSetupStream(&ssn.client, 1);
3256  FLOW_INITIALIZE(&f);
3257 
3259 
3260  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3261  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3262  if (p == NULL) {
3263  printf("couldn't get a packet: ");
3264  goto end;
3265  }
3266  p->tcph->th_seq = htonl(12);
3267  p->flow = &f;
3269 
3270  FLOWLOCK_WRLOCK(&f);
3271  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3272  printf("failed to add segment 1: ");
3273  goto end;
3274  }
3275  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3276  printf("failed to add segment 2: ");
3277  goto end;
3278  }
3279  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3280  printf("failed to add segment 3: ");
3281  goto end;
3282  }
3283  ssn.client.next_seq = 17;
3284  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3285  printf("failed to add segment 4: ");
3286  goto end;
3287  }
3288  ssn.client.next_seq = 22;
3289 
3290  p->tcph->th_seq = htonl(17);
3291  ret = 1;
3292 end:
3293  FLOWLOCK_UNLOCK(&f);
3294  FLOW_DESTROY(&f);
3295  UTHFreePacket(p);
3297  StreamTcpUTDeinit(ra_ctx);
3298  return ret;
3299 }
3300 
3301 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3302  * test the sliding window reassembly with a small window size so that we
3303  * cutting off at the start (left edge) with small packet overlap.
3304  */
3305 static int StreamTcpReassembleInlineTest04(void)
3306 {
3307  int ret = 0;
3308  TcpReassemblyThreadCtx *ra_ctx = NULL;
3309  ThreadVars tv;
3310  TcpSession ssn;
3311  Flow f;
3312 
3313  memset(&tv, 0x00, sizeof(tv));
3314 
3315  StreamTcpUTInit(&ra_ctx);
3318  StreamTcpUTSetupStream(&ssn.client, 1);
3319  FLOW_INITIALIZE(&f);
3320 
3322 
3323  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3324  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3325  if (p == NULL) {
3326  printf("couldn't get a packet: ");
3327  goto end;
3328  }
3329  p->tcph->th_seq = htonl(12);
3330  p->flow = &f;
3332 
3333  FLOWLOCK_WRLOCK(&f);
3334  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3335  printf("failed to add segment 1: ");
3336  goto end;
3337  }
3338  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3339  printf("failed to add segment 2: ");
3340  goto end;
3341  }
3342  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3343  printf("failed to add segment 3: ");
3344  goto end;
3345  }
3346  ssn.client.next_seq = 17;
3347  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3348  printf("failed to add segment 4: ");
3349  goto end;
3350  }
3351  ssn.client.next_seq = 22;
3352 
3353  p->tcph->th_seq = htonl(17);
3354  ret = 1;
3355 end:
3356  FLOWLOCK_UNLOCK(&f);
3357  FLOW_DESTROY(&f);
3358  UTHFreePacket(p);
3360  StreamTcpUTDeinit(ra_ctx);
3361  return ret;
3362 }
3363 
3364 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3365  * test the sliding window reassembly with a small window size so that we
3366  * cutting off at the start (left edge). Test if the first segment is
3367  * removed from the list.
3368  */
3369 static int StreamTcpReassembleInlineTest08(void)
3370 {
3371  TcpReassemblyThreadCtx *ra_ctx = NULL;
3372  ThreadVars tv;
3373  memset(&tv, 0x00, sizeof(tv));
3374  TcpSession ssn;
3375  Flow f;
3376  StreamTcpUTInit(&ra_ctx);
3379  StreamTcpUTSetupStream(&ssn.client, 1);
3380  FLOW_INITIALIZE(&f);
3381 
3384  f.protoctx = &ssn;
3385 
3386  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3387  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3388  FAIL_IF(p == NULL);
3389  p->tcph->th_seq = htonl(12);
3390  p->flow = &f;
3392 
3393  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3394  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3395  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3396  ssn.client.next_seq = 17;
3397  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3398  ssn.client.next_seq = 22;
3399  p->tcph->th_seq = htonl(17);
3401 
3402  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3403  FAIL_IF_NULL(seg);
3404  FAIL_IF_NOT(seg->seq == 2);
3405 
3406  FLOW_DESTROY(&f);
3407  UTHFreePacket(p);
3409  StreamTcpUTDeinit(ra_ctx);
3410  PASS;
3411 }
3412 
3413 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3414  * test the sliding window reassembly with a small window size so that we
3415  * cutting off at the start (left edge). Test if the first segment is
3416  * removed from the list.
3417  */
3418 static int StreamTcpReassembleInlineTest09(void)
3419 {
3420  int ret = 0;
3421  TcpReassemblyThreadCtx *ra_ctx = NULL;
3422  ThreadVars tv;
3423  TcpSession ssn;
3424  Flow f;
3425 
3426  memset(&tv, 0x00, sizeof(tv));
3427 
3428  StreamTcpUTInit(&ra_ctx);
3431  StreamTcpUTSetupStream(&ssn.client, 1);
3432  FLOW_INITIALIZE(&f);
3433 
3436 
3437  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3438  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3439  if (p == NULL) {
3440  printf("couldn't get a packet: ");
3441  goto end;
3442  }
3443  p->tcph->th_seq = htonl(17);
3444  p->flow = &f;
3446 
3447  FLOWLOCK_WRLOCK(&f);
3448  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3449  printf("failed to add segment 1: ");
3450  goto end;
3451  }
3452  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3453  printf("failed to add segment 2: ");
3454  goto end;
3455  }
3456  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3457  printf("failed to add segment 3: ");
3458  goto end;
3459  }
3460  ssn.client.next_seq = 12;
3461  ssn.client.last_ack = 10;
3462 
3463  /* close the GAP and see if we properly reassemble and update base_seq */
3464  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3465  printf("failed to add segment 4: ");
3466  goto end;
3467  }
3468  ssn.client.next_seq = 22;
3469 
3470  p->tcph->th_seq = htonl(12);
3471 
3472  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3473  FAIL_IF_NULL(seg);
3474  FAIL_IF_NOT(seg->seq == 2);
3475 
3476  ret = 1;
3477 end:
3478  FLOWLOCK_UNLOCK(&f);
3479  FLOW_DESTROY(&f);
3480  UTHFreePacket(p);
3482  StreamTcpUTDeinit(ra_ctx);
3483  return ret;
3484 }
3485 
3486 /** \test App Layer reassembly.
3487  */
3488 static int StreamTcpReassembleInlineTest10(void)
3489 {
3490  int ret = 0;
3491  TcpReassemblyThreadCtx *ra_ctx = NULL;
3492  ThreadVars tv;
3493  TcpSession ssn;
3494  Flow *f = NULL;
3495  Packet *p = NULL;
3496 
3497  memset(&tv, 0x00, sizeof(tv));
3498 
3499  StreamTcpUTInit(&ra_ctx);
3502  StreamTcpUTSetupStream(&ssn.server, 1);
3503  ssn.server.last_ack = 2;
3504  StreamTcpUTSetupStream(&ssn.client, 1);
3505  ssn.client.last_ack = 2;
3507 
3508  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3509  if (f == NULL)
3510  goto end;
3511  f->protoctx = &ssn;
3512  f->proto = IPPROTO_TCP;
3513 
3514  uint8_t stream_payload1[] = "GE";
3515  uint8_t stream_payload2[] = "T /";
3516  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3517 
3518  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3519  if (p == NULL) {
3520  printf("couldn't get a packet: ");
3521  goto end;
3522  }
3523  p->tcph->th_seq = htonl(7);
3524  p->flow = f;
3526 
3527  FLOWLOCK_WRLOCK(f);
3528  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3529  printf("failed to add segment 1: ");
3530  goto end;
3531  }
3532  ssn.client.next_seq = 4;
3533 
3534  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3535  if (r < 0) {
3536  printf("StreamTcpReassembleAppLayer failed: ");
3537  goto end;
3538  }
3539 
3540  /* ssn.server.ra_app_base_seq should be isn here. */
3541  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3542  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3543  goto end;
3544  }
3545 
3546  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3547  printf("failed to add segment 2: ");
3548  goto end;
3549  }
3550  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3551  printf("failed to add segment 3: ");
3552  goto end;
3553  }
3554  ssn.client.next_seq = 19;
3555 
3556  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3557  if (r < 0) {
3558  printf("StreamTcpReassembleAppLayer failed: ");
3559  goto end;
3560  }
3561 
3562  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3563 
3564  ret = 1;
3565 end:
3566  UTHFreePacket(p);
3568  StreamTcpUTDeinit(ra_ctx);
3569  FLOWLOCK_UNLOCK(f);
3570  UTHFreeFlow(f);
3571  return ret;
3572 }
3573 
3574 /** \test test insert with overlap
3575  */
3576 static int StreamTcpReassembleInsertTest01(void)
3577 {
3578  TcpReassemblyThreadCtx *ra_ctx = NULL;
3579  ThreadVars tv;
3580  TcpSession ssn;
3581  Flow f;
3582 
3583  memset(&tv, 0x00, sizeof(tv));
3584 
3585  StreamTcpUTInit(&ra_ctx);
3587  StreamTcpUTSetupStream(&ssn.client, 1);
3589  FLOW_INITIALIZE(&f);
3590 
3591  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3592  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3593  FAIL_IF(p == NULL);
3594  p->tcph->th_seq = htonl(12);
3595  p->flow = &f;
3596 
3597  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3598  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3599  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3600  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3601  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3602  ssn.client.next_seq = 21;
3603 
3604  FLOW_DESTROY(&f);
3605  UTHFreePacket(p);
3607  StreamTcpUTDeinit(ra_ctx);
3608  PASS;
3609 }
3610 
3611 /** \test test insert with overlaps
3612  */
3613 static int StreamTcpReassembleInsertTest02(void)
3614 {
3615  int ret = 0;
3616  TcpReassemblyThreadCtx *ra_ctx = NULL;
3617  ThreadVars tv;
3618  TcpSession ssn;
3619 
3620  memset(&tv, 0x00, sizeof(tv));
3621 
3622  StreamTcpUTInit(&ra_ctx);
3624  StreamTcpUTSetupStream(&ssn.client, 1);
3625 
3626  int i;
3627  for (i = 2; i < 10; i++) {
3628  int len;
3629  len = i % 2;
3630  if (len == 0)
3631  len = 1;
3632  int seq;
3633  seq = i * 10;
3634  if (seq < 2)
3635  seq = 2;
3636 
3637  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3638  printf("failed to add segment 1: ");
3639  goto end;
3640  }
3641  }
3642  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3643  printf("failed to add segment 2: ");
3644  goto end;
3645  }
3646 
3647  ret = 1;
3648 end:
3650  StreamTcpUTDeinit(ra_ctx);
3651  return ret;
3652 }
3653 
3654 /** \test test insert with overlaps
3655  */
3656 static int StreamTcpReassembleInsertTest03(void)
3657 {
3658  int ret = 0;
3659  TcpReassemblyThreadCtx *ra_ctx = NULL;
3660  ThreadVars tv;
3661  TcpSession ssn;
3662 
3663  memset(&tv, 0x00, sizeof(tv));
3664 
3665  StreamTcpUTInit(&ra_ctx);
3667  StreamTcpUTSetupStream(&ssn.client, 1);
3668 
3669  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3670  printf("failed to add segment 2: ");
3671  goto end;
3672  }
3673 
3674  int i;
3675  for (i = 2; i < 10; i++) {
3676  int len;
3677  len = i % 2;
3678  if (len == 0)
3679  len = 1;
3680  int seq;
3681  seq = i * 10;
3682  if (seq < 2)
3683  seq = 2;
3684 
3685  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3686  printf("failed to add segment 2: ");
3687  goto end;
3688  }
3689  }
3690  ret = 1;
3691 end:
3693  StreamTcpUTDeinit(ra_ctx);
3694  return ret;
3695 }
3696 
3698 #endif /* UNITTESTS */
3699 
3700 /** \brief The Function Register the Unit tests to test the reassembly engine
3701  * for various OS policies.
3702  */
3703 
3705 {
3706 #ifdef UNITTESTS
3707  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3708  StreamTcpReassembleTest25);
3709  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3710  StreamTcpReassembleTest26);
3711  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3712  StreamTcpReassembleTest27);
3713  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3714  StreamTcpReassembleTest28);
3715  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3716  StreamTcpReassembleTest29);
3717  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3718  StreamTcpReassembleTest33);
3719  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3720  StreamTcpReassembleTest34);
3721  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3722  StreamTcpReassembleTest37);
3723  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3724  StreamTcpReassembleTest39);
3725  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3726  StreamTcpReassembleTest40);
3727  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3728  StreamTcpReassembleTest44);
3729  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3730  StreamTcpReassembleTest45);
3731  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3732  StreamTcpReassembleTest46);
3733  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3734  StreamTcpReassembleTest47);
3735 
3736  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3737  StreamTcpReassembleInlineTest01);
3738  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3739  StreamTcpReassembleInlineTest02);
3740  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3741  StreamTcpReassembleInlineTest03);
3742  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3743  StreamTcpReassembleInlineTest04);
3744  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3745  StreamTcpReassembleInlineTest08);
3746  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3747  StreamTcpReassembleInlineTest09);
3748 
3749  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3750  StreamTcpReassembleInlineTest10);
3751 
3752  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3753  StreamTcpReassembleInsertTest01);
3754  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3755  StreamTcpReassembleInsertTest02);
3756  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3757  StreamTcpReassembleInsertTest03);
3758 
3762  StreamTcpReassembleRawRegisterTests();
3763 #endif /* UNITTESTS */
3764 }
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:250
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:57
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:55
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:434
TcpStream_
Definition: stream-tcp-private.h:94
len
uint8_t len
Definition: app-layer-dnp3.h:2
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:149
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:101
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:101
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:1907
StreamingBufferConfig_::Malloc
void *(* Malloc)(size_t size)
Definition: util-streaming-buffer.h:71
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:169
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:302
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:92
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:70
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1753
STREAM_MIDSTREAM
#define STREAM_MIDSTREAM
Definition: stream.h:35
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:95
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:82
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:303
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:72
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:104
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:516
stream-tcp.h
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:375
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:491
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
Definition: stream-tcp-reassemble.c:175
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:557
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:123
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:1971
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:568
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:997
StreamTcpThread_
Definition: stream-tcp.h:70
Flow_::proto
uint8_t proto
Definition: flow.h:365
Packet_::payload
uint8_t * payload
Definition: decode.h:547
StreamTcpInlineMode
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6244
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:1951
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:61
Packet_::flags
uint32_t flags
Definition: decode.h:447
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:175
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:98
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:60
Flow_
Flow data structure.
Definition: flow.h:347
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:66
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:68
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:1934
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:321
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:144
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:38
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1349
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:849
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:219
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:143
StreamNeedsReassembly
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:787
MIN
#define MIN(x, y)
Definition: suricata-common.h:377
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:64
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:95
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
StreamingBufferConfig_::flags
uint32_t flags
Definition: util-streaming-buffer.h:68
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:150
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:866
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:443
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:119
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:41
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:242
Flow_::protoctx
void * protoctx
Definition: flow.h:441
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:711
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:95
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:548
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:32
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:205
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:574
util-unittest.h
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2043
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
TcpSession_::flags
uint16_t flags
Definition: stream-tcp-private.h:270
FLOWLOCK_UNLOCK
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:264
STREAM_START
#define STREAM_START
Definition: stream.h:29
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:158
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:269
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:103
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:475
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:434
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:521
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1192
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:271
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:353
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:572
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:261
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:92
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:270
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1101
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1105
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:131
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
FLOWLOCK_WRLOCK
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:261
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:118
StreamReassembleLog
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
Definition: stream-tcp-reassemble.c:1781
util-print.h
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:50
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:832
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:174
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:4748
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1305
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:263
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:87
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:1985
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:801
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
Definition: stream-tcp-reassemble.c:146
TRUE
#define TRUE
Definition: suricata-common.h:33
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueueNoLock *pq)
Definition: stream-tcp-reassemble.c:1809
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:282
TCP_GET_SEQ
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:112
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:330
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
FALSE
#define FALSE
Definition: suricata-common.h:34
SCReturn
#define SCReturn
Definition: util-debug.h:302
stream.h
TcpSegment
Definition: stream-tcp-private.h:61
StreamTcpReassembleFree
void StreamTcpReassembleFree(char quiet)
Definition: stream-tcp-reassemble.c:415
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:279
Packet_
Definition: decode.h:412
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:216
STREAM_DEPTH
#define STREAM_DEPTH
Definition: stream.h:34
StreamTcpFreeConfig
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:668
stream-tcp-private.h
PoolThreadReturn
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
Definition: util-pool-thread.c:187
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:42
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:306
conf.h
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:277
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:889
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:105
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:475
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
STREAM_GAP
#define STREAM_GAP
Definition: stream.h:33
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:315
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:220
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:264
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
StreamTcpReassembleInit
int StreamTcpReassembleInit(char quiet)
Definition: stream-tcp-reassemble.c:399
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:70
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:115
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:79
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:526
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:56
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:103
StreamingBuffer_
Definition: util-streaming-buffer.h:95
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:598
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:63
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:150
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:769
StringParseUint32
int StringParseUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:313
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:168
Packet_::flow
struct Flow_ * flow
Definition: decode.h:449
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:242
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:337
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:241
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:529
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:112
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:89
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2024
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:122
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:218
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:273
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:149
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:102
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:144
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:272
StreamTcpReassembleIncrMemuse
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:98
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:451
ConfNode_
Definition: conf.h:32
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:485
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:854
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:239
Flow_::flags
uint32_t flags
Definition: flow.h:421
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:766
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:74
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:50
ALPROTO_HTTP
@ ALPROTO_HTTP
Definition: app-layer-protos.h:30
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:132
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:283
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2038
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:146
StreamTcpInitConfig
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
STREAM_EOF
#define STREAM_EOF
Definition: stream.h:30
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:60
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:57
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:168
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:188
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:201
suricata.h
Address_::family
char family
Definition: decode.h:115
Packet_::dst
Address dst
Definition: decode.h:417
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:6259
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:261
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:240
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint8_t flag)
Definition: app-layer-parser.c:1606
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:220
STREAMING_BUFFER_NOFLAGS
#define STREAMING_BUFFER_NOFLAGS
Definition: util-streaming-buffer.h:64
STREAMTCP_STREAM_FLAG_GAP
#define STREAMTCP_STREAM_FLAG_GAP
Definition: stream-tcp-private.h:199
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3704
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:90
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:171
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:132
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:212
StreamTcpReassembleDecrMemuse
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:111
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:364
TcpSession_
Definition: stream-tcp-private.h:261
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:266
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:160
flow.h
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:452
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:207
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:143
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:260
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:85
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:83
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:66
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:743
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:82
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:130
Packet_::src
Address src
Definition: decode.h:416
debug.h
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:73
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:214
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:133
app-layer.h
StreamTcpCheckStreamContents
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
Definition: stream-tcp-reassemble.c:2002