suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 
64 #include "detect-engine-state.h"
65 
66 #include "util-profiling.h"
67 #include "util-validate.h"
68 
69 #ifdef DEBUG
70 static SCMutex segment_pool_memuse_mutex;
71 static uint64_t segment_pool_memuse = 0;
72 static uint64_t segment_pool_memcnt = 0;
73 #endif
74 
75 static PoolThread *segment_thread_pool = NULL;
76 /* init only, protect initializing and growing pool */
77 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
78 
79 /* Memory use counter */
80 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
81 
82 /* prototypes */
84 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346 }
347 
348 /** \param f locked flow */
350 {
351  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
352  return 0;
353 
354  TcpSession *ssn = (TcpSession *)f->protoctx;
356 }
357 
358 static int StreamTcpReassemblyConfig(char quiet)
359 {
360  uint32_t segment_prealloc = 2048;
361  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
362  if (seg) {
363  uint32_t prealloc = 0;
364  if (ByteExtractStringUint32(&prealloc, 10, strlen(seg->val), seg->val) == -1)
365  {
366  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
367  "%s is invalid", seg->val);
368  return -1;
369  }
370  segment_prealloc = prealloc;
371  }
372  if (!quiet)
373  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
374  stream_config.prealloc_segments = segment_prealloc;
375 
376  int overlap_diff_data = 0;
377  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
378  if (overlap_diff_data) {
380  }
381  if (StreamTcpInlineMode() == TRUE) {
383  }
384 
387  stream_config.sbcnf.Malloc = ReassembleMalloc;
388  stream_config.sbcnf.Calloc = ReassembleCalloc;
389  stream_config.sbcnf.Realloc = ReassembleRealloc;
390  stream_config.sbcnf.Free = ReassembleFree;
391 
392  return 0;
393 }
394 
395 int StreamTcpReassembleInit(char quiet)
396 {
397  /* init the memcap/use tracker */
399 
400  if (StreamTcpReassemblyConfig(quiet) < 0)
401  return -1;
402 
403 #ifdef DEBUG
404  SCMutexInit(&segment_pool_memuse_mutex, NULL);
405 #endif
406  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
408  return 0;
409 }
410 
411 void StreamTcpReassembleFree(char quiet)
412 {
413  SCMutexLock(&segment_thread_pool_mutex);
414  if (segment_thread_pool != NULL) {
415  PoolThreadFree(segment_thread_pool);
416  segment_thread_pool = NULL;
417  }
418  SCMutexUnlock(&segment_thread_pool_mutex);
419  SCMutexDestroy(&segment_thread_pool_mutex);
420 
421 #ifdef DEBUG
422  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
423  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
424  SCMutexDestroy(&segment_pool_memuse_mutex);
425 #endif
426 }
427 
429 {
430  SCEnter();
432  if (unlikely(ra_ctx == NULL))
433  return NULL;
434 
435  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
436 
437  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
438 
439  SCMutexLock(&segment_thread_pool_mutex);
440  if (segment_thread_pool == NULL) {
441  segment_thread_pool = PoolThreadInit(1, /* thread */
442  0, /* unlimited */
444  sizeof(TcpSegment),
445  TcpSegmentPoolAlloc,
446  TcpSegmentPoolInit, NULL,
447  TcpSegmentPoolCleanup, NULL);
448  ra_ctx->segment_thread_pool_id = 0;
449  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
450  PoolThreadSize(segment_thread_pool),
451  ra_ctx->segment_thread_pool_id);
452  } else {
453  /* grow segment_thread_pool until we have a element for our thread id */
454  ra_ctx->segment_thread_pool_id = PoolThreadGrow(segment_thread_pool,
455  0, /* unlimited */
457  sizeof(TcpSegment),
458  TcpSegmentPoolAlloc,
459  TcpSegmentPoolInit, NULL,
460  TcpSegmentPoolCleanup, NULL);
461  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
462  PoolThreadSize(segment_thread_pool),
463  ra_ctx->segment_thread_pool_id);
464  }
465  SCMutexUnlock(&segment_thread_pool_mutex);
466  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
467  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
469  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
470  }
471 
472  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
473 }
474 
476 {
477  SCEnter();
479  SCFree(ra_ctx);
480  SCReturn;
481 }
482 
483 /**
484  * \brief check if stream in pkt direction has depth reached
485  *
486  * \param p packet with *LOCKED* flow
487  *
488  * \retval 1 stream has depth reached
489  * \retval 0 stream does not have depth reached
490  */
492 {
493  if (p->flow != NULL && p->flow->protoctx != NULL) {
494  TcpSession *ssn = p->flow->protoctx;
495  TcpStream *stream;
496  if (p->flowflags & FLOW_PKT_TOSERVER) {
497  stream = &ssn->client;
498  } else {
499  stream = &ssn->server;
500  }
501 
502  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
503  }
504 
505  return 0;
506 }
507 
508 /**
509  * \internal
510  * \brief Function to Check the reassembly depth valuer against the
511  * allowed max depth of the stream reassmbly for TCP streams.
512  *
513  * \param stream stream direction
514  * \param seq sequence number where "size" starts
515  * \param size size of the segment that is added
516  *
517  * \retval size Part of the size that fits in the depth, 0 if none
518  */
519 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
520  uint32_t seq, uint32_t size)
521 {
522  SCEnter();
523 
524  /* if the configured depth value is 0, it means there is no limit on
525  reassembly depth. Otherwise carry on my boy ;) */
526  if (ssn->reassembly_depth == 0) {
527  SCReturnUInt(size);
528  }
529 
530  /* if the final flag is set, we're not accepting anymore */
532  SCReturnUInt(0);
533  }
534 
535  uint64_t seg_depth;
536  if (SEQ_GT(stream->base_seq, seq)) {
537  if (SEQ_LEQ(seq+size, stream->base_seq)) {
538  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
539  stream->base_seq, seq, seq+size);
540  SCReturnUInt(0);
541  }
542 
543  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
544  } else {
545  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
546  }
547 
548  /* if the base_seq has moved passed the depth window we stop
549  * checking and just reject the rest of the packets including
550  * retransmissions. Saves us the hassle of dealing with sequence
551  * wraps as well */
552  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
553  stream->base_seq, seg_depth,
554  ssn->reassembly_depth);
555 
556  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
557  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
559  SCReturnUInt(0);
560  }
561  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
562  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
563 #if 0
564  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
565  (stream->base_seq_offset + stream->base_seq + size),
566  (stream->isn + ssn->reassembly_depth));
567 #endif
568  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
569  /* packet (partly?) fits the depth window */
570 
571  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
572  /* complete fit */
573  SCReturnUInt(size);
574  } else {
576  /* partial fit, return only what fits */
577  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
578  DEBUG_VALIDATE_BUG_ON(part > size);
579  if (part > size)
580  part = size;
581  SCReturnUInt(part);
582  }
583  }
584 
585  SCReturnUInt(0);
586 }
587 
588 /**
589  * \brief Insert a packets TCP data into the stream reassembly engine.
590  *
591  * \retval 0 good segment, as far as we checked.
592  * \retval -1 badness, reason to drop in inline mode
593  *
594  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
595  * or it wasn't added because of reassembly depth.
596  *
597  */
599  TcpSession *ssn, TcpStream *stream, Packet *p)
600 {
601  SCEnter();
602 
603  if (ssn->data_first_seen_dir == 0) {
604  if (PKT_IS_TOSERVER(p)) {
606  } else {
608  }
609  }
610 
611  /* If the OS policy is not set then set the OS policy for this stream */
612  if (stream->os_policy == 0) {
613  StreamTcpSetOSPolicy(stream, p);
614  }
615 
618  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
619  SCReturnInt(0);
620  }
621 
622  /* If we have reached the defined depth for either of the stream, then stop
623  reassembling the TCP session */
624  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
625  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
626 
628  /* increment stream depth counter */
629  StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
630  }
631  if (size == 0) {
632  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
633  SCReturnInt(0);
634  }
635 
637  if (size > p->payload_len)
638  size = p->payload_len;
639 
640  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
641  if (seg == NULL) {
642  SCLogDebug("segment_pool is empty");
644  SCReturnInt(-1);
645  }
646 
647  TCP_SEG_LEN(seg) = size;
648  seg->seq = TCP_GET_SEQ(p);
649 
650  /* proto detection skipped, but now we do get data. Set event. */
651  if (RB_EMPTY(&stream->seg_tree) &&
653 
656  }
657 
658  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
659  SCLogDebug("StreamTcpReassembleInsertSegment failed");
660  SCReturnInt(-1);
661  }
662  SCReturnInt(0);
663 }
664 
665 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
666  Packet *p, enum StreamUpdateDir dir)
667 {
668  uint8_t flag = 0;
669 
671  flag |= STREAM_START;
672  }
673 
674  if (ssn->state == TCP_CLOSED) {
675  flag |= STREAM_EOF;
676  }
677 
678  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
679  flag |= STREAM_MIDSTREAM;
680  }
681 
682  if (p->flags & PKT_PSEUDO_STREAM_END) {
683  flag |= STREAM_EOF;
684  }
685 
686  if (dir == UPDATE_DIR_OPPOSING) {
687  if (p->flowflags & FLOW_PKT_TOSERVER) {
688  flag |= STREAM_TOCLIENT;
689  } else {
690  flag |= STREAM_TOSERVER;
691  }
692  } else {
693  if (p->flowflags & FLOW_PKT_TOSERVER) {
694  flag |= STREAM_TOSERVER;
695  } else {
696  flag |= STREAM_TOCLIENT;
697  }
698  }
699 
701  flag |= STREAM_DEPTH;
702  }
703  return flag;
704 }
705 
706 /**
707  * \brief Check the minimum size limits for reassembly.
708  *
709  * \retval 0 don't reassemble yet
710  * \retval 1 do reassemble
711  */
712 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
713  const TcpStream *stream, const Packet *p)
714 {
715  SCEnter();
716 
717  /* if any of these flags is set we always inspect immediately */
718 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
719  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
720  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
721  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
722 
725  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
726  "is set, so not expecting any new data segments");
727  }
729  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
730  }
732  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
733  "so no new segments will be considered");
734  }
735  SCReturnInt(1);
736  }
737 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
738 
739  /* some states mean we reassemble no matter how much data we have */
740  if (ssn->state > TCP_TIME_WAIT)
741  SCReturnInt(1);
742 
743  if (p->flags & PKT_PSEUDO_STREAM_END)
744  SCReturnInt(1);
745 
746  /* check if we have enough data to do raw reassembly */
747  if (PKT_IS_TOSERVER(p)) {
748  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
749  uint32_t delta = stream->last_ack - stream->base_seq;
750  /* get max absolute offset */
751  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
752 
753  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
754  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
755  SCReturnInt(1);
756  } else {
757  SCLogDebug("toserver min chunk len not yet reached: "
758  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
759  "%"PRIu32"", stream->last_ack, stream->base_seq,
760  (stream->last_ack - stream->base_seq),
762  SCReturnInt(0);
763  }
764  }
765  } else {
766  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
767  uint32_t delta = stream->last_ack - stream->base_seq;
768  /* get max absolute offset */
769  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
770 
771  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
772 
773  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
774  SCReturnInt(1);
775  } else {
776  SCLogDebug("toclient min chunk len not yet reached: "
777  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
778  "%"PRIu32"", stream->last_ack, stream->base_seq,
779  (stream->last_ack - stream->base_seq),
781  SCReturnInt(0);
782  }
783  }
784  }
785 
786  SCReturnInt(0);
787 }
788 
789 /**
790  * \brief see what if any work the TCP session still needs
791  */
792 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
793 {
794  const TcpStream *stream = NULL;
795 #ifdef DEBUG
796  const char *dirstr = NULL;
797 #endif
798  if (direction == STREAM_TOSERVER) {
799  stream = &ssn->client;
800 #ifdef DEBUG
801  dirstr = "client";
802 #endif
803  } else {
804  stream = &ssn->server;
805 #ifdef DEBUG
806  dirstr = "server";
807 #endif
808  }
809 
810  int use_app = 1;
811  int use_raw = 1;
812 
814  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
815  {
816  // app is dead
817  use_app = 0;
818  }
819 
821  // raw is dead
822  use_raw = 0;
823  }
824 
825  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
826 
827  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
828  dirstr,
829  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
830  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
831  right_edge);
832  if (use_raw) {
833  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
834  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
836  }
837  }
838  if (use_app) {
839  if (right_edge > STREAM_APP_PROGRESS(stream)) {
840  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
842  }
843  }
844 
845  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
847 }
848 
849 #ifdef DEBUG
850 static uint64_t GetStreamSize(TcpStream *stream)
851 {
852  if (stream) {
853  uint64_t size = 0;
854  uint32_t cnt = 0;
855 
856  TcpSegment *seg;
857  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
858  cnt++;
859  size += (uint64_t)TCP_SEG_LEN(seg);
860  }
861 
862  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
863  return size;
864  }
865  return (uint64_t)0;
866 }
867 
868 static void GetSessionSize(TcpSession *ssn, Packet *p)
869 {
870  uint64_t size = 0;
871  if (ssn) {
872  size = GetStreamSize(&ssn->client);
873  size += GetStreamSize(&ssn->server);
874 
875  //if (size > 900000)
876  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
877  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
878  }
879 }
880 #endif
881 
882 /** \internal
883  *
884  * Get buffer, or first part of the buffer if data gaps exist.
885  *
886  * \brief get stream data from offset
887  * \param offset stream offset */
888 static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset)
889 {
890  const uint8_t *mydata;
891  uint32_t mydata_len;
892 
893  if (RB_EMPTY(&stream->sb.sbb_tree)) {
894  SCLogDebug("getting one blob");
895 
896  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
897 
898  *data = mydata;
899  *data_len = mydata_len;
900  } else {
901  StreamingBufferBlock *blk = stream->sb.head;
902 
903  if (blk->offset > offset) {
904  SCLogDebug("gap, want data at offset %"PRIu64", "
905  "got data at %"PRIu64". GAP of size %"PRIu64,
906  offset, blk->offset, blk->offset - offset);
907  *data = NULL;
908  *data_len = blk->offset - offset;
909 
910  } else if (offset >= (blk->offset + blk->len)) {
911 
912  *data = NULL;
913  StreamingBufferBlock *nblk = SBB_RB_NEXT(blk);
914  *data_len = nblk ? nblk->offset - offset : 0;
915  if (nblk) {
916  SCLogDebug("gap, want data at offset %"PRIu64", "
917  "got data at %"PRIu64". GAP of size %"PRIu64,
918  offset, nblk->offset, nblk->offset - offset);
919  }
920 
921  } else if (offset > blk->offset && offset < (blk->offset + blk->len)) {
922  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
923  offset, blk->offset, blk->len);
924  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
925  SCLogDebug("data %p, data_len %u", *data, *data_len);
926  } else {
927  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
928  }
929  }
930 }
931 
932 /** \internal
933  * \brief check to see if we should declare a GAP
934  * Call this when the app layer didn't get data at the requested
935  * offset.
936  */
937 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
938 {
939  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
940  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
941 
942  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
943  /* get window of data that is acked */
944  const uint32_t delta = stream->last_ack - stream->base_seq;
945  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
946  /* get max absolute offset */
947  last_ack_abs += delta;
948 
949  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
950  last_ack_abs -= ackadded;
951 
952  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
953  SCLogDebug("next_seq %u", stream->next_seq);
954 
955  /* if last_ack_abs is beyond the app_progress data that we haven't seen
956  * has been ack'd. This looks like a GAP. */
957  if (last_ack_abs > app_progress) {
958  /* however, we can accept ACKs a bit too liberally. If last_ack
959  * is beyond next_seq, we only consider it a gap now if we do
960  * already have data beyond the gap. */
961  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
962  if (RB_EMPTY(&stream->sb.sbb_tree)) {
963  SCLogDebug("packet %"PRIu64": no GAP. "
964  "next_seq %u < last_ack %u, but no data in list",
965  p->pcap_cnt, stream->next_seq, stream->last_ack);
966  return false;
967  } else {
968  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
969  const StreamingBufferBlock *blk = stream->sb.head;
970  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
971  /* ack'd data after the gap */
972  SCLogDebug("packet %"PRIu64": GAP. "
973  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
974  p->pcap_cnt, stream->next_seq, stream->last_ack);
975  return true;
976  }
977  }
978  }
979 
980  SCLogDebug("packet %"PRIu64": GAP! "
981  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
982  "but we have no data.",
983  p->pcap_cnt, last_ack_abs, app_progress);
984  return true;
985  }
986  }
987  SCLogDebug("packet %"PRIu64": no GAP. "
988  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
989  p->pcap_cnt, last_ack_abs, app_progress);
990  return false;
991 }
992 
993 /** \internal
994  * \brief get stream buffer and update the app-layer
995  * \retval 0 success
996  */
997 static int ReassembleUpdateAppLayer (ThreadVars *tv,
998  TcpReassemblyThreadCtx *ra_ctx,
999  TcpSession *ssn, TcpStream *stream,
1000  Packet *p, enum StreamUpdateDir dir)
1001 {
1002  uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1003 
1004  SCLogDebug("app progress %"PRIu64, app_progress);
1005  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1006 
1007  const uint8_t *mydata;
1008  uint32_t mydata_len;
1009 
1010  while (1) {
1011  GetAppBuffer(stream, &mydata, &mydata_len, app_progress);
1012  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, stream, p)) {
1013  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1014 
1015  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1016  NULL, mydata_len,
1017  StreamGetAppLayerFlags(ssn, stream, p, dir)|STREAM_GAP);
1018  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1019 
1021  StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
1022 
1023  stream->app_progress_rel += mydata_len;
1024  app_progress += mydata_len;
1025  if (r < 0)
1026  break;
1027 
1028  continue;
1029  } else if (mydata == NULL || mydata_len == 0) {
1030  /* Possibly a gap, but no new data. */
1031  return 0;
1032  }
1033  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1034  break;
1035  }
1036 
1037  //PrintRawDataFp(stdout, mydata, mydata_len);
1038 
1039  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1040  stream, &stream->sb, mydata_len, app_progress);
1041 
1042  /* get window of data that is acked */
1043  if (StreamTcpInlineMode() == FALSE) {
1044  if (p->flags & PKT_PSEUDO_STREAM_END) {
1045  // fall through, we use all available data
1046  } else {
1047  uint64_t last_ack_abs = app_progress; /* absolute right edge of ack'd data */
1048  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1049  /* get window of data that is acked */
1050  uint32_t delta = stream->last_ack - stream->base_seq;
1051  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1052  /* get max absolute offset */
1053  last_ack_abs += delta;
1054  }
1055 
1056  /* see if the buffer contains unack'd data as well */
1057  if (app_progress + mydata_len > last_ack_abs) {
1058  uint32_t check = mydata_len;
1059  mydata_len = last_ack_abs - app_progress;
1060  BUG_ON(mydata_len > check);
1061  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1062  "data is considered", mydata_len);
1063  }
1064  }
1065  }
1066 
1067  /* update the app-layer */
1068  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1069  (uint8_t *)mydata, mydata_len,
1070  StreamGetAppLayerFlags(ssn, stream, p, dir));
1071  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1072 
1073  /* see if we can update the progress */
1074  if (r == 0 && mydata_len > 0 &&
1076  {
1077  SCLogDebug("app progress %"PRIu64" increasing with data len %u to %"PRIu64,
1078  app_progress, mydata_len, app_progress + mydata_len);
1079 
1080  stream->app_progress_rel += mydata_len;
1081  SCLogDebug("app progress now %"PRIu64, STREAM_APP_PROGRESS(stream));
1082  } else {
1083  SCLogDebug("NOT UPDATED app progress still %"PRIu64, app_progress);
1084  }
1085 
1086  SCReturnInt(0);
1087 }
1088 
1089 /**
1090  * \brief Update the stream reassembly upon receiving a packet.
1091  *
1092  * For IDS mode, the stream is in the opposite direction of the packet,
1093  * as the ACK-packet is ACK'ing the stream.
1094  *
1095  * One of the utilities call by this function AppLayerHandleTCPData(),
1096  * has a feature where it will call this very same function for the
1097  * stream opposing the stream it is called with. This shouldn't cause
1098  * any issues, since processing of each stream is independent of the
1099  * other stream.
1100  */
1102  TcpSession *ssn, TcpStream *stream,
1103  Packet *p, enum StreamUpdateDir dir)
1104 {
1105  SCEnter();
1106 
1107  /* this function can be directly called by app layer protocol
1108  * detection. */
1111  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1112  SCReturnInt(0);
1113  }
1114 
1115 #ifdef DEBUG
1116  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1117  GetSessionSize(ssn, p);
1118 #endif
1119  /* if no segments are in the list or all are already processed,
1120  * and state is beyond established, we send an empty msg */
1121  if (STREAM_HAS_SEEN_DATA(stream) && STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1122  {
1123  /* send an empty EOF msg if we have no segments but TCP state
1124  * is beyond ESTABLISHED */
1125  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1126  SCLogDebug("sending empty eof message");
1127  /* send EOF to app layer */
1128  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1129  NULL, 0,
1130  StreamGetAppLayerFlags(ssn, stream, p, dir));
1131  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1132 
1133  SCReturnInt(0);
1134  }
1135  }
1136 
1137  /* with all that out of the way, lets update the app-layer */
1138  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, stream, p, dir);
1139 }
1140 
1141 /** \internal
1142  * \brief get stream data from offset
1143  * \param offset stream offset */
1144 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1145  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1146 {
1147  const uint8_t *mydata;
1148  uint32_t mydata_len;
1149  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1150  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1151 
1152  uint64_t roffset = offset;
1153  if (offset)
1154  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1155  else {
1156  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1157  }
1158 
1159  *data = mydata;
1160  *data_len = mydata_len;
1161  *data_offset = roffset;
1162  } else {
1163  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1164  *iter == NULL ? "starting" : "continuing", offset);
1165  if (*iter == NULL) {
1166  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1167  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1168  SCLogDebug("*iter %p", *iter);
1169  }
1170  if (*iter == NULL) {
1171  SCLogDebug("no data");
1172  *data = NULL;
1173  *data_len = 0;
1174  *data_offset = 0;
1175  return 0;
1176  }
1177  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1178 
1179  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1180  SCLogDebug("mydata %p", mydata);
1181 
1182  if ((*iter)->offset < offset) {
1183  uint64_t delta = offset - (*iter)->offset;
1184  if (delta < mydata_len) {
1185  *data = mydata + delta;
1186  *data_len = mydata_len - delta;
1187  *data_offset = offset;
1188  } else {
1189  SCLogDebug("no data (yet)");
1190  *data = NULL;
1191  *data_len = 0;
1192  *data_offset = 0;
1193  }
1194 
1195  } else {
1196  *data = mydata;
1197  *data_len = mydata_len;
1198  *data_offset = (*iter)->offset;
1199  }
1200 
1201  *iter = SBB_RB_NEXT(*iter);
1202  SCLogDebug("*iter %p", *iter);
1203  }
1204  return 0;
1205 }
1206 
1207 /** \brief does the stream engine have data to inspect?
1208  *
1209  * Returns true if there is data to inspect. In IDS case this is
1210  * about ACK'd data in the packet's direction.
1211  *
1212  * In the IPS case this is about the packet itself.
1213  */
1215 {
1216  TcpStream *stream;
1217  if (PKT_IS_TOSERVER(p)) {
1218  stream = &ssn->client;
1219  } else {
1220  stream = &ssn->server;
1221  }
1222 
1223  if (RB_EMPTY(&stream->seg_tree)) {
1224  return false;
1225  }
1226 
1229  return false;
1230 
1231  if (StreamTcpInlineMode() == FALSE) {
1232  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1233  return false;
1234  }
1235  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1236  return true;
1237  }
1238  } else {
1239  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1240  return true;
1241  }
1242  }
1243  return false;
1244 }
1245 
1246 /** \brief update stream engine after detection
1247  *
1248  * Tasked with progressing the 'progress' for Raw reassembly.
1249  * 2 main scenario's:
1250  * 1. progress is != 0, so we use this
1251  * 2. progress is 0, meaning the detect engine didn't touch
1252  * raw at all. In this case we need to look into progressing
1253  * raw anyway.
1254  *
1255  * Additionally, this function is tasked with disabling raw
1256  * reassembly if the app-layer requested to disable it.
1257  */
1258 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1259 {
1260  TcpStream *stream;
1261  if (PKT_IS_TOSERVER(p)) {
1262  stream = &ssn->client;
1263  } else {
1264  stream = &ssn->server;
1265  }
1266 
1267  if (progress > STREAM_RAW_PROGRESS(stream)) {
1268  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1269  stream->raw_progress_rel += slide;
1271 
1272  /* if app is active and beyond raw, sync raw to app */
1273  } else if (progress == 0 &&
1274  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1276  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1277  {
1278  /* if trigger raw is set we sync the 2 trackers */
1280  {
1281  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1282  stream->raw_progress_rel += slide;
1284 
1285  /* otherwise mix in the tcp window */
1286  } else {
1287  uint64_t tcp_window = stream->window;
1288  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1289  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1290  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1291  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1292  stream->raw_progress_rel += slide;
1293  }
1294  }
1295  }
1296  /* app is dead */
1297  } else if (progress == 0) {
1298  uint64_t tcp_window = stream->window;
1299  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1300  if (tcp_window < stream_right_edge) {
1301  uint64_t new_raw = stream_right_edge - tcp_window;
1302  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1303  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1304  stream->raw_progress_rel += slide;
1305  }
1306  }
1308 
1309  } else {
1310  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1311  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1312  STREAM_RAW_PROGRESS(stream), stream->window);
1313  }
1314 
1315  /* if we were told to accept no more raw data, we can mark raw as
1316  * disabled now. */
1319  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1320  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1321  }
1322 
1323  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1324 }
1325 
1326 /** \internal
1327  * \brief get a buffer around the current packet and run the callback on it
1328  *
1329  * The inline/IPS scanning method takes the current payload and wraps it in
1330  * data from other segments.
1331  *
1332  * How much data is inspected is controlled by the available data, chunk_size
1333  * and the payload size of the packet.
1334  *
1335  * Large packets: if payload size is close to the chunk_size, where close is
1336  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1337  * used: payload_len + 33% of the chunk_size.
1338  * If the payload size if equal to or bigger than the chunk_size, we use
1339  * payload len + 33% of the chunk size.
1340  */
1341 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1342  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1343 {
1344  SCEnter();
1345  int r = 0;
1346 
1347  TcpStream *stream;
1348  if (PKT_IS_TOSERVER(p)) {
1349  stream = &ssn->client;
1350  } else {
1351  stream = &ssn->server;
1352  }
1353 
1354  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1356  {
1357  *progress_out = STREAM_RAW_PROGRESS(stream);
1358  return 0;
1359  }
1360 
1361  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1364  if (chunk_size <= p->payload_len) {
1365  chunk_size = p->payload_len + (chunk_size / 3);
1366  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1367  p->payload_len, chunk_size);
1368  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1369  chunk_size = p->payload_len + ((chunk_size / 3));
1370  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1371  p->payload_len, chunk_size);
1372  }
1373 
1374  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1375  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1376  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1377  packet_leftedge_abs, packet_rightedge_abs);
1378 
1379  const uint8_t *mydata = NULL;
1380  uint32_t mydata_len = 0;
1381  uint64_t mydata_offset = 0;
1382  /* simply return progress from the block we inspected. */
1383  bool return_progress = false;
1384 
1385  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1386  /* continues block */
1387  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1388  return_progress = true;
1389 
1390  } else {
1391  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1392  /* find our block */
1393  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1394  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1395  if (sbb) {
1396  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1397  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1398  mydata_offset = sbb->offset;
1399  }
1400  }
1401 
1402  /* this can only happen if the segment insert of our current 'p' failed */
1403  if (mydata == NULL || mydata_len == 0) {
1404  SCLogDebug("no data: %p/%u", mydata, mydata_len);
1405  *progress_out = STREAM_RAW_PROGRESS(stream);
1406  return 0;
1407  }
1408  if (mydata_offset >= packet_rightedge_abs ||
1409  (packet_leftedge_abs >= (mydata_offset + mydata_len))) {
1410  SCLogDebug("no data overlap: %p/%u", mydata, mydata_len);
1411  *progress_out = STREAM_RAW_PROGRESS(stream);
1412  return 0;
1413  }
1414 
1415  /* adjust buffer to match chunk_size */
1416 
1417  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1418  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1419  if (mydata_len > chunk_size) {
1420  uint32_t excess = mydata_len - chunk_size;
1421  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1422 
1423  if (mydata_rightedge_abs == packet_rightedge_abs) {
1424  mydata += excess;
1425  mydata_len -= excess;
1426  mydata_offset += excess;
1427  SCLogDebug("cutting front of the buffer with %u", excess);
1428  } else if (mydata_offset == packet_leftedge_abs) {
1429  mydata_len -= excess;
1430  SCLogDebug("cutting tail of the buffer with %u", excess);
1431  } else {
1432  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1433  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1434  SCLogDebug("before %u after %u", before, after);
1435 
1436  if (after >= (chunk_size - p->payload_len) / 2) {
1437  // more trailing data than we need
1438 
1439  if (before >= (chunk_size - p->payload_len) / 2) {
1440  // also more heading data, devide evenly
1441  before = after = (chunk_size - p->payload_len) / 2;
1442  } else {
1443  // heading data is less than requested, give the
1444  // rest to the trailing data
1445  after = (chunk_size - p->payload_len) - before;
1446  }
1447  } else {
1448  // less trailing data than requested
1449 
1450  if (before >= (chunk_size - p->payload_len) / 2) {
1451  before = (chunk_size - p->payload_len) - after;
1452  } else {
1453  // both smaller than their requested size
1454  }
1455  }
1456 
1457  /* adjust the buffer */
1458  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1459  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1460  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1461  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1462  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1463  mydata += skip;
1464  mydata_len -= (skip + cut);
1465  mydata_offset += skip;
1466  }
1467  }
1468 
1469  /* run the callback */
1470  r = Callback(cb_data, mydata, mydata_len);
1471  BUG_ON(r < 0);
1472 
1473  if (return_progress) {
1474  *progress_out = (mydata_offset + mydata_len);
1475  } else {
1476  /* several blocks of data, so we need to be a bit more careful:
1477  * - if last_ack is beyond last progress, move progress forward to last_ack
1478  * - if our block matches or starts before last ack, return right edge of
1479  * our block.
1480  */
1481  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1482  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1483  uint32_t delta = stream->last_ack - stream->base_seq;
1484  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1485  /* get max absolute offset */
1486  last_ack_abs += delta;
1487  }
1488  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1489 
1490  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1491  if (mydata_offset > last_ack_abs) {
1492  /* gap between us and last ack, set progress to last ack */
1493  *progress_out = last_ack_abs;
1494  } else {
1495  *progress_out = (mydata_offset + mydata_len);
1496  }
1497  } else {
1498  *progress_out = STREAM_RAW_PROGRESS(stream);
1499  }
1500  }
1501  return r;
1502 }
1503 
1504 /** \brief access 'raw' reassembly data.
1505  *
1506  * Access data as tracked by 'raw' tracker. Data is made available to
1507  * callback that is passed to this function.
1508  *
1509  * In the case of IDS the callback may be run multiple times if data
1510  * contains gaps. It will then be run for each block of data that is
1511  * continuous.
1512  *
1513  * The callback should give on of 2 return values:
1514  * - 0 ok
1515  * - 1 done
1516  * The value 1 will break the loop if there is a block list that is
1517  * inspected.
1518  *
1519  * This function will return the 'progress' value that has been
1520  * consumed until now.
1521  *
1522  * \param ssn tcp session
1523  * \param stream tcp stream
1524  * \param Callback the function pointer to the callback function
1525  * \param cb_data callback data
1526  * \param[in] progress_in progress to work from
1527  * \param[out] progress_out absolute progress value of the data this
1528  * call handled.
1529  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1530  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1531  *
1532  * `respect_inspect_depth` is used to avoid useless inspection of too
1533  * much data.
1534  */
1535 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1536  StreamReassembleRawFunc Callback, void *cb_data,
1537  const uint64_t progress_in,
1538  uint64_t *progress_out, bool eof,
1539  bool respect_inspect_depth)
1540 {
1541  SCEnter();
1542  int r = 0;
1543 
1544  StreamingBufferBlock *iter = NULL;
1545  uint64_t progress = progress_in;
1546  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1547 
1548  /* if the app layer triggered a flush, and we're supposed to
1549  * use a minimal inspect depth, we actually take the app progress
1550  * as that is the right edge of the data. Then we take the window
1551  * of 'min_inspect_depth' before that. */
1552  if (respect_inspect_depth &&
1554  && stream->min_inspect_depth)
1555  {
1556  progress = STREAM_APP_PROGRESS(stream);
1557  if (stream->min_inspect_depth >= progress) {
1558  progress = 0;
1559  } else {
1560  progress -= stream->min_inspect_depth;
1561  }
1562  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1563 
1564  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1565  }
1566 
1567  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1568 
1569  /* get window of data that is acked */
1570  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1571  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1572  uint32_t delta = stream->last_ack - stream->base_seq;
1573  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1574  /* get max absolute offset */
1575  last_ack_abs += delta;
1576  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1577  }
1578 
1579  /* loop through available buffers. On no packet loss we'll have a single
1580  * iteration. On missing data we'll walk the blocks */
1581  while (1) {
1582  const uint8_t *mydata;
1583  uint32_t mydata_len;
1584  uint64_t mydata_offset = 0;
1585 
1586  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1587  if (mydata_len == 0) {
1588  SCLogDebug("no data");
1589  break;
1590  }
1591  //PrintRawDataFp(stdout, mydata, mydata_len);
1592 
1593  SCLogDebug("raw progress %"PRIu64, progress);
1594  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1595  stream, &stream->sb, mydata_len, progress);
1596 
1597  if (eof) {
1598  // inspect all remaining data, ack'd or not
1599  } else {
1600  if (last_ack_abs < progress) {
1601  SCLogDebug("nothing to do");
1602  goto end;
1603  }
1604 
1605  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1606  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1607 
1608  /* see if the buffer contains unack'd data as well */
1609  if (progress + mydata_len > last_ack_abs) {
1610  uint32_t check = mydata_len;
1611  mydata_len = last_ack_abs - progress;
1612  BUG_ON(check < mydata_len);
1613  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1614  "data is considered", mydata_len);
1615  }
1616 
1617  }
1618  if (mydata_len == 0)
1619  break;
1620 
1621  SCLogDebug("data %p len %u", mydata, mydata_len);
1622 
1623  /* we have data. */
1624  r = Callback(cb_data, mydata, mydata_len);
1625  BUG_ON(r < 0);
1626 
1627  if (mydata_offset == progress) {
1628  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1629  progress, mydata_len, progress_in + mydata_len);
1630 
1631  progress += mydata_len;
1632  SCLogDebug("raw progress now %"PRIu64, progress);
1633 
1634  /* data is beyond the progress we'd like, and before last ack. Gap. */
1635  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1636  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1637  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1638  progress = mydata_offset;
1639  SCLogDebug("raw progress now %"PRIu64, progress);
1640 
1641  } else {
1642  SCLogDebug("not increasing progress, data gap => mydata_offset "
1643  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1644  }
1645 
1646  if (iter == NULL || r == 1)
1647  break;
1648  }
1649 end:
1650  *progress_out = progress;
1651  return r;
1652 }
1653 
1655  StreamReassembleRawFunc Callback, void *cb_data,
1656  uint64_t *progress_out, bool respect_inspect_depth)
1657 {
1658  /* handle inline seperately as the logic is very different */
1659  if (StreamTcpInlineMode() == TRUE) {
1660  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1661  }
1662 
1663  TcpStream *stream;
1664  if (PKT_IS_TOSERVER(p)) {
1665  stream = &ssn->client;
1666  } else {
1667  stream = &ssn->server;
1668  }
1669 
1671  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1672  {
1673  *progress_out = STREAM_RAW_PROGRESS(stream);
1674  return 0;
1675  }
1676 
1677  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1678  STREAM_RAW_PROGRESS(stream), progress_out,
1679  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1680 }
1681 
1683  StreamReassembleRawFunc Callback, void *cb_data,
1684  uint64_t progress_in,
1685  uint64_t *progress_out, bool eof)
1686 {
1687  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1688  return 0;
1689 
1690  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1691  progress_in, progress_out, eof, false);
1692 }
1693 
1694 /** \internal
1695  * \brief update app layer based on received ACK
1696  *
1697  * \retval r 0 on success, -1 on error
1698  */
1699 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1700  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1701 {
1702  SCEnter();
1703 
1704  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1705  SCReturnInt(-1);
1706 
1707  SCReturnInt(0);
1708 }
1709 
1711  TcpSession *ssn, TcpStream *stream,
1712  Packet *p, PacketQueue *pq)
1713 {
1714  SCEnter();
1715 
1716  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1717 
1718  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1719  ssn, stream, p, p->payload_len);
1720 
1721  /* we need to update the opposing stream in
1722  * StreamTcpReassembleHandleSegmentUpdateACK */
1723  TcpStream *opposing_stream = NULL;
1724  if (stream == &ssn->client) {
1725  opposing_stream = &ssn->server;
1726  } else {
1727  opposing_stream = &ssn->client;
1728  }
1729 
1730  /* default IDS: update opposing side (triggered by ACK) */
1732  /* inline and stream end and flow timeout packets trigger same dir handling */
1733  if (StreamTcpInlineMode()) {
1734  dir = UPDATE_DIR_PACKET;
1735  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1736  dir = UPDATE_DIR_PACKET;
1737  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1738  dir = UPDATE_DIR_PACKET;
1739  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1740  dir = UPDATE_DIR_PACKET;
1741  } else if (ssn->state == TCP_CLOSED) {
1742  dir = UPDATE_DIR_BOTH;
1743  }
1744 
1745  /* handle ack received */
1746  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1747  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1748  {
1749  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1750  SCReturnInt(-1);
1751  }
1752 
1753  /* if this segment contains data, insert it */
1754  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1755  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1756 
1757  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1758  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1759  SCReturnInt(-1);
1760  }
1761 
1762  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1763  p->flags |= PKT_STREAM_ADD;
1764  } else {
1765  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1766  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1767  ssn, stream, p->payload_len,
1768  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1769 
1770  }
1771 
1772  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1773  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1774  * was *just* set. In this case we trigger the AppLayer Truncate
1775  * logic, to inform the applayer no more data in this direction is
1776  * to be expected. */
1777  if ((stream->flags &
1780  {
1781  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1782  if (dir != UPDATE_DIR_PACKET) {
1783  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1784  "can trigger Truncate");
1785  dir = UPDATE_DIR_PACKET;
1786  }
1787  }
1788 
1789  /* in stream inline mode even if we have no data we call the reassembly
1790  * functions to handle EOF */
1791  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1792  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1793  StreamTcpInlineMode()?"true":"false",
1794  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1795  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1796  SCReturnInt(-1);
1797  }
1798  }
1799 
1800  SCReturnInt(0);
1801 }
1802 
1803 /**
1804  * \brief get a segment from the pool
1805  *
1806  * \retval seg Segment from the pool or NULL
1807  */
1809 {
1810  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1811  SCLogDebug("seg we return is %p", seg);
1812  if (seg == NULL) {
1813  /* Increment the counter to show that we are not able to serve the
1814  segment request due to memcap limit */
1816  } else {
1817  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1818  }
1819 
1820  return seg;
1821 }
1822 
1823 /**
1824  * \brief Trigger RAW stream reassembly
1825  *
1826  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1827  * reassembly from the applayer, for example upon completion of a
1828  * HTTP request.
1829  *
1830  * It sets a flag in the stream so that the next Raw call will return
1831  * the data.
1832  *
1833  * \param ssn TcpSession
1834  */
1836 {
1837 #ifdef DEBUG
1838  BUG_ON(ssn == NULL);
1839 #endif
1840 
1841  if (ssn != NULL) {
1842  if (direction == STREAM_TOSERVER) {
1844  } else {
1846  }
1847 
1848  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1849  }
1850 }
1851 
1852 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1853 {
1854 #ifdef DEBUG
1855  BUG_ON(ssn == NULL);
1856 #endif
1857 
1858  if (ssn != NULL) {
1859  if (direction == STREAM_TOSERVER) {
1860  ssn->client.min_inspect_depth = depth;
1861  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1862  } else {
1863  ssn->server.min_inspect_depth = depth;
1864  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1865  }
1866  }
1867 }
1868 
1869 #ifdef UNITTESTS
1870 /** unit tests and it's support functions below */
1871 
1872 #define SET_ISN(stream, setseq) \
1873  (stream)->isn = (setseq); \
1874  (stream)->base_seq = (setseq) + 1
1875 
1876 /** \brief The Function to create the packet with given payload, which is used
1877  * to test the reassembly of the engine.
1878  *
1879  * \param payload The variable used to store the payload contents of the
1880  * current packet.
1881  * \param value The value which current payload will have for this packet
1882  * \param payload_len The length of the filed payload for current packet.
1883  * \param len Length of the payload array
1884  */
1885 
1886 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1887  uint8_t payload_len, uint8_t len)
1888 {
1889  uint8_t i;
1890  for (i = 0; i < payload_len; i++)
1891  payload[i] = value;
1892  for (; i < len; i++)
1893  payload = NULL;
1894 }
1895 
1896 /** \brief The Function Checks the reassembled stream contents against predefined
1897  * stream contents according to OS policy used.
1898  *
1899  * \param stream_policy Predefined value of stream for different OS policies
1900  * \param stream Reassembled stream returned from the reassembly functions
1901  */
1902 
1903 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
1904 {
1905  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
1906  {
1907  //PrintRawDataFp(stdout, stream_policy, sp_size);
1908  return 0;
1909  }
1910  return 1;
1911 }
1912 
1913 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
1914 {
1915  if (StreamingBufferCompareRawData(&stream->sb,
1916  data, data_len) == 0)
1917  {
1918  SCReturnInt(0);
1919  }
1920  SCLogInfo("OK");
1921  PrintRawDataFp(stdout, data, data_len);
1922  return 1;
1923 }
1924 
1925 #define MISSED_START(isn) \
1926  TcpReassemblyThreadCtx *ra_ctx = NULL; \
1927  TcpSession ssn; \
1928  ThreadVars tv; \
1929  memset(&tv, 0, sizeof(tv)); \
1930  \
1931  StreamTcpUTInit(&ra_ctx); \
1932  \
1933  StreamTcpUTSetupSession(&ssn); \
1934  StreamTcpUTSetupStream(&ssn.server, (isn)); \
1935  StreamTcpUTSetupStream(&ssn.client, (isn)); \
1936  \
1937  TcpStream *stream = &ssn.client;
1938 
1939 #define MISSED_END \
1940  StreamTcpUTClearSession(&ssn); \
1941  StreamTcpUTDeinit(ra_ctx); \
1942  PASS
1943 
1944 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
1945  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
1946  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
1947 
1948 /**
1949  * \test Test the handling of packets missed by both IDS and the end host.
1950  * The packet is missed in the starting of the stream.
1951  *
1952  * \retval On success it returns 1 and on failure 0.
1953  */
1954 
1955 static int StreamTcpReassembleTest25 (void)
1956 {
1957  MISSED_START(6);
1958  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
1959  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
1960  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
1961  MISSED_END;
1962 }
1963 
1964 /**
1965  * \test Test the handling of packets missed by both IDS and the end host.
1966  * The packet is missed in the middle of the stream.
1967  *
1968  * \retval On success it returns 1 and on failure 0.
1969  */
1970 
1971 static int StreamTcpReassembleTest26 (void)
1972 {
1973  MISSED_START(9);
1974  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1975  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
1976  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
1977  MISSED_END;
1978 }
1979 
1980 /**
1981  * \test Test the handling of packets missed by both IDS and the end host.
1982  * The packet is missed in the end of the stream.
1983  *
1984  * \retval On success it returns 1 and on failure 0.
1985  */
1986 
1987 static int StreamTcpReassembleTest27 (void)
1988 {
1989  MISSED_START(9);
1990  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1991  MISSED_STEP(13, "BB", 2, "AAABB", 5);
1992  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
1993  MISSED_END;
1994 }
1995 
1996 /**
1997  * \test Test the handling of packets missed by IDS, but the end host has
1998  * received it and send the acknowledgment of it. The packet is missed
1999  * in the starting of the stream.
2000  *
2001  * \retval On success it returns 1 and on failure 0.
2002  */
2003 
2004 static int StreamTcpReassembleTest28 (void)
2005 {
2006  MISSED_START(6);
2007  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2008  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2009  ssn.state = TCP_TIME_WAIT;
2010  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2011  MISSED_END;
2012 }
2013 
2014 /**
2015  * \test Test the handling of packets missed by IDS, but the end host has
2016  * received it and send the acknowledgment of it. The packet is missed
2017  * in the middle of the stream.
2018  *
2019  * \retval On success it returns 1 and on failure 0.
2020  */
2021 
2022 static int StreamTcpReassembleTest29 (void)
2023 {
2024  MISSED_START(9);
2025  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2026  ssn.state = TCP_TIME_WAIT;
2027  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2028  MISSED_END;
2029 }
2030 
2031 static int StreamTcpReassembleTest33(void)
2032 {
2033  TcpSession ssn;
2034  Packet *p = PacketGetFromAlloc();
2035  FAIL_IF(unlikely(p == NULL));
2036  Flow f;
2037  TCPHdr tcph;
2038  TcpReassemblyThreadCtx *ra_ctx = NULL;
2040  uint8_t packet[1460] = "";
2041 
2042  StreamTcpUTInit(&ra_ctx);
2044 
2045  PacketQueue pq;
2046  memset(&pq,0,sizeof(PacketQueue));
2047  memset(&f, 0, sizeof (Flow));
2048  memset(&tcph, 0, sizeof (TCPHdr));
2049  ThreadVars tv;
2050  memset(&tv, 0, sizeof (ThreadVars));
2051  FLOW_INITIALIZE(&f);
2052  f.protoctx = &ssn;
2053  f.proto = IPPROTO_TCP;
2054  p->src.family = AF_INET;
2055  p->dst.family = AF_INET;
2056  p->proto = IPPROTO_TCP;
2057  p->flow = &f;
2058  tcph.th_win = 5480;
2059  tcph.th_flags = TH_PUSH | TH_ACK;
2060  p->tcph = &tcph;
2062  p->payload = packet;
2063 
2064  p->tcph->th_seq = htonl(10);
2065  p->tcph->th_ack = htonl(31);
2066  p->payload_len = 10;
2067 
2068  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2069 
2070  p->tcph->th_seq = htonl(20);
2071  p->tcph->th_ack = htonl(31);
2072  p->payload_len = 10;
2073 
2074  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2075 
2076  p->tcph->th_seq = htonl(40);
2077  p->tcph->th_ack = htonl(31);
2078  p->payload_len = 10;
2079 
2080  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2081 
2082  p->tcph->th_seq = htonl(5);
2083  p->tcph->th_ack = htonl(31);
2084  p->payload_len = 30;
2085 
2086  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2087 
2089  StreamTcpUTDeinit(ra_ctx);
2090  SCFree(p);
2091  PASS;
2092 }
2093 
2094 static int StreamTcpReassembleTest34(void)
2095 {
2096  TcpSession ssn;
2097  Packet *p = PacketGetFromAlloc();
2098  FAIL_IF(unlikely(p == NULL));
2099  Flow f;
2100  TCPHdr tcph;
2101  TcpReassemblyThreadCtx *ra_ctx = NULL;
2103  uint8_t packet[1460] = "";
2104 
2105  StreamTcpUTInit(&ra_ctx);
2107  PacketQueue pq;
2108  memset(&pq,0,sizeof(PacketQueue));
2109  memset(&f, 0, sizeof (Flow));
2110  memset(&tcph, 0, sizeof (TCPHdr));
2111  ThreadVars tv;
2112  memset(&tv, 0, sizeof (ThreadVars));
2113  FLOW_INITIALIZE(&f);
2114  f.protoctx = &ssn;
2115  f.proto = IPPROTO_TCP;
2116  p->src.family = AF_INET;
2117  p->dst.family = AF_INET;
2118  p->proto = IPPROTO_TCP;
2119  p->flow = &f;
2120  tcph.th_win = 5480;
2121  tcph.th_flags = TH_PUSH | TH_ACK;
2122  p->tcph = &tcph;
2124  p->payload = packet;
2125  SET_ISN(&ssn.client, 857961230);
2126 
2127  p->tcph->th_seq = htonl(857961230);
2128  p->tcph->th_ack = htonl(31);
2129  p->payload_len = 304;
2130 
2131  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2132 
2133  p->tcph->th_seq = htonl(857961534);
2134  p->tcph->th_ack = htonl(31);
2135  p->payload_len = 1460;
2136 
2137  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2138 
2139  p->tcph->th_seq = htonl(857963582);
2140  p->tcph->th_ack = htonl(31);
2141  p->payload_len = 1460;
2142 
2143  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2144 
2145  p->tcph->th_seq = htonl(857960946);
2146  p->tcph->th_ack = htonl(31);
2147  p->payload_len = 1460;
2148 
2149  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2150 
2152  StreamTcpUTDeinit(ra_ctx);
2153  SCFree(p);
2154  PASS;
2155 }
2156 
2157 /** \test Test the bug 76 condition */
2158 static int StreamTcpReassembleTest37(void)
2159 {
2160  TcpSession ssn;
2161  Flow f;
2162  TCPHdr tcph;
2163  TcpReassemblyThreadCtx *ra_ctx = NULL;
2164  uint8_t packet[1460] = "";
2165  PacketQueue pq;
2166  ThreadVars tv;
2167 
2168  Packet *p = PacketGetFromAlloc();
2169  FAIL_IF(unlikely(p == NULL));
2170 
2171  StreamTcpUTInit(&ra_ctx);
2173  memset(&pq,0,sizeof(PacketQueue));
2174  memset(&f, 0, sizeof (Flow));
2175  memset(&tcph, 0, sizeof (TCPHdr));
2176  memset(&tv, 0, sizeof (ThreadVars));
2177 
2178  FLOW_INITIALIZE(&f);
2179  f.protoctx = &ssn;
2180  f.proto = IPPROTO_TCP;
2181  p->src.family = AF_INET;
2182  p->dst.family = AF_INET;
2183  p->proto = IPPROTO_TCP;
2184  p->flow = &f;
2185  tcph.th_win = 5480;
2186  tcph.th_flags = TH_PUSH | TH_ACK;
2187  p->tcph = &tcph;
2189  p->payload = packet;
2191 
2192  p->tcph->th_seq = htonl(3061088537UL);
2193  p->tcph->th_ack = htonl(1729548549UL);
2194  p->payload_len = 1391;
2195  ssn.client.last_ack = 3061091137UL;
2196  SET_ISN(&ssn.client, 3061091309UL);
2197 
2198  /* pre base_seq, so should be rejected */
2199  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2200 
2201  p->tcph->th_seq = htonl(3061089928UL);
2202  p->tcph->th_ack = htonl(1729548549UL);
2203  p->payload_len = 1391;
2204  ssn.client.last_ack = 3061091137UL;
2205 
2206  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2207 
2208  p->tcph->th_seq = htonl(3061091319UL);
2209  p->tcph->th_ack = htonl(1729548549UL);
2210  p->payload_len = 1391;
2211  ssn.client.last_ack = 3061091137UL;
2212 
2213  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2214 
2216  StreamTcpUTDeinit(ra_ctx);
2217  SCFree(p);
2218  PASS;
2219 }
2220 
2221 /**
2222  * \test Test to make sure that we don't return the segments until the app
2223  * layer proto has been detected and after that remove the processed
2224  * segments.
2225  *
2226  * \retval On success it returns 1 and on failure 0.
2227  */
2228 
2229 static int StreamTcpReassembleTest39 (void)
2230 {
2231  Packet *p = PacketGetFromAlloc();
2232  FAIL_IF(unlikely(p == NULL));
2233  Flow f;
2234  ThreadVars tv;
2235  StreamTcpThread stt;
2236  TCPHdr tcph;
2237  PacketQueue pq;
2238  memset(&pq,0,sizeof(PacketQueue));
2239  memset (&f, 0, sizeof(Flow));
2240  memset(&tv, 0, sizeof (ThreadVars));
2241  memset(&stt, 0, sizeof (stt));
2242  memset(&tcph, 0, sizeof (TCPHdr));
2243 
2244  FLOW_INITIALIZE(&f);
2245  f.flags = FLOW_IPV4;
2246  f.proto = IPPROTO_TCP;
2247  p->flow = &f;
2248  p->tcph = &tcph;
2249 
2250  StreamTcpUTInit(&stt.ra_ctx);
2251 
2252  /* handshake */
2253  tcph.th_win = htons(5480);
2254  tcph.th_flags = TH_SYN;
2256  p->payload_len = 0;
2257  p->payload = NULL;
2258  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2259 
2260  TcpSession *ssn = (TcpSession *)f.protoctx;
2261  FAIL_IF_NULL(ssn);
2262 
2273  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2274  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2275  FAIL_IF(ssn->data_first_seen_dir != 0);
2276 
2277  /* handshake */
2278  p->tcph->th_ack = htonl(1);
2279  p->tcph->th_flags = TH_SYN | TH_ACK;
2281  p->payload_len = 0;
2282  p->payload = NULL;
2283  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2294  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2295  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2296  FAIL_IF(ssn->data_first_seen_dir != 0);
2297 
2298  /* handshake */
2299  p->tcph->th_ack = htonl(1);
2300  p->tcph->th_seq = htonl(1);
2301  p->tcph->th_flags = TH_ACK;
2303  p->payload_len = 0;
2304  p->payload = NULL;
2305  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2316  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2317  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2318  FAIL_IF(ssn->data_first_seen_dir != 0);
2319 
2320  /* partial request */
2321  uint8_t request1[] = { 0x47, 0x45, };
2322  p->tcph->th_ack = htonl(1);
2323  p->tcph->th_seq = htonl(1);
2324  p->tcph->th_flags = TH_PUSH | TH_ACK;
2326  p->payload_len = sizeof(request1);
2327  p->payload = request1;
2328  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2339  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2340  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2341  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2343 
2344  /* response ack against partial request */
2345  p->tcph->th_ack = htonl(3);
2346  p->tcph->th_seq = htonl(1);
2347  p->tcph->th_flags = TH_ACK;
2349  p->payload_len = 0;
2350  p->payload = NULL;
2351  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2362  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2363  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2364  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2366 
2367  /* complete partial request */
2368  uint8_t request2[] = {
2369  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2370  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2371  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2372  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2373  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2374  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2375  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2376  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2377  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2378  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2379  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2380  p->tcph->th_ack = htonl(1);
2381  p->tcph->th_seq = htonl(3);
2382  p->tcph->th_flags = TH_PUSH | TH_ACK;
2384  p->payload_len = sizeof(request2);
2385  p->payload = request2;
2386  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2397  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2398  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2399  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2400  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2402 
2403  /* response - request ack */
2404  uint8_t response[] = {
2405  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2406  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2407  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2408  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2409  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2410  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2411  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2412  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2413  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2414  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2415  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2416  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2417  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2418  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2419  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2420  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2421  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2422  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2423  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2424  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2425  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2426  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2427  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2428  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2429  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2430  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2431  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2432  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2433  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2434  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2435  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2436  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2437  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2438  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2439  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2440  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2441  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2442  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2443  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2444  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2445  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2446  p->tcph->th_ack = htonl(88);
2447  p->tcph->th_seq = htonl(1);
2448  p->tcph->th_flags = TH_PUSH | TH_ACK;
2450  p->payload_len = sizeof(response);
2451  p->payload = response;
2452 
2453  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2465  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2466  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2467  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2468  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2469 
2470  /* response ack from request */
2471  p->tcph->th_ack = htonl(328);
2472  p->tcph->th_seq = htonl(88);
2473  p->tcph->th_flags = TH_ACK;
2475  p->payload_len = 0;
2476  p->payload = NULL;
2477  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2489  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2490  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2491  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2492  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2493  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2494 
2495  /* response - acking */
2496  p->tcph->th_ack = htonl(88);
2497  p->tcph->th_seq = htonl(328);
2498  p->tcph->th_flags = TH_PUSH | TH_ACK;
2500  p->payload_len = 0;
2501  p->payload = NULL;
2502  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2514  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2515  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2516  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2517  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2518  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2519 
2520  /* response ack from request */
2521  p->tcph->th_ack = htonl(328);
2522  p->tcph->th_seq = htonl(88);
2523  p->tcph->th_flags = TH_ACK;
2525  p->payload_len = 0;
2526  p->payload = NULL;
2527  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2539  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2540  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2541  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2542  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2543 
2544  /* response - acking the request again*/
2545  p->tcph->th_ack = htonl(88);
2546  p->tcph->th_seq = htonl(328);
2547  p->tcph->th_flags = TH_PUSH | TH_ACK;
2549  p->payload_len = 0;
2550  p->payload = NULL;
2551  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2563  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2564  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2565  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2566  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2567 
2568  /*** New Request ***/
2569 
2570  /* partial request */
2571  p->tcph->th_ack = htonl(328);
2572  p->tcph->th_seq = htonl(88);
2573  p->tcph->th_flags = TH_PUSH | TH_ACK;
2575  p->payload_len = sizeof(request1);
2576  p->payload = request1;
2577  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2589  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2590  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2591  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2592  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2593  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2594 
2595  /* response ack against partial request */
2596  p->tcph->th_ack = htonl(90);
2597  p->tcph->th_seq = htonl(328);
2598  p->tcph->th_flags = TH_ACK;
2600  p->payload_len = 0;
2601  p->payload = NULL;
2602  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2603  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2615  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2616  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2617  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2618  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2619  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2620 
2621  /* complete request */
2622  p->tcph->th_ack = htonl(328);
2623  p->tcph->th_seq = htonl(90);
2624  p->tcph->th_flags = TH_PUSH | TH_ACK;
2626  p->payload_len = sizeof(request2);
2627  p->payload = request2;
2628  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2640  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2641  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2642  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2643  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2644  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2645  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2646 
2647  /* response ack against second partial request */
2648  p->tcph->th_ack = htonl(175);
2649  p->tcph->th_seq = htonl(328);
2650  p->tcph->th_flags = TH_ACK;
2652  p->payload_len = 0;
2653  p->payload = NULL;
2654 
2655  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2667  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2668  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2669  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2670  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2671  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2672  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2673 
2674  /* response acking a request */
2675  p->tcph->th_ack = htonl(175);
2676  p->tcph->th_seq = htonl(328);
2677  p->tcph->th_flags = TH_ACK;
2679  p->payload_len = 0;
2680  p->payload = NULL;
2681  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2687 
2690 
2691  /* request acking a response */
2692  p->tcph->th_ack = htonl(328);
2693  p->tcph->th_seq = htonl(175);
2694  p->tcph->th_flags = TH_ACK;
2696  p->payload_len = 0;
2697  p->payload = NULL;
2698  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2699 
2700  StreamTcpSessionClear(ssn);
2702  SCFree(p);
2703  PASS;
2704 }
2705 
2706 /**
2707  * \test Test to make sure that we sent all the segments from the initial
2708  * segments to app layer until we have detected the app layer proto.
2709  *
2710  * \retval On success it returns 1 and on failure 0.
2711  */
2712 
2713 static int StreamTcpReassembleTest40 (void)
2714 {
2715  int ret = 0;
2716  Packet *p = PacketGetFromAlloc();
2717  if (unlikely(p == NULL))
2718  return 0;
2719  Flow *f = NULL;
2720  TCPHdr tcph;
2721  TcpSession ssn;
2722  PacketQueue pq;
2723  memset(&pq,0,sizeof(PacketQueue));
2724  memset(&tcph, 0, sizeof (TCPHdr));
2725  ThreadVars tv;
2726  memset(&tv, 0, sizeof (ThreadVars));
2727 
2730 
2732 
2733  uint8_t httpbuf1[] = "P";
2734  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2735  uint8_t httpbuf3[] = "O";
2736  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2737  uint8_t httpbuf4[] = "S";
2738  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2739  uint8_t httpbuf5[] = "T \r\n";
2740  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2741 
2742  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2743  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2744 
2745  SET_ISN(&ssn.server, 9);
2746  ssn.server.last_ack = 10;
2747  SET_ISN(&ssn.client, 9);
2748  ssn.client.isn = 9;
2749 
2750  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2751  if (f == NULL)
2752  goto end;
2753  f->protoctx = &ssn;
2754  f->proto = IPPROTO_TCP;
2755  p->flow = f;
2756 
2757  tcph.th_win = htons(5480);
2758  tcph.th_seq = htonl(10);
2759  tcph.th_ack = htonl(10);
2760  tcph.th_flags = TH_ACK|TH_PUSH;
2761  p->tcph = &tcph;
2763 
2764  p->payload = httpbuf1;
2765  p->payload_len = httplen1;
2766  ssn.state = TCP_ESTABLISHED;
2767 
2768  TcpStream *s = NULL;
2769  s = &ssn.client;
2770 
2771  FLOWLOCK_WRLOCK(f);
2772  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2773  printf("failed in segments reassembly, while processing toserver packet (1): ");
2774  goto end;
2775  }
2777  p->payload = httpbuf2;
2778  p->payload_len = httplen2;
2779  tcph.th_seq = htonl(10);
2780  tcph.th_ack = htonl(11);
2781  s = &ssn.server;
2782  ssn.server.last_ack = 11;
2783 
2784  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2785  printf("failed in segments reassembly, while processing toserver packet (3): ");
2786  goto end;
2787  }
2788 
2790  p->payload = httpbuf3;
2791  p->payload_len = httplen3;
2792  tcph.th_seq = htonl(11);
2793  tcph.th_ack = htonl(55);
2794  s = &ssn.client;
2795  ssn.client.last_ack = 55;
2796 
2797  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2798  printf("failed in segments reassembly, while processing toserver packet (5): ");
2799  goto end;
2800  }
2801 
2803  p->payload = httpbuf2;
2804  p->payload_len = httplen2;
2805  tcph.th_seq = htonl(55);
2806  tcph.th_ack = htonl(12);
2807  s = &ssn.server;
2808  ssn.server.last_ack = 12;
2809 
2810  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2811  printf("failed in segments reassembly, while processing toserver packet (6): ");
2812  goto end;
2813  }
2814 
2815  /* check is have the segment in the list and flagged or not */
2816  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2817  FAIL_IF_NULL(seg);
2818  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2819 
2821  p->payload = httpbuf4;
2822  p->payload_len = httplen4;
2823  tcph.th_seq = htonl(12);
2824  tcph.th_ack = htonl(100);
2825  s = &ssn.client;
2826  ssn.client.last_ack = 100;
2827 
2828  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2829  printf("failed in segments reassembly, while processing toserver packet (10): ");
2830  goto end;
2831  }
2832 
2834  p->payload = httpbuf2;
2835  p->payload_len = httplen2;
2836  tcph.th_seq = htonl(100);
2837  tcph.th_ack = htonl(13);
2838  s = &ssn.server;
2839  ssn.server.last_ack = 13;
2840 
2841  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2842  printf("failed in segments reassembly, while processing toserver packet (11): ");
2843  goto end;
2844  }
2845 
2847  p->payload = httpbuf5;
2848  p->payload_len = httplen5;
2849  tcph.th_seq = htonl(13);
2850  tcph.th_ack = htonl(145);
2851  s = &ssn.client;
2852  ssn.client.last_ack = 145;
2853 
2854  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2855  printf("failed in segments reassembly, while processing toserver packet (14): ");
2856  goto end;
2857  }
2858 
2860  p->payload = httpbuf2;
2861  p->payload_len = httplen2;
2862  tcph.th_seq = htonl(145);
2863  tcph.th_ack = htonl(16);
2864  s = &ssn.server;
2865  ssn.server.last_ack = 16;
2866 
2867  if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
2868  printf("failed in segments reassembly, while processing toserver packet (15): ");
2869  goto end;
2870  }
2871  if (f->alproto != ALPROTO_HTTP) {
2872  printf("app layer proto has not been detected (18): ");
2873  goto end;
2874  }
2875 
2876  ret = 1;
2877 end:
2881  SCFree(p);
2882  FLOWLOCK_UNLOCK(f);
2883  UTHFreeFlow(f);
2884  return ret;
2885 }
2886 
2887 /** \test Test the memcap incrementing/decrementing and memcap check */
2888 static int StreamTcpReassembleTest44(void)
2889 {
2891  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2893  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2895  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2897  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2899  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2900  PASS;
2901 }
2902 
2903 /**
2904  * \test Test to make sure that reassembly_depth is enforced.
2905  *
2906  * \retval On success it returns 1 and on failure 0.
2907  */
2908 
2909 static int StreamTcpReassembleTest45 (void)
2910 {
2911  TcpReassemblyThreadCtx *ra_ctx = NULL;
2912  TcpSession ssn;
2913  ThreadVars tv;
2914  memset(&tv, 0, sizeof(tv));
2915  uint8_t payload[100] = {0};
2916  uint16_t payload_size = 100;
2917 
2918  StreamTcpUTInit(&ra_ctx);
2920 
2922  ssn.reassembly_depth = 100;
2923  StreamTcpUTSetupStream(&ssn.server, 100);
2924  StreamTcpUTSetupStream(&ssn.client, 100);
2925 
2926  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2927  FAIL_IF(r != 0);
2929 
2930  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2931  FAIL_IF(r != 0);
2933 
2937  StreamTcpUTDeinit(ra_ctx);
2938  PASS;
2939 }
2940 
2941 /**
2942  * \test Test the unlimited config value of reassembly depth.
2943  *
2944  * \retval On success it returns 1 and on failure 0.
2945  */
2946 
2947 static int StreamTcpReassembleTest46 (void)
2948 {
2949  int result = 0;
2950  TcpReassemblyThreadCtx *ra_ctx = NULL;
2951  TcpSession ssn;
2952  ThreadVars tv;
2953  memset(&tv, 0, sizeof(tv));
2954  uint8_t payload[100] = {0};
2955  uint16_t payload_size = 100;
2956 
2957  StreamTcpUTInit(&ra_ctx);
2959 
2961  StreamTcpUTSetupStream(&ssn.server, 100);
2962  StreamTcpUTSetupStream(&ssn.client, 100);
2963 
2964  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2965  if (r != 0)
2966  goto end;
2968  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2969  goto end;
2970  }
2971 
2972  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2973  if (r != 0)
2974  goto end;
2976  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2977  goto end;
2978  }
2979 
2980  result = 1;
2981 end:
2985  StreamTcpUTDeinit(ra_ctx);
2986  return result;
2987 }
2988 
2989 /**
2990  * \test Test to make sure we detect the sequence wrap around and continue
2991  * stream reassembly properly.
2992  *
2993  * \retval On success it returns 1 and on failure 0.
2994  */
2995 
2996 static int StreamTcpReassembleTest47 (void)
2997 {
2998  Packet *p = PacketGetFromAlloc();
2999  FAIL_IF(unlikely(p == NULL));
3000  Flow *f = NULL;
3001  TCPHdr tcph;
3002  TcpSession ssn;
3003  ThreadVars tv;
3004  PacketQueue pq;
3005  memset(&pq,0,sizeof(PacketQueue));
3006  memset(&tcph, 0, sizeof (TCPHdr));
3007  memset(&tv, 0, sizeof (ThreadVars));
3011 
3012  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3013  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3014 
3015  SET_ISN(&ssn.server, 572799781UL);
3016  ssn.server.last_ack = 572799782UL;
3017 
3018  SET_ISN(&ssn.client, 4294967289UL);
3019  ssn.client.last_ack = 21;
3020 
3021  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3022  FAIL_IF(f == NULL);
3023  f->protoctx = &ssn;
3024  f->proto = IPPROTO_TCP;
3025  p->flow = f;
3026 
3027  tcph.th_win = htons(5480);
3028  ssn.state = TCP_ESTABLISHED;
3029  TcpStream *s = NULL;
3030  uint8_t cnt = 0;
3031 
3032  for (cnt=0; cnt < httplen1; cnt++) {
3033  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3034  tcph.th_ack = htonl(572799782UL);
3035  tcph.th_flags = TH_ACK|TH_PUSH;
3036  p->tcph = &tcph;
3038  p->payload = &httpbuf1[cnt];
3039  p->payload_len = 1;
3040  s = &ssn.client;
3041 
3042  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3043 
3045  p->payload = NULL;
3046  p->payload_len = 0;
3047  tcph.th_seq = htonl(572799782UL);
3048  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3049  tcph.th_flags = TH_ACK;
3050  p->tcph = &tcph;
3051  s = &ssn.server;
3052 
3053  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3054  }
3055 
3056  FAIL_IF(f->alproto != ALPROTO_HTTP);
3057 
3061  SCFree(p);
3062  UTHFreeFlow(f);
3063  PASS;
3064 }
3065 
3066 /** \test 3 in order segments in inline reassembly */
3067 static int StreamTcpReassembleInlineTest01(void)
3068 {
3069  int ret = 0;
3070  TcpReassemblyThreadCtx *ra_ctx = NULL;
3071  ThreadVars tv;
3072  TcpSession ssn;
3073  Flow f;
3074 
3075  memset(&tv, 0x00, sizeof(tv));
3076 
3077  StreamTcpUTInit(&ra_ctx);
3080  StreamTcpUTSetupStream(&ssn.client, 1);
3081  FLOW_INITIALIZE(&f);
3082 
3083  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3084  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3085  if (p == NULL) {
3086  printf("couldn't get a packet: ");
3087  goto end;
3088  }
3089  p->tcph->th_seq = htonl(12);
3090  p->flow = &f;
3091 
3092  FLOWLOCK_WRLOCK(&f);
3093  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3094  printf("failed to add segment 1: ");
3095  goto end;
3096  }
3097  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3098  printf("failed to add segment 2: ");
3099  goto end;
3100  }
3101  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3102  printf("failed to add segment 3: ");
3103  goto end;
3104  }
3105  ssn.client.next_seq = 17;
3106  ret = 1;
3107 end:
3108  FLOWLOCK_UNLOCK(&f);
3109  FLOW_DESTROY(&f);
3110  UTHFreePacket(p);
3112  StreamTcpUTDeinit(ra_ctx);
3113  return ret;
3114 }
3115 
3116 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3117  * test the sliding window reassembly.
3118  */
3119 static int StreamTcpReassembleInlineTest02(void)
3120 {
3121  int ret = 0;
3122  TcpReassemblyThreadCtx *ra_ctx = NULL;
3123  ThreadVars tv;
3124  TcpSession ssn;
3125  Flow f;
3126 
3127  memset(&tv, 0x00, sizeof(tv));
3128 
3129  StreamTcpUTInit(&ra_ctx);
3132  StreamTcpUTSetupStream(&ssn.client, 1);
3133  FLOW_INITIALIZE(&f);
3134 
3135  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3136  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3137  if (p == NULL) {
3138  printf("couldn't get a packet: ");
3139  goto end;
3140  }
3141  p->tcph->th_seq = htonl(12);
3142  p->flow = &f;
3143 
3144  FLOWLOCK_WRLOCK(&f);
3145  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3146  printf("failed to add segment 1: ");
3147  goto end;
3148  }
3149  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3150  printf("failed to add segment 2: ");
3151  goto end;
3152  }
3153  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3154  printf("failed to add segment 3: ");
3155  goto end;
3156  }
3157  ssn.client.next_seq = 17;
3158  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3159  printf("failed to add segment 4: ");
3160  goto end;
3161  }
3162  ssn.client.next_seq = 22;
3163  ret = 1;
3164 end:
3165  FLOWLOCK_UNLOCK(&f);
3166  FLOW_DESTROY(&f);
3167  UTHFreePacket(p);
3169  StreamTcpUTDeinit(ra_ctx);
3170  return ret;
3171 }
3172 
3173 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3174  * test the sliding window reassembly with a small window size so that we
3175  * cutting off at the start (left edge)
3176  */
3177 static int StreamTcpReassembleInlineTest03(void)
3178 {
3179  int ret = 0;
3180  TcpReassemblyThreadCtx *ra_ctx = NULL;
3181  ThreadVars tv;
3182  TcpSession ssn;
3183  Flow f;
3184 
3185  memset(&tv, 0x00, sizeof(tv));
3186 
3187  StreamTcpUTInit(&ra_ctx);
3190  StreamTcpUTSetupStream(&ssn.client, 1);
3191  FLOW_INITIALIZE(&f);
3192 
3194 
3195  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3196  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3197  if (p == NULL) {
3198  printf("couldn't get a packet: ");
3199  goto end;
3200  }
3201  p->tcph->th_seq = htonl(12);
3202  p->flow = &f;
3204 
3205  FLOWLOCK_WRLOCK(&f);
3206  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3207  printf("failed to add segment 1: ");
3208  goto end;
3209  }
3210  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3211  printf("failed to add segment 2: ");
3212  goto end;
3213  }
3214  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3215  printf("failed to add segment 3: ");
3216  goto end;
3217  }
3218  ssn.client.next_seq = 17;
3219  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3220  printf("failed to add segment 4: ");
3221  goto end;
3222  }
3223  ssn.client.next_seq = 22;
3224 
3225  p->tcph->th_seq = htonl(17);
3226  ret = 1;
3227 end:
3228  FLOWLOCK_UNLOCK(&f);
3229  FLOW_DESTROY(&f);
3230  UTHFreePacket(p);
3232  StreamTcpUTDeinit(ra_ctx);
3233  return ret;
3234 }
3235 
3236 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3237  * test the sliding window reassembly with a small window size so that we
3238  * cutting off at the start (left edge) with small packet overlap.
3239  */
3240 static int StreamTcpReassembleInlineTest04(void)
3241 {
3242  int ret = 0;
3243  TcpReassemblyThreadCtx *ra_ctx = NULL;
3244  ThreadVars tv;
3245  TcpSession ssn;
3246  Flow f;
3247 
3248  memset(&tv, 0x00, sizeof(tv));
3249 
3250  StreamTcpUTInit(&ra_ctx);
3253  StreamTcpUTSetupStream(&ssn.client, 1);
3254  FLOW_INITIALIZE(&f);
3255 
3257 
3258  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3259  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3260  if (p == NULL) {
3261  printf("couldn't get a packet: ");
3262  goto end;
3263  }
3264  p->tcph->th_seq = htonl(12);
3265  p->flow = &f;
3267 
3268  FLOWLOCK_WRLOCK(&f);
3269  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3270  printf("failed to add segment 1: ");
3271  goto end;
3272  }
3273  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3274  printf("failed to add segment 2: ");
3275  goto end;
3276  }
3277  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3278  printf("failed to add segment 3: ");
3279  goto end;
3280  }
3281  ssn.client.next_seq = 17;
3282  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3283  printf("failed to add segment 4: ");
3284  goto end;
3285  }
3286  ssn.client.next_seq = 22;
3287 
3288  p->tcph->th_seq = htonl(17);
3289  ret = 1;
3290 end:
3291  FLOWLOCK_UNLOCK(&f);
3292  FLOW_DESTROY(&f);
3293  UTHFreePacket(p);
3295  StreamTcpUTDeinit(ra_ctx);
3296  return ret;
3297 }
3298 
3299 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3300  * test the sliding window reassembly with a small window size so that we
3301  * cutting off at the start (left edge). Test if the first segment is
3302  * removed from the list.
3303  */
3304 static int StreamTcpReassembleInlineTest08(void)
3305 {
3306  TcpReassemblyThreadCtx *ra_ctx = NULL;
3307  ThreadVars tv;
3308  memset(&tv, 0x00, sizeof(tv));
3309  TcpSession ssn;
3310  Flow f;
3311  StreamTcpUTInit(&ra_ctx);
3314  StreamTcpUTSetupStream(&ssn.client, 1);
3315  FLOW_INITIALIZE(&f);
3316 
3319  f.protoctx = &ssn;
3320 
3321  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3322  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3323  FAIL_IF(p == NULL);
3324  p->tcph->th_seq = htonl(12);
3325  p->flow = &f;
3327 
3328  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3329  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3330  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3331  ssn.client.next_seq = 17;
3332  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3333  ssn.client.next_seq = 22;
3334  p->tcph->th_seq = htonl(17);
3336 
3337  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3338  FAIL_IF_NULL(seg);
3339  FAIL_IF_NOT(seg->seq == 2);
3340 
3341  FLOW_DESTROY(&f);
3342  UTHFreePacket(p);
3344  StreamTcpUTDeinit(ra_ctx);
3345  PASS;
3346 }
3347 
3348 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3349  * test the sliding window reassembly with a small window size so that we
3350  * cutting off at the start (left edge). Test if the first segment is
3351  * removed from the list.
3352  */
3353 static int StreamTcpReassembleInlineTest09(void)
3354 {
3355  int ret = 0;
3356  TcpReassemblyThreadCtx *ra_ctx = NULL;
3357  ThreadVars tv;
3358  TcpSession ssn;
3359  Flow f;
3360 
3361  memset(&tv, 0x00, sizeof(tv));
3362 
3363  StreamTcpUTInit(&ra_ctx);
3366  StreamTcpUTSetupStream(&ssn.client, 1);
3367  FLOW_INITIALIZE(&f);
3368 
3371 
3372  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3373  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3374  if (p == NULL) {
3375  printf("couldn't get a packet: ");
3376  goto end;
3377  }
3378  p->tcph->th_seq = htonl(17);
3379  p->flow = &f;
3381 
3382  FLOWLOCK_WRLOCK(&f);
3383  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3384  printf("failed to add segment 1: ");
3385  goto end;
3386  }
3387  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3388  printf("failed to add segment 2: ");
3389  goto end;
3390  }
3391  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3392  printf("failed to add segment 3: ");
3393  goto end;
3394  }
3395  ssn.client.next_seq = 12;
3396  ssn.client.last_ack = 10;
3397 
3398  /* close the GAP and see if we properly reassemble and update base_seq */
3399  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3400  printf("failed to add segment 4: ");
3401  goto end;
3402  }
3403  ssn.client.next_seq = 22;
3404 
3405  p->tcph->th_seq = htonl(12);
3406 
3407  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3408  FAIL_IF_NULL(seg);
3409  FAIL_IF_NOT(seg->seq == 2);
3410 
3411  ret = 1;
3412 end:
3413  FLOWLOCK_UNLOCK(&f);
3414  FLOW_DESTROY(&f);
3415  UTHFreePacket(p);
3417  StreamTcpUTDeinit(ra_ctx);
3418  return ret;
3419 }
3420 
3421 /** \test App Layer reassembly.
3422  */
3423 static int StreamTcpReassembleInlineTest10(void)
3424 {
3425  int ret = 0;
3426  TcpReassemblyThreadCtx *ra_ctx = NULL;
3427  ThreadVars tv;
3428  TcpSession ssn;
3429  Flow *f = NULL;
3430  Packet *p = NULL;
3431 
3432  memset(&tv, 0x00, sizeof(tv));
3433 
3434  StreamTcpUTInit(&ra_ctx);
3437  StreamTcpUTSetupStream(&ssn.server, 1);
3438  ssn.server.last_ack = 2;
3439  StreamTcpUTSetupStream(&ssn.client, 1);
3440  ssn.client.last_ack = 2;
3442 
3443  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3444  if (f == NULL)
3445  goto end;
3446  f->protoctx = &ssn;
3447  f->proto = IPPROTO_TCP;
3448 
3449  uint8_t stream_payload1[] = "GE";
3450  uint8_t stream_payload2[] = "T /";
3451  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3452 
3453  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3454  if (p == NULL) {
3455  printf("couldn't get a packet: ");
3456  goto end;
3457  }
3458  p->tcph->th_seq = htonl(7);
3459  p->flow = f;
3461 
3462  FLOWLOCK_WRLOCK(f);
3463  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3464  printf("failed to add segment 1: ");
3465  goto end;
3466  }
3467  ssn.client.next_seq = 4;
3468 
3469  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3470  if (r < 0) {
3471  printf("StreamTcpReassembleAppLayer failed: ");
3472  goto end;
3473  }
3474 
3475  /* ssn.server.ra_app_base_seq should be isn here. */
3476  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3477  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3478  goto end;
3479  }
3480 
3481  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3482  printf("failed to add segment 2: ");
3483  goto end;
3484  }
3485  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3486  printf("failed to add segment 3: ");
3487  goto end;
3488  }
3489  ssn.client.next_seq = 19;
3490 
3491  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3492  if (r < 0) {
3493  printf("StreamTcpReassembleAppLayer failed: ");
3494  goto end;
3495  }
3496 
3497  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3498 
3499  ret = 1;
3500 end:
3501  UTHFreePacket(p);
3503  StreamTcpUTDeinit(ra_ctx);
3504  FLOWLOCK_UNLOCK(f);
3505  UTHFreeFlow(f);
3506  return ret;
3507 }
3508 
3509 /** \test test insert with overlap
3510  */
3511 static int StreamTcpReassembleInsertTest01(void)
3512 {
3513  TcpReassemblyThreadCtx *ra_ctx = NULL;
3514  ThreadVars tv;
3515  TcpSession ssn;
3516  Flow f;
3517 
3518  memset(&tv, 0x00, sizeof(tv));
3519 
3520  StreamTcpUTInit(&ra_ctx);
3522  StreamTcpUTSetupStream(&ssn.client, 1);
3524  FLOW_INITIALIZE(&f);
3525 
3526  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3527  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3528  FAIL_IF(p == NULL);
3529  p->tcph->th_seq = htonl(12);
3530  p->flow = &f;
3531 
3532  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3533  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3534  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3535  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3536  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3537  ssn.client.next_seq = 21;
3538 
3539  FLOW_DESTROY(&f);
3540  UTHFreePacket(p);
3542  StreamTcpUTDeinit(ra_ctx);
3543  PASS;
3544 }
3545 
3546 /** \test test insert with overlaps
3547  */
3548 static int StreamTcpReassembleInsertTest02(void)
3549 {
3550  int ret = 0;
3551  TcpReassemblyThreadCtx *ra_ctx = NULL;
3552  ThreadVars tv;
3553  TcpSession ssn;
3554 
3555  memset(&tv, 0x00, sizeof(tv));
3556 
3557  StreamTcpUTInit(&ra_ctx);
3559  StreamTcpUTSetupStream(&ssn.client, 1);
3560 
3561  int i;
3562  for (i = 2; i < 10; i++) {
3563  int len;
3564  len = i % 2;
3565  if (len == 0)
3566  len = 1;
3567  int seq;
3568  seq = i * 10;
3569  if (seq < 2)
3570  seq = 2;
3571 
3572  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3573  printf("failed to add segment 1: ");
3574  goto end;
3575  }
3576  }
3577  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3578  printf("failed to add segment 2: ");
3579  goto end;
3580  }
3581 
3582  ret = 1;
3583 end:
3585  StreamTcpUTDeinit(ra_ctx);
3586  return ret;
3587 }
3588 
3589 /** \test test insert with overlaps
3590  */
3591 static int StreamTcpReassembleInsertTest03(void)
3592 {
3593  int ret = 0;
3594  TcpReassemblyThreadCtx *ra_ctx = NULL;
3595  ThreadVars tv;
3596  TcpSession ssn;
3597 
3598  memset(&tv, 0x00, sizeof(tv));
3599 
3600  StreamTcpUTInit(&ra_ctx);
3602  StreamTcpUTSetupStream(&ssn.client, 1);
3603 
3604  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3605  printf("failed to add segment 2: ");
3606  goto end;
3607  }
3608 
3609  int i;
3610  for (i = 2; i < 10; i++) {
3611  int len;
3612  len = i % 2;
3613  if (len == 0)
3614  len = 1;
3615  int seq;
3616  seq = i * 10;
3617  if (seq < 2)
3618  seq = 2;
3619 
3620  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3621  printf("failed to add segment 2: ");
3622  goto end;
3623  }
3624  }
3625  ret = 1;
3626 end:
3628  StreamTcpUTDeinit(ra_ctx);
3629  return ret;
3630 }
3631 
3633 #endif /* UNITTESTS */
3634 
3635 /** \brief The Function Register the Unit tests to test the reassembly engine
3636  * for various OS policies.
3637  */
3638 
3640 {
3641 #ifdef UNITTESTS
3642  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3643  StreamTcpReassembleTest25);
3644  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3645  StreamTcpReassembleTest26);
3646  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3647  StreamTcpReassembleTest27);
3648  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3649  StreamTcpReassembleTest28);
3650  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3651  StreamTcpReassembleTest29);
3652  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3653  StreamTcpReassembleTest33);
3654  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3655  StreamTcpReassembleTest34);
3656  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3657  StreamTcpReassembleTest37);
3658  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3659  StreamTcpReassembleTest39);
3660  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3661  StreamTcpReassembleTest40);
3662  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3663  StreamTcpReassembleTest44);
3664  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3665  StreamTcpReassembleTest45);
3666  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3667  StreamTcpReassembleTest46);
3668  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3669  StreamTcpReassembleTest47);
3670 
3671  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3672  StreamTcpReassembleInlineTest01);
3673  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3674  StreamTcpReassembleInlineTest02);
3675  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3676  StreamTcpReassembleInlineTest03);
3677  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3678  StreamTcpReassembleInlineTest04);
3679  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3680  StreamTcpReassembleInlineTest08);
3681  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3682  StreamTcpReassembleInlineTest09);
3683 
3684  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3685  StreamTcpReassembleInlineTest10);
3686 
3687  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3688  StreamTcpReassembleInsertTest01);
3689  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3690  StreamTcpReassembleInsertTest02);
3691  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3692  StreamTcpReassembleInsertTest03);
3693 
3697  StreamTcpReassembleRawRegisterTests();
3698 #endif /* UNITTESTS */
3699 }
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
StreamingBufferSegment sbseg
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
#define SCLogDebug(...)
Definition: util-debug.h:335
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
void *(* Malloc)(size_t size)
int StreamTcpAppLayerIsDisabled(Flow *f)
#define SCMutexDestroy(x)
struct Flow_ * flow
Definition: decode.h:444
AppProto alproto_tc
Definition: flow.h:409
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:572
#define STREAM_BASE_OFFSET(stream)
int StreamTcpReassembleInit(char quiet)
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:346
#define FALSE
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:235
StreamingBuffer sb
#define PASS
Pass the test.
#define unlikely(expr)
Definition: util-optimize.h:35
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
#define STREAMTCP_STREAM_FLAG_GAP
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void StreamTcpListRegisterTests(void)
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define TH_RST
Definition: decode-tcp.h:37
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet...
Definition: stream-tcp.c:770
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream *stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:540
void StreamTcpDisableAppLayer(Flow *f)
#define TH_FIN
Definition: decode-tcp.h:35
#define STREAMING_BUFFER_NOFLAGS
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:108
uint64_t offset
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Address dst
Definition: decode.h:412
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
#define SCReturnUInt(x)
Definition: util-debug.h:343
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
int ByteExtractStringUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:244
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:669
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:232
uint64_t pcap_cnt
Definition: decode.h:566
#define STREAM_RAW_PROGRESS(stream)
void StreamTcpReassembleInitMemuse(void)
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:125
char * val
Definition: conf.h:34
TCPHdr * tcph
Definition: decode.h:525
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define TRUE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
StreamingBufferBlock * head
#define STREAMTCP_FLAG_MIDSTREAM
void * protoctx
Definition: flow.h:398
void *(* Calloc)(size_t n, size_t size)
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
#define STREAM_DEPTH
Definition: stream.h:34
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
uint32_t min_inspect_depth
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueue *pq)
Definition: stream-tcp.c:4670
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:82
#define StreamTcpSetEvent(p, e)
AppProto alproto_ts
Definition: flow.h:408
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
#define SCCalloc(nm, a)
Definition: util-mem.h:205
#define SEQ_GEQ(a, b)
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:106
#define STREAM_GAP
Definition: stream.h:33
#define SCMutexUnlock(mut)
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:783
uint32_t seq
uint32_t raw_progress_rel
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:240
char family
Definition: decode.h:110
#define SCMutexInit(mut, mutattr)
uint8_t proto
Definition: decode.h:429
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
#define TCP_SEG_LEN(seg)
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
#define STREAM_HAS_SEEN_DATA(stream)
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
uint32_t app_progress_rel
Data structures and function prototypes for keeping state for the detection engine.
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define FLOW_DESTROY(f)
Definition: flow-util.h:115
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies...
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
#define STREAM_EOF
Definition: stream.h:30
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define StreamTcpDisableAppLayerReassembly(ssn)
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
#define SCEnter(...)
Definition: util-debug.h:337
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
void StreamTcpReassembleFree(char quiet)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:438
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PKT_IS_TOSERVER(p)
Definition: decode.h:256
#define FLOW_PKT_TOSERVER
Definition: flow.h:193
uint32_t reassembly_depth
#define TH_PUSH
Definition: decode-tcp.h:38
#define SEQ_GT(a, b)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:163
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1102
int RunmodeIsUnittests(void)
Definition: suricata.c:261
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1000
int8_t data_first_seen_dir
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define SCReturnInt(x)
Definition: util-debug.h:341
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueue *pq)
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
void StreamTcpUTInitInline(void)
#define SEQ_LEQ(a, b)
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
uint32_t reassembly_depth
Definition: stream-tcp.h:60
#define SCRealloc(x, a)
Definition: util-mem.h:190
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define RB_MIN(name, x)
Definition: tree.h:780
Definition: conf.h:32
int PoolThreadGrow(PoolThread *pt, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
grow a thread pool by one
#define SCMalloc(a)
Definition: util-mem.h:174
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:208
struct TCPSEG seg_tree
uint16_t payload_len
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define TH_SYN
Definition: decode-tcp.h:36
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
block of continues data
#define SCFree(a)
Definition: util-mem.h:236
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:241
#define RB_REMOVE(name, x, y)
Definition: tree.h:775
void StreamTcpUtilRegisterTests(void)
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:805
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
#define SCMutex
#define STREAM_START
Definition: stream.h:29
#define SCMutexLock(mut)
uint32_t prealloc_segments
Definition: stream-tcp.h:57
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
#define PKT_STREAM_ADD
Definition: decode.h:1098
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself...
void StreamTcpInlineRegisterTests(void)
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:193
void StreamTcpReassemblePseudoPacketCreate(TcpStream *, Packet *, PacketQueue *)
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
TCPHdr tcph
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
#define SCReturn
Definition: util-debug.h:339
void StreamTcpUTSetupSession(TcpSession *ssn)
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define MISSED_END
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
#define FLOW_PKT_TOCLIENT
Definition: flow.h:194
AppProto alproto
application level protocol
Definition: flow.h:407
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of &#39;threads&#39;, so array elements)
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
uint32_t flags
Definition: decode.h:442
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
void StreamTcpReassembleConfigEnableOverlapCheck(void)
uint16_t payload_len
Definition: decode.h:546
#define STREAM_APP_PROGRESS(stream)
#define SEQ_LT(a, b)
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6283
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
#define STREAM_RIGHT_EDGE(stream)
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
void StreamTcpUTClearSession(TcpSession *ssn)
#define MISSED_START(isn)
Flow data structure.
Definition: flow.h:327
#define FLOW_IPV4
Definition: flow.h:93
uint8_t * payload
Definition: decode.h:545
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
initialize a thread pool
uint32_t flags
Definition: flow.h:377
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
void StreamTcpUTClearStream(TcpStream *s)
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
void(* Free)(void *ptr, size_t size)
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:140
Address src
Definition: decode.h:411
void UTHFreeFlow(Flow *flow)
#define STREAM_MIDSTREAM
Definition: stream.h:35
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SET_ISN(stream, setseq)
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
#define SCMUTEX_INITIALIZER