suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 
64 #include "detect-engine-state.h"
65 
66 #include "util-profiling.h"
67 #include "util-validate.h"
68 
69 #ifdef DEBUG
70 static SCMutex segment_pool_memuse_mutex;
71 static uint64_t segment_pool_memuse = 0;
72 static uint64_t segment_pool_memcnt = 0;
73 #endif
74 
75 static PoolThread *segment_thread_pool = NULL;
76 /* init only, protect initializing and growing pool */
77 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
78 
79 /* Memory use counter */
80 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
81 
82 /* prototypes */
84 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346 }
347 
348 /** \param f locked flow */
350 {
351  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
352  return 0;
353 
354  TcpSession *ssn = (TcpSession *)f->protoctx;
356 }
357 
358 static int StreamTcpReassemblyConfig(char quiet)
359 {
360  uint32_t segment_prealloc = 2048;
361  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
362  if (seg) {
363  uint32_t prealloc = 0;
364  if (ByteExtractStringUint32(&prealloc, 10, strlen(seg->val), seg->val) == -1)
365  {
366  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
367  "%s is invalid", seg->val);
368  return -1;
369  }
370  segment_prealloc = prealloc;
371  }
372  if (!quiet)
373  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
374  stream_config.prealloc_segments = segment_prealloc;
375 
376  int overlap_diff_data = 0;
377  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
378  if (overlap_diff_data) {
380  }
381  if (StreamTcpInlineMode() == TRUE) {
383  }
384 
387  stream_config.sbcnf.Malloc = ReassembleMalloc;
388  stream_config.sbcnf.Calloc = ReassembleCalloc;
389  stream_config.sbcnf.Realloc = ReassembleRealloc;
390  stream_config.sbcnf.Free = ReassembleFree;
391 
392  return 0;
393 }
394 
395 int StreamTcpReassembleInit(char quiet)
396 {
397  /* init the memcap/use tracker */
399 
400  if (StreamTcpReassemblyConfig(quiet) < 0)
401  return -1;
402 
403 #ifdef DEBUG
404  SCMutexInit(&segment_pool_memuse_mutex, NULL);
405 #endif
406  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
408  return 0;
409 }
410 
411 void StreamTcpReassembleFree(char quiet)
412 {
413  SCMutexLock(&segment_thread_pool_mutex);
414  if (segment_thread_pool != NULL) {
415  PoolThreadFree(segment_thread_pool);
416  segment_thread_pool = NULL;
417  }
418  SCMutexUnlock(&segment_thread_pool_mutex);
419  SCMutexDestroy(&segment_thread_pool_mutex);
420 
421 #ifdef DEBUG
422  if (segment_pool_memuse > 0)
423  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
424  if (segment_pool_memcnt > 0)
425  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
426  SCMutexDestroy(&segment_pool_memuse_mutex);
427 #endif
428 }
429 
431 {
432  SCEnter();
434  if (unlikely(ra_ctx == NULL))
435  return NULL;
436 
437  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
438 
439  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
440 
441  SCMutexLock(&segment_thread_pool_mutex);
442  if (segment_thread_pool == NULL) {
443  segment_thread_pool = PoolThreadInit(1, /* thread */
444  0, /* unlimited */
446  sizeof(TcpSegment),
447  TcpSegmentPoolAlloc,
448  TcpSegmentPoolInit, NULL,
449  TcpSegmentPoolCleanup, NULL);
450  ra_ctx->segment_thread_pool_id = 0;
451  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
452  PoolThreadSize(segment_thread_pool),
453  ra_ctx->segment_thread_pool_id);
454  } else {
455  /* grow segment_thread_pool until we have a element for our thread id */
456  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
457  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
458  PoolThreadSize(segment_thread_pool),
459  ra_ctx->segment_thread_pool_id);
460  }
461  SCMutexUnlock(&segment_thread_pool_mutex);
462  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
463  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
465  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
466  }
467 
468  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
469 }
470 
472 {
473  SCEnter();
475  SCFree(ra_ctx);
476  SCReturn;
477 }
478 
479 /**
480  * \brief check if stream in pkt direction has depth reached
481  *
482  * \param p packet with *LOCKED* flow
483  *
484  * \retval 1 stream has depth reached
485  * \retval 0 stream does not have depth reached
486  */
488 {
489  if (p->flow != NULL && p->flow->protoctx != NULL) {
490  TcpSession *ssn = p->flow->protoctx;
491  TcpStream *stream;
492  if (p->flowflags & FLOW_PKT_TOSERVER) {
493  stream = &ssn->client;
494  } else {
495  stream = &ssn->server;
496  }
497 
498  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
499  }
500 
501  return 0;
502 }
503 
504 /**
505  * \internal
506  * \brief Function to Check the reassembly depth valuer against the
507  * allowed max depth of the stream reassmbly for TCP streams.
508  *
509  * \param stream stream direction
510  * \param seq sequence number where "size" starts
511  * \param size size of the segment that is added
512  *
513  * \retval size Part of the size that fits in the depth, 0 if none
514  */
515 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
516  uint32_t seq, uint32_t size)
517 {
518  SCEnter();
519 
520  /* if the configured depth value is 0, it means there is no limit on
521  reassembly depth. Otherwise carry on my boy ;) */
522  if (ssn->reassembly_depth == 0) {
523  SCReturnUInt(size);
524  }
525 
526  /* if the final flag is set, we're not accepting anymore */
528  SCReturnUInt(0);
529  }
530 
531  uint64_t seg_depth;
532  if (SEQ_GT(stream->base_seq, seq)) {
533  if (SEQ_LEQ(seq+size, stream->base_seq)) {
534  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
535  stream->base_seq, seq, seq+size);
536  SCReturnUInt(0);
537  }
538 
539  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
540  } else {
541  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
542  }
543 
544  /* if the base_seq has moved passed the depth window we stop
545  * checking and just reject the rest of the packets including
546  * retransmissions. Saves us the hassle of dealing with sequence
547  * wraps as well */
548  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
549  stream->base_seq, seg_depth,
550  ssn->reassembly_depth);
551 
552  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
553  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
555  SCReturnUInt(0);
556  }
557  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
558  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
559 #if 0
560  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
561  (stream->base_seq_offset + stream->base_seq + size),
562  (stream->isn + ssn->reassembly_depth));
563 #endif
564  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
565  /* packet (partly?) fits the depth window */
566 
567  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
568  /* complete fit */
569  SCReturnUInt(size);
570  } else {
572  /* partial fit, return only what fits */
573  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
574  DEBUG_VALIDATE_BUG_ON(part > size);
575  if (part > size)
576  part = size;
577  SCReturnUInt(part);
578  }
579  }
580 
581  SCReturnUInt(0);
582 }
583 
584 /**
585  * \brief Insert a packets TCP data into the stream reassembly engine.
586  *
587  * \retval 0 good segment, as far as we checked.
588  * \retval -1 badness, reason to drop in inline mode
589  *
590  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
591  * or it wasn't added because of reassembly depth.
592  *
593  */
595  TcpSession *ssn, TcpStream *stream, Packet *p)
596 {
597  SCEnter();
598 
599  if (ssn->data_first_seen_dir == 0) {
600  if (PKT_IS_TOSERVER(p)) {
602  } else {
604  }
605  }
606 
607  /* If the OS policy is not set then set the OS policy for this stream */
608  if (stream->os_policy == 0) {
609  StreamTcpSetOSPolicy(stream, p);
610  }
611 
614  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
615  SCReturnInt(0);
616  }
617 
618  /* If we have reached the defined depth for either of the stream, then stop
619  reassembling the TCP session */
620  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
621  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
622 
624  /* increment stream depth counter */
625  StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
626  }
627  if (size == 0) {
628  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
629  SCReturnInt(0);
630  }
631 
633  if (size > p->payload_len)
634  size = p->payload_len;
635 
636  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
637  if (seg == NULL) {
638  SCLogDebug("segment_pool is empty");
640  SCReturnInt(-1);
641  }
642 
643  TCP_SEG_LEN(seg) = size;
644  seg->seq = TCP_GET_SEQ(p);
645 
646  /* proto detection skipped, but now we do get data. Set event. */
647  if (RB_EMPTY(&stream->seg_tree) &&
649 
652  }
653 
654  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
655  SCLogDebug("StreamTcpReassembleInsertSegment failed");
656  SCReturnInt(-1);
657  }
658  SCReturnInt(0);
659 }
660 
661 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
662  Packet *p, enum StreamUpdateDir dir)
663 {
664  uint8_t flag = 0;
665 
667  flag |= STREAM_START;
668  }
669 
670  if (ssn->state == TCP_CLOSED) {
671  flag |= STREAM_EOF;
672  }
673 
674  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
675  flag |= STREAM_MIDSTREAM;
676  }
677 
678  if (p->flags & PKT_PSEUDO_STREAM_END) {
679  flag |= STREAM_EOF;
680  }
681 
682  if (dir == UPDATE_DIR_OPPOSING) {
683  if (p->flowflags & FLOW_PKT_TOSERVER) {
684  flag |= STREAM_TOCLIENT;
685  } else {
686  flag |= STREAM_TOSERVER;
687  }
688  } else {
689  if (p->flowflags & FLOW_PKT_TOSERVER) {
690  flag |= STREAM_TOSERVER;
691  } else {
692  flag |= STREAM_TOCLIENT;
693  }
694  }
695 
697  flag |= STREAM_DEPTH;
698  }
699  return flag;
700 }
701 
702 /**
703  * \brief Check the minimum size limits for reassembly.
704  *
705  * \retval 0 don't reassemble yet
706  * \retval 1 do reassemble
707  */
708 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
709  const TcpStream *stream, const Packet *p)
710 {
711  SCEnter();
712 
713  /* if any of these flags is set we always inspect immediately */
714 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
715  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
716  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
717  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
718 
721  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
722  "is set, so not expecting any new data segments");
723  }
725  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
726  }
728  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
729  "so no new segments will be considered");
730  }
731  SCReturnInt(1);
732  }
733 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
734 
735  /* some states mean we reassemble no matter how much data we have */
736  if (ssn->state > TCP_TIME_WAIT)
737  SCReturnInt(1);
738 
739  if (p->flags & PKT_PSEUDO_STREAM_END)
740  SCReturnInt(1);
741 
742  /* check if we have enough data to do raw reassembly */
743  if (PKT_IS_TOSERVER(p)) {
744  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
745  uint32_t delta = stream->last_ack - stream->base_seq;
746  /* get max absolute offset */
747  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
748 
749  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
750  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
751  SCReturnInt(1);
752  } else {
753  SCLogDebug("toserver min chunk len not yet reached: "
754  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
755  "%"PRIu32"", stream->last_ack, stream->base_seq,
756  (stream->last_ack - stream->base_seq),
758  SCReturnInt(0);
759  }
760  }
761  } else {
762  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
763  uint32_t delta = stream->last_ack - stream->base_seq;
764  /* get max absolute offset */
765  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
766 
767  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
768 
769  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
770  SCReturnInt(1);
771  } else {
772  SCLogDebug("toclient min chunk len not yet reached: "
773  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
774  "%"PRIu32"", stream->last_ack, stream->base_seq,
775  (stream->last_ack - stream->base_seq),
777  SCReturnInt(0);
778  }
779  }
780  }
781 
782  SCReturnInt(0);
783 }
784 
785 /**
786  * \brief see what if any work the TCP session still needs
787  */
788 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
789 {
790  const TcpStream *stream = NULL;
791 #ifdef DEBUG
792  const char *dirstr = NULL;
793 #endif
794  if (direction == STREAM_TOSERVER) {
795  stream = &ssn->client;
796 #ifdef DEBUG
797  dirstr = "client";
798 #endif
799  } else {
800  stream = &ssn->server;
801 #ifdef DEBUG
802  dirstr = "server";
803 #endif
804  }
805 
806  int use_app = 1;
807  int use_raw = 1;
808 
810  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
811  {
812  // app is dead
813  use_app = 0;
814  }
815 
817  // raw is dead
818  use_raw = 0;
819  }
820 
821  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
822 
823  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
824  dirstr,
825  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
826  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
827  right_edge);
828  if (use_raw) {
829  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
830  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
832  }
833  }
834  if (use_app) {
835  if (right_edge > STREAM_APP_PROGRESS(stream)) {
836  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
838  }
839  }
840 
841  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
843 }
844 
845 #ifdef DEBUG
846 static uint64_t GetStreamSize(TcpStream *stream)
847 {
848  if (stream) {
849  uint64_t size = 0;
850  uint32_t cnt = 0;
851 
852  TcpSegment *seg;
853  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
854  cnt++;
855  size += (uint64_t)TCP_SEG_LEN(seg);
856  }
857 
858  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
859  return size;
860  }
861  return (uint64_t)0;
862 }
863 
864 static void GetSessionSize(TcpSession *ssn, Packet *p)
865 {
866  uint64_t size = 0;
867  if (ssn) {
868  size = GetStreamSize(&ssn->client);
869  size += GetStreamSize(&ssn->server);
870 
871  //if (size > 900000)
872  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
873  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
874  }
875 }
876 #endif
877 
878 /** \internal
879  *
880  * Get buffer, or first part of the buffer if data gaps exist.
881  *
882  * \brief get stream data from offset
883  * \param offset stream offset */
884 static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset)
885 {
886  const uint8_t *mydata;
887  uint32_t mydata_len;
888 
889  if (RB_EMPTY(&stream->sb.sbb_tree)) {
890  SCLogDebug("getting one blob");
891 
892  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
893 
894  *data = mydata;
895  *data_len = mydata_len;
896  } else {
897  StreamingBufferBlock *blk = stream->sb.head;
898 
899  if (blk->offset > offset) {
900  SCLogDebug("gap, want data at offset %"PRIu64", "
901  "got data at %"PRIu64". GAP of size %"PRIu64,
902  offset, blk->offset, blk->offset - offset);
903  *data = NULL;
904  *data_len = blk->offset - offset;
905 
906  } else if (offset >= (blk->offset + blk->len)) {
907 
908  *data = NULL;
909  StreamingBufferBlock *nblk = SBB_RB_NEXT(blk);
910  *data_len = nblk ? nblk->offset - offset : 0;
911  if (nblk) {
912  SCLogDebug("gap, want data at offset %"PRIu64", "
913  "got data at %"PRIu64". GAP of size %"PRIu64,
914  offset, nblk->offset, nblk->offset - offset);
915  }
916 
917  } else if (offset > blk->offset && offset < (blk->offset + blk->len)) {
918  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
919  offset, blk->offset, blk->len);
920  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
921  SCLogDebug("data %p, data_len %u", *data, *data_len);
922  } else {
923  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
924  }
925  }
926 }
927 
928 /** \internal
929  * \brief check to see if we should declare a GAP
930  * Call this when the app layer didn't get data at the requested
931  * offset.
932  */
933 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
934 {
935  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
936  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
937 
938  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
939  /* get window of data that is acked */
940  const uint32_t delta = stream->last_ack - stream->base_seq;
941  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
942  /* get max absolute offset */
943  last_ack_abs += delta;
944 
945  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
946  last_ack_abs -= ackadded;
947 
948  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
949  SCLogDebug("next_seq %u", stream->next_seq);
950 
951  /* if last_ack_abs is beyond the app_progress data that we haven't seen
952  * has been ack'd. This looks like a GAP. */
953  if (last_ack_abs > app_progress) {
954  /* however, we can accept ACKs a bit too liberally. If last_ack
955  * is beyond next_seq, we only consider it a gap now if we do
956  * already have data beyond the gap. */
957  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
958  if (RB_EMPTY(&stream->sb.sbb_tree)) {
959  SCLogDebug("packet %"PRIu64": no GAP. "
960  "next_seq %u < last_ack %u, but no data in list",
961  p->pcap_cnt, stream->next_seq, stream->last_ack);
962  return false;
963  } else {
964  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
965  const StreamingBufferBlock *blk = stream->sb.head;
966  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
967  /* ack'd data after the gap */
968  SCLogDebug("packet %"PRIu64": GAP. "
969  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
970  p->pcap_cnt, stream->next_seq, stream->last_ack);
971  return true;
972  }
973  }
974  }
975 
976  SCLogDebug("packet %"PRIu64": GAP! "
977  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
978  "but we have no data.",
979  p->pcap_cnt, last_ack_abs, app_progress);
980  return true;
981  }
982  }
983  SCLogDebug("packet %"PRIu64": no GAP. "
984  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
985  p->pcap_cnt, last_ack_abs, app_progress);
986  return false;
987 }
988 
989 /** \internal
990  * \brief get stream buffer and update the app-layer
991  * \retval 0 success
992  */
993 static int ReassembleUpdateAppLayer (ThreadVars *tv,
994  TcpReassemblyThreadCtx *ra_ctx,
995  TcpSession *ssn, TcpStream *stream,
996  Packet *p, enum StreamUpdateDir dir)
997 {
998  uint64_t app_progress = STREAM_APP_PROGRESS(stream);
999 
1000  SCLogDebug("app progress %"PRIu64, app_progress);
1001  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1002 
1003  const uint8_t *mydata;
1004  uint32_t mydata_len;
1005 
1006  while (1) {
1007  GetAppBuffer(stream, &mydata, &mydata_len, app_progress);
1008  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, stream, p)) {
1009  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1010 
1011  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1012  NULL, mydata_len,
1013  StreamGetAppLayerFlags(ssn, stream, p, dir)|STREAM_GAP);
1014  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1015 
1017  StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
1018 
1019  stream->app_progress_rel += mydata_len;
1020  app_progress += mydata_len;
1021  if (r < 0)
1022  break;
1023 
1024  continue;
1025  } else if (mydata == NULL || mydata_len == 0) {
1026  /* Possibly a gap, but no new data. */
1027  return 0;
1028  }
1029  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1030  break;
1031  }
1032 
1033  //PrintRawDataFp(stdout, mydata, mydata_len);
1034 
1035  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1036  stream, &stream->sb, mydata_len, app_progress);
1037 
1038  /* get window of data that is acked */
1039  if (StreamTcpInlineMode() == FALSE) {
1040  if (p->flags & PKT_PSEUDO_STREAM_END) {
1041  // fall through, we use all available data
1042  } else {
1043  uint64_t last_ack_abs = app_progress; /* absolute right edge of ack'd data */
1044  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1045  /* get window of data that is acked */
1046  uint32_t delta = stream->last_ack - stream->base_seq;
1047  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1048  /* get max absolute offset */
1049  last_ack_abs += delta;
1050  }
1051 
1052  /* see if the buffer contains unack'd data as well */
1053  if (app_progress + mydata_len > last_ack_abs) {
1054  uint32_t check = mydata_len;
1055  mydata_len = last_ack_abs - app_progress;
1056  BUG_ON(mydata_len > check);
1057  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1058  "data is considered", mydata_len);
1059  }
1060  }
1061  }
1062 
1063  /* update the app-layer */
1064  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1065  (uint8_t *)mydata, mydata_len,
1066  StreamGetAppLayerFlags(ssn, stream, p, dir));
1067  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1068 
1069  /* see if we can update the progress */
1070  if (r == 0 && mydata_len > 0 &&
1072  {
1073  SCLogDebug("app progress %"PRIu64" increasing with data len %u to %"PRIu64,
1074  app_progress, mydata_len, app_progress + mydata_len);
1075 
1076  stream->app_progress_rel += mydata_len;
1077  SCLogDebug("app progress now %"PRIu64, STREAM_APP_PROGRESS(stream));
1078  } else {
1079  SCLogDebug("NOT UPDATED app progress still %"PRIu64, app_progress);
1080  }
1081 
1082  SCReturnInt(0);
1083 }
1084 
1085 /**
1086  * \brief Update the stream reassembly upon receiving a packet.
1087  *
1088  * For IDS mode, the stream is in the opposite direction of the packet,
1089  * as the ACK-packet is ACK'ing the stream.
1090  *
1091  * One of the utilities call by this function AppLayerHandleTCPData(),
1092  * has a feature where it will call this very same function for the
1093  * stream opposing the stream it is called with. This shouldn't cause
1094  * any issues, since processing of each stream is independent of the
1095  * other stream.
1096  */
1098  TcpSession *ssn, TcpStream *stream,
1099  Packet *p, enum StreamUpdateDir dir)
1100 {
1101  SCEnter();
1102 
1103  /* this function can be directly called by app layer protocol
1104  * detection. */
1107  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1108  SCReturnInt(0);
1109  }
1110 
1111 #ifdef DEBUG
1112  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1113  GetSessionSize(ssn, p);
1114 #endif
1115  /* if no segments are in the list or all are already processed,
1116  * and state is beyond established, we send an empty msg */
1117  if (STREAM_HAS_SEEN_DATA(stream) && STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1118  {
1119  /* send an empty EOF msg if we have no segments but TCP state
1120  * is beyond ESTABLISHED */
1121  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1122  SCLogDebug("sending empty eof message");
1123  /* send EOF to app layer */
1124  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1125  NULL, 0,
1126  StreamGetAppLayerFlags(ssn, stream, p, dir));
1127  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1128 
1129  SCReturnInt(0);
1130  }
1131  }
1132 
1133  /* with all that out of the way, lets update the app-layer */
1134  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, stream, p, dir);
1135 }
1136 
1137 /** \internal
1138  * \brief get stream data from offset
1139  * \param offset stream offset */
1140 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1141  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1142 {
1143  const uint8_t *mydata;
1144  uint32_t mydata_len;
1145  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1146  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1147 
1148  uint64_t roffset = offset;
1149  if (offset)
1150  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1151  else {
1152  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1153  }
1154 
1155  *data = mydata;
1156  *data_len = mydata_len;
1157  *data_offset = roffset;
1158  } else {
1159  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1160  *iter == NULL ? "starting" : "continuing", offset);
1161  if (*iter == NULL) {
1162  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1163  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1164  SCLogDebug("*iter %p", *iter);
1165  }
1166  if (*iter == NULL) {
1167  SCLogDebug("no data");
1168  *data = NULL;
1169  *data_len = 0;
1170  *data_offset = 0;
1171  return 0;
1172  }
1173  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1174 
1175  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1176  SCLogDebug("mydata %p", mydata);
1177 
1178  if ((*iter)->offset < offset) {
1179  uint64_t delta = offset - (*iter)->offset;
1180  if (delta < mydata_len) {
1181  *data = mydata + delta;
1182  *data_len = mydata_len - delta;
1183  *data_offset = offset;
1184  } else {
1185  SCLogDebug("no data (yet)");
1186  *data = NULL;
1187  *data_len = 0;
1188  *data_offset = 0;
1189  }
1190 
1191  } else {
1192  *data = mydata;
1193  *data_len = mydata_len;
1194  *data_offset = (*iter)->offset;
1195  }
1196 
1197  *iter = SBB_RB_NEXT(*iter);
1198  SCLogDebug("*iter %p", *iter);
1199  }
1200  return 0;
1201 }
1202 
1203 /** \brief does the stream engine have data to inspect?
1204  *
1205  * Returns true if there is data to inspect. In IDS case this is
1206  * about ACK'd data in the packet's direction.
1207  *
1208  * In the IPS case this is about the packet itself.
1209  */
1211 {
1212  TcpStream *stream;
1213  if (PKT_IS_TOSERVER(p)) {
1214  stream = &ssn->client;
1215  } else {
1216  stream = &ssn->server;
1217  }
1218 
1219  if (RB_EMPTY(&stream->seg_tree)) {
1220  return false;
1221  }
1222 
1225  return false;
1226 
1227  if (StreamTcpInlineMode() == FALSE) {
1228  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1229  return false;
1230  }
1231  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1232  return true;
1233  }
1234  } else {
1235  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1236  return true;
1237  }
1238  }
1239  return false;
1240 }
1241 
1242 /** \brief update stream engine after detection
1243  *
1244  * Tasked with progressing the 'progress' for Raw reassembly.
1245  * 2 main scenario's:
1246  * 1. progress is != 0, so we use this
1247  * 2. progress is 0, meaning the detect engine didn't touch
1248  * raw at all. In this case we need to look into progressing
1249  * raw anyway.
1250  *
1251  * Additionally, this function is tasked with disabling raw
1252  * reassembly if the app-layer requested to disable it.
1253  */
1254 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1255 {
1256  TcpStream *stream;
1257  if (PKT_IS_TOSERVER(p)) {
1258  stream = &ssn->client;
1259  } else {
1260  stream = &ssn->server;
1261  }
1262 
1263  if (progress > STREAM_RAW_PROGRESS(stream)) {
1264  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1265  stream->raw_progress_rel += slide;
1267 
1268  /* if app is active and beyond raw, sync raw to app */
1269  } else if (progress == 0 &&
1270  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1272  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1273  {
1274  /* if trigger raw is set we sync the 2 trackers */
1276  {
1277  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1278  stream->raw_progress_rel += slide;
1280 
1281  /* otherwise mix in the tcp window */
1282  } else {
1283  uint64_t tcp_window = stream->window;
1284  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1285  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1286  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1287  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1288  stream->raw_progress_rel += slide;
1289  }
1290  }
1291  }
1292  /* app is dead */
1293  } else if (progress == 0) {
1294  uint64_t tcp_window = stream->window;
1295  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1296  if (tcp_window < stream_right_edge) {
1297  uint64_t new_raw = stream_right_edge - tcp_window;
1298  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1299  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1300  stream->raw_progress_rel += slide;
1301  }
1302  }
1304 
1305  } else {
1306  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1307  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1308  STREAM_RAW_PROGRESS(stream), stream->window);
1309  }
1310 
1311  /* if we were told to accept no more raw data, we can mark raw as
1312  * disabled now. */
1315  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1316  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1317  }
1318 
1319  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1320 }
1321 
1322 /** \internal
1323  * \brief get a buffer around the current packet and run the callback on it
1324  *
1325  * The inline/IPS scanning method takes the current payload and wraps it in
1326  * data from other segments.
1327  *
1328  * How much data is inspected is controlled by the available data, chunk_size
1329  * and the payload size of the packet.
1330  *
1331  * Large packets: if payload size is close to the chunk_size, where close is
1332  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1333  * used: payload_len + 33% of the chunk_size.
1334  * If the payload size if equal to or bigger than the chunk_size, we use
1335  * payload len + 33% of the chunk size.
1336  */
1337 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1338  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1339 {
1340  SCEnter();
1341  int r = 0;
1342 
1343  TcpStream *stream;
1344  if (PKT_IS_TOSERVER(p)) {
1345  stream = &ssn->client;
1346  } else {
1347  stream = &ssn->server;
1348  }
1349 
1350  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1352  {
1353  *progress_out = STREAM_RAW_PROGRESS(stream);
1354  return 0;
1355  }
1356 
1357  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1360  if (chunk_size <= p->payload_len) {
1361  chunk_size = p->payload_len + (chunk_size / 3);
1362  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1363  p->payload_len, chunk_size);
1364  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1365  chunk_size = p->payload_len + ((chunk_size / 3));
1366  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1367  p->payload_len, chunk_size);
1368  }
1369 
1370  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1371  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1372  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1373  packet_leftedge_abs, packet_rightedge_abs);
1374 
1375  const uint8_t *mydata = NULL;
1376  uint32_t mydata_len = 0;
1377  uint64_t mydata_offset = 0;
1378  /* simply return progress from the block we inspected. */
1379  bool return_progress = false;
1380 
1381  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1382  /* continues block */
1383  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1384  return_progress = true;
1385 
1386  } else {
1387  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1388  /* find our block */
1389  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1390  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1391  if (sbb) {
1392  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1393  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1394  mydata_offset = sbb->offset;
1395  }
1396  }
1397 
1398  /* this can only happen if the segment insert of our current 'p' failed */
1399  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1400  if ((mydata == NULL || mydata_len == 0) || /* no data */
1401  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1402  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1403  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1404  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1405  {
1406  /* no data, or data is incomplete or wrong: use packet data */
1407  mydata = p->payload;
1408  mydata_len = p->payload_len;
1409  mydata_offset = packet_leftedge_abs;
1410  //mydata_rightedge_abs = packet_rightedge_abs;
1411  } else {
1412  /* adjust buffer to match chunk_size */
1413  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1414  if (mydata_len > chunk_size) {
1415  uint32_t excess = mydata_len - chunk_size;
1416  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1417 
1418  if (mydata_rightedge_abs == packet_rightedge_abs) {
1419  mydata += excess;
1420  mydata_len -= excess;
1421  mydata_offset += excess;
1422  SCLogDebug("cutting front of the buffer with %u", excess);
1423  } else if (mydata_offset == packet_leftedge_abs) {
1424  mydata_len -= excess;
1425  SCLogDebug("cutting tail of the buffer with %u", excess);
1426  } else {
1427  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1428  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1429  SCLogDebug("before %u after %u", before, after);
1430 
1431  if (after >= (chunk_size - p->payload_len) / 2) {
1432  // more trailing data than we need
1433 
1434  if (before >= (chunk_size - p->payload_len) / 2) {
1435  // also more heading data, devide evenly
1436  before = after = (chunk_size - p->payload_len) / 2;
1437  } else {
1438  // heading data is less than requested, give the
1439  // rest to the trailing data
1440  after = (chunk_size - p->payload_len) - before;
1441  }
1442  } else {
1443  // less trailing data than requested
1444 
1445  if (before >= (chunk_size - p->payload_len) / 2) {
1446  before = (chunk_size - p->payload_len) - after;
1447  } else {
1448  // both smaller than their requested size
1449  }
1450  }
1451 
1452  /* adjust the buffer */
1453  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1454  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1455  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1456  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1457  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1458  mydata += skip;
1459  mydata_len -= (skip + cut);
1460  mydata_offset += skip;
1461  }
1462  }
1463  }
1464 
1465  /* run the callback */
1466  r = Callback(cb_data, mydata, mydata_len);
1467  BUG_ON(r < 0);
1468 
1469  if (return_progress) {
1470  *progress_out = (mydata_offset + mydata_len);
1471  } else {
1472  /* several blocks of data, so we need to be a bit more careful:
1473  * - if last_ack is beyond last progress, move progress forward to last_ack
1474  * - if our block matches or starts before last ack, return right edge of
1475  * our block.
1476  */
1477  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1478  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1479  uint32_t delta = stream->last_ack - stream->base_seq;
1480  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1481  /* get max absolute offset */
1482  last_ack_abs += delta;
1483  }
1484  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1485 
1486  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1487  if (mydata_offset > last_ack_abs) {
1488  /* gap between us and last ack, set progress to last ack */
1489  *progress_out = last_ack_abs;
1490  } else {
1491  *progress_out = (mydata_offset + mydata_len);
1492  }
1493  } else {
1494  *progress_out = STREAM_RAW_PROGRESS(stream);
1495  }
1496  }
1497  return r;
1498 }
1499 
1500 /** \brief access 'raw' reassembly data.
1501  *
1502  * Access data as tracked by 'raw' tracker. Data is made available to
1503  * callback that is passed to this function.
1504  *
1505  * In the case of IDS the callback may be run multiple times if data
1506  * contains gaps. It will then be run for each block of data that is
1507  * continuous.
1508  *
1509  * The callback should give on of 2 return values:
1510  * - 0 ok
1511  * - 1 done
1512  * The value 1 will break the loop if there is a block list that is
1513  * inspected.
1514  *
1515  * This function will return the 'progress' value that has been
1516  * consumed until now.
1517  *
1518  * \param ssn tcp session
1519  * \param stream tcp stream
1520  * \param Callback the function pointer to the callback function
1521  * \param cb_data callback data
1522  * \param[in] progress_in progress to work from
1523  * \param[out] progress_out absolute progress value of the data this
1524  * call handled.
1525  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1526  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1527  *
1528  * `respect_inspect_depth` is used to avoid useless inspection of too
1529  * much data.
1530  */
1531 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1532  StreamReassembleRawFunc Callback, void *cb_data,
1533  const uint64_t progress_in,
1534  uint64_t *progress_out, bool eof,
1535  bool respect_inspect_depth)
1536 {
1537  SCEnter();
1538  int r = 0;
1539 
1540  StreamingBufferBlock *iter = NULL;
1541  uint64_t progress = progress_in;
1542  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1543 
1544  /* if the app layer triggered a flush, and we're supposed to
1545  * use a minimal inspect depth, we actually take the app progress
1546  * as that is the right edge of the data. Then we take the window
1547  * of 'min_inspect_depth' before that. */
1548  if (respect_inspect_depth &&
1550  && stream->min_inspect_depth)
1551  {
1552  progress = STREAM_APP_PROGRESS(stream);
1553  if (stream->min_inspect_depth >= progress) {
1554  progress = 0;
1555  } else {
1556  progress -= stream->min_inspect_depth;
1557  }
1558  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1559 
1560  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1561  }
1562 
1563  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1564 
1565  /* get window of data that is acked */
1566  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1567  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1568  uint32_t delta = stream->last_ack - stream->base_seq;
1569  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1570  /* get max absolute offset */
1571  last_ack_abs += delta;
1572  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1573  }
1574 
1575  /* loop through available buffers. On no packet loss we'll have a single
1576  * iteration. On missing data we'll walk the blocks */
1577  while (1) {
1578  const uint8_t *mydata;
1579  uint32_t mydata_len;
1580  uint64_t mydata_offset = 0;
1581 
1582  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1583  if (mydata_len == 0) {
1584  SCLogDebug("no data");
1585  break;
1586  }
1587  //PrintRawDataFp(stdout, mydata, mydata_len);
1588 
1589  SCLogDebug("raw progress %"PRIu64, progress);
1590  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1591  stream, &stream->sb, mydata_len, progress);
1592 
1593  if (eof) {
1594  // inspect all remaining data, ack'd or not
1595  } else {
1596  if (last_ack_abs < progress) {
1597  SCLogDebug("nothing to do");
1598  goto end;
1599  }
1600 
1601  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1602  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1603 
1604  /* see if the buffer contains unack'd data as well */
1605  if (progress + mydata_len > last_ack_abs) {
1606  uint32_t check = mydata_len;
1607  mydata_len = last_ack_abs - progress;
1608  BUG_ON(check < mydata_len);
1609  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1610  "data is considered", mydata_len);
1611  }
1612 
1613  }
1614  if (mydata_len == 0)
1615  break;
1616 
1617  SCLogDebug("data %p len %u", mydata, mydata_len);
1618 
1619  /* we have data. */
1620  r = Callback(cb_data, mydata, mydata_len);
1621  BUG_ON(r < 0);
1622 
1623  if (mydata_offset == progress) {
1624  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1625  progress, mydata_len, progress_in + mydata_len);
1626 
1627  progress += mydata_len;
1628  SCLogDebug("raw progress now %"PRIu64, progress);
1629 
1630  /* data is beyond the progress we'd like, and before last ack. Gap. */
1631  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1632  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1633  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1634  progress = mydata_offset;
1635  SCLogDebug("raw progress now %"PRIu64, progress);
1636 
1637  } else {
1638  SCLogDebug("not increasing progress, data gap => mydata_offset "
1639  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1640  }
1641 
1642  if (iter == NULL || r == 1)
1643  break;
1644  }
1645 end:
1646  *progress_out = progress;
1647  return r;
1648 }
1649 
1651  StreamReassembleRawFunc Callback, void *cb_data,
1652  uint64_t *progress_out, bool respect_inspect_depth)
1653 {
1654  /* handle inline seperately as the logic is very different */
1655  if (StreamTcpInlineMode() == TRUE) {
1656  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1657  }
1658 
1659  TcpStream *stream;
1660  if (PKT_IS_TOSERVER(p)) {
1661  stream = &ssn->client;
1662  } else {
1663  stream = &ssn->server;
1664  }
1665 
1667  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1668  {
1669  *progress_out = STREAM_RAW_PROGRESS(stream);
1670  return 0;
1671  }
1672 
1673  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1674  STREAM_RAW_PROGRESS(stream), progress_out,
1675  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1676 }
1677 
1679  StreamReassembleRawFunc Callback, void *cb_data,
1680  uint64_t progress_in,
1681  uint64_t *progress_out, bool eof)
1682 {
1683  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1684  return 0;
1685 
1686  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1687  progress_in, progress_out, eof, false);
1688 }
1689 
1690 /** \internal
1691  * \brief update app layer based on received ACK
1692  *
1693  * \retval r 0 on success, -1 on error
1694  */
1695 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1696  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1697 {
1698  SCEnter();
1699 
1700  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1701  SCReturnInt(-1);
1702 
1703  SCReturnInt(0);
1704 }
1705 
1707  TcpSession *ssn, TcpStream *stream,
1708  Packet *p, PacketQueue *pq)
1709 {
1710  SCEnter();
1711 
1712  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1713 
1714  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1715  ssn, stream, p, p->payload_len);
1716 
1717  /* we need to update the opposing stream in
1718  * StreamTcpReassembleHandleSegmentUpdateACK */
1719  TcpStream *opposing_stream = NULL;
1720  if (stream == &ssn->client) {
1721  opposing_stream = &ssn->server;
1722  } else {
1723  opposing_stream = &ssn->client;
1724  }
1725 
1726  /* default IDS: update opposing side (triggered by ACK) */
1728  /* inline and stream end and flow timeout packets trigger same dir handling */
1729  if (StreamTcpInlineMode()) {
1730  dir = UPDATE_DIR_PACKET;
1731  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1732  dir = UPDATE_DIR_PACKET;
1733  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1734  dir = UPDATE_DIR_PACKET;
1735  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1736  dir = UPDATE_DIR_PACKET;
1737  } else if (ssn->state == TCP_CLOSED) {
1738  dir = UPDATE_DIR_BOTH;
1739  }
1740 
1741  /* handle ack received */
1742  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1743  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1744  {
1745  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1746  SCReturnInt(-1);
1747  }
1748 
1749  /* if this segment contains data, insert it */
1750  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1751  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1752 
1753  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1754  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1755  SCReturnInt(-1);
1756  }
1757 
1758  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1759  p->flags |= PKT_STREAM_ADD;
1760  } else {
1761  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1762  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1763  ssn, stream, p->payload_len,
1764  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1765 
1766  }
1767 
1768  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1769  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1770  * was *just* set. In this case we trigger the AppLayer Truncate
1771  * logic, to inform the applayer no more data in this direction is
1772  * to be expected. */
1773  if ((stream->flags &
1776  {
1777  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1778  if (dir != UPDATE_DIR_PACKET) {
1779  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1780  "can trigger Truncate");
1781  dir = UPDATE_DIR_PACKET;
1782  }
1783  }
1784 
1785  /* in stream inline mode even if we have no data we call the reassembly
1786  * functions to handle EOF */
1787  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1788  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1789  StreamTcpInlineMode()?"true":"false",
1790  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1791  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1792  SCReturnInt(-1);
1793  }
1794  }
1795 
1796  SCReturnInt(0);
1797 }
1798 
1799 /**
1800  * \brief get a segment from the pool
1801  *
1802  * \retval seg Segment from the pool or NULL
1803  */
1805 {
1806  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1807  SCLogDebug("seg we return is %p", seg);
1808  if (seg == NULL) {
1809  /* Increment the counter to show that we are not able to serve the
1810  segment request due to memcap limit */
1812  } else {
1813  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1814  }
1815 
1816  return seg;
1817 }
1818 
1819 /**
1820  * \brief Trigger RAW stream reassembly
1821  *
1822  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1823  * reassembly from the applayer, for example upon completion of a
1824  * HTTP request.
1825  *
1826  * It sets a flag in the stream so that the next Raw call will return
1827  * the data.
1828  *
1829  * \param ssn TcpSession
1830  */
1832 {
1833 #ifdef DEBUG
1834  BUG_ON(ssn == NULL);
1835 #endif
1836 
1837  if (ssn != NULL) {
1838  if (direction == STREAM_TOSERVER) {
1840  } else {
1842  }
1843 
1844  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1845  }
1846 }
1847 
1848 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1849 {
1850 #ifdef DEBUG
1851  BUG_ON(ssn == NULL);
1852 #endif
1853 
1854  if (ssn != NULL) {
1855  if (direction == STREAM_TOSERVER) {
1856  ssn->client.min_inspect_depth = depth;
1857  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1858  } else {
1859  ssn->server.min_inspect_depth = depth;
1860  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1861  }
1862  }
1863 }
1864 
1865 #ifdef UNITTESTS
1866 /** unit tests and it's support functions below */
1867 
1868 #define SET_ISN(stream, setseq) \
1869  (stream)->isn = (setseq); \
1870  (stream)->base_seq = (setseq) + 1
1871 
1872 /** \brief The Function to create the packet with given payload, which is used
1873  * to test the reassembly of the engine.
1874  *
1875  * \param payload The variable used to store the payload contents of the
1876  * current packet.
1877  * \param value The value which current payload will have for this packet
1878  * \param payload_len The length of the filed payload for current packet.
1879  * \param len Length of the payload array
1880  */
1881 
1882 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1883  uint8_t payload_len, uint8_t len)
1884 {
1885  uint8_t i;
1886  for (i = 0; i < payload_len; i++)
1887  payload[i] = value;
1888  for (; i < len; i++)
1889  payload = NULL;
1890 }
1891 
1892 /** \brief The Function Checks the reassembled stream contents against predefined
1893  * stream contents according to OS policy used.
1894  *
1895  * \param stream_policy Predefined value of stream for different OS policies
1896  * \param stream Reassembled stream returned from the reassembly functions
1897  */
1898 
1899 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
1900 {
1901  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
1902  {
1903  //PrintRawDataFp(stdout, stream_policy, sp_size);
1904  return 0;
1905  }
1906  return 1;
1907 }
1908 
1909 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
1910 {
1911  if (StreamingBufferCompareRawData(&stream->sb,
1912  data, data_len) == 0)
1913  {
1914  SCReturnInt(0);
1915  }
1916  SCLogInfo("OK");
1917  PrintRawDataFp(stdout, data, data_len);
1918  return 1;
1919 }
1920 
1921 #define MISSED_START(isn) \
1922  TcpReassemblyThreadCtx *ra_ctx = NULL; \
1923  TcpSession ssn; \
1924  ThreadVars tv; \
1925  memset(&tv, 0, sizeof(tv)); \
1926  \
1927  StreamTcpUTInit(&ra_ctx); \
1928  \
1929  StreamTcpUTSetupSession(&ssn); \
1930  StreamTcpUTSetupStream(&ssn.server, (isn)); \
1931  StreamTcpUTSetupStream(&ssn.client, (isn)); \
1932  \
1933  TcpStream *stream = &ssn.client;
1934 
1935 #define MISSED_END \
1936  StreamTcpUTClearSession(&ssn); \
1937  StreamTcpUTDeinit(ra_ctx); \
1938  PASS
1939 
1940 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
1941  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
1942  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
1943 
1944 /**
1945  * \test Test the handling of packets missed by both IDS and the end host.
1946  * The packet is missed in the starting of the stream.
1947  *
1948  * \retval On success it returns 1 and on failure 0.
1949  */
1950 
1951 static int StreamTcpReassembleTest25 (void)
1952 {
1953  MISSED_START(6);
1954  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
1955  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
1956  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
1957  MISSED_END;
1958 }
1959 
1960 /**
1961  * \test Test the handling of packets missed by both IDS and the end host.
1962  * The packet is missed in the middle of the stream.
1963  *
1964  * \retval On success it returns 1 and on failure 0.
1965  */
1966 
1967 static int StreamTcpReassembleTest26 (void)
1968 {
1969  MISSED_START(9);
1970  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1971  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
1972  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
1973  MISSED_END;
1974 }
1975 
1976 /**
1977  * \test Test the handling of packets missed by both IDS and the end host.
1978  * The packet is missed in the end of the stream.
1979  *
1980  * \retval On success it returns 1 and on failure 0.
1981  */
1982 
1983 static int StreamTcpReassembleTest27 (void)
1984 {
1985  MISSED_START(9);
1986  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1987  MISSED_STEP(13, "BB", 2, "AAABB", 5);
1988  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
1989  MISSED_END;
1990 }
1991 
1992 /**
1993  * \test Test the handling of packets missed by IDS, but the end host has
1994  * received it and send the acknowledgment of it. The packet is missed
1995  * in the starting of the stream.
1996  *
1997  * \retval On success it returns 1 and on failure 0.
1998  */
1999 
2000 static int StreamTcpReassembleTest28 (void)
2001 {
2002  MISSED_START(6);
2003  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2004  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2005  ssn.state = TCP_TIME_WAIT;
2006  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2007  MISSED_END;
2008 }
2009 
2010 /**
2011  * \test Test the handling of packets missed by IDS, but the end host has
2012  * received it and send the acknowledgment of it. The packet is missed
2013  * in the middle of the stream.
2014  *
2015  * \retval On success it returns 1 and on failure 0.
2016  */
2017 
2018 static int StreamTcpReassembleTest29 (void)
2019 {
2020  MISSED_START(9);
2021  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2022  ssn.state = TCP_TIME_WAIT;
2023  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2024  MISSED_END;
2025 }
2026 
2027 static int StreamTcpReassembleTest33(void)
2028 {
2029  TcpSession ssn;
2030  Packet *p = PacketGetFromAlloc();
2031  FAIL_IF(unlikely(p == NULL));
2032  Flow f;
2033  TCPHdr tcph;
2034  TcpReassemblyThreadCtx *ra_ctx = NULL;
2036  uint8_t packet[1460] = "";
2037 
2038  StreamTcpUTInit(&ra_ctx);
2040 
2041  PacketQueue pq;
2042  memset(&pq,0,sizeof(PacketQueue));
2043  memset(&f, 0, sizeof (Flow));
2044  memset(&tcph, 0, sizeof (TCPHdr));
2045  ThreadVars tv;
2046  memset(&tv, 0, sizeof (ThreadVars));
2047  FLOW_INITIALIZE(&f);
2048  f.protoctx = &ssn;
2049  f.proto = IPPROTO_TCP;
2050  p->src.family = AF_INET;
2051  p->dst.family = AF_INET;
2052  p->proto = IPPROTO_TCP;
2053  p->flow = &f;
2054  tcph.th_win = 5480;
2055  tcph.th_flags = TH_PUSH | TH_ACK;
2056  p->tcph = &tcph;
2058  p->payload = packet;
2059 
2060  p->tcph->th_seq = htonl(10);
2061  p->tcph->th_ack = htonl(31);
2062  p->payload_len = 10;
2063 
2064  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2065 
2066  p->tcph->th_seq = htonl(20);
2067  p->tcph->th_ack = htonl(31);
2068  p->payload_len = 10;
2069 
2070  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2071 
2072  p->tcph->th_seq = htonl(40);
2073  p->tcph->th_ack = htonl(31);
2074  p->payload_len = 10;
2075 
2076  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2077 
2078  p->tcph->th_seq = htonl(5);
2079  p->tcph->th_ack = htonl(31);
2080  p->payload_len = 30;
2081 
2082  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2083 
2085  StreamTcpUTDeinit(ra_ctx);
2086  SCFree(p);
2087  PASS;
2088 }
2089 
2090 static int StreamTcpReassembleTest34(void)
2091 {
2092  TcpSession ssn;
2093  Packet *p = PacketGetFromAlloc();
2094  FAIL_IF(unlikely(p == NULL));
2095  Flow f;
2096  TCPHdr tcph;
2097  TcpReassemblyThreadCtx *ra_ctx = NULL;
2099  uint8_t packet[1460] = "";
2100 
2101  StreamTcpUTInit(&ra_ctx);
2103  PacketQueue pq;
2104  memset(&pq,0,sizeof(PacketQueue));
2105  memset(&f, 0, sizeof (Flow));
2106  memset(&tcph, 0, sizeof (TCPHdr));
2107  ThreadVars tv;
2108  memset(&tv, 0, sizeof (ThreadVars));
2109  FLOW_INITIALIZE(&f);
2110  f.protoctx = &ssn;
2111  f.proto = IPPROTO_TCP;
2112  p->src.family = AF_INET;
2113  p->dst.family = AF_INET;
2114  p->proto = IPPROTO_TCP;
2115  p->flow = &f;
2116  tcph.th_win = 5480;
2117  tcph.th_flags = TH_PUSH | TH_ACK;
2118  p->tcph = &tcph;
2120  p->payload = packet;
2121  SET_ISN(&ssn.client, 857961230);
2122 
2123  p->tcph->th_seq = htonl(857961230);
2124  p->tcph->th_ack = htonl(31);
2125  p->payload_len = 304;
2126 
2127  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2128 
2129  p->tcph->th_seq = htonl(857961534);
2130  p->tcph->th_ack = htonl(31);
2131  p->payload_len = 1460;
2132 
2133  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2134 
2135  p->tcph->th_seq = htonl(857963582);
2136  p->tcph->th_ack = htonl(31);
2137  p->payload_len = 1460;
2138 
2139  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2140 
2141  p->tcph->th_seq = htonl(857960946);
2142  p->tcph->th_ack = htonl(31);
2143  p->payload_len = 1460;
2144 
2145  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2146 
2148  StreamTcpUTDeinit(ra_ctx);
2149  SCFree(p);
2150  PASS;
2151 }
2152 
2153 /** \test Test the bug 76 condition */
2154 static int StreamTcpReassembleTest37(void)
2155 {
2156  TcpSession ssn;
2157  Flow f;
2158  TCPHdr tcph;
2159  TcpReassemblyThreadCtx *ra_ctx = NULL;
2160  uint8_t packet[1460] = "";
2161  PacketQueue pq;
2162  ThreadVars tv;
2163  memset(&tv, 0, sizeof (ThreadVars));
2164 
2165  Packet *p = PacketGetFromAlloc();
2166  FAIL_IF(unlikely(p == NULL));
2167 
2168  StreamTcpUTInit(&ra_ctx);
2170  memset(&pq,0,sizeof(PacketQueue));
2171  memset(&f, 0, sizeof (Flow));
2172  memset(&tcph, 0, sizeof (TCPHdr));
2173  memset(&tv, 0, sizeof (ThreadVars));
2174 
2175  FLOW_INITIALIZE(&f);
2176  f.protoctx = &ssn;
2177  f.proto = IPPROTO_TCP;
2178  p->src.family = AF_INET;
2179  p->dst.family = AF_INET;
2180  p->proto = IPPROTO_TCP;
2181  p->flow = &f;
2182  tcph.th_win = 5480;
2183  tcph.th_flags = TH_PUSH | TH_ACK;
2184  p->tcph = &tcph;
2186  p->payload = packet;
2188 
2189  p->tcph->th_seq = htonl(3061088537UL);
2190  p->tcph->th_ack = htonl(1729548549UL);
2191  p->payload_len = 1391;
2192  ssn.client.last_ack = 3061091137UL;
2193  SET_ISN(&ssn.client, 3061091309UL);
2194 
2195  /* pre base_seq, so should be rejected */
2196  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2197 
2198  p->tcph->th_seq = htonl(3061089928UL);
2199  p->tcph->th_ack = htonl(1729548549UL);
2200  p->payload_len = 1391;
2201  ssn.client.last_ack = 3061091137UL;
2202 
2203  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2204 
2205  p->tcph->th_seq = htonl(3061091319UL);
2206  p->tcph->th_ack = htonl(1729548549UL);
2207  p->payload_len = 1391;
2208  ssn.client.last_ack = 3061091137UL;
2209 
2210  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2211 
2213  StreamTcpUTDeinit(ra_ctx);
2214  SCFree(p);
2215  PASS;
2216 }
2217 
2218 /**
2219  * \test Test to make sure that we don't return the segments until the app
2220  * layer proto has been detected and after that remove the processed
2221  * segments.
2222  *
2223  * \retval On success it returns 1 and on failure 0.
2224  */
2225 
2226 static int StreamTcpReassembleTest39 (void)
2227 {
2228  Packet *p = PacketGetFromAlloc();
2229  FAIL_IF(unlikely(p == NULL));
2230  Flow f;
2231  ThreadVars tv;
2232  StreamTcpThread stt;
2233  TCPHdr tcph;
2234  PacketQueue pq;
2235  memset(&pq,0,sizeof(PacketQueue));
2236  memset (&f, 0, sizeof(Flow));
2237  memset(&tv, 0, sizeof (ThreadVars));
2238  memset(&stt, 0, sizeof (stt));
2239  memset(&tcph, 0, sizeof (TCPHdr));
2240 
2241  FLOW_INITIALIZE(&f);
2242  f.flags = FLOW_IPV4;
2243  f.proto = IPPROTO_TCP;
2244  p->flow = &f;
2245  p->tcph = &tcph;
2246 
2247  StreamTcpUTInit(&stt.ra_ctx);
2248 
2249  /* handshake */
2250  tcph.th_win = htons(5480);
2251  tcph.th_flags = TH_SYN;
2253  p->payload_len = 0;
2254  p->payload = NULL;
2255  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2256 
2257  TcpSession *ssn = (TcpSession *)f.protoctx;
2258  FAIL_IF_NULL(ssn);
2259 
2270  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2271  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2272  FAIL_IF(ssn->data_first_seen_dir != 0);
2273 
2274  /* handshake */
2275  p->tcph->th_ack = htonl(1);
2276  p->tcph->th_flags = TH_SYN | TH_ACK;
2278  p->payload_len = 0;
2279  p->payload = NULL;
2280  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2291  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2292  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2293  FAIL_IF(ssn->data_first_seen_dir != 0);
2294 
2295  /* handshake */
2296  p->tcph->th_ack = htonl(1);
2297  p->tcph->th_seq = htonl(1);
2298  p->tcph->th_flags = TH_ACK;
2300  p->payload_len = 0;
2301  p->payload = NULL;
2302  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2313  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2314  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2315  FAIL_IF(ssn->data_first_seen_dir != 0);
2316 
2317  /* partial request */
2318  uint8_t request1[] = { 0x47, 0x45, };
2319  p->tcph->th_ack = htonl(1);
2320  p->tcph->th_seq = htonl(1);
2321  p->tcph->th_flags = TH_PUSH | TH_ACK;
2323  p->payload_len = sizeof(request1);
2324  p->payload = request1;
2325  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2336  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2337  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2338  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2340 
2341  /* response ack against partial request */
2342  p->tcph->th_ack = htonl(3);
2343  p->tcph->th_seq = htonl(1);
2344  p->tcph->th_flags = TH_ACK;
2346  p->payload_len = 0;
2347  p->payload = NULL;
2348  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2359  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2360  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2361  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2363 
2364  /* complete partial request */
2365  uint8_t request2[] = {
2366  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2367  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2368  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2369  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2370  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2371  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2372  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2373  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2374  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2375  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2376  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2377  p->tcph->th_ack = htonl(1);
2378  p->tcph->th_seq = htonl(3);
2379  p->tcph->th_flags = TH_PUSH | TH_ACK;
2381  p->payload_len = sizeof(request2);
2382  p->payload = request2;
2383  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2394  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2395  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2396  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2397  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2399 
2400  /* response - request ack */
2401  uint8_t response[] = {
2402  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2403  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2404  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2405  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2406  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2407  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2408  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2409  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2410  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2411  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2412  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2413  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2414  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2415  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2416  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2417  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2418  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2419  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2420  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2421  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2422  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2423  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2424  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2425  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2426  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2427  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2428  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2429  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2430  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2431  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2432  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2433  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2434  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2435  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2436  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2437  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2438  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2439  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2440  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2441  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2442  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2443  p->tcph->th_ack = htonl(88);
2444  p->tcph->th_seq = htonl(1);
2445  p->tcph->th_flags = TH_PUSH | TH_ACK;
2447  p->payload_len = sizeof(response);
2448  p->payload = response;
2449 
2450  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2462  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2463  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2464  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2465  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2466 
2467  /* response ack from request */
2468  p->tcph->th_ack = htonl(328);
2469  p->tcph->th_seq = htonl(88);
2470  p->tcph->th_flags = TH_ACK;
2472  p->payload_len = 0;
2473  p->payload = NULL;
2474  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2486  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2487  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2488  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2489  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2490  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2491 
2492  /* response - acking */
2493  p->tcph->th_ack = htonl(88);
2494  p->tcph->th_seq = htonl(328);
2495  p->tcph->th_flags = TH_PUSH | TH_ACK;
2497  p->payload_len = 0;
2498  p->payload = NULL;
2499  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2511  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2512  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2513  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2514  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2515  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2516 
2517  /* response ack from request */
2518  p->tcph->th_ack = htonl(328);
2519  p->tcph->th_seq = htonl(88);
2520  p->tcph->th_flags = TH_ACK;
2522  p->payload_len = 0;
2523  p->payload = NULL;
2524  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2536  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2537  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2538  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2539  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2540 
2541  /* response - acking the request again*/
2542  p->tcph->th_ack = htonl(88);
2543  p->tcph->th_seq = htonl(328);
2544  p->tcph->th_flags = TH_PUSH | TH_ACK;
2546  p->payload_len = 0;
2547  p->payload = NULL;
2548  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2560  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2561  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2562  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2563  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2564 
2565  /*** New Request ***/
2566 
2567  /* partial request */
2568  p->tcph->th_ack = htonl(328);
2569  p->tcph->th_seq = htonl(88);
2570  p->tcph->th_flags = TH_PUSH | TH_ACK;
2572  p->payload_len = sizeof(request1);
2573  p->payload = request1;
2574  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2586  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2587  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2588  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2589  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2590  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2591 
2592  /* response ack against partial request */
2593  p->tcph->th_ack = htonl(90);
2594  p->tcph->th_seq = htonl(328);
2595  p->tcph->th_flags = TH_ACK;
2597  p->payload_len = 0;
2598  p->payload = NULL;
2599  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2600  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2612  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2613  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2614  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2615  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2616  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2617 
2618  /* complete request */
2619  p->tcph->th_ack = htonl(328);
2620  p->tcph->th_seq = htonl(90);
2621  p->tcph->th_flags = TH_PUSH | TH_ACK;
2623  p->payload_len = sizeof(request2);
2624  p->payload = request2;
2625  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2637  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2638  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2639  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2640  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2641  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2642  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2643 
2644  /* response ack against second partial request */
2645  p->tcph->th_ack = htonl(175);
2646  p->tcph->th_seq = htonl(328);
2647  p->tcph->th_flags = TH_ACK;
2649  p->payload_len = 0;
2650  p->payload = NULL;
2651 
2652  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2664  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2665  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2666  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2667  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2668  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2669  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2670 
2671  /* response acking a request */
2672  p->tcph->th_ack = htonl(175);
2673  p->tcph->th_seq = htonl(328);
2674  p->tcph->th_flags = TH_ACK;
2676  p->payload_len = 0;
2677  p->payload = NULL;
2678  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2684 
2687 
2688  /* request acking a response */
2689  p->tcph->th_ack = htonl(328);
2690  p->tcph->th_seq = htonl(175);
2691  p->tcph->th_flags = TH_ACK;
2693  p->payload_len = 0;
2694  p->payload = NULL;
2695  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2696 
2697  StreamTcpSessionClear(ssn);
2699  SCFree(p);
2700  PASS;
2701 }
2702 
2703 /**
2704  * \test Test to make sure that we sent all the segments from the initial
2705  * segments to app layer until we have detected the app layer proto.
2706  *
2707  * \retval On success it returns 1 and on failure 0.
2708  */
2709 
2710 static int StreamTcpReassembleTest40 (void)
2711 {
2712  Packet *p = PacketGetFromAlloc();
2713  FAIL_IF_NULL(p);
2714  Flow *f = NULL;
2715  TCPHdr tcph;
2716  TcpSession ssn;
2717  PacketQueue pq;
2718  memset(&pq,0,sizeof(PacketQueue));
2719  memset(&tcph, 0, sizeof (TCPHdr));
2720  ThreadVars tv;
2721  memset(&tv, 0, sizeof (ThreadVars));
2722 
2725 
2727  FAIL_IF_NULL(ra_ctx);
2728 
2729  uint8_t httpbuf1[] = "P";
2730  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2731  uint8_t httpbuf3[] = "O";
2732  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2733  uint8_t httpbuf4[] = "S";
2734  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2735  uint8_t httpbuf5[] = "T \r\n";
2736  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2737 
2738  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2739  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2740 
2741  SET_ISN(&ssn.server, 9);
2742  ssn.server.last_ack = 10;
2743  SET_ISN(&ssn.client, 9);
2744  ssn.client.isn = 9;
2745 
2746  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2747  FAIL_IF_NULL(f);
2748  f->protoctx = &ssn;
2749  f->proto = IPPROTO_TCP;
2750  p->flow = f;
2751 
2752  tcph.th_win = htons(5480);
2753  tcph.th_seq = htonl(10);
2754  tcph.th_ack = htonl(10);
2755  tcph.th_flags = TH_ACK|TH_PUSH;
2756  p->tcph = &tcph;
2758  p->payload = httpbuf1;
2759  p->payload_len = httplen1;
2760  ssn.state = TCP_ESTABLISHED;
2761  TcpStream *s = &ssn.client;
2762  SCLogDebug("1 -- start");
2763  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2764 
2766  p->payload = httpbuf2;
2767  p->payload_len = httplen2;
2768  tcph.th_seq = htonl(10);
2769  tcph.th_ack = htonl(11);
2770  s = &ssn.server;
2771  ssn.server.last_ack = 11;
2772  SCLogDebug("2 -- start");
2773  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2774 
2776  p->payload = httpbuf3;
2777  p->payload_len = httplen3;
2778  tcph.th_seq = htonl(11);
2779  tcph.th_ack = htonl(55);
2780  s = &ssn.client;
2781  ssn.client.last_ack = 55;
2782  SCLogDebug("3 -- start");
2783  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2784 
2786  p->payload = httpbuf2;
2787  p->payload_len = httplen2;
2788  tcph.th_seq = htonl(55);
2789  tcph.th_ack = htonl(12);
2790  s = &ssn.server;
2791  ssn.server.last_ack = 12;
2792  SCLogDebug("4 -- start");
2793  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2794 
2795  /* check is have the segment in the list and flagged or not */
2796  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2797  FAIL_IF_NULL(seg);
2798  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2799 
2801  p->payload = httpbuf4;
2802  p->payload_len = httplen4;
2803  tcph.th_seq = htonl(12);
2804  tcph.th_ack = htonl(100);
2805  s = &ssn.client;
2806  ssn.client.last_ack = 100;
2807  SCLogDebug("5 -- start");
2808  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2809 
2811  p->payload = httpbuf2;
2812  p->payload_len = httplen2;
2813  tcph.th_seq = htonl(100);
2814  tcph.th_ack = htonl(13);
2815  s = &ssn.server;
2816  ssn.server.last_ack = 13;
2817  SCLogDebug("6 -- start");
2818  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2819 
2821  p->payload = httpbuf5;
2822  p->payload_len = httplen5;
2823  tcph.th_seq = htonl(13);
2824  tcph.th_ack = htonl(145);
2825  s = &ssn.client;
2826  ssn.client.last_ack = 145;
2827  SCLogDebug("7 -- start");
2828  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2829 
2831  p->payload = httpbuf2;
2832  p->payload_len = httplen2;
2833  tcph.th_seq = htonl(145);
2834  tcph.th_ack = htonl(16);
2835  s = &ssn.server;
2836  ssn.server.last_ack = 16;
2837  SCLogDebug("8 -- start");
2838  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2839  FAIL_IF(f->alproto != ALPROTO_HTTP);
2840 
2844  SCFree(p);
2845  UTHFreeFlow(f);
2846  PASS;
2847 }
2848 
2849 /** \test Test the memcap incrementing/decrementing and memcap check */
2850 static int StreamTcpReassembleTest44(void)
2851 {
2853  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2855  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2857  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2859  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2861  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2862  PASS;
2863 }
2864 
2865 /**
2866  * \test Test to make sure that reassembly_depth is enforced.
2867  *
2868  * \retval On success it returns 1 and on failure 0.
2869  */
2870 
2871 static int StreamTcpReassembleTest45 (void)
2872 {
2873  TcpReassemblyThreadCtx *ra_ctx = NULL;
2874  TcpSession ssn;
2875  ThreadVars tv;
2876  memset(&tv, 0, sizeof(tv));
2877  uint8_t payload[100] = {0};
2878  uint16_t payload_size = 100;
2879 
2880  StreamTcpUTInit(&ra_ctx);
2882 
2884  ssn.reassembly_depth = 100;
2885  StreamTcpUTSetupStream(&ssn.server, 100);
2886  StreamTcpUTSetupStream(&ssn.client, 100);
2887 
2888  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2889  FAIL_IF(r != 0);
2891 
2892  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2893  FAIL_IF(r != 0);
2895 
2899  StreamTcpUTDeinit(ra_ctx);
2900  PASS;
2901 }
2902 
2903 /**
2904  * \test Test the unlimited config value of reassembly depth.
2905  *
2906  * \retval On success it returns 1 and on failure 0.
2907  */
2908 
2909 static int StreamTcpReassembleTest46 (void)
2910 {
2911  int result = 0;
2912  TcpReassemblyThreadCtx *ra_ctx = NULL;
2913  TcpSession ssn;
2914  ThreadVars tv;
2915  memset(&tv, 0, sizeof(tv));
2916  uint8_t payload[100] = {0};
2917  uint16_t payload_size = 100;
2918 
2919  StreamTcpUTInit(&ra_ctx);
2921 
2923  StreamTcpUTSetupStream(&ssn.server, 100);
2924  StreamTcpUTSetupStream(&ssn.client, 100);
2925 
2926  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2927  if (r != 0)
2928  goto end;
2930  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2931  goto end;
2932  }
2933 
2934  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2935  if (r != 0)
2936  goto end;
2938  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2939  goto end;
2940  }
2941 
2942  result = 1;
2943 end:
2947  StreamTcpUTDeinit(ra_ctx);
2948  return result;
2949 }
2950 
2951 /**
2952  * \test Test to make sure we detect the sequence wrap around and continue
2953  * stream reassembly properly.
2954  *
2955  * \retval On success it returns 1 and on failure 0.
2956  */
2957 
2958 static int StreamTcpReassembleTest47 (void)
2959 {
2960  Packet *p = PacketGetFromAlloc();
2961  FAIL_IF(unlikely(p == NULL));
2962  Flow *f = NULL;
2963  TCPHdr tcph;
2964  TcpSession ssn;
2965  ThreadVars tv;
2966  PacketQueue pq;
2967  memset(&pq,0,sizeof(PacketQueue));
2968  memset(&tcph, 0, sizeof (TCPHdr));
2969  memset(&tv, 0, sizeof (ThreadVars));
2973 
2974  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
2975  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2976 
2977  SET_ISN(&ssn.server, 572799781UL);
2978  ssn.server.last_ack = 572799782UL;
2979 
2980  SET_ISN(&ssn.client, 4294967289UL);
2981  ssn.client.last_ack = 21;
2982 
2983  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2984  FAIL_IF(f == NULL);
2985  f->protoctx = &ssn;
2986  f->proto = IPPROTO_TCP;
2987  p->flow = f;
2988 
2989  tcph.th_win = htons(5480);
2990  ssn.state = TCP_ESTABLISHED;
2991  TcpStream *s = NULL;
2992  uint8_t cnt = 0;
2993 
2994  for (cnt=0; cnt < httplen1; cnt++) {
2995  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
2996  tcph.th_ack = htonl(572799782UL);
2997  tcph.th_flags = TH_ACK|TH_PUSH;
2998  p->tcph = &tcph;
3000  p->payload = &httpbuf1[cnt];
3001  p->payload_len = 1;
3002  s = &ssn.client;
3003 
3004  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3005 
3007  p->payload = NULL;
3008  p->payload_len = 0;
3009  tcph.th_seq = htonl(572799782UL);
3010  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3011  tcph.th_flags = TH_ACK;
3012  p->tcph = &tcph;
3013  s = &ssn.server;
3014 
3015  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3016  }
3017 
3018  FAIL_IF(f->alproto != ALPROTO_HTTP);
3019 
3023  SCFree(p);
3024  UTHFreeFlow(f);
3025  PASS;
3026 }
3027 
3028 /** \test 3 in order segments in inline reassembly */
3029 static int StreamTcpReassembleInlineTest01(void)
3030 {
3031  int ret = 0;
3032  TcpReassemblyThreadCtx *ra_ctx = NULL;
3033  ThreadVars tv;
3034  TcpSession ssn;
3035  Flow f;
3036 
3037  memset(&tv, 0x00, sizeof(tv));
3038 
3039  StreamTcpUTInit(&ra_ctx);
3042  StreamTcpUTSetupStream(&ssn.client, 1);
3043  FLOW_INITIALIZE(&f);
3044 
3045  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3046  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3047  if (p == NULL) {
3048  printf("couldn't get a packet: ");
3049  goto end;
3050  }
3051  p->tcph->th_seq = htonl(12);
3052  p->flow = &f;
3053 
3054  FLOWLOCK_WRLOCK(&f);
3055  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3056  printf("failed to add segment 1: ");
3057  goto end;
3058  }
3059  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3060  printf("failed to add segment 2: ");
3061  goto end;
3062  }
3063  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3064  printf("failed to add segment 3: ");
3065  goto end;
3066  }
3067  ssn.client.next_seq = 17;
3068  ret = 1;
3069 end:
3070  FLOWLOCK_UNLOCK(&f);
3071  FLOW_DESTROY(&f);
3072  UTHFreePacket(p);
3074  StreamTcpUTDeinit(ra_ctx);
3075  return ret;
3076 }
3077 
3078 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3079  * test the sliding window reassembly.
3080  */
3081 static int StreamTcpReassembleInlineTest02(void)
3082 {
3083  int ret = 0;
3084  TcpReassemblyThreadCtx *ra_ctx = NULL;
3085  ThreadVars tv;
3086  TcpSession ssn;
3087  Flow f;
3088 
3089  memset(&tv, 0x00, sizeof(tv));
3090 
3091  StreamTcpUTInit(&ra_ctx);
3094  StreamTcpUTSetupStream(&ssn.client, 1);
3095  FLOW_INITIALIZE(&f);
3096 
3097  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3098  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3099  if (p == NULL) {
3100  printf("couldn't get a packet: ");
3101  goto end;
3102  }
3103  p->tcph->th_seq = htonl(12);
3104  p->flow = &f;
3105 
3106  FLOWLOCK_WRLOCK(&f);
3107  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3108  printf("failed to add segment 1: ");
3109  goto end;
3110  }
3111  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3112  printf("failed to add segment 2: ");
3113  goto end;
3114  }
3115  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3116  printf("failed to add segment 3: ");
3117  goto end;
3118  }
3119  ssn.client.next_seq = 17;
3120  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3121  printf("failed to add segment 4: ");
3122  goto end;
3123  }
3124  ssn.client.next_seq = 22;
3125  ret = 1;
3126 end:
3127  FLOWLOCK_UNLOCK(&f);
3128  FLOW_DESTROY(&f);
3129  UTHFreePacket(p);
3131  StreamTcpUTDeinit(ra_ctx);
3132  return ret;
3133 }
3134 
3135 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3136  * test the sliding window reassembly with a small window size so that we
3137  * cutting off at the start (left edge)
3138  */
3139 static int StreamTcpReassembleInlineTest03(void)
3140 {
3141  int ret = 0;
3142  TcpReassemblyThreadCtx *ra_ctx = NULL;
3143  ThreadVars tv;
3144  TcpSession ssn;
3145  Flow f;
3146 
3147  memset(&tv, 0x00, sizeof(tv));
3148 
3149  StreamTcpUTInit(&ra_ctx);
3152  StreamTcpUTSetupStream(&ssn.client, 1);
3153  FLOW_INITIALIZE(&f);
3154 
3156 
3157  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3158  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3159  if (p == NULL) {
3160  printf("couldn't get a packet: ");
3161  goto end;
3162  }
3163  p->tcph->th_seq = htonl(12);
3164  p->flow = &f;
3166 
3167  FLOWLOCK_WRLOCK(&f);
3168  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3169  printf("failed to add segment 1: ");
3170  goto end;
3171  }
3172  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3173  printf("failed to add segment 2: ");
3174  goto end;
3175  }
3176  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3177  printf("failed to add segment 3: ");
3178  goto end;
3179  }
3180  ssn.client.next_seq = 17;
3181  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3182  printf("failed to add segment 4: ");
3183  goto end;
3184  }
3185  ssn.client.next_seq = 22;
3186 
3187  p->tcph->th_seq = htonl(17);
3188  ret = 1;
3189 end:
3190  FLOWLOCK_UNLOCK(&f);
3191  FLOW_DESTROY(&f);
3192  UTHFreePacket(p);
3194  StreamTcpUTDeinit(ra_ctx);
3195  return ret;
3196 }
3197 
3198 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3199  * test the sliding window reassembly with a small window size so that we
3200  * cutting off at the start (left edge) with small packet overlap.
3201  */
3202 static int StreamTcpReassembleInlineTest04(void)
3203 {
3204  int ret = 0;
3205  TcpReassemblyThreadCtx *ra_ctx = NULL;
3206  ThreadVars tv;
3207  TcpSession ssn;
3208  Flow f;
3209 
3210  memset(&tv, 0x00, sizeof(tv));
3211 
3212  StreamTcpUTInit(&ra_ctx);
3215  StreamTcpUTSetupStream(&ssn.client, 1);
3216  FLOW_INITIALIZE(&f);
3217 
3219 
3220  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3221  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3222  if (p == NULL) {
3223  printf("couldn't get a packet: ");
3224  goto end;
3225  }
3226  p->tcph->th_seq = htonl(12);
3227  p->flow = &f;
3229 
3230  FLOWLOCK_WRLOCK(&f);
3231  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3232  printf("failed to add segment 1: ");
3233  goto end;
3234  }
3235  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3236  printf("failed to add segment 2: ");
3237  goto end;
3238  }
3239  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3240  printf("failed to add segment 3: ");
3241  goto end;
3242  }
3243  ssn.client.next_seq = 17;
3244  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3245  printf("failed to add segment 4: ");
3246  goto end;
3247  }
3248  ssn.client.next_seq = 22;
3249 
3250  p->tcph->th_seq = htonl(17);
3251  ret = 1;
3252 end:
3253  FLOWLOCK_UNLOCK(&f);
3254  FLOW_DESTROY(&f);
3255  UTHFreePacket(p);
3257  StreamTcpUTDeinit(ra_ctx);
3258  return ret;
3259 }
3260 
3261 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3262  * test the sliding window reassembly with a small window size so that we
3263  * cutting off at the start (left edge). Test if the first segment is
3264  * removed from the list.
3265  */
3266 static int StreamTcpReassembleInlineTest08(void)
3267 {
3268  TcpReassemblyThreadCtx *ra_ctx = NULL;
3269  ThreadVars tv;
3270  memset(&tv, 0x00, sizeof(tv));
3271  TcpSession ssn;
3272  Flow f;
3273  StreamTcpUTInit(&ra_ctx);
3276  StreamTcpUTSetupStream(&ssn.client, 1);
3277  FLOW_INITIALIZE(&f);
3278 
3281  f.protoctx = &ssn;
3282 
3283  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3284  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3285  FAIL_IF(p == NULL);
3286  p->tcph->th_seq = htonl(12);
3287  p->flow = &f;
3289 
3290  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3291  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3292  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3293  ssn.client.next_seq = 17;
3294  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3295  ssn.client.next_seq = 22;
3296  p->tcph->th_seq = htonl(17);
3298 
3299  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3300  FAIL_IF_NULL(seg);
3301  FAIL_IF_NOT(seg->seq == 2);
3302 
3303  FLOW_DESTROY(&f);
3304  UTHFreePacket(p);
3306  StreamTcpUTDeinit(ra_ctx);
3307  PASS;
3308 }
3309 
3310 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3311  * test the sliding window reassembly with a small window size so that we
3312  * cutting off at the start (left edge). Test if the first segment is
3313  * removed from the list.
3314  */
3315 static int StreamTcpReassembleInlineTest09(void)
3316 {
3317  int ret = 0;
3318  TcpReassemblyThreadCtx *ra_ctx = NULL;
3319  ThreadVars tv;
3320  TcpSession ssn;
3321  Flow f;
3322 
3323  memset(&tv, 0x00, sizeof(tv));
3324 
3325  StreamTcpUTInit(&ra_ctx);
3328  StreamTcpUTSetupStream(&ssn.client, 1);
3329  FLOW_INITIALIZE(&f);
3330 
3333 
3334  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3335  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3336  if (p == NULL) {
3337  printf("couldn't get a packet: ");
3338  goto end;
3339  }
3340  p->tcph->th_seq = htonl(17);
3341  p->flow = &f;
3343 
3344  FLOWLOCK_WRLOCK(&f);
3345  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3346  printf("failed to add segment 1: ");
3347  goto end;
3348  }
3349  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3350  printf("failed to add segment 2: ");
3351  goto end;
3352  }
3353  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3354  printf("failed to add segment 3: ");
3355  goto end;
3356  }
3357  ssn.client.next_seq = 12;
3358  ssn.client.last_ack = 10;
3359 
3360  /* close the GAP and see if we properly reassemble and update base_seq */
3361  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3362  printf("failed to add segment 4: ");
3363  goto end;
3364  }
3365  ssn.client.next_seq = 22;
3366 
3367  p->tcph->th_seq = htonl(12);
3368 
3369  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3370  FAIL_IF_NULL(seg);
3371  FAIL_IF_NOT(seg->seq == 2);
3372 
3373  ret = 1;
3374 end:
3375  FLOWLOCK_UNLOCK(&f);
3376  FLOW_DESTROY(&f);
3377  UTHFreePacket(p);
3379  StreamTcpUTDeinit(ra_ctx);
3380  return ret;
3381 }
3382 
3383 /** \test App Layer reassembly.
3384  */
3385 static int StreamTcpReassembleInlineTest10(void)
3386 {
3387  int ret = 0;
3388  TcpReassemblyThreadCtx *ra_ctx = NULL;
3389  ThreadVars tv;
3390  TcpSession ssn;
3391  Flow *f = NULL;
3392  Packet *p = NULL;
3393 
3394  memset(&tv, 0x00, sizeof(tv));
3395 
3396  StreamTcpUTInit(&ra_ctx);
3399  StreamTcpUTSetupStream(&ssn.server, 1);
3400  ssn.server.last_ack = 2;
3401  StreamTcpUTSetupStream(&ssn.client, 1);
3402  ssn.client.last_ack = 2;
3404 
3405  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3406  if (f == NULL)
3407  goto end;
3408  f->protoctx = &ssn;
3409  f->proto = IPPROTO_TCP;
3410 
3411  uint8_t stream_payload1[] = "GE";
3412  uint8_t stream_payload2[] = "T /";
3413  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3414 
3415  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3416  if (p == NULL) {
3417  printf("couldn't get a packet: ");
3418  goto end;
3419  }
3420  p->tcph->th_seq = htonl(7);
3421  p->flow = f;
3423 
3424  FLOWLOCK_WRLOCK(f);
3425  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3426  printf("failed to add segment 1: ");
3427  goto end;
3428  }
3429  ssn.client.next_seq = 4;
3430 
3431  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3432  if (r < 0) {
3433  printf("StreamTcpReassembleAppLayer failed: ");
3434  goto end;
3435  }
3436 
3437  /* ssn.server.ra_app_base_seq should be isn here. */
3438  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3439  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3440  goto end;
3441  }
3442 
3443  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3444  printf("failed to add segment 2: ");
3445  goto end;
3446  }
3447  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3448  printf("failed to add segment 3: ");
3449  goto end;
3450  }
3451  ssn.client.next_seq = 19;
3452 
3453  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3454  if (r < 0) {
3455  printf("StreamTcpReassembleAppLayer failed: ");
3456  goto end;
3457  }
3458 
3459  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3460 
3461  ret = 1;
3462 end:
3463  UTHFreePacket(p);
3465  StreamTcpUTDeinit(ra_ctx);
3466  FLOWLOCK_UNLOCK(f);
3467  UTHFreeFlow(f);
3468  return ret;
3469 }
3470 
3471 /** \test test insert with overlap
3472  */
3473 static int StreamTcpReassembleInsertTest01(void)
3474 {
3475  TcpReassemblyThreadCtx *ra_ctx = NULL;
3476  ThreadVars tv;
3477  TcpSession ssn;
3478  Flow f;
3479 
3480  memset(&tv, 0x00, sizeof(tv));
3481 
3482  StreamTcpUTInit(&ra_ctx);
3484  StreamTcpUTSetupStream(&ssn.client, 1);
3486  FLOW_INITIALIZE(&f);
3487 
3488  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3489  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3490  FAIL_IF(p == NULL);
3491  p->tcph->th_seq = htonl(12);
3492  p->flow = &f;
3493 
3494  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3495  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3496  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3497  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3498  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3499  ssn.client.next_seq = 21;
3500 
3501  FLOW_DESTROY(&f);
3502  UTHFreePacket(p);
3504  StreamTcpUTDeinit(ra_ctx);
3505  PASS;
3506 }
3507 
3508 /** \test test insert with overlaps
3509  */
3510 static int StreamTcpReassembleInsertTest02(void)
3511 {
3512  int ret = 0;
3513  TcpReassemblyThreadCtx *ra_ctx = NULL;
3514  ThreadVars tv;
3515  TcpSession ssn;
3516 
3517  memset(&tv, 0x00, sizeof(tv));
3518 
3519  StreamTcpUTInit(&ra_ctx);
3521  StreamTcpUTSetupStream(&ssn.client, 1);
3522 
3523  int i;
3524  for (i = 2; i < 10; i++) {
3525  int len;
3526  len = i % 2;
3527  if (len == 0)
3528  len = 1;
3529  int seq;
3530  seq = i * 10;
3531  if (seq < 2)
3532  seq = 2;
3533 
3534  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3535  printf("failed to add segment 1: ");
3536  goto end;
3537  }
3538  }
3539  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3540  printf("failed to add segment 2: ");
3541  goto end;
3542  }
3543 
3544  ret = 1;
3545 end:
3547  StreamTcpUTDeinit(ra_ctx);
3548  return ret;
3549 }
3550 
3551 /** \test test insert with overlaps
3552  */
3553 static int StreamTcpReassembleInsertTest03(void)
3554 {
3555  int ret = 0;
3556  TcpReassemblyThreadCtx *ra_ctx = NULL;
3557  ThreadVars tv;
3558  TcpSession ssn;
3559 
3560  memset(&tv, 0x00, sizeof(tv));
3561 
3562  StreamTcpUTInit(&ra_ctx);
3564  StreamTcpUTSetupStream(&ssn.client, 1);
3565 
3566  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3567  printf("failed to add segment 2: ");
3568  goto end;
3569  }
3570 
3571  int i;
3572  for (i = 2; i < 10; i++) {
3573  int len;
3574  len = i % 2;
3575  if (len == 0)
3576  len = 1;
3577  int seq;
3578  seq = i * 10;
3579  if (seq < 2)
3580  seq = 2;
3581 
3582  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3583  printf("failed to add segment 2: ");
3584  goto end;
3585  }
3586  }
3587  ret = 1;
3588 end:
3590  StreamTcpUTDeinit(ra_ctx);
3591  return ret;
3592 }
3593 
3595 #endif /* UNITTESTS */
3596 
3597 /** \brief The Function Register the Unit tests to test the reassembly engine
3598  * for various OS policies.
3599  */
3600 
3602 {
3603 #ifdef UNITTESTS
3604  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3605  StreamTcpReassembleTest25);
3606  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3607  StreamTcpReassembleTest26);
3608  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3609  StreamTcpReassembleTest27);
3610  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3611  StreamTcpReassembleTest28);
3612  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3613  StreamTcpReassembleTest29);
3614  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3615  StreamTcpReassembleTest33);
3616  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3617  StreamTcpReassembleTest34);
3618  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3619  StreamTcpReassembleTest37);
3620  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3621  StreamTcpReassembleTest39);
3622  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3623  StreamTcpReassembleTest40);
3624  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3625  StreamTcpReassembleTest44);
3626  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3627  StreamTcpReassembleTest45);
3628  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3629  StreamTcpReassembleTest46);
3630  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3631  StreamTcpReassembleTest47);
3632 
3633  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3634  StreamTcpReassembleInlineTest01);
3635  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3636  StreamTcpReassembleInlineTest02);
3637  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3638  StreamTcpReassembleInlineTest03);
3639  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3640  StreamTcpReassembleInlineTest04);
3641  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3642  StreamTcpReassembleInlineTest08);
3643  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3644  StreamTcpReassembleInlineTest09);
3645 
3646  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3647  StreamTcpReassembleInlineTest10);
3648 
3649  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3650  StreamTcpReassembleInsertTest01);
3651  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3652  StreamTcpReassembleInsertTest02);
3653  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3654  StreamTcpReassembleInsertTest03);
3655 
3659  StreamTcpReassembleRawRegisterTests();
3660 #endif /* UNITTESTS */
3661 }
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
#define SCMutex
StreamingBufferSegment sbseg
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
#define SCLogDebug(...)
Definition: util-debug.h:335
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
void *(* Malloc)(size_t size)
int StreamTcpAppLayerIsDisabled(Flow *f)
struct Flow_ * flow
Definition: decode.h:443
AppProto alproto_tc
Definition: flow.h:411
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
#define STREAM_BASE_OFFSET(stream)
int StreamTcpReassembleInit(char quiet)
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:344
#define FALSE
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:243
StreamingBuffer sb
#define PASS
Pass the test.
#define unlikely(expr)
Definition: util-optimize.h:35
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
#define STREAMTCP_STREAM_FLAG_GAP
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void StreamTcpListRegisterTests(void)
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define TH_RST
Definition: decode-tcp.h:37
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet...
Definition: stream-tcp.c:770
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream *stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:551
void StreamTcpDisableAppLayer(Flow *f)
#define TH_FIN
Definition: decode-tcp.h:35
#define STREAMING_BUFFER_NOFLAGS
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
uint64_t offset
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Address dst
Definition: decode.h:411
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
#define SCReturnUInt(x)
Definition: util-debug.h:343
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
int ByteExtractStringUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:244
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:669
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:240
uint64_t pcap_cnt
Definition: decode.h:561
#define STREAM_RAW_PROGRESS(stream)
void StreamTcpReassembleInitMemuse(void)
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:124
char * val
Definition: conf.h:34
TCPHdr * tcph
Definition: decode.h:520
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define TRUE
#define SCMutexLock(mut)
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
StreamingBufferBlock * head
#define STREAMTCP_FLAG_MIDSTREAM
void * protoctx
Definition: flow.h:400
void *(* Calloc)(size_t n, size_t size)
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
#define STREAM_DEPTH
Definition: stream.h:34
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
uint32_t min_inspect_depth
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueue *pq)
Definition: stream-tcp.c:4674
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define StreamTcpSetEvent(p, e)
AppProto alproto_ts
Definition: flow.h:410
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
#define SCCalloc(nm, a)
Definition: util-mem.h:197
#define SEQ_GEQ(a, b)
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:106
#define STREAM_GAP
Definition: stream.h:33
#define SCMutexUnlock(mut)
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:803
uint32_t seq
uint32_t raw_progress_rel
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:248
char family
Definition: decode.h:109
uint8_t proto
Definition: decode.h:428
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
#define SCMUTEX_INITIALIZER
#define TCP_SEG_LEN(seg)
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
#define STREAM_HAS_SEEN_DATA(stream)
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
uint32_t app_progress_rel
Data structures and function prototypes for keeping state for the detection engine.
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define FLOW_DESTROY(f)
Definition: flow-util.h:119
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies...
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
#define STREAM_EOF
Definition: stream.h:30
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define StreamTcpDisableAppLayerReassembly(ssn)
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
#define SCEnter(...)
Definition: util-debug.h:337
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
void StreamTcpReassembleFree(char quiet)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:437
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PKT_IS_TOSERVER(p)
Definition: decode.h:255
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
uint32_t reassembly_depth
#define TH_PUSH
Definition: decode-tcp.h:38
#define SEQ_GT(a, b)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:163
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1093
int RunmodeIsUnittests(void)
Definition: suricata.c:261
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:989
#define SCMutexInit(mut, mutattrs)
int8_t data_first_seen_dir
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define SCReturnInt(x)
Definition: util-debug.h:341
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueue *pq)
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
void StreamTcpUTInitInline(void)
#define SEQ_LEQ(a, b)
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
uint32_t reassembly_depth
Definition: stream-tcp.h:60
#define SCRealloc(x, a)
Definition: util-mem.h:182
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define RB_MIN(name, x)
Definition: tree.h:780
Definition: conf.h:32
#define SCMalloc(a)
Definition: util-mem.h:166
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:207
struct TCPSEG seg_tree
uint16_t payload_len
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define TH_SYN
Definition: decode-tcp.h:36
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
block of continues data
#define SCFree(a)
Definition: util-mem.h:228
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:249
#define RB_REMOVE(name, x, y)
Definition: tree.h:775
void StreamTcpUtilRegisterTests(void)
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:825
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
#define STREAM_START
Definition: stream.h:29
uint32_t prealloc_segments
Definition: stream-tcp.h:57
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
#define PKT_STREAM_ADD
Definition: decode.h:1089
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself...
void StreamTcpInlineRegisterTests(void)
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
void StreamTcpReassemblePseudoPacketCreate(TcpStream *, Packet *, PacketQueue *)
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
#define SCReturn
Definition: util-debug.h:339
void StreamTcpUTSetupSession(TcpSession *ssn)
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define MISSED_END
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of &#39;threads&#39;, so array elements)
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
uint32_t flags
Definition: decode.h:441
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
void StreamTcpReassembleConfigEnableOverlapCheck(void)
uint16_t payload_len
Definition: decode.h:541
#define STREAM_APP_PROGRESS(stream)
#define SEQ_LT(a, b)
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6280
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
#define STREAM_RIGHT_EDGE(stream)
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
void StreamTcpUTClearSession(TcpSession *ssn)
#define MISSED_START(isn)
Flow data structure.
Definition: flow.h:325
#define FLOW_IPV4
Definition: flow.h:94
uint8_t * payload
Definition: decode.h:540
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
uint32_t flags
Definition: flow.h:379
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
void StreamTcpUTClearStream(TcpStream *s)
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
void(* Free)(void *ptr, size_t size)
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:141
Address src
Definition: decode.h:410
void UTHFreeFlow(Flow *flow)
#define STREAM_MIDSTREAM
Definition: stream.h:35
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SET_ISN(stream, setseq)
#define SCMutexDestroy
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.