suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 
64 #include "detect-engine-state.h"
65 
66 #include "util-profiling.h"
67 #include "util-validate.h"
68 
69 #ifdef DEBUG
70 static SCMutex segment_pool_memuse_mutex;
71 static uint64_t segment_pool_memuse = 0;
72 static uint64_t segment_pool_memcnt = 0;
73 #endif
74 
75 static PoolThread *segment_thread_pool = NULL;
76 /* init only, protect initializing and growing pool */
77 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
78 
79 /* Memory use counter */
80 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
81 
82 /* prototypes */
84 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346 }
347 
348 /** \param f locked flow */
350 {
351  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
352  return 0;
353 
354  TcpSession *ssn = (TcpSession *)f->protoctx;
356 }
357 
358 static int StreamTcpReassemblyConfig(char quiet)
359 {
360  uint32_t segment_prealloc = 2048;
361  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
362  if (seg) {
363  uint32_t prealloc = 0;
364  if (ByteExtractStringUint32(&prealloc, 10, strlen(seg->val), seg->val) == -1)
365  {
366  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
367  "%s is invalid", seg->val);
368  return -1;
369  }
370  segment_prealloc = prealloc;
371  }
372  if (!quiet)
373  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
374  stream_config.prealloc_segments = segment_prealloc;
375 
376  int overlap_diff_data = 0;
377  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
378  if (overlap_diff_data) {
380  }
381  if (StreamTcpInlineMode() == TRUE) {
383  }
384 
387  stream_config.sbcnf.Malloc = ReassembleMalloc;
388  stream_config.sbcnf.Calloc = ReassembleCalloc;
389  stream_config.sbcnf.Realloc = ReassembleRealloc;
390  stream_config.sbcnf.Free = ReassembleFree;
391 
392  return 0;
393 }
394 
395 int StreamTcpReassembleInit(char quiet)
396 {
397  /* init the memcap/use tracker */
399 
400  if (StreamTcpReassemblyConfig(quiet) < 0)
401  return -1;
402 
403 #ifdef DEBUG
404  SCMutexInit(&segment_pool_memuse_mutex, NULL);
405 #endif
406  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
408  return 0;
409 }
410 
411 void StreamTcpReassembleFree(char quiet)
412 {
413  SCMutexLock(&segment_thread_pool_mutex);
414  if (segment_thread_pool != NULL) {
415  PoolThreadFree(segment_thread_pool);
416  segment_thread_pool = NULL;
417  }
418  SCMutexUnlock(&segment_thread_pool_mutex);
419  SCMutexDestroy(&segment_thread_pool_mutex);
420 
421 #ifdef DEBUG
422  if (segment_pool_memuse > 0)
423  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
424  if (segment_pool_memcnt > 0)
425  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
426  SCMutexDestroy(&segment_pool_memuse_mutex);
427 #endif
428 }
429 
431 {
432  SCEnter();
434  if (unlikely(ra_ctx == NULL))
435  return NULL;
436 
437  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
438 
439  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
440 
441  SCMutexLock(&segment_thread_pool_mutex);
442  if (segment_thread_pool == NULL) {
443  segment_thread_pool = PoolThreadInit(1, /* thread */
444  0, /* unlimited */
446  sizeof(TcpSegment),
447  TcpSegmentPoolAlloc,
448  TcpSegmentPoolInit, NULL,
449  TcpSegmentPoolCleanup, NULL);
450  ra_ctx->segment_thread_pool_id = 0;
451  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
452  PoolThreadSize(segment_thread_pool),
453  ra_ctx->segment_thread_pool_id);
454  } else {
455  /* grow segment_thread_pool until we have a element for our thread id */
456  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
457  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
458  PoolThreadSize(segment_thread_pool),
459  ra_ctx->segment_thread_pool_id);
460  }
461  SCMutexUnlock(&segment_thread_pool_mutex);
462  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
463  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
465  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
466  }
467 
468  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
469 }
470 
472 {
473  SCEnter();
475  SCFree(ra_ctx);
476  SCReturn;
477 }
478 
479 /**
480  * \brief check if stream in pkt direction has depth reached
481  *
482  * \param p packet with *LOCKED* flow
483  *
484  * \retval 1 stream has depth reached
485  * \retval 0 stream does not have depth reached
486  */
488 {
489  if (p->flow != NULL && p->flow->protoctx != NULL) {
490  TcpSession *ssn = p->flow->protoctx;
491  TcpStream *stream;
492  if (p->flowflags & FLOW_PKT_TOSERVER) {
493  stream = &ssn->client;
494  } else {
495  stream = &ssn->server;
496  }
497 
498  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
499  }
500 
501  return 0;
502 }
503 
504 /**
505  * \internal
506  * \brief Function to Check the reassembly depth valuer against the
507  * allowed max depth of the stream reassmbly for TCP streams.
508  *
509  * \param stream stream direction
510  * \param seq sequence number where "size" starts
511  * \param size size of the segment that is added
512  *
513  * \retval size Part of the size that fits in the depth, 0 if none
514  */
515 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
516  uint32_t seq, uint32_t size)
517 {
518  SCEnter();
519 
520  /* if the configured depth value is 0, it means there is no limit on
521  reassembly depth. Otherwise carry on my boy ;) */
522  if (ssn->reassembly_depth == 0) {
523  SCReturnUInt(size);
524  }
525 
526  /* if the final flag is set, we're not accepting anymore */
528  SCReturnUInt(0);
529  }
530 
531  uint64_t seg_depth;
532  if (SEQ_GT(stream->base_seq, seq)) {
533  if (SEQ_LEQ(seq+size, stream->base_seq)) {
534  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
535  stream->base_seq, seq, seq+size);
536  SCReturnUInt(0);
537  }
538 
539  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
540  } else {
541  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
542  }
543 
544  /* if the base_seq has moved passed the depth window we stop
545  * checking and just reject the rest of the packets including
546  * retransmissions. Saves us the hassle of dealing with sequence
547  * wraps as well */
548  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
549  stream->base_seq, seg_depth,
550  ssn->reassembly_depth);
551 
552  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
553  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
555  SCReturnUInt(0);
556  }
557  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
558  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
559 #if 0
560  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
561  (stream->base_seq_offset + stream->base_seq + size),
562  (stream->isn + ssn->reassembly_depth));
563 #endif
564  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
565  /* packet (partly?) fits the depth window */
566 
567  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
568  /* complete fit */
569  SCReturnUInt(size);
570  } else {
572  /* partial fit, return only what fits */
573  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
574  DEBUG_VALIDATE_BUG_ON(part > size);
575  if (part > size)
576  part = size;
577  SCReturnUInt(part);
578  }
579  }
580 
581  SCReturnUInt(0);
582 }
583 
584 /**
585  * \brief Insert a packets TCP data into the stream reassembly engine.
586  *
587  * \retval 0 good segment, as far as we checked.
588  * \retval -1 badness, reason to drop in inline mode
589  *
590  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
591  * or it wasn't added because of reassembly depth.
592  *
593  */
595  TcpSession *ssn, TcpStream *stream, Packet *p)
596 {
597  SCEnter();
598 
599  if (ssn->data_first_seen_dir == 0) {
600  if (PKT_IS_TOSERVER(p)) {
602  } else {
604  }
605  }
606 
607  /* If the OS policy is not set then set the OS policy for this stream */
608  if (stream->os_policy == 0) {
609  StreamTcpSetOSPolicy(stream, p);
610  }
611 
614  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
615  SCReturnInt(0);
616  }
617 
618  /* If we have reached the defined depth for either of the stream, then stop
619  reassembling the TCP session */
620  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
621  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
622 
624  /* increment stream depth counter */
625  StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
626  }
627  if (size == 0) {
628  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
629  SCReturnInt(0);
630  }
631 
633  if (size > p->payload_len)
634  size = p->payload_len;
635 
636  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
637  if (seg == NULL) {
638  SCLogDebug("segment_pool is empty");
640  SCReturnInt(-1);
641  }
642 
643  TCP_SEG_LEN(seg) = size;
644  seg->seq = TCP_GET_SEQ(p);
645 
646  /* proto detection skipped, but now we do get data. Set event. */
647  if (RB_EMPTY(&stream->seg_tree) &&
649 
652  }
653 
654  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
655  SCLogDebug("StreamTcpReassembleInsertSegment failed");
656  SCReturnInt(-1);
657  }
658  SCReturnInt(0);
659 }
660 
661 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
662  Packet *p, enum StreamUpdateDir dir)
663 {
664  uint8_t flag = 0;
665 
667  flag |= STREAM_START;
668  }
669 
670  if (ssn->state == TCP_CLOSED) {
671  flag |= STREAM_EOF;
672  }
673 
674  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
675  flag |= STREAM_MIDSTREAM;
676  }
677 
678  if (p->flags & PKT_PSEUDO_STREAM_END) {
679  flag |= STREAM_EOF;
680  }
681 
682  if (dir == UPDATE_DIR_OPPOSING) {
683  if (p->flowflags & FLOW_PKT_TOSERVER) {
684  flag |= STREAM_TOCLIENT;
685  } else {
686  flag |= STREAM_TOSERVER;
687  }
688  } else {
689  if (p->flowflags & FLOW_PKT_TOSERVER) {
690  flag |= STREAM_TOSERVER;
691  } else {
692  flag |= STREAM_TOCLIENT;
693  }
694  }
695 
697  flag |= STREAM_DEPTH;
698  }
699  return flag;
700 }
701 
702 /**
703  * \brief Check the minimum size limits for reassembly.
704  *
705  * \retval 0 don't reassemble yet
706  * \retval 1 do reassemble
707  */
708 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
709  const TcpStream *stream, const Packet *p)
710 {
711  SCEnter();
712 
713  /* if any of these flags is set we always inspect immediately */
714 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
715  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
716  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
717  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
718 
721  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
722  "is set, so not expecting any new data segments");
723  }
725  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
726  }
728  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
729  "so no new segments will be considered");
730  }
731  SCReturnInt(1);
732  }
733 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
734 
735  /* some states mean we reassemble no matter how much data we have */
736  if (ssn->state > TCP_TIME_WAIT)
737  SCReturnInt(1);
738 
739  if (p->flags & PKT_PSEUDO_STREAM_END)
740  SCReturnInt(1);
741 
742  /* check if we have enough data to do raw reassembly */
743  if (PKT_IS_TOSERVER(p)) {
744  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
745  uint32_t delta = stream->last_ack - stream->base_seq;
746  /* get max absolute offset */
747  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
748 
749  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
750  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
751  SCReturnInt(1);
752  } else {
753  SCLogDebug("toserver min chunk len not yet reached: "
754  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
755  "%"PRIu32"", stream->last_ack, stream->base_seq,
756  (stream->last_ack - stream->base_seq),
758  SCReturnInt(0);
759  }
760  }
761  } else {
762  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
763  uint32_t delta = stream->last_ack - stream->base_seq;
764  /* get max absolute offset */
765  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
766 
767  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
768 
769  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
770  SCReturnInt(1);
771  } else {
772  SCLogDebug("toclient min chunk len not yet reached: "
773  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
774  "%"PRIu32"", stream->last_ack, stream->base_seq,
775  (stream->last_ack - stream->base_seq),
777  SCReturnInt(0);
778  }
779  }
780  }
781 
782  SCReturnInt(0);
783 }
784 
785 /**
786  * \brief see what if any work the TCP session still needs
787  */
788 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
789 {
790  const TcpStream *stream = NULL;
791 #ifdef DEBUG
792  const char *dirstr = NULL;
793 #endif
794  if (direction == STREAM_TOSERVER) {
795  stream = &ssn->client;
796 #ifdef DEBUG
797  dirstr = "client";
798 #endif
799  } else {
800  stream = &ssn->server;
801 #ifdef DEBUG
802  dirstr = "server";
803 #endif
804  }
805 
806  int use_app = 1;
807  int use_raw = 1;
808 
810  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
811  {
812  // app is dead
813  use_app = 0;
814  }
815 
817  // raw is dead
818  use_raw = 0;
819  }
820 
821  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
822 
823  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
824  dirstr,
825  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
826  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
827  right_edge);
828  if (use_raw) {
829  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
830  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
832  }
833  }
834  if (use_app) {
835  if (right_edge > STREAM_APP_PROGRESS(stream)) {
836  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
838  }
839  }
840 
841  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
843 }
844 
845 #ifdef DEBUG
846 static uint64_t GetStreamSize(TcpStream *stream)
847 {
848  if (stream) {
849  uint64_t size = 0;
850  uint32_t cnt = 0;
851 
852  TcpSegment *seg;
853  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
854  cnt++;
855  size += (uint64_t)TCP_SEG_LEN(seg);
856  }
857 
858  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
859  return size;
860  }
861  return (uint64_t)0;
862 }
863 
864 static void GetSessionSize(TcpSession *ssn, Packet *p)
865 {
866  uint64_t size = 0;
867  if (ssn) {
868  size = GetStreamSize(&ssn->client);
869  size += GetStreamSize(&ssn->server);
870 
871  //if (size > 900000)
872  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
873  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
874  }
875 }
876 #endif
877 
878 /** \internal
879  *
880  * Get buffer, or first part of the buffer if data gaps exist.
881  *
882  * \brief get stream data from offset
883  * \param offset stream offset */
884 static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset)
885 {
886  const uint8_t *mydata;
887  uint32_t mydata_len;
888 
889  if (RB_EMPTY(&stream->sb.sbb_tree)) {
890  SCLogDebug("getting one blob");
891 
892  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
893 
894  *data = mydata;
895  *data_len = mydata_len;
896  } else {
897  StreamingBufferBlock *blk = stream->sb.head;
898 
899  if (blk->offset > offset) {
900  SCLogDebug("gap, want data at offset %"PRIu64", "
901  "got data at %"PRIu64". GAP of size %"PRIu64,
902  offset, blk->offset, blk->offset - offset);
903  *data = NULL;
904  *data_len = blk->offset - offset;
905 
906  } else if (offset >= (blk->offset + blk->len)) {
907 
908  *data = NULL;
909  StreamingBufferBlock *nblk = SBB_RB_NEXT(blk);
910  *data_len = nblk ? nblk->offset - offset : 0;
911  if (nblk) {
912  SCLogDebug("gap, want data at offset %"PRIu64", "
913  "got data at %"PRIu64". GAP of size %"PRIu64,
914  offset, nblk->offset, nblk->offset - offset);
915  }
916 
917  } else if (offset > blk->offset && offset < (blk->offset + blk->len)) {
918  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
919  offset, blk->offset, blk->len);
920  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
921  SCLogDebug("data %p, data_len %u", *data, *data_len);
922  } else {
923  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
924  }
925  }
926 }
927 
928 /** \internal
929  * \brief check to see if we should declare a GAP
930  * Call this when the app layer didn't get data at the requested
931  * offset.
932  */
933 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
934 {
935  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
936  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
937 
938  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
939  /* get window of data that is acked */
940  const uint32_t delta = stream->last_ack - stream->base_seq;
941  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
942  /* get max absolute offset */
943  last_ack_abs += delta;
944 
945  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
946  last_ack_abs -= ackadded;
947 
948  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
949  SCLogDebug("next_seq %u", stream->next_seq);
950 
951  /* if last_ack_abs is beyond the app_progress data that we haven't seen
952  * has been ack'd. This looks like a GAP. */
953  if (last_ack_abs > app_progress) {
954  /* however, we can accept ACKs a bit too liberally. If last_ack
955  * is beyond next_seq, we only consider it a gap now if we do
956  * already have data beyond the gap. */
957  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
958  if (RB_EMPTY(&stream->sb.sbb_tree)) {
959  SCLogDebug("packet %"PRIu64": no GAP. "
960  "next_seq %u < last_ack %u, but no data in list",
961  p->pcap_cnt, stream->next_seq, stream->last_ack);
962  return false;
963  } else {
964  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
965  const StreamingBufferBlock *blk = stream->sb.head;
966  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
967  /* ack'd data after the gap */
968  SCLogDebug("packet %"PRIu64": GAP. "
969  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
970  p->pcap_cnt, stream->next_seq, stream->last_ack);
971  return true;
972  }
973  }
974  }
975 
976  SCLogDebug("packet %"PRIu64": GAP! "
977  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
978  "but we have no data.",
979  p->pcap_cnt, last_ack_abs, app_progress);
980  return true;
981  }
982  }
983  SCLogDebug("packet %"PRIu64": no GAP. "
984  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
985  p->pcap_cnt, last_ack_abs, app_progress);
986  return false;
987 }
988 
989 /** \internal
990  * \brief get stream buffer and update the app-layer
991  * \param stream pointer to pointer as app-layer can switch flow dir
992  * \retval 0 success
993  */
994 static int ReassembleUpdateAppLayer (ThreadVars *tv,
995  TcpReassemblyThreadCtx *ra_ctx,
996  TcpSession *ssn, TcpStream **stream,
997  Packet *p, enum StreamUpdateDir dir)
998 {
999  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1000 
1001  SCLogDebug("app progress %"PRIu64, app_progress);
1002  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1003 
1004  const uint8_t *mydata;
1005  uint32_t mydata_len;
1006 
1007  while (1) {
1008  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress);
1009  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1010  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1011 
1012  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1013  NULL, mydata_len,
1014  StreamGetAppLayerFlags(ssn, *stream, p, dir)|STREAM_GAP);
1015  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1016 
1018  StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
1019 
1020  /* AppLayerHandleTCPData has likely updated progress. */
1021  app_progress = STREAM_APP_PROGRESS(*stream);
1022 
1023  if (r < 0)
1024  return 0;
1025 
1026  continue;
1027  } else if (mydata == NULL || mydata_len == 0) {
1028  /* Possibly a gap, but no new data. */
1029  return 0;
1030  }
1031  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1032  break;
1033  }
1034 
1035  //PrintRawDataFp(stdout, mydata, mydata_len);
1036 
1037  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1038  *stream, &(*stream)->sb, mydata_len, app_progress);
1039 
1040  /* get window of data that is acked */
1041  if (StreamTcpInlineMode() == FALSE) {
1042  if (p->flags & PKT_PSEUDO_STREAM_END) {
1043  // fall through, we use all available data
1044  } else {
1045  uint64_t last_ack_abs = app_progress; /* absolute right edge of ack'd data */
1046  if (STREAM_LASTACK_GT_BASESEQ(*stream)) {
1047  /* get window of data that is acked */
1048  uint32_t delta = (*stream)->last_ack - (*stream)->base_seq;
1049  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > (*stream)->window);
1050  /* get max absolute offset */
1051  last_ack_abs += delta;
1052  }
1053 
1054  /* see if the buffer contains unack'd data as well */
1055  if (app_progress + mydata_len > last_ack_abs) {
1056  uint32_t check = mydata_len;
1057  mydata_len = last_ack_abs - app_progress;
1058  BUG_ON(mydata_len > check);
1059  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1060  "data is considered", mydata_len);
1061  }
1062  }
1063  }
1064 
1065  /* update the app-layer */
1066  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1067  (uint8_t *)mydata, mydata_len,
1068  StreamGetAppLayerFlags(ssn, *stream, p, dir));
1069  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1070 
1071  SCReturnInt(0);
1072 }
1073 
1074 /**
1075  * \brief Update the stream reassembly upon receiving a packet.
1076  *
1077  * For IDS mode, the stream is in the opposite direction of the packet,
1078  * as the ACK-packet is ACK'ing the stream.
1079  *
1080  * One of the utilities call by this function AppLayerHandleTCPData(),
1081  * has a feature where it will call this very same function for the
1082  * stream opposing the stream it is called with. This shouldn't cause
1083  * any issues, since processing of each stream is independent of the
1084  * other stream.
1085  */
1087  TcpSession *ssn, TcpStream *stream,
1088  Packet *p, enum StreamUpdateDir dir)
1089 {
1090  SCEnter();
1091 
1092  /* this function can be directly called by app layer protocol
1093  * detection. */
1096  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1097  SCReturnInt(0);
1098  }
1099 
1100 #ifdef DEBUG
1101  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1102  GetSessionSize(ssn, p);
1103 #endif
1104  /* if no segments are in the list or all are already processed,
1105  * and state is beyond established, we send an empty msg */
1106  if (STREAM_HAS_SEEN_DATA(stream) && STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1107  {
1108  /* send an empty EOF msg if we have no segments but TCP state
1109  * is beyond ESTABLISHED */
1110  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1111  SCLogDebug("sending empty eof message");
1112  /* send EOF to app layer */
1113  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1114  NULL, 0,
1115  StreamGetAppLayerFlags(ssn, stream, p, dir));
1116  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1117 
1118  SCReturnInt(0);
1119  }
1120  }
1121 
1122  /* with all that out of the way, lets update the app-layer */
1123  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1124 }
1125 
1126 /** \internal
1127  * \brief get stream data from offset
1128  * \param offset stream offset */
1129 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1130  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1131 {
1132  const uint8_t *mydata;
1133  uint32_t mydata_len;
1134  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1135  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1136 
1137  uint64_t roffset = offset;
1138  if (offset)
1139  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1140  else {
1141  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1142  }
1143 
1144  *data = mydata;
1145  *data_len = mydata_len;
1146  *data_offset = roffset;
1147  } else {
1148  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1149  *iter == NULL ? "starting" : "continuing", offset);
1150  if (*iter == NULL) {
1151  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1152  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1153  SCLogDebug("*iter %p", *iter);
1154  }
1155  if (*iter == NULL) {
1156  SCLogDebug("no data");
1157  *data = NULL;
1158  *data_len = 0;
1159  *data_offset = 0;
1160  return 0;
1161  }
1162  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1163 
1164  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1165  SCLogDebug("mydata %p", mydata);
1166 
1167  if ((*iter)->offset < offset) {
1168  uint64_t delta = offset - (*iter)->offset;
1169  if (delta < mydata_len) {
1170  *data = mydata + delta;
1171  *data_len = mydata_len - delta;
1172  *data_offset = offset;
1173  } else {
1174  SCLogDebug("no data (yet)");
1175  *data = NULL;
1176  *data_len = 0;
1177  *data_offset = 0;
1178  }
1179 
1180  } else {
1181  *data = mydata;
1182  *data_len = mydata_len;
1183  *data_offset = (*iter)->offset;
1184  }
1185 
1186  *iter = SBB_RB_NEXT(*iter);
1187  SCLogDebug("*iter %p", *iter);
1188  }
1189  return 0;
1190 }
1191 
1192 /** \brief does the stream engine have data to inspect?
1193  *
1194  * Returns true if there is data to inspect. In IDS case this is
1195  * about ACK'd data in the packet's direction.
1196  *
1197  * In the IPS case this is about the packet itself.
1198  */
1200 {
1201  TcpStream *stream;
1202  if (PKT_IS_TOSERVER(p)) {
1203  stream = &ssn->client;
1204  } else {
1205  stream = &ssn->server;
1206  }
1207 
1208  if (RB_EMPTY(&stream->seg_tree)) {
1209  return false;
1210  }
1211 
1214  return false;
1215 
1216  if (StreamTcpInlineMode() == FALSE) {
1217  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1218  return false;
1219  }
1220  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1221  return true;
1222  }
1223  } else {
1224  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1225  return true;
1226  }
1227  }
1228  return false;
1229 }
1230 
1231 /** \brief update stream engine after detection
1232  *
1233  * Tasked with progressing the 'progress' for Raw reassembly.
1234  * 2 main scenario's:
1235  * 1. progress is != 0, so we use this
1236  * 2. progress is 0, meaning the detect engine didn't touch
1237  * raw at all. In this case we need to look into progressing
1238  * raw anyway.
1239  *
1240  * Additionally, this function is tasked with disabling raw
1241  * reassembly if the app-layer requested to disable it.
1242  */
1243 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1244 {
1245  TcpStream *stream;
1246  if (PKT_IS_TOSERVER(p)) {
1247  stream = &ssn->client;
1248  } else {
1249  stream = &ssn->server;
1250  }
1251 
1252  if (progress > STREAM_RAW_PROGRESS(stream)) {
1253  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1254  stream->raw_progress_rel += slide;
1256 
1257  /* if app is active and beyond raw, sync raw to app */
1258  } else if (progress == 0 &&
1259  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1261  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1262  {
1263  /* if trigger raw is set we sync the 2 trackers */
1265  {
1266  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1267  stream->raw_progress_rel += slide;
1269 
1270  /* otherwise mix in the tcp window */
1271  } else {
1272  uint64_t tcp_window = stream->window;
1273  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1274  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1275  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1276  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1277  stream->raw_progress_rel += slide;
1278  }
1279  }
1280  }
1281  /* app is dead */
1282  } else if (progress == 0) {
1283  uint64_t tcp_window = stream->window;
1284  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1285  if (tcp_window < stream_right_edge) {
1286  uint64_t new_raw = stream_right_edge - tcp_window;
1287  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1288  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1289  stream->raw_progress_rel += slide;
1290  }
1291  }
1293 
1294  } else {
1295  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1296  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1297  STREAM_RAW_PROGRESS(stream), stream->window);
1298  }
1299 
1300  /* if we were told to accept no more raw data, we can mark raw as
1301  * disabled now. */
1304  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1305  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1306  }
1307 
1308  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1309 }
1310 
1311 /** \internal
1312  * \brief get a buffer around the current packet and run the callback on it
1313  *
1314  * The inline/IPS scanning method takes the current payload and wraps it in
1315  * data from other segments.
1316  *
1317  * How much data is inspected is controlled by the available data, chunk_size
1318  * and the payload size of the packet.
1319  *
1320  * Large packets: if payload size is close to the chunk_size, where close is
1321  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1322  * used: payload_len + 33% of the chunk_size.
1323  * If the payload size if equal to or bigger than the chunk_size, we use
1324  * payload len + 33% of the chunk size.
1325  */
1326 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1327  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1328 {
1329  SCEnter();
1330  int r = 0;
1331 
1332  TcpStream *stream;
1333  if (PKT_IS_TOSERVER(p)) {
1334  stream = &ssn->client;
1335  } else {
1336  stream = &ssn->server;
1337  }
1338 
1339  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1341  {
1342  *progress_out = STREAM_RAW_PROGRESS(stream);
1343  return 0;
1344  }
1345 
1346  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1349  if (chunk_size <= p->payload_len) {
1350  chunk_size = p->payload_len + (chunk_size / 3);
1351  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1352  p->payload_len, chunk_size);
1353  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1354  chunk_size = p->payload_len + ((chunk_size / 3));
1355  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1356  p->payload_len, chunk_size);
1357  }
1358 
1359  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1360  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1361  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1362  packet_leftedge_abs, packet_rightedge_abs);
1363 
1364  const uint8_t *mydata = NULL;
1365  uint32_t mydata_len = 0;
1366  uint64_t mydata_offset = 0;
1367  /* simply return progress from the block we inspected. */
1368  bool return_progress = false;
1369 
1370  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1371  /* continues block */
1372  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1373  return_progress = true;
1374 
1375  } else {
1376  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1377  /* find our block */
1378  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1379  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1380  if (sbb) {
1381  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1382  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1383  mydata_offset = sbb->offset;
1384  }
1385  }
1386 
1387  /* this can only happen if the segment insert of our current 'p' failed */
1388  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1389  if ((mydata == NULL || mydata_len == 0) || /* no data */
1390  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1391  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1392  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1393  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1394  {
1395  /* no data, or data is incomplete or wrong: use packet data */
1396  mydata = p->payload;
1397  mydata_len = p->payload_len;
1398  mydata_offset = packet_leftedge_abs;
1399  //mydata_rightedge_abs = packet_rightedge_abs;
1400  } else {
1401  /* adjust buffer to match chunk_size */
1402  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1403  if (mydata_len > chunk_size) {
1404  uint32_t excess = mydata_len - chunk_size;
1405  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1406 
1407  if (mydata_rightedge_abs == packet_rightedge_abs) {
1408  mydata += excess;
1409  mydata_len -= excess;
1410  mydata_offset += excess;
1411  SCLogDebug("cutting front of the buffer with %u", excess);
1412  } else if (mydata_offset == packet_leftedge_abs) {
1413  mydata_len -= excess;
1414  SCLogDebug("cutting tail of the buffer with %u", excess);
1415  } else {
1416  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1417  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1418  SCLogDebug("before %u after %u", before, after);
1419 
1420  if (after >= (chunk_size - p->payload_len) / 2) {
1421  // more trailing data than we need
1422 
1423  if (before >= (chunk_size - p->payload_len) / 2) {
1424  // also more heading data, devide evenly
1425  before = after = (chunk_size - p->payload_len) / 2;
1426  } else {
1427  // heading data is less than requested, give the
1428  // rest to the trailing data
1429  after = (chunk_size - p->payload_len) - before;
1430  }
1431  } else {
1432  // less trailing data than requested
1433 
1434  if (before >= (chunk_size - p->payload_len) / 2) {
1435  before = (chunk_size - p->payload_len) - after;
1436  } else {
1437  // both smaller than their requested size
1438  }
1439  }
1440 
1441  /* adjust the buffer */
1442  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1443  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1444  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1445  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1446  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1447  mydata += skip;
1448  mydata_len -= (skip + cut);
1449  mydata_offset += skip;
1450  }
1451  }
1452  }
1453 
1454  /* run the callback */
1455  r = Callback(cb_data, mydata, mydata_len);
1456  BUG_ON(r < 0);
1457 
1458  if (return_progress) {
1459  *progress_out = (mydata_offset + mydata_len);
1460  } else {
1461  /* several blocks of data, so we need to be a bit more careful:
1462  * - if last_ack is beyond last progress, move progress forward to last_ack
1463  * - if our block matches or starts before last ack, return right edge of
1464  * our block.
1465  */
1466  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1467  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1468  uint32_t delta = stream->last_ack - stream->base_seq;
1469  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1470  /* get max absolute offset */
1471  last_ack_abs += delta;
1472  }
1473  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1474 
1475  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1476  if (mydata_offset > last_ack_abs) {
1477  /* gap between us and last ack, set progress to last ack */
1478  *progress_out = last_ack_abs;
1479  } else {
1480  *progress_out = (mydata_offset + mydata_len);
1481  }
1482  } else {
1483  *progress_out = STREAM_RAW_PROGRESS(stream);
1484  }
1485  }
1486  return r;
1487 }
1488 
1489 /** \brief access 'raw' reassembly data.
1490  *
1491  * Access data as tracked by 'raw' tracker. Data is made available to
1492  * callback that is passed to this function.
1493  *
1494  * In the case of IDS the callback may be run multiple times if data
1495  * contains gaps. It will then be run for each block of data that is
1496  * continuous.
1497  *
1498  * The callback should give on of 2 return values:
1499  * - 0 ok
1500  * - 1 done
1501  * The value 1 will break the loop if there is a block list that is
1502  * inspected.
1503  *
1504  * This function will return the 'progress' value that has been
1505  * consumed until now.
1506  *
1507  * \param ssn tcp session
1508  * \param stream tcp stream
1509  * \param Callback the function pointer to the callback function
1510  * \param cb_data callback data
1511  * \param[in] progress_in progress to work from
1512  * \param[out] progress_out absolute progress value of the data this
1513  * call handled.
1514  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1515  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1516  *
1517  * `respect_inspect_depth` is used to avoid useless inspection of too
1518  * much data.
1519  */
1520 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1521  StreamReassembleRawFunc Callback, void *cb_data,
1522  const uint64_t progress_in,
1523  uint64_t *progress_out, bool eof,
1524  bool respect_inspect_depth)
1525 {
1526  SCEnter();
1527  int r = 0;
1528 
1529  StreamingBufferBlock *iter = NULL;
1530  uint64_t progress = progress_in;
1531  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1532 
1533  /* if the app layer triggered a flush, and we're supposed to
1534  * use a minimal inspect depth, we actually take the app progress
1535  * as that is the right edge of the data. Then we take the window
1536  * of 'min_inspect_depth' before that. */
1537  if (respect_inspect_depth &&
1539  && stream->min_inspect_depth)
1540  {
1541  progress = STREAM_APP_PROGRESS(stream);
1542  if (stream->min_inspect_depth >= progress) {
1543  progress = 0;
1544  } else {
1545  progress -= stream->min_inspect_depth;
1546  }
1547  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1548 
1549  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1550  }
1551 
1552  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1553 
1554  /* get window of data that is acked */
1555  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1556  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1557  uint32_t delta = stream->last_ack - stream->base_seq;
1558  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1559  /* get max absolute offset */
1560  last_ack_abs += delta;
1561  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1562  }
1563 
1564  /* loop through available buffers. On no packet loss we'll have a single
1565  * iteration. On missing data we'll walk the blocks */
1566  while (1) {
1567  const uint8_t *mydata;
1568  uint32_t mydata_len;
1569  uint64_t mydata_offset = 0;
1570 
1571  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1572  if (mydata_len == 0) {
1573  SCLogDebug("no data");
1574  break;
1575  }
1576  //PrintRawDataFp(stdout, mydata, mydata_len);
1577 
1578  SCLogDebug("raw progress %"PRIu64, progress);
1579  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1580  stream, &stream->sb, mydata_len, progress);
1581 
1582  if (eof) {
1583  // inspect all remaining data, ack'd or not
1584  } else {
1585  if (last_ack_abs < progress) {
1586  SCLogDebug("nothing to do");
1587  goto end;
1588  }
1589 
1590  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1591  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1592 
1593  /* see if the buffer contains unack'd data as well */
1594  if (progress + mydata_len > last_ack_abs) {
1595  uint32_t check = mydata_len;
1596  mydata_len = last_ack_abs - progress;
1597  BUG_ON(check < mydata_len);
1598  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1599  "data is considered", mydata_len);
1600  }
1601 
1602  }
1603  if (mydata_len == 0)
1604  break;
1605 
1606  SCLogDebug("data %p len %u", mydata, mydata_len);
1607 
1608  /* we have data. */
1609  r = Callback(cb_data, mydata, mydata_len);
1610  BUG_ON(r < 0);
1611 
1612  if (mydata_offset == progress) {
1613  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1614  progress, mydata_len, progress_in + mydata_len);
1615 
1616  progress += mydata_len;
1617  SCLogDebug("raw progress now %"PRIu64, progress);
1618 
1619  /* data is beyond the progress we'd like, and before last ack. Gap. */
1620  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1621  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1622  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1623  progress = mydata_offset;
1624  SCLogDebug("raw progress now %"PRIu64, progress);
1625 
1626  } else {
1627  SCLogDebug("not increasing progress, data gap => mydata_offset "
1628  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1629  }
1630 
1631  if (iter == NULL || r == 1)
1632  break;
1633  }
1634 end:
1635  *progress_out = progress;
1636  return r;
1637 }
1638 
1640  StreamReassembleRawFunc Callback, void *cb_data,
1641  uint64_t *progress_out, bool respect_inspect_depth)
1642 {
1643  /* handle inline seperately as the logic is very different */
1644  if (StreamTcpInlineMode() == TRUE) {
1645  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1646  }
1647 
1648  TcpStream *stream;
1649  if (PKT_IS_TOSERVER(p)) {
1650  stream = &ssn->client;
1651  } else {
1652  stream = &ssn->server;
1653  }
1654 
1656  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1657  {
1658  *progress_out = STREAM_RAW_PROGRESS(stream);
1659  return 0;
1660  }
1661 
1662  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1663  STREAM_RAW_PROGRESS(stream), progress_out,
1664  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1665 }
1666 
1668  StreamReassembleRawFunc Callback, void *cb_data,
1669  uint64_t progress_in,
1670  uint64_t *progress_out, bool eof)
1671 {
1672  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1673  return 0;
1674 
1675  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1676  progress_in, progress_out, eof, false);
1677 }
1678 
1679 /** \internal
1680  * \brief update app layer based on received ACK
1681  *
1682  * \retval r 0 on success, -1 on error
1683  */
1684 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1685  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1686 {
1687  SCEnter();
1688 
1689  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1690  SCReturnInt(-1);
1691 
1692  SCReturnInt(0);
1693 }
1694 
1696  TcpSession *ssn, TcpStream *stream,
1697  Packet *p, PacketQueue *pq)
1698 {
1699  SCEnter();
1700 
1701  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1702 
1703  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1704  ssn, stream, p, p->payload_len);
1705 
1706  /* we need to update the opposing stream in
1707  * StreamTcpReassembleHandleSegmentUpdateACK */
1708  TcpStream *opposing_stream = NULL;
1709  if (stream == &ssn->client) {
1710  opposing_stream = &ssn->server;
1711  } else {
1712  opposing_stream = &ssn->client;
1713  }
1714 
1715  /* default IDS: update opposing side (triggered by ACK) */
1717  /* inline and stream end and flow timeout packets trigger same dir handling */
1718  if (StreamTcpInlineMode()) {
1719  dir = UPDATE_DIR_PACKET;
1720  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1721  dir = UPDATE_DIR_PACKET;
1722  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1723  dir = UPDATE_DIR_PACKET;
1724  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1725  dir = UPDATE_DIR_PACKET;
1726  } else if (ssn->state == TCP_CLOSED) {
1727  dir = UPDATE_DIR_BOTH;
1728  }
1729 
1730  /* handle ack received */
1731  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1732  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1733  {
1734  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1735  SCReturnInt(-1);
1736  }
1737 
1738  /* if this segment contains data, insert it */
1739  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1740  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1741 
1742  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1743  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1744  SCReturnInt(-1);
1745  }
1746 
1747  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1748  p->flags |= PKT_STREAM_ADD;
1749  } else {
1750  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1751  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1752  ssn, stream, p->payload_len,
1753  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1754 
1755  }
1756 
1757  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1758  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1759  * was *just* set. In this case we trigger the AppLayer Truncate
1760  * logic, to inform the applayer no more data in this direction is
1761  * to be expected. */
1762  if ((stream->flags &
1765  {
1766  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1767  if (dir != UPDATE_DIR_PACKET) {
1768  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1769  "can trigger Truncate");
1770  dir = UPDATE_DIR_PACKET;
1771  }
1772  }
1773 
1774  /* in stream inline mode even if we have no data we call the reassembly
1775  * functions to handle EOF */
1776  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1777  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1778  StreamTcpInlineMode()?"true":"false",
1779  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1780  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1781  SCReturnInt(-1);
1782  }
1783  }
1784 
1785  SCReturnInt(0);
1786 }
1787 
1788 /**
1789  * \brief get a segment from the pool
1790  *
1791  * \retval seg Segment from the pool or NULL
1792  */
1794 {
1795  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1796  SCLogDebug("seg we return is %p", seg);
1797  if (seg == NULL) {
1798  /* Increment the counter to show that we are not able to serve the
1799  segment request due to memcap limit */
1801  } else {
1802  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1803  }
1804 
1805  return seg;
1806 }
1807 
1808 /**
1809  * \brief Trigger RAW stream reassembly
1810  *
1811  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1812  * reassembly from the applayer, for example upon completion of a
1813  * HTTP request.
1814  *
1815  * It sets a flag in the stream so that the next Raw call will return
1816  * the data.
1817  *
1818  * \param ssn TcpSession
1819  */
1821 {
1822 #ifdef DEBUG
1823  BUG_ON(ssn == NULL);
1824 #endif
1825 
1826  if (ssn != NULL) {
1827  if (direction == STREAM_TOSERVER) {
1829  } else {
1831  }
1832 
1833  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1834  }
1835 }
1836 
1837 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1838 {
1839 #ifdef DEBUG
1840  BUG_ON(ssn == NULL);
1841 #endif
1842 
1843  if (ssn != NULL) {
1844  if (direction == STREAM_TOSERVER) {
1845  ssn->client.min_inspect_depth = depth;
1846  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1847  } else {
1848  ssn->server.min_inspect_depth = depth;
1849  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1850  }
1851  }
1852 }
1853 
1854 #ifdef UNITTESTS
1855 /** unit tests and it's support functions below */
1856 
1857 #define SET_ISN(stream, setseq) \
1858  (stream)->isn = (setseq); \
1859  (stream)->base_seq = (setseq) + 1
1860 
1861 /** \brief The Function to create the packet with given payload, which is used
1862  * to test the reassembly of the engine.
1863  *
1864  * \param payload The variable used to store the payload contents of the
1865  * current packet.
1866  * \param value The value which current payload will have for this packet
1867  * \param payload_len The length of the filed payload for current packet.
1868  * \param len Length of the payload array
1869  */
1870 
1871 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1872  uint8_t payload_len, uint8_t len)
1873 {
1874  uint8_t i;
1875  for (i = 0; i < payload_len; i++)
1876  payload[i] = value;
1877  for (; i < len; i++)
1878  payload = NULL;
1879 }
1880 
1881 /** \brief The Function Checks the reassembled stream contents against predefined
1882  * stream contents according to OS policy used.
1883  *
1884  * \param stream_policy Predefined value of stream for different OS policies
1885  * \param stream Reassembled stream returned from the reassembly functions
1886  */
1887 
1888 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
1889 {
1890  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
1891  {
1892  //PrintRawDataFp(stdout, stream_policy, sp_size);
1893  return 0;
1894  }
1895  return 1;
1896 }
1897 
1898 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
1899 {
1900  if (StreamingBufferCompareRawData(&stream->sb,
1901  data, data_len) == 0)
1902  {
1903  SCReturnInt(0);
1904  }
1905  SCLogInfo("OK");
1906  PrintRawDataFp(stdout, data, data_len);
1907  return 1;
1908 }
1909 
1910 #define MISSED_START(isn) \
1911  TcpReassemblyThreadCtx *ra_ctx = NULL; \
1912  TcpSession ssn; \
1913  ThreadVars tv; \
1914  memset(&tv, 0, sizeof(tv)); \
1915  \
1916  StreamTcpUTInit(&ra_ctx); \
1917  \
1918  StreamTcpUTSetupSession(&ssn); \
1919  StreamTcpUTSetupStream(&ssn.server, (isn)); \
1920  StreamTcpUTSetupStream(&ssn.client, (isn)); \
1921  \
1922  TcpStream *stream = &ssn.client;
1923 
1924 #define MISSED_END \
1925  StreamTcpUTClearSession(&ssn); \
1926  StreamTcpUTDeinit(ra_ctx); \
1927  PASS
1928 
1929 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
1930  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
1931  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
1932 
1933 /**
1934  * \test Test the handling of packets missed by both IDS and the end host.
1935  * The packet is missed in the starting of the stream.
1936  *
1937  * \retval On success it returns 1 and on failure 0.
1938  */
1939 
1940 static int StreamTcpReassembleTest25 (void)
1941 {
1942  MISSED_START(6);
1943  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
1944  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
1945  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
1946  MISSED_END;
1947 }
1948 
1949 /**
1950  * \test Test the handling of packets missed by both IDS and the end host.
1951  * The packet is missed in the middle of the stream.
1952  *
1953  * \retval On success it returns 1 and on failure 0.
1954  */
1955 
1956 static int StreamTcpReassembleTest26 (void)
1957 {
1958  MISSED_START(9);
1959  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1960  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
1961  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
1962  MISSED_END;
1963 }
1964 
1965 /**
1966  * \test Test the handling of packets missed by both IDS and the end host.
1967  * The packet is missed in the end of the stream.
1968  *
1969  * \retval On success it returns 1 and on failure 0.
1970  */
1971 
1972 static int StreamTcpReassembleTest27 (void)
1973 {
1974  MISSED_START(9);
1975  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1976  MISSED_STEP(13, "BB", 2, "AAABB", 5);
1977  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
1978  MISSED_END;
1979 }
1980 
1981 /**
1982  * \test Test the handling of packets missed by IDS, but the end host has
1983  * received it and send the acknowledgment of it. The packet is missed
1984  * in the starting of the stream.
1985  *
1986  * \retval On success it returns 1 and on failure 0.
1987  */
1988 
1989 static int StreamTcpReassembleTest28 (void)
1990 {
1991  MISSED_START(6);
1992  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
1993  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
1994  ssn.state = TCP_TIME_WAIT;
1995  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
1996  MISSED_END;
1997 }
1998 
1999 /**
2000  * \test Test the handling of packets missed by IDS, but the end host has
2001  * received it and send the acknowledgment of it. The packet is missed
2002  * in the middle of the stream.
2003  *
2004  * \retval On success it returns 1 and on failure 0.
2005  */
2006 
2007 static int StreamTcpReassembleTest29 (void)
2008 {
2009  MISSED_START(9);
2010  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2011  ssn.state = TCP_TIME_WAIT;
2012  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2013  MISSED_END;
2014 }
2015 
2016 static int StreamTcpReassembleTest33(void)
2017 {
2018  TcpSession ssn;
2019  Packet *p = PacketGetFromAlloc();
2020  FAIL_IF(unlikely(p == NULL));
2021  Flow f;
2022  TCPHdr tcph;
2023  TcpReassemblyThreadCtx *ra_ctx = NULL;
2025  uint8_t packet[1460] = "";
2026 
2027  StreamTcpUTInit(&ra_ctx);
2029 
2030  PacketQueue pq;
2031  memset(&pq,0,sizeof(PacketQueue));
2032  memset(&f, 0, sizeof (Flow));
2033  memset(&tcph, 0, sizeof (TCPHdr));
2034  ThreadVars tv;
2035  memset(&tv, 0, sizeof (ThreadVars));
2036  FLOW_INITIALIZE(&f);
2037  f.protoctx = &ssn;
2038  f.proto = IPPROTO_TCP;
2039  p->src.family = AF_INET;
2040  p->dst.family = AF_INET;
2041  p->proto = IPPROTO_TCP;
2042  p->flow = &f;
2043  tcph.th_win = 5480;
2044  tcph.th_flags = TH_PUSH | TH_ACK;
2045  p->tcph = &tcph;
2047  p->payload = packet;
2048 
2049  p->tcph->th_seq = htonl(10);
2050  p->tcph->th_ack = htonl(31);
2051  p->payload_len = 10;
2052 
2053  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2054 
2055  p->tcph->th_seq = htonl(20);
2056  p->tcph->th_ack = htonl(31);
2057  p->payload_len = 10;
2058 
2059  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2060 
2061  p->tcph->th_seq = htonl(40);
2062  p->tcph->th_ack = htonl(31);
2063  p->payload_len = 10;
2064 
2065  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2066 
2067  p->tcph->th_seq = htonl(5);
2068  p->tcph->th_ack = htonl(31);
2069  p->payload_len = 30;
2070 
2071  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2072 
2074  StreamTcpUTDeinit(ra_ctx);
2075  SCFree(p);
2076  PASS;
2077 }
2078 
2079 static int StreamTcpReassembleTest34(void)
2080 {
2081  TcpSession ssn;
2082  Packet *p = PacketGetFromAlloc();
2083  FAIL_IF(unlikely(p == NULL));
2084  Flow f;
2085  TCPHdr tcph;
2086  TcpReassemblyThreadCtx *ra_ctx = NULL;
2088  uint8_t packet[1460] = "";
2089 
2090  StreamTcpUTInit(&ra_ctx);
2092  PacketQueue pq;
2093  memset(&pq,0,sizeof(PacketQueue));
2094  memset(&f, 0, sizeof (Flow));
2095  memset(&tcph, 0, sizeof (TCPHdr));
2096  ThreadVars tv;
2097  memset(&tv, 0, sizeof (ThreadVars));
2098  FLOW_INITIALIZE(&f);
2099  f.protoctx = &ssn;
2100  f.proto = IPPROTO_TCP;
2101  p->src.family = AF_INET;
2102  p->dst.family = AF_INET;
2103  p->proto = IPPROTO_TCP;
2104  p->flow = &f;
2105  tcph.th_win = 5480;
2106  tcph.th_flags = TH_PUSH | TH_ACK;
2107  p->tcph = &tcph;
2109  p->payload = packet;
2110  SET_ISN(&ssn.client, 857961230);
2111 
2112  p->tcph->th_seq = htonl(857961230);
2113  p->tcph->th_ack = htonl(31);
2114  p->payload_len = 304;
2115 
2116  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2117 
2118  p->tcph->th_seq = htonl(857961534);
2119  p->tcph->th_ack = htonl(31);
2120  p->payload_len = 1460;
2121 
2122  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2123 
2124  p->tcph->th_seq = htonl(857963582);
2125  p->tcph->th_ack = htonl(31);
2126  p->payload_len = 1460;
2127 
2128  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2129 
2130  p->tcph->th_seq = htonl(857960946);
2131  p->tcph->th_ack = htonl(31);
2132  p->payload_len = 1460;
2133 
2134  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2135 
2137  StreamTcpUTDeinit(ra_ctx);
2138  SCFree(p);
2139  PASS;
2140 }
2141 
2142 /** \test Test the bug 76 condition */
2143 static int StreamTcpReassembleTest37(void)
2144 {
2145  TcpSession ssn;
2146  Flow f;
2147  TCPHdr tcph;
2148  TcpReassemblyThreadCtx *ra_ctx = NULL;
2149  uint8_t packet[1460] = "";
2150  PacketQueue pq;
2151  ThreadVars tv;
2152  memset(&tv, 0, sizeof (ThreadVars));
2153 
2154  Packet *p = PacketGetFromAlloc();
2155  FAIL_IF(unlikely(p == NULL));
2156 
2157  StreamTcpUTInit(&ra_ctx);
2159  memset(&pq,0,sizeof(PacketQueue));
2160  memset(&f, 0, sizeof (Flow));
2161  memset(&tcph, 0, sizeof (TCPHdr));
2162  memset(&tv, 0, sizeof (ThreadVars));
2163 
2164  FLOW_INITIALIZE(&f);
2165  f.protoctx = &ssn;
2166  f.proto = IPPROTO_TCP;
2167  p->src.family = AF_INET;
2168  p->dst.family = AF_INET;
2169  p->proto = IPPROTO_TCP;
2170  p->flow = &f;
2171  tcph.th_win = 5480;
2172  tcph.th_flags = TH_PUSH | TH_ACK;
2173  p->tcph = &tcph;
2175  p->payload = packet;
2177 
2178  p->tcph->th_seq = htonl(3061088537UL);
2179  p->tcph->th_ack = htonl(1729548549UL);
2180  p->payload_len = 1391;
2181  ssn.client.last_ack = 3061091137UL;
2182  SET_ISN(&ssn.client, 3061091309UL);
2183 
2184  /* pre base_seq, so should be rejected */
2185  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2186 
2187  p->tcph->th_seq = htonl(3061089928UL);
2188  p->tcph->th_ack = htonl(1729548549UL);
2189  p->payload_len = 1391;
2190  ssn.client.last_ack = 3061091137UL;
2191 
2192  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2193 
2194  p->tcph->th_seq = htonl(3061091319UL);
2195  p->tcph->th_ack = htonl(1729548549UL);
2196  p->payload_len = 1391;
2197  ssn.client.last_ack = 3061091137UL;
2198 
2199  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2200 
2202  StreamTcpUTDeinit(ra_ctx);
2203  SCFree(p);
2204  PASS;
2205 }
2206 
2207 /**
2208  * \test Test to make sure that we don't return the segments until the app
2209  * layer proto has been detected and after that remove the processed
2210  * segments.
2211  *
2212  * \retval On success it returns 1 and on failure 0.
2213  */
2214 
2215 static int StreamTcpReassembleTest39 (void)
2216 {
2217  Packet *p = PacketGetFromAlloc();
2218  FAIL_IF(unlikely(p == NULL));
2219  Flow f;
2220  ThreadVars tv;
2221  StreamTcpThread stt;
2222  TCPHdr tcph;
2223  PacketQueue pq;
2224  memset(&pq,0,sizeof(PacketQueue));
2225  memset (&f, 0, sizeof(Flow));
2226  memset(&tv, 0, sizeof (ThreadVars));
2227  memset(&stt, 0, sizeof (stt));
2228  memset(&tcph, 0, sizeof (TCPHdr));
2229 
2230  FLOW_INITIALIZE(&f);
2231  f.flags = FLOW_IPV4;
2232  f.proto = IPPROTO_TCP;
2233  p->flow = &f;
2234  p->tcph = &tcph;
2235 
2236  StreamTcpUTInit(&stt.ra_ctx);
2237 
2238  /* handshake */
2239  tcph.th_win = htons(5480);
2240  tcph.th_flags = TH_SYN;
2242  p->payload_len = 0;
2243  p->payload = NULL;
2244  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2245 
2246  TcpSession *ssn = (TcpSession *)f.protoctx;
2247  FAIL_IF_NULL(ssn);
2248 
2259  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2260  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2261  FAIL_IF(ssn->data_first_seen_dir != 0);
2262 
2263  /* handshake */
2264  p->tcph->th_ack = htonl(1);
2265  p->tcph->th_flags = TH_SYN | TH_ACK;
2267  p->payload_len = 0;
2268  p->payload = NULL;
2269  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2280  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2281  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2282  FAIL_IF(ssn->data_first_seen_dir != 0);
2283 
2284  /* handshake */
2285  p->tcph->th_ack = htonl(1);
2286  p->tcph->th_seq = htonl(1);
2287  p->tcph->th_flags = TH_ACK;
2289  p->payload_len = 0;
2290  p->payload = NULL;
2291  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2302  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2303  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2304  FAIL_IF(ssn->data_first_seen_dir != 0);
2305 
2306  /* partial request */
2307  uint8_t request1[] = { 0x47, 0x45, };
2308  p->tcph->th_ack = htonl(1);
2309  p->tcph->th_seq = htonl(1);
2310  p->tcph->th_flags = TH_PUSH | TH_ACK;
2312  p->payload_len = sizeof(request1);
2313  p->payload = request1;
2314  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2325  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2326  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2327  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2329 
2330  /* response ack against partial request */
2331  p->tcph->th_ack = htonl(3);
2332  p->tcph->th_seq = htonl(1);
2333  p->tcph->th_flags = TH_ACK;
2335  p->payload_len = 0;
2336  p->payload = NULL;
2337  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2348  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2349  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2350  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2352 
2353  /* complete partial request */
2354  uint8_t request2[] = {
2355  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2356  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2357  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2358  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2359  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2360  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2361  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2362  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2363  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2364  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2365  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2366  p->tcph->th_ack = htonl(1);
2367  p->tcph->th_seq = htonl(3);
2368  p->tcph->th_flags = TH_PUSH | TH_ACK;
2370  p->payload_len = sizeof(request2);
2371  p->payload = request2;
2372  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2383  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2384  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2385  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2386  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2388 
2389  /* response - request ack */
2390  uint8_t response[] = {
2391  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2392  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2393  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2394  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2395  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2396  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2397  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2398  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2399  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2400  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2401  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2402  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2403  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2404  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2405  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2406  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2407  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2408  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2409  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2410  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2411  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2412  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2413  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2414  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2415  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2416  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2417  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2418  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2419  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2420  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2421  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2422  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2423  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2424  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2425  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2426  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2427  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2428  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2429  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2430  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2431  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2432  p->tcph->th_ack = htonl(88);
2433  p->tcph->th_seq = htonl(1);
2434  p->tcph->th_flags = TH_PUSH | TH_ACK;
2436  p->payload_len = sizeof(response);
2437  p->payload = response;
2438 
2439  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2451  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2452  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2453  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2454  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2455 
2456  /* response ack from request */
2457  p->tcph->th_ack = htonl(328);
2458  p->tcph->th_seq = htonl(88);
2459  p->tcph->th_flags = TH_ACK;
2461  p->payload_len = 0;
2462  p->payload = NULL;
2463  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2475  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2476  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2477  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2478  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2479  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2480 
2481  /* response - acking */
2482  p->tcph->th_ack = htonl(88);
2483  p->tcph->th_seq = htonl(328);
2484  p->tcph->th_flags = TH_PUSH | TH_ACK;
2486  p->payload_len = 0;
2487  p->payload = NULL;
2488  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2500  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2501  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2502  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2503  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2504  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2505 
2506  /* response ack from request */
2507  p->tcph->th_ack = htonl(328);
2508  p->tcph->th_seq = htonl(88);
2509  p->tcph->th_flags = TH_ACK;
2511  p->payload_len = 0;
2512  p->payload = NULL;
2513  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2525  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2526  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2527  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2528  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2529 
2530  /* response - acking the request again*/
2531  p->tcph->th_ack = htonl(88);
2532  p->tcph->th_seq = htonl(328);
2533  p->tcph->th_flags = TH_PUSH | TH_ACK;
2535  p->payload_len = 0;
2536  p->payload = NULL;
2537  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2549  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2550  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2551  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2552  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2553 
2554  /*** New Request ***/
2555 
2556  /* partial request */
2557  p->tcph->th_ack = htonl(328);
2558  p->tcph->th_seq = htonl(88);
2559  p->tcph->th_flags = TH_PUSH | TH_ACK;
2561  p->payload_len = sizeof(request1);
2562  p->payload = request1;
2563  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2575  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2576  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2577  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2578  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2579  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2580 
2581  /* response ack against partial request */
2582  p->tcph->th_ack = htonl(90);
2583  p->tcph->th_seq = htonl(328);
2584  p->tcph->th_flags = TH_ACK;
2586  p->payload_len = 0;
2587  p->payload = NULL;
2588  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2589  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2601  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2602  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2603  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2604  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2605  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2606 
2607  /* complete request */
2608  p->tcph->th_ack = htonl(328);
2609  p->tcph->th_seq = htonl(90);
2610  p->tcph->th_flags = TH_PUSH | TH_ACK;
2612  p->payload_len = sizeof(request2);
2613  p->payload = request2;
2614  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2626  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2627  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2628  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2629  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2630  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2631  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2632 
2633  /* response ack against second partial request */
2634  p->tcph->th_ack = htonl(175);
2635  p->tcph->th_seq = htonl(328);
2636  p->tcph->th_flags = TH_ACK;
2638  p->payload_len = 0;
2639  p->payload = NULL;
2640 
2641  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2653  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2654  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2655  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2656  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2657  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2658  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2659 
2660  /* response acking a request */
2661  p->tcph->th_ack = htonl(175);
2662  p->tcph->th_seq = htonl(328);
2663  p->tcph->th_flags = TH_ACK;
2665  p->payload_len = 0;
2666  p->payload = NULL;
2667  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2673 
2676 
2677  /* request acking a response */
2678  p->tcph->th_ack = htonl(328);
2679  p->tcph->th_seq = htonl(175);
2680  p->tcph->th_flags = TH_ACK;
2682  p->payload_len = 0;
2683  p->payload = NULL;
2684  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2685 
2686  StreamTcpSessionClear(ssn);
2688  SCFree(p);
2689  PASS;
2690 }
2691 
2692 /**
2693  * \test Test to make sure that we sent all the segments from the initial
2694  * segments to app layer until we have detected the app layer proto.
2695  *
2696  * \retval On success it returns 1 and on failure 0.
2697  */
2698 
2699 static int StreamTcpReassembleTest40 (void)
2700 {
2701  Packet *p = PacketGetFromAlloc();
2702  FAIL_IF_NULL(p);
2703  Flow *f = NULL;
2704  TCPHdr tcph;
2705  TcpSession ssn;
2706  PacketQueue pq;
2707  memset(&pq,0,sizeof(PacketQueue));
2708  memset(&tcph, 0, sizeof (TCPHdr));
2709  ThreadVars tv;
2710  memset(&tv, 0, sizeof (ThreadVars));
2711 
2714 
2716  FAIL_IF_NULL(ra_ctx);
2717 
2718  uint8_t httpbuf1[] = "P";
2719  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2720  uint8_t httpbuf3[] = "O";
2721  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2722  uint8_t httpbuf4[] = "S";
2723  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2724  uint8_t httpbuf5[] = "T \r\n";
2725  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2726 
2727  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2728  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2729 
2730  SET_ISN(&ssn.server, 9);
2731  ssn.server.last_ack = 10;
2732  SET_ISN(&ssn.client, 9);
2733  ssn.client.isn = 9;
2734 
2735  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2736  FAIL_IF_NULL(f);
2737  f->protoctx = &ssn;
2738  f->proto = IPPROTO_TCP;
2739  p->flow = f;
2740 
2741  tcph.th_win = htons(5480);
2742  tcph.th_seq = htonl(10);
2743  tcph.th_ack = htonl(10);
2744  tcph.th_flags = TH_ACK|TH_PUSH;
2745  p->tcph = &tcph;
2747  p->payload = httpbuf1;
2748  p->payload_len = httplen1;
2749  ssn.state = TCP_ESTABLISHED;
2750  TcpStream *s = &ssn.client;
2751  SCLogDebug("1 -- start");
2752  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2753 
2755  p->payload = httpbuf2;
2756  p->payload_len = httplen2;
2757  tcph.th_seq = htonl(10);
2758  tcph.th_ack = htonl(11);
2759  s = &ssn.server;
2760  ssn.server.last_ack = 11;
2761  SCLogDebug("2 -- start");
2762  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2763 
2765  p->payload = httpbuf3;
2766  p->payload_len = httplen3;
2767  tcph.th_seq = htonl(11);
2768  tcph.th_ack = htonl(55);
2769  s = &ssn.client;
2770  ssn.client.last_ack = 55;
2771  SCLogDebug("3 -- start");
2772  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2773 
2775  p->payload = httpbuf2;
2776  p->payload_len = httplen2;
2777  tcph.th_seq = htonl(55);
2778  tcph.th_ack = htonl(12);
2779  s = &ssn.server;
2780  ssn.server.last_ack = 12;
2781  SCLogDebug("4 -- start");
2782  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2783 
2784  /* check is have the segment in the list and flagged or not */
2785  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2786  FAIL_IF_NULL(seg);
2787  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2788 
2790  p->payload = httpbuf4;
2791  p->payload_len = httplen4;
2792  tcph.th_seq = htonl(12);
2793  tcph.th_ack = htonl(100);
2794  s = &ssn.client;
2795  ssn.client.last_ack = 100;
2796  SCLogDebug("5 -- start");
2797  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2798 
2800  p->payload = httpbuf2;
2801  p->payload_len = httplen2;
2802  tcph.th_seq = htonl(100);
2803  tcph.th_ack = htonl(13);
2804  s = &ssn.server;
2805  ssn.server.last_ack = 13;
2806  SCLogDebug("6 -- start");
2807  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2808 
2810  p->payload = httpbuf5;
2811  p->payload_len = httplen5;
2812  tcph.th_seq = htonl(13);
2813  tcph.th_ack = htonl(145);
2814  s = &ssn.client;
2815  ssn.client.last_ack = 145;
2816  SCLogDebug("7 -- start");
2817  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2818 
2820  p->payload = httpbuf2;
2821  p->payload_len = httplen2;
2822  tcph.th_seq = htonl(145);
2823  tcph.th_ack = htonl(16);
2824  s = &ssn.server;
2825  ssn.server.last_ack = 16;
2826  SCLogDebug("8 -- start");
2827  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2828  FAIL_IF(f->alproto != ALPROTO_HTTP);
2829 
2833  SCFree(p);
2834  UTHFreeFlow(f);
2835  PASS;
2836 }
2837 
2838 /** \test Test the memcap incrementing/decrementing and memcap check */
2839 static int StreamTcpReassembleTest44(void)
2840 {
2842  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2844  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2846  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2848  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2850  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2851  PASS;
2852 }
2853 
2854 /**
2855  * \test Test to make sure that reassembly_depth is enforced.
2856  *
2857  * \retval On success it returns 1 and on failure 0.
2858  */
2859 
2860 static int StreamTcpReassembleTest45 (void)
2861 {
2862  TcpReassemblyThreadCtx *ra_ctx = NULL;
2863  TcpSession ssn;
2864  ThreadVars tv;
2865  memset(&tv, 0, sizeof(tv));
2866  uint8_t payload[100] = {0};
2867  uint16_t payload_size = 100;
2868 
2869  StreamTcpUTInit(&ra_ctx);
2871 
2873  ssn.reassembly_depth = 100;
2874  StreamTcpUTSetupStream(&ssn.server, 100);
2875  StreamTcpUTSetupStream(&ssn.client, 100);
2876 
2877  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2878  FAIL_IF(r != 0);
2880 
2881  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2882  FAIL_IF(r != 0);
2884 
2888  StreamTcpUTDeinit(ra_ctx);
2889  PASS;
2890 }
2891 
2892 /**
2893  * \test Test the unlimited config value of reassembly depth.
2894  *
2895  * \retval On success it returns 1 and on failure 0.
2896  */
2897 
2898 static int StreamTcpReassembleTest46 (void)
2899 {
2900  int result = 0;
2901  TcpReassemblyThreadCtx *ra_ctx = NULL;
2902  TcpSession ssn;
2903  ThreadVars tv;
2904  memset(&tv, 0, sizeof(tv));
2905  uint8_t payload[100] = {0};
2906  uint16_t payload_size = 100;
2907 
2908  StreamTcpUTInit(&ra_ctx);
2910 
2912  StreamTcpUTSetupStream(&ssn.server, 100);
2913  StreamTcpUTSetupStream(&ssn.client, 100);
2914 
2915  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2916  if (r != 0)
2917  goto end;
2919  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2920  goto end;
2921  }
2922 
2923  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2924  if (r != 0)
2925  goto end;
2927  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2928  goto end;
2929  }
2930 
2931  result = 1;
2932 end:
2936  StreamTcpUTDeinit(ra_ctx);
2937  return result;
2938 }
2939 
2940 /**
2941  * \test Test to make sure we detect the sequence wrap around and continue
2942  * stream reassembly properly.
2943  *
2944  * \retval On success it returns 1 and on failure 0.
2945  */
2946 
2947 static int StreamTcpReassembleTest47 (void)
2948 {
2949  Packet *p = PacketGetFromAlloc();
2950  FAIL_IF(unlikely(p == NULL));
2951  Flow *f = NULL;
2952  TCPHdr tcph;
2953  TcpSession ssn;
2954  ThreadVars tv;
2955  PacketQueue pq;
2956  memset(&pq,0,sizeof(PacketQueue));
2957  memset(&tcph, 0, sizeof (TCPHdr));
2958  memset(&tv, 0, sizeof (ThreadVars));
2962 
2963  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
2964  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2965 
2966  SET_ISN(&ssn.server, 572799781UL);
2967  ssn.server.last_ack = 572799782UL;
2968 
2969  SET_ISN(&ssn.client, 4294967289UL);
2970  ssn.client.last_ack = 21;
2971 
2972  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2973  FAIL_IF(f == NULL);
2974  f->protoctx = &ssn;
2975  f->proto = IPPROTO_TCP;
2976  p->flow = f;
2977 
2978  tcph.th_win = htons(5480);
2979  ssn.state = TCP_ESTABLISHED;
2980  TcpStream *s = NULL;
2981  uint8_t cnt = 0;
2982 
2983  for (cnt=0; cnt < httplen1; cnt++) {
2984  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
2985  tcph.th_ack = htonl(572799782UL);
2986  tcph.th_flags = TH_ACK|TH_PUSH;
2987  p->tcph = &tcph;
2989  p->payload = &httpbuf1[cnt];
2990  p->payload_len = 1;
2991  s = &ssn.client;
2992 
2993  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2994 
2996  p->payload = NULL;
2997  p->payload_len = 0;
2998  tcph.th_seq = htonl(572799782UL);
2999  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3000  tcph.th_flags = TH_ACK;
3001  p->tcph = &tcph;
3002  s = &ssn.server;
3003 
3004  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3005  }
3006 
3007  FAIL_IF(f->alproto != ALPROTO_HTTP);
3008 
3012  SCFree(p);
3013  UTHFreeFlow(f);
3014  PASS;
3015 }
3016 
3017 /** \test 3 in order segments in inline reassembly */
3018 static int StreamTcpReassembleInlineTest01(void)
3019 {
3020  int ret = 0;
3021  TcpReassemblyThreadCtx *ra_ctx = NULL;
3022  ThreadVars tv;
3023  TcpSession ssn;
3024  Flow f;
3025 
3026  memset(&tv, 0x00, sizeof(tv));
3027 
3028  StreamTcpUTInit(&ra_ctx);
3031  StreamTcpUTSetupStream(&ssn.client, 1);
3032  FLOW_INITIALIZE(&f);
3033 
3034  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3035  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3036  if (p == NULL) {
3037  printf("couldn't get a packet: ");
3038  goto end;
3039  }
3040  p->tcph->th_seq = htonl(12);
3041  p->flow = &f;
3042 
3043  FLOWLOCK_WRLOCK(&f);
3044  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3045  printf("failed to add segment 1: ");
3046  goto end;
3047  }
3048  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3049  printf("failed to add segment 2: ");
3050  goto end;
3051  }
3052  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3053  printf("failed to add segment 3: ");
3054  goto end;
3055  }
3056  ssn.client.next_seq = 17;
3057  ret = 1;
3058 end:
3059  FLOWLOCK_UNLOCK(&f);
3060  FLOW_DESTROY(&f);
3061  UTHFreePacket(p);
3063  StreamTcpUTDeinit(ra_ctx);
3064  return ret;
3065 }
3066 
3067 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3068  * test the sliding window reassembly.
3069  */
3070 static int StreamTcpReassembleInlineTest02(void)
3071 {
3072  int ret = 0;
3073  TcpReassemblyThreadCtx *ra_ctx = NULL;
3074  ThreadVars tv;
3075  TcpSession ssn;
3076  Flow f;
3077 
3078  memset(&tv, 0x00, sizeof(tv));
3079 
3080  StreamTcpUTInit(&ra_ctx);
3083  StreamTcpUTSetupStream(&ssn.client, 1);
3084  FLOW_INITIALIZE(&f);
3085 
3086  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3087  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3088  if (p == NULL) {
3089  printf("couldn't get a packet: ");
3090  goto end;
3091  }
3092  p->tcph->th_seq = htonl(12);
3093  p->flow = &f;
3094 
3095  FLOWLOCK_WRLOCK(&f);
3096  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3097  printf("failed to add segment 1: ");
3098  goto end;
3099  }
3100  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3101  printf("failed to add segment 2: ");
3102  goto end;
3103  }
3104  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3105  printf("failed to add segment 3: ");
3106  goto end;
3107  }
3108  ssn.client.next_seq = 17;
3109  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3110  printf("failed to add segment 4: ");
3111  goto end;
3112  }
3113  ssn.client.next_seq = 22;
3114  ret = 1;
3115 end:
3116  FLOWLOCK_UNLOCK(&f);
3117  FLOW_DESTROY(&f);
3118  UTHFreePacket(p);
3120  StreamTcpUTDeinit(ra_ctx);
3121  return ret;
3122 }
3123 
3124 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3125  * test the sliding window reassembly with a small window size so that we
3126  * cutting off at the start (left edge)
3127  */
3128 static int StreamTcpReassembleInlineTest03(void)
3129 {
3130  int ret = 0;
3131  TcpReassemblyThreadCtx *ra_ctx = NULL;
3132  ThreadVars tv;
3133  TcpSession ssn;
3134  Flow f;
3135 
3136  memset(&tv, 0x00, sizeof(tv));
3137 
3138  StreamTcpUTInit(&ra_ctx);
3141  StreamTcpUTSetupStream(&ssn.client, 1);
3142  FLOW_INITIALIZE(&f);
3143 
3145 
3146  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3147  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3148  if (p == NULL) {
3149  printf("couldn't get a packet: ");
3150  goto end;
3151  }
3152  p->tcph->th_seq = htonl(12);
3153  p->flow = &f;
3155 
3156  FLOWLOCK_WRLOCK(&f);
3157  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3158  printf("failed to add segment 1: ");
3159  goto end;
3160  }
3161  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3162  printf("failed to add segment 2: ");
3163  goto end;
3164  }
3165  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3166  printf("failed to add segment 3: ");
3167  goto end;
3168  }
3169  ssn.client.next_seq = 17;
3170  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3171  printf("failed to add segment 4: ");
3172  goto end;
3173  }
3174  ssn.client.next_seq = 22;
3175 
3176  p->tcph->th_seq = htonl(17);
3177  ret = 1;
3178 end:
3179  FLOWLOCK_UNLOCK(&f);
3180  FLOW_DESTROY(&f);
3181  UTHFreePacket(p);
3183  StreamTcpUTDeinit(ra_ctx);
3184  return ret;
3185 }
3186 
3187 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3188  * test the sliding window reassembly with a small window size so that we
3189  * cutting off at the start (left edge) with small packet overlap.
3190  */
3191 static int StreamTcpReassembleInlineTest04(void)
3192 {
3193  int ret = 0;
3194  TcpReassemblyThreadCtx *ra_ctx = NULL;
3195  ThreadVars tv;
3196  TcpSession ssn;
3197  Flow f;
3198 
3199  memset(&tv, 0x00, sizeof(tv));
3200 
3201  StreamTcpUTInit(&ra_ctx);
3204  StreamTcpUTSetupStream(&ssn.client, 1);
3205  FLOW_INITIALIZE(&f);
3206 
3208 
3209  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3210  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3211  if (p == NULL) {
3212  printf("couldn't get a packet: ");
3213  goto end;
3214  }
3215  p->tcph->th_seq = htonl(12);
3216  p->flow = &f;
3218 
3219  FLOWLOCK_WRLOCK(&f);
3220  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3221  printf("failed to add segment 1: ");
3222  goto end;
3223  }
3224  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3225  printf("failed to add segment 2: ");
3226  goto end;
3227  }
3228  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3229  printf("failed to add segment 3: ");
3230  goto end;
3231  }
3232  ssn.client.next_seq = 17;
3233  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3234  printf("failed to add segment 4: ");
3235  goto end;
3236  }
3237  ssn.client.next_seq = 22;
3238 
3239  p->tcph->th_seq = htonl(17);
3240  ret = 1;
3241 end:
3242  FLOWLOCK_UNLOCK(&f);
3243  FLOW_DESTROY(&f);
3244  UTHFreePacket(p);
3246  StreamTcpUTDeinit(ra_ctx);
3247  return ret;
3248 }
3249 
3250 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3251  * test the sliding window reassembly with a small window size so that we
3252  * cutting off at the start (left edge). Test if the first segment is
3253  * removed from the list.
3254  */
3255 static int StreamTcpReassembleInlineTest08(void)
3256 {
3257  TcpReassemblyThreadCtx *ra_ctx = NULL;
3258  ThreadVars tv;
3259  memset(&tv, 0x00, sizeof(tv));
3260  TcpSession ssn;
3261  Flow f;
3262  StreamTcpUTInit(&ra_ctx);
3265  StreamTcpUTSetupStream(&ssn.client, 1);
3266  FLOW_INITIALIZE(&f);
3267 
3270  f.protoctx = &ssn;
3271 
3272  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3273  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3274  FAIL_IF(p == NULL);
3275  p->tcph->th_seq = htonl(12);
3276  p->flow = &f;
3278 
3279  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3280  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3281  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3282  ssn.client.next_seq = 17;
3283  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3284  ssn.client.next_seq = 22;
3285  p->tcph->th_seq = htonl(17);
3287 
3288  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3289  FAIL_IF_NULL(seg);
3290  FAIL_IF_NOT(seg->seq == 2);
3291 
3292  FLOW_DESTROY(&f);
3293  UTHFreePacket(p);
3295  StreamTcpUTDeinit(ra_ctx);
3296  PASS;
3297 }
3298 
3299 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3300  * test the sliding window reassembly with a small window size so that we
3301  * cutting off at the start (left edge). Test if the first segment is
3302  * removed from the list.
3303  */
3304 static int StreamTcpReassembleInlineTest09(void)
3305 {
3306  int ret = 0;
3307  TcpReassemblyThreadCtx *ra_ctx = NULL;
3308  ThreadVars tv;
3309  TcpSession ssn;
3310  Flow f;
3311 
3312  memset(&tv, 0x00, sizeof(tv));
3313 
3314  StreamTcpUTInit(&ra_ctx);
3317  StreamTcpUTSetupStream(&ssn.client, 1);
3318  FLOW_INITIALIZE(&f);
3319 
3322 
3323  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3324  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3325  if (p == NULL) {
3326  printf("couldn't get a packet: ");
3327  goto end;
3328  }
3329  p->tcph->th_seq = htonl(17);
3330  p->flow = &f;
3332 
3333  FLOWLOCK_WRLOCK(&f);
3334  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3335  printf("failed to add segment 1: ");
3336  goto end;
3337  }
3338  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3339  printf("failed to add segment 2: ");
3340  goto end;
3341  }
3342  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3343  printf("failed to add segment 3: ");
3344  goto end;
3345  }
3346  ssn.client.next_seq = 12;
3347  ssn.client.last_ack = 10;
3348 
3349  /* close the GAP and see if we properly reassemble and update base_seq */
3350  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3351  printf("failed to add segment 4: ");
3352  goto end;
3353  }
3354  ssn.client.next_seq = 22;
3355 
3356  p->tcph->th_seq = htonl(12);
3357 
3358  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3359  FAIL_IF_NULL(seg);
3360  FAIL_IF_NOT(seg->seq == 2);
3361 
3362  ret = 1;
3363 end:
3364  FLOWLOCK_UNLOCK(&f);
3365  FLOW_DESTROY(&f);
3366  UTHFreePacket(p);
3368  StreamTcpUTDeinit(ra_ctx);
3369  return ret;
3370 }
3371 
3372 /** \test App Layer reassembly.
3373  */
3374 static int StreamTcpReassembleInlineTest10(void)
3375 {
3376  int ret = 0;
3377  TcpReassemblyThreadCtx *ra_ctx = NULL;
3378  ThreadVars tv;
3379  TcpSession ssn;
3380  Flow *f = NULL;
3381  Packet *p = NULL;
3382 
3383  memset(&tv, 0x00, sizeof(tv));
3384 
3385  StreamTcpUTInit(&ra_ctx);
3388  StreamTcpUTSetupStream(&ssn.server, 1);
3389  ssn.server.last_ack = 2;
3390  StreamTcpUTSetupStream(&ssn.client, 1);
3391  ssn.client.last_ack = 2;
3393 
3394  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3395  if (f == NULL)
3396  goto end;
3397  f->protoctx = &ssn;
3398  f->proto = IPPROTO_TCP;
3399 
3400  uint8_t stream_payload1[] = "GE";
3401  uint8_t stream_payload2[] = "T /";
3402  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3403 
3404  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3405  if (p == NULL) {
3406  printf("couldn't get a packet: ");
3407  goto end;
3408  }
3409  p->tcph->th_seq = htonl(7);
3410  p->flow = f;
3412 
3413  FLOWLOCK_WRLOCK(f);
3414  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3415  printf("failed to add segment 1: ");
3416  goto end;
3417  }
3418  ssn.client.next_seq = 4;
3419 
3420  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3421  if (r < 0) {
3422  printf("StreamTcpReassembleAppLayer failed: ");
3423  goto end;
3424  }
3425 
3426  /* ssn.server.ra_app_base_seq should be isn here. */
3427  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3428  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3429  goto end;
3430  }
3431 
3432  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3433  printf("failed to add segment 2: ");
3434  goto end;
3435  }
3436  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3437  printf("failed to add segment 3: ");
3438  goto end;
3439  }
3440  ssn.client.next_seq = 19;
3441 
3442  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3443  if (r < 0) {
3444  printf("StreamTcpReassembleAppLayer failed: ");
3445  goto end;
3446  }
3447 
3448  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3449 
3450  ret = 1;
3451 end:
3452  UTHFreePacket(p);
3454  StreamTcpUTDeinit(ra_ctx);
3455  FLOWLOCK_UNLOCK(f);
3456  UTHFreeFlow(f);
3457  return ret;
3458 }
3459 
3460 /** \test test insert with overlap
3461  */
3462 static int StreamTcpReassembleInsertTest01(void)
3463 {
3464  TcpReassemblyThreadCtx *ra_ctx = NULL;
3465  ThreadVars tv;
3466  TcpSession ssn;
3467  Flow f;
3468 
3469  memset(&tv, 0x00, sizeof(tv));
3470 
3471  StreamTcpUTInit(&ra_ctx);
3473  StreamTcpUTSetupStream(&ssn.client, 1);
3475  FLOW_INITIALIZE(&f);
3476 
3477  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3478  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3479  FAIL_IF(p == NULL);
3480  p->tcph->th_seq = htonl(12);
3481  p->flow = &f;
3482 
3483  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3484  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3485  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3486  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3487  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3488  ssn.client.next_seq = 21;
3489 
3490  FLOW_DESTROY(&f);
3491  UTHFreePacket(p);
3493  StreamTcpUTDeinit(ra_ctx);
3494  PASS;
3495 }
3496 
3497 /** \test test insert with overlaps
3498  */
3499 static int StreamTcpReassembleInsertTest02(void)
3500 {
3501  int ret = 0;
3502  TcpReassemblyThreadCtx *ra_ctx = NULL;
3503  ThreadVars tv;
3504  TcpSession ssn;
3505 
3506  memset(&tv, 0x00, sizeof(tv));
3507 
3508  StreamTcpUTInit(&ra_ctx);
3510  StreamTcpUTSetupStream(&ssn.client, 1);
3511 
3512  int i;
3513  for (i = 2; i < 10; i++) {
3514  int len;
3515  len = i % 2;
3516  if (len == 0)
3517  len = 1;
3518  int seq;
3519  seq = i * 10;
3520  if (seq < 2)
3521  seq = 2;
3522 
3523  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3524  printf("failed to add segment 1: ");
3525  goto end;
3526  }
3527  }
3528  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3529  printf("failed to add segment 2: ");
3530  goto end;
3531  }
3532 
3533  ret = 1;
3534 end:
3536  StreamTcpUTDeinit(ra_ctx);
3537  return ret;
3538 }
3539 
3540 /** \test test insert with overlaps
3541  */
3542 static int StreamTcpReassembleInsertTest03(void)
3543 {
3544  int ret = 0;
3545  TcpReassemblyThreadCtx *ra_ctx = NULL;
3546  ThreadVars tv;
3547  TcpSession ssn;
3548 
3549  memset(&tv, 0x00, sizeof(tv));
3550 
3551  StreamTcpUTInit(&ra_ctx);
3553  StreamTcpUTSetupStream(&ssn.client, 1);
3554 
3555  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3556  printf("failed to add segment 2: ");
3557  goto end;
3558  }
3559 
3560  int i;
3561  for (i = 2; i < 10; i++) {
3562  int len;
3563  len = i % 2;
3564  if (len == 0)
3565  len = 1;
3566  int seq;
3567  seq = i * 10;
3568  if (seq < 2)
3569  seq = 2;
3570 
3571  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3572  printf("failed to add segment 2: ");
3573  goto end;
3574  }
3575  }
3576  ret = 1;
3577 end:
3579  StreamTcpUTDeinit(ra_ctx);
3580  return ret;
3581 }
3582 
3584 #endif /* UNITTESTS */
3585 
3586 /** \brief The Function Register the Unit tests to test the reassembly engine
3587  * for various OS policies.
3588  */
3589 
3591 {
3592 #ifdef UNITTESTS
3593  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3594  StreamTcpReassembleTest25);
3595  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3596  StreamTcpReassembleTest26);
3597  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3598  StreamTcpReassembleTest27);
3599  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3600  StreamTcpReassembleTest28);
3601  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3602  StreamTcpReassembleTest29);
3603  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3604  StreamTcpReassembleTest33);
3605  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3606  StreamTcpReassembleTest34);
3607  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3608  StreamTcpReassembleTest37);
3609  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3610  StreamTcpReassembleTest39);
3611  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3612  StreamTcpReassembleTest40);
3613  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3614  StreamTcpReassembleTest44);
3615  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3616  StreamTcpReassembleTest45);
3617  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3618  StreamTcpReassembleTest46);
3619  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3620  StreamTcpReassembleTest47);
3621 
3622  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3623  StreamTcpReassembleInlineTest01);
3624  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3625  StreamTcpReassembleInlineTest02);
3626  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3627  StreamTcpReassembleInlineTest03);
3628  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3629  StreamTcpReassembleInlineTest04);
3630  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3631  StreamTcpReassembleInlineTest08);
3632  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3633  StreamTcpReassembleInlineTest09);
3634 
3635  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3636  StreamTcpReassembleInlineTest10);
3637 
3638  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3639  StreamTcpReassembleInsertTest01);
3640  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3641  StreamTcpReassembleInsertTest02);
3642  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3643  StreamTcpReassembleInsertTest03);
3644 
3648  StreamTcpReassembleRawRegisterTests();
3649 #endif /* UNITTESTS */
3650 }
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
#define SCMutex
StreamingBufferSegment sbseg
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
#define SCLogDebug(...)
Definition: util-debug.h:335
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
void *(* Malloc)(size_t size)
int StreamTcpAppLayerIsDisabled(Flow *f)
struct Flow_ * flow
Definition: decode.h:445
AppProto alproto_tc
Definition: flow.h:411
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
#define STREAM_BASE_OFFSET(stream)
int StreamTcpReassembleInit(char quiet)
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:344
#define FALSE
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:243
StreamingBuffer sb
#define PASS
Pass the test.
#define unlikely(expr)
Definition: util-optimize.h:35
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
#define STREAMTCP_STREAM_FLAG_GAP
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void StreamTcpListRegisterTests(void)
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define TH_RST
Definition: decode-tcp.h:37
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet...
Definition: stream-tcp.c:770
void StreamTcpDisableAppLayer(Flow *f)
#define TH_FIN
Definition: decode-tcp.h:35
#define STREAMING_BUFFER_NOFLAGS
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
uint64_t offset
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Address dst
Definition: decode.h:413
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
#define SCReturnUInt(x)
Definition: util-debug.h:343
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
int ByteExtractStringUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:244
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:669
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:240
uint64_t pcap_cnt
Definition: decode.h:561
#define STREAM_RAW_PROGRESS(stream)
void StreamTcpReassembleInitMemuse(void)
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:124
char * val
Definition: conf.h:34
TCPHdr * tcph
Definition: decode.h:522
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define TRUE
#define SCMutexLock(mut)
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
StreamingBufferBlock * head
#define STREAMTCP_FLAG_MIDSTREAM
void * protoctx
Definition: flow.h:400
void *(* Calloc)(size_t n, size_t size)
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
#define STREAM_DEPTH
Definition: stream.h:34
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
uint32_t min_inspect_depth
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueue *pq)
Definition: stream-tcp.c:4674
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define StreamTcpSetEvent(p, e)
AppProto alproto_ts
Definition: flow.h:410
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
#define SCCalloc(nm, a)
Definition: util-mem.h:253
#define SEQ_GEQ(a, b)
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:107
#define STREAM_GAP
Definition: stream.h:33
#define SCMutexUnlock(mut)
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:820
uint32_t seq
uint32_t raw_progress_rel
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:248
char family
Definition: decode.h:111
uint8_t proto
Definition: decode.h:430
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
#define SCMUTEX_INITIALIZER
#define TCP_SEG_LEN(seg)
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
#define STREAM_HAS_SEEN_DATA(stream)
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Data structures and function prototypes for keeping state for the detection engine.
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define FLOW_DESTROY(f)
Definition: flow-util.h:119
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies...
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
#define STREAM_EOF
Definition: stream.h:30
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define StreamTcpDisableAppLayerReassembly(ssn)
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
#define SCEnter(...)
Definition: util-debug.h:337
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
void StreamTcpReassembleFree(char quiet)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:439
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PKT_IS_TOSERVER(p)
Definition: decode.h:257
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
uint32_t reassembly_depth
#define TH_PUSH
Definition: decode-tcp.h:38
#define SEQ_GT(a, b)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:163
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1091
int RunmodeIsUnittests(void)
Definition: suricata.c:265
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:989
#define SCMutexInit(mut, mutattrs)
int8_t data_first_seen_dir
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define SCReturnInt(x)
Definition: util-debug.h:341
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueue *pq)
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
void StreamTcpUTInitInline(void)
#define SEQ_LEQ(a, b)
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
uint32_t reassembly_depth
Definition: stream-tcp.h:60
#define SCRealloc(x, a)
Definition: util-mem.h:238
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:563
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define RB_MIN(name, x)
Definition: tree.h:780
Definition: conf.h:32
#define SCMalloc(a)
Definition: util-mem.h:222
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:207
struct TCPSEG seg_tree
uint16_t payload_len
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define TH_SYN
Definition: decode-tcp.h:36
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
block of continues data
#define SCFree(a)
Definition: util-mem.h:322
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:249
#define RB_REMOVE(name, x, y)
Definition: tree.h:775
void StreamTcpUtilRegisterTests(void)
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:842
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
#define STREAM_START
Definition: stream.h:29
uint32_t prealloc_segments
Definition: stream-tcp.h:57
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
#define PKT_STREAM_ADD
Definition: decode.h:1087
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself...
void StreamTcpInlineRegisterTests(void)
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
void StreamTcpReassemblePseudoPacketCreate(TcpStream *, Packet *, PacketQueue *)
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
#define SCReturn
Definition: util-debug.h:339
void StreamTcpUTSetupSession(TcpSession *ssn)
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define MISSED_END
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of &#39;threads&#39;, so array elements)
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
uint32_t flags
Definition: decode.h:443
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
void StreamTcpReassembleConfigEnableOverlapCheck(void)
uint16_t payload_len
Definition: decode.h:541
#define STREAM_APP_PROGRESS(stream)
#define SEQ_LT(a, b)
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6280
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
#define STREAM_RIGHT_EDGE(stream)
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
void StreamTcpUTClearSession(TcpSession *ssn)
#define MISSED_START(isn)
Flow data structure.
Definition: flow.h:325
#define FLOW_IPV4
Definition: flow.h:94
uint8_t * payload
Definition: decode.h:540
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
uint32_t flags
Definition: flow.h:379
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
void StreamTcpUTClearStream(TcpStream *s)
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
void(* Free)(void *ptr, size_t size)
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:141
Address src
Definition: decode.h:412
void UTHFreeFlow(Flow *flow)
#define STREAM_MIDSTREAM
Definition: stream.h:35
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SET_ISN(stream, setseq)
#define SCMutexDestroy
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.