suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 
64 #include "detect-engine-state.h"
65 
66 #include "util-profiling.h"
67 #include "util-validate.h"
68 
69 #ifdef DEBUG
70 static SCMutex segment_pool_memuse_mutex;
71 static uint64_t segment_pool_memuse = 0;
72 static uint64_t segment_pool_memcnt = 0;
73 #endif
74 
75 static PoolThread *segment_thread_pool = NULL;
76 /* init only, protect initializing and growing pool */
77 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
78 
79 /* Memory use counter */
80 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
81 
82 /* prototypes */
84 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346 }
347 
348 /** \param f locked flow */
350 {
351  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
352  return 0;
353 
354  TcpSession *ssn = (TcpSession *)f->protoctx;
356 }
357 
358 static int StreamTcpReassemblyConfig(char quiet)
359 {
360  uint32_t segment_prealloc = 2048;
361  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
362  if (seg) {
363  uint32_t prealloc = 0;
364  if (ByteExtractStringUint32(&prealloc, 10, strlen(seg->val), seg->val) == -1)
365  {
366  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
367  "%s is invalid", seg->val);
368  return -1;
369  }
370  segment_prealloc = prealloc;
371  }
372  if (!quiet)
373  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
374  stream_config.prealloc_segments = segment_prealloc;
375 
376  int overlap_diff_data = 0;
377  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
378  if (overlap_diff_data) {
380  }
381  if (StreamTcpInlineMode() == TRUE) {
383  }
384 
387  stream_config.sbcnf.Malloc = ReassembleMalloc;
388  stream_config.sbcnf.Calloc = ReassembleCalloc;
389  stream_config.sbcnf.Realloc = ReassembleRealloc;
390  stream_config.sbcnf.Free = ReassembleFree;
391 
392  return 0;
393 }
394 
395 int StreamTcpReassembleInit(char quiet)
396 {
397  /* init the memcap/use tracker */
399 
400  if (StreamTcpReassemblyConfig(quiet) < 0)
401  return -1;
402 
403 #ifdef DEBUG
404  SCMutexInit(&segment_pool_memuse_mutex, NULL);
405 #endif
406  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
408  return 0;
409 }
410 
411 void StreamTcpReassembleFree(char quiet)
412 {
413  SCMutexLock(&segment_thread_pool_mutex);
414  if (segment_thread_pool != NULL) {
415  PoolThreadFree(segment_thread_pool);
416  segment_thread_pool = NULL;
417  }
418  SCMutexUnlock(&segment_thread_pool_mutex);
419  SCMutexDestroy(&segment_thread_pool_mutex);
420 
421 #ifdef DEBUG
422  if (segment_pool_memuse > 0)
423  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
424  if (segment_pool_memcnt > 0)
425  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
426  SCMutexDestroy(&segment_pool_memuse_mutex);
427 #endif
428 }
429 
431 {
432  SCEnter();
434  if (unlikely(ra_ctx == NULL))
435  return NULL;
436 
437  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
438 
439  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
440 
441  SCMutexLock(&segment_thread_pool_mutex);
442  if (segment_thread_pool == NULL) {
443  segment_thread_pool = PoolThreadInit(1, /* thread */
444  0, /* unlimited */
446  sizeof(TcpSegment),
447  TcpSegmentPoolAlloc,
448  TcpSegmentPoolInit, NULL,
449  TcpSegmentPoolCleanup, NULL);
450  ra_ctx->segment_thread_pool_id = 0;
451  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
452  PoolThreadSize(segment_thread_pool),
453  ra_ctx->segment_thread_pool_id);
454  } else {
455  /* grow segment_thread_pool until we have a element for our thread id */
456  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
457  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
458  PoolThreadSize(segment_thread_pool),
459  ra_ctx->segment_thread_pool_id);
460  }
461  SCMutexUnlock(&segment_thread_pool_mutex);
462  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
463  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
465  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
466  }
467 
468  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
469 }
470 
472 {
473  SCEnter();
475  SCFree(ra_ctx);
476  SCReturn;
477 }
478 
479 /**
480  * \brief check if stream in pkt direction has depth reached
481  *
482  * \param p packet with *LOCKED* flow
483  *
484  * \retval 1 stream has depth reached
485  * \retval 0 stream does not have depth reached
486  */
488 {
489  if (p->flow != NULL && p->flow->protoctx != NULL) {
490  TcpSession *ssn = p->flow->protoctx;
491  TcpStream *stream;
492  if (p->flowflags & FLOW_PKT_TOSERVER) {
493  stream = &ssn->client;
494  } else {
495  stream = &ssn->server;
496  }
497 
498  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
499  }
500 
501  return 0;
502 }
503 
504 /**
505  * \internal
506  * \brief Function to Check the reassembly depth valuer against the
507  * allowed max depth of the stream reassembly for TCP streams.
508  *
509  * \param stream stream direction
510  * \param seq sequence number where "size" starts
511  * \param size size of the segment that is added
512  *
513  * \retval size Part of the size that fits in the depth, 0 if none
514  */
515 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
516  uint32_t seq, uint32_t size)
517 {
518  SCEnter();
519 
520  /* if the configured depth value is 0, it means there is no limit on
521  reassembly depth. Otherwise carry on my boy ;) */
522  if (ssn->reassembly_depth == 0) {
523  SCReturnUInt(size);
524  }
525 
526  /* if the final flag is set, we're not accepting anymore */
528  SCReturnUInt(0);
529  }
530 
531  uint64_t seg_depth;
532  if (SEQ_GT(stream->base_seq, seq)) {
533  if (SEQ_LEQ(seq+size, stream->base_seq)) {
534  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
535  stream->base_seq, seq, seq+size);
536  SCReturnUInt(0);
537  }
538 
539  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
540  } else {
541  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
542  }
543 
544  /* if the base_seq has moved passed the depth window we stop
545  * checking and just reject the rest of the packets including
546  * retransmissions. Saves us the hassle of dealing with sequence
547  * wraps as well */
548  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
549  stream->base_seq, seg_depth,
550  ssn->reassembly_depth);
551 
552  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
553  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
555  SCReturnUInt(0);
556  }
557  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
558  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
559 #if 0
560  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
561  (stream->base_seq_offset + stream->base_seq + size),
562  (stream->isn + ssn->reassembly_depth));
563 #endif
564  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
565  /* packet (partly?) fits the depth window */
566 
567  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
568  /* complete fit */
569  SCReturnUInt(size);
570  } else {
572  /* partial fit, return only what fits */
573  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
574  DEBUG_VALIDATE_BUG_ON(part > size);
575  if (part > size)
576  part = size;
577  SCReturnUInt(part);
578  }
579  }
580 
581  SCReturnUInt(0);
582 }
583 
584 /**
585  * \brief Insert a packets TCP data into the stream reassembly engine.
586  *
587  * \retval 0 good segment, as far as we checked.
588  * \retval -1 badness, reason to drop in inline mode
589  *
590  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
591  * or it wasn't added because of reassembly depth.
592  *
593  */
595  TcpSession *ssn, TcpStream *stream, Packet *p)
596 {
597  SCEnter();
598 
599  if (ssn->data_first_seen_dir == 0) {
600  if (PKT_IS_TOSERVER(p)) {
602  } else {
604  }
605  }
606 
607  /* If the OS policy is not set then set the OS policy for this stream */
608  if (stream->os_policy == 0) {
609  StreamTcpSetOSPolicy(stream, p);
610  }
611 
614  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
615  SCReturnInt(0);
616  }
617 
618  /* If we have reached the defined depth for either of the stream, then stop
619  reassembling the TCP session */
620  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
621  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
622 
624  /* increment stream depth counter */
625  StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
626  }
627  if (size == 0) {
628  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
629  SCReturnInt(0);
630  }
631 
633  if (size > p->payload_len)
634  size = p->payload_len;
635 
636  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
637  if (seg == NULL) {
638  SCLogDebug("segment_pool is empty");
640  SCReturnInt(-1);
641  }
642 
643  TCP_SEG_LEN(seg) = size;
644  seg->seq = TCP_GET_SEQ(p);
645 
646  /* proto detection skipped, but now we do get data. Set event. */
647  if (RB_EMPTY(&stream->seg_tree) &&
649 
652  }
653 
654  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
655  SCLogDebug("StreamTcpReassembleInsertSegment failed");
656  SCReturnInt(-1);
657  }
658  SCReturnInt(0);
659 }
660 
661 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
662  Packet *p, enum StreamUpdateDir dir)
663 {
664  uint8_t flag = 0;
665 
667  flag |= STREAM_START;
668  }
669 
670  if (ssn->state == TCP_CLOSED) {
671  flag |= STREAM_EOF;
672  }
673 
674  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
675  flag |= STREAM_MIDSTREAM;
676  }
677 
678  if (p->flags & PKT_PSEUDO_STREAM_END) {
679  flag |= STREAM_EOF;
680  }
681 
682  if (dir == UPDATE_DIR_OPPOSING) {
683  if (p->flowflags & FLOW_PKT_TOSERVER) {
684  flag |= STREAM_TOCLIENT;
685  } else {
686  flag |= STREAM_TOSERVER;
687  }
688  } else {
689  if (p->flowflags & FLOW_PKT_TOSERVER) {
690  flag |= STREAM_TOSERVER;
691  } else {
692  flag |= STREAM_TOCLIENT;
693  }
694  }
695 
697  flag |= STREAM_DEPTH;
698  }
699  return flag;
700 }
701 
702 /**
703  * \brief Check the minimum size limits for reassembly.
704  *
705  * \retval 0 don't reassemble yet
706  * \retval 1 do reassemble
707  */
708 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
709  const TcpStream *stream, const Packet *p)
710 {
711  SCEnter();
712 
713  /* if any of these flags is set we always inspect immediately */
714 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
715  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
716  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
717  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
718 
721  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
722  "is set, so not expecting any new data segments");
723  }
725  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
726  }
728  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
729  "so no new segments will be considered");
730  }
731  SCReturnInt(1);
732  }
733 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
734 
735  /* some states mean we reassemble no matter how much data we have */
736  if (ssn->state > TCP_TIME_WAIT)
737  SCReturnInt(1);
738 
739  if (p->flags & PKT_PSEUDO_STREAM_END)
740  SCReturnInt(1);
741 
742  /* check if we have enough data to do raw reassembly */
743  if (PKT_IS_TOSERVER(p)) {
744  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
745  uint32_t delta = stream->last_ack - stream->base_seq;
746  /* get max absolute offset */
747  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
748 
749  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
750  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
751  SCReturnInt(1);
752  } else {
753  SCLogDebug("toserver min chunk len not yet reached: "
754  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
755  "%"PRIu32"", stream->last_ack, stream->base_seq,
756  (stream->last_ack - stream->base_seq),
758  SCReturnInt(0);
759  }
760  }
761  } else {
762  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
763  uint32_t delta = stream->last_ack - stream->base_seq;
764  /* get max absolute offset */
765  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
766 
767  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
768 
769  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
770  SCReturnInt(1);
771  } else {
772  SCLogDebug("toclient min chunk len not yet reached: "
773  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
774  "%"PRIu32"", stream->last_ack, stream->base_seq,
775  (stream->last_ack - stream->base_seq),
777  SCReturnInt(0);
778  }
779  }
780  }
781 
782  SCReturnInt(0);
783 }
784 
785 /**
786  * \brief see what if any work the TCP session still needs
787  */
788 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
789 {
790  const TcpStream *stream = NULL;
791 #ifdef DEBUG
792  const char *dirstr = NULL;
793 #endif
794  if (direction == STREAM_TOSERVER) {
795  stream = &ssn->client;
796 #ifdef DEBUG
797  dirstr = "client";
798 #endif
799  } else {
800  stream = &ssn->server;
801 #ifdef DEBUG
802  dirstr = "server";
803 #endif
804  }
805 
806  int use_app = 1;
807  int use_raw = 1;
808 
810  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
811  {
812  // app is dead
813  use_app = 0;
814  }
815 
817  // raw is dead
818  use_raw = 0;
819  }
820 
821  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
822 
823  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
824  dirstr,
825  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
826  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
827  right_edge);
828  if (use_raw) {
829  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
830  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
832  }
833  }
834  if (use_app) {
835  if (right_edge > STREAM_APP_PROGRESS(stream)) {
836  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
838  }
839  }
840 
841  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
843 }
844 
845 #ifdef DEBUG
846 static uint64_t GetStreamSize(TcpStream *stream)
847 {
848  if (stream) {
849  uint64_t size = 0;
850  uint32_t cnt = 0;
851 
852  TcpSegment *seg;
853  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
854  cnt++;
855  size += (uint64_t)TCP_SEG_LEN(seg);
856  }
857 
858  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
859  return size;
860  }
861  return (uint64_t)0;
862 }
863 
864 static void GetSessionSize(TcpSession *ssn, Packet *p)
865 {
866  uint64_t size = 0;
867  if (ssn) {
868  size = GetStreamSize(&ssn->client);
869  size += GetStreamSize(&ssn->server);
870 
871  //if (size > 900000)
872  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
873  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
874  }
875 }
876 #endif
877 
878 /** \internal
879  *
880  * Get buffer, or first part of the buffer if data gaps exist.
881  *
882  * \brief get stream data from offset
883  * \param offset stream offset */
884 static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset)
885 {
886  const uint8_t *mydata;
887  uint32_t mydata_len;
888 
889  if (RB_EMPTY(&stream->sb.sbb_tree)) {
890  SCLogDebug("getting one blob");
891 
892  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
893 
894  *data = mydata;
895  *data_len = mydata_len;
896  } else {
897  StreamingBufferBlock *blk = stream->sb.head;
898 
899  if (blk->offset > offset) {
900  SCLogDebug("gap, want data at offset %"PRIu64", "
901  "got data at %"PRIu64". GAP of size %"PRIu64,
902  offset, blk->offset, blk->offset - offset);
903  *data = NULL;
904  *data_len = blk->offset - offset;
905 
906  } else if (offset >= (blk->offset + blk->len)) {
907 
908  *data = NULL;
909  StreamingBufferBlock *nblk = SBB_RB_NEXT(blk);
910  *data_len = nblk ? nblk->offset - offset : 0;
911  if (nblk) {
912  SCLogDebug("gap, want data at offset %"PRIu64", "
913  "got data at %"PRIu64". GAP of size %"PRIu64,
914  offset, nblk->offset, nblk->offset - offset);
915  }
916 
917  } else if (offset > blk->offset && offset < (blk->offset + blk->len)) {
918  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
919  offset, blk->offset, blk->len);
920  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
921  SCLogDebug("data %p, data_len %u", *data, *data_len);
922  } else {
923  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
924  }
925  }
926 }
927 
928 /** \internal
929  * \brief check to see if we should declare a GAP
930  * Call this when the app layer didn't get data at the requested
931  * offset.
932  */
933 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
934 {
935  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
936  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
937 
938  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
939  /* get window of data that is acked */
940  const uint32_t delta = stream->last_ack - stream->base_seq;
941  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
942  /* get max absolute offset */
943  last_ack_abs += delta;
944 
945  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
946  last_ack_abs -= ackadded;
947 
948  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
949  SCLogDebug("next_seq %u", stream->next_seq);
950 
951  /* if last_ack_abs is beyond the app_progress data that we haven't seen
952  * has been ack'd. This looks like a GAP. */
953  if (last_ack_abs > app_progress) {
954  /* however, we can accept ACKs a bit too liberally. If last_ack
955  * is beyond next_seq, we only consider it a gap now if we do
956  * already have data beyond the gap. */
957  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
958  if (RB_EMPTY(&stream->sb.sbb_tree)) {
959  SCLogDebug("packet %"PRIu64": no GAP. "
960  "next_seq %u < last_ack %u, but no data in list",
961  p->pcap_cnt, stream->next_seq, stream->last_ack);
962  return false;
963  } else {
964  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
965  const StreamingBufferBlock *blk = stream->sb.head;
966  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
967  /* ack'd data after the gap */
968  SCLogDebug("packet %"PRIu64": GAP. "
969  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
970  p->pcap_cnt, stream->next_seq, stream->last_ack);
971  return true;
972  }
973  }
974  }
975 
976  SCLogDebug("packet %"PRIu64": GAP! "
977  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
978  "but we have no data.",
979  p->pcap_cnt, last_ack_abs, app_progress);
980  return true;
981  }
982  }
983  SCLogDebug("packet %"PRIu64": no GAP. "
984  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
985  p->pcap_cnt, last_ack_abs, app_progress);
986  return false;
987 }
988 
989 /** \internal
990  * \brief get stream buffer and update the app-layer
991  * \param stream pointer to pointer as app-layer can switch flow dir
992  * \retval 0 success
993  */
994 static int ReassembleUpdateAppLayer (ThreadVars *tv,
995  TcpReassemblyThreadCtx *ra_ctx,
996  TcpSession *ssn, TcpStream **stream,
997  Packet *p, enum StreamUpdateDir dir)
998 {
999  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1000 
1001  SCLogDebug("app progress %"PRIu64, app_progress);
1002  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1003 
1004  const uint8_t *mydata;
1005  uint32_t mydata_len;
1006 
1007  while (1) {
1008  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress);
1009  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1010  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1011 
1012  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1013  NULL, mydata_len,
1014  StreamGetAppLayerFlags(ssn, *stream, p, dir)|STREAM_GAP);
1015  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1016 
1018  StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
1019 
1020  /* AppLayerHandleTCPData has likely updated progress. */
1021  app_progress = STREAM_APP_PROGRESS(*stream);
1022 
1023  if (r < 0)
1024  return 0;
1025 
1026  continue;
1027  } else if (mydata == NULL || mydata_len == 0) {
1028  /* Possibly a gap, but no new data. */
1029  return 0;
1030  }
1031  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1032  break;
1033  }
1034 
1035  //PrintRawDataFp(stdout, mydata, mydata_len);
1036 
1037  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1038  *stream, &(*stream)->sb, mydata_len, app_progress);
1039 
1040  /* get window of data that is acked */
1041  if (StreamTcpInlineMode() == FALSE) {
1042  if (p->flags & PKT_PSEUDO_STREAM_END) {
1043  // fall through, we use all available data
1044  } else {
1045  uint64_t last_ack_abs = app_progress; /* absolute right edge of ack'd data */
1046  if (STREAM_LASTACK_GT_BASESEQ(*stream)) {
1047  /* get window of data that is acked */
1048  uint32_t delta = (*stream)->last_ack - (*stream)->base_seq;
1049  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > (*stream)->window);
1050  /* get max absolute offset */
1051  last_ack_abs += delta;
1052  }
1053 
1054  /* see if the buffer contains unack'd data as well */
1055  if (app_progress + mydata_len > last_ack_abs) {
1056  uint32_t check = mydata_len;
1057  mydata_len = last_ack_abs - app_progress;
1058  BUG_ON(mydata_len > check);
1059  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1060  "data is considered", mydata_len);
1061  }
1062  }
1063  }
1064 
1065  /* update the app-layer */
1066  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1067  (uint8_t *)mydata, mydata_len,
1068  StreamGetAppLayerFlags(ssn, *stream, p, dir));
1069  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1070 
1071  SCReturnInt(0);
1072 }
1073 
1074 /**
1075  * \brief Update the stream reassembly upon receiving a packet.
1076  *
1077  * For IDS mode, the stream is in the opposite direction of the packet,
1078  * as the ACK-packet is ACK'ing the stream.
1079  *
1080  * One of the utilities call by this function AppLayerHandleTCPData(),
1081  * has a feature where it will call this very same function for the
1082  * stream opposing the stream it is called with. This shouldn't cause
1083  * any issues, since processing of each stream is independent of the
1084  * other stream.
1085  */
1087  TcpSession *ssn, TcpStream *stream,
1088  Packet *p, enum StreamUpdateDir dir)
1089 {
1090  SCEnter();
1091 
1092  /* this function can be directly called by app layer protocol
1093  * detection. */
1096  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1097  SCReturnInt(0);
1098  }
1099 
1100 #ifdef DEBUG
1101  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1102  GetSessionSize(ssn, p);
1103 #endif
1104  /* if no segments are in the list or all are already processed,
1105  * and state is beyond established, we send an empty msg */
1106  if (STREAM_HAS_SEEN_DATA(stream) && STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1107  {
1108  /* send an empty EOF msg if we have no segments but TCP state
1109  * is beyond ESTABLISHED */
1110  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1111  SCLogDebug("sending empty eof message");
1112  /* send EOF to app layer */
1113  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1114  NULL, 0,
1115  StreamGetAppLayerFlags(ssn, stream, p, dir));
1116  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1117 
1118  SCReturnInt(0);
1119  }
1120  }
1121 
1122  /* with all that out of the way, lets update the app-layer */
1123  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1124 }
1125 
1126 /** \internal
1127  * \brief get stream data from offset
1128  * \param offset stream offset */
1129 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1130  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1131 {
1132  const uint8_t *mydata;
1133  uint32_t mydata_len;
1134  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1135  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1136 
1137  uint64_t roffset = offset;
1138  if (offset)
1139  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1140  else {
1141  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1142  }
1143 
1144  *data = mydata;
1145  *data_len = mydata_len;
1146  *data_offset = roffset;
1147  } else {
1148  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1149  *iter == NULL ? "starting" : "continuing", offset);
1150  if (*iter == NULL) {
1151  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1152  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1153  SCLogDebug("*iter %p", *iter);
1154  }
1155  if (*iter == NULL) {
1156  SCLogDebug("no data");
1157  *data = NULL;
1158  *data_len = 0;
1159  *data_offset = 0;
1160  return 0;
1161  }
1162  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1163 
1164  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1165  SCLogDebug("mydata %p", mydata);
1166 
1167  if ((*iter)->offset < offset) {
1168  uint64_t delta = offset - (*iter)->offset;
1169  if (delta < mydata_len) {
1170  *data = mydata + delta;
1171  *data_len = mydata_len - delta;
1172  *data_offset = offset;
1173  } else {
1174  SCLogDebug("no data (yet)");
1175  *data = NULL;
1176  *data_len = 0;
1177  *data_offset = 0;
1178  }
1179 
1180  } else {
1181  *data = mydata;
1182  *data_len = mydata_len;
1183  *data_offset = (*iter)->offset;
1184  }
1185 
1186  *iter = SBB_RB_NEXT(*iter);
1187  SCLogDebug("*iter %p", *iter);
1188  }
1189  return 0;
1190 }
1191 
1192 /** \brief does the stream engine have data to inspect?
1193  *
1194  * Returns true if there is data to inspect. In IDS case this is
1195  * about ACK'd data in the packet's direction.
1196  *
1197  * In the IPS case this is about the packet itself.
1198  */
1200 {
1201  TcpStream *stream;
1202  if (PKT_IS_TOSERVER(p)) {
1203  stream = &ssn->client;
1204  } else {
1205  stream = &ssn->server;
1206  }
1207 
1208  if (RB_EMPTY(&stream->seg_tree)) {
1209  return false;
1210  }
1211 
1214  return false;
1215 
1216  if (StreamTcpInlineMode() == FALSE) {
1217  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1218  return false;
1219  }
1220  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1221  return true;
1222  }
1223  } else {
1224  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1225  return true;
1226  }
1227  }
1228  return false;
1229 }
1230 
1231 /** \brief update stream engine after detection
1232  *
1233  * Tasked with progressing the 'progress' for Raw reassembly.
1234  * 2 main scenario's:
1235  * 1. progress is != 0, so we use this
1236  * 2. progress is 0, meaning the detect engine didn't touch
1237  * raw at all. In this case we need to look into progressing
1238  * raw anyway.
1239  *
1240  * Additionally, this function is tasked with disabling raw
1241  * reassembly if the app-layer requested to disable it.
1242  */
1243 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1244 {
1245  TcpStream *stream;
1246  if (PKT_IS_TOSERVER(p)) {
1247  stream = &ssn->client;
1248  } else {
1249  stream = &ssn->server;
1250  }
1251 
1252  if (progress > STREAM_RAW_PROGRESS(stream)) {
1253  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1254  stream->raw_progress_rel += slide;
1256 
1257  /* if app is active and beyond raw, sync raw to app */
1258  } else if (progress == 0 &&
1259  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1261  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1262  {
1263  /* if trigger raw is set we sync the 2 trackers */
1265  {
1266  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1267  stream->raw_progress_rel += slide;
1269 
1270  /* otherwise mix in the tcp window */
1271  } else {
1272  uint64_t tcp_window = stream->window;
1273  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1274  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1275  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1276  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1277  stream->raw_progress_rel += slide;
1278  }
1279  }
1280  }
1281  /* app is dead */
1282  } else if (progress == 0) {
1283  uint64_t tcp_window = stream->window;
1284  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1285  if (tcp_window < stream_right_edge) {
1286  uint64_t new_raw = stream_right_edge - tcp_window;
1287  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1288  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1289  stream->raw_progress_rel += slide;
1290  }
1291  }
1293 
1294  } else {
1295  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1296  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1297  STREAM_RAW_PROGRESS(stream), stream->window);
1298  }
1299 
1300  /* if we were told to accept no more raw data, we can mark raw as
1301  * disabled now. */
1304  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1305  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1306  }
1307 
1308  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1309 }
1310 
1311 /** \internal
1312  * \brief get a buffer around the current packet and run the callback on it
1313  *
1314  * The inline/IPS scanning method takes the current payload and wraps it in
1315  * data from other segments.
1316  *
1317  * How much data is inspected is controlled by the available data, chunk_size
1318  * and the payload size of the packet.
1319  *
1320  * Large packets: if payload size is close to the chunk_size, where close is
1321  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1322  * used: payload_len + 33% of the chunk_size.
1323  * If the payload size if equal to or bigger than the chunk_size, we use
1324  * payload len + 33% of the chunk size.
1325  */
1326 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1327  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1328 {
1329  SCEnter();
1330  int r = 0;
1331 
1332  TcpStream *stream;
1333  if (PKT_IS_TOSERVER(p)) {
1334  stream = &ssn->client;
1335  } else {
1336  stream = &ssn->server;
1337  }
1338 
1339  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1341  {
1342  *progress_out = STREAM_RAW_PROGRESS(stream);
1343  return 0;
1344  }
1345 
1346  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1349  if (chunk_size <= p->payload_len) {
1350  chunk_size = p->payload_len + (chunk_size / 3);
1351  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1352  p->payload_len, chunk_size);
1353  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1354  chunk_size = p->payload_len + ((chunk_size / 3));
1355  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1356  p->payload_len, chunk_size);
1357  }
1358 
1359  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1360  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1361  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1362  packet_leftedge_abs, packet_rightedge_abs);
1363 
1364  const uint8_t *mydata = NULL;
1365  uint32_t mydata_len = 0;
1366  uint64_t mydata_offset = 0;
1367  /* simply return progress from the block we inspected. */
1368  bool return_progress = false;
1369 
1370  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1371  /* continues block */
1372  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1373  return_progress = true;
1374 
1375  } else {
1376  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1377  /* find our block */
1378  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1379  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1380  if (sbb) {
1381  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1382  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1383  mydata_offset = sbb->offset;
1384  }
1385  }
1386 
1387  /* this can only happen if the segment insert of our current 'p' failed */
1388  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1389  if ((mydata == NULL || mydata_len == 0) || /* no data */
1390  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1391  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1392  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1393  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1394  {
1395  /* no data, or data is incomplete or wrong: use packet data */
1396  mydata = p->payload;
1397  mydata_len = p->payload_len;
1398  mydata_offset = packet_leftedge_abs;
1399  //mydata_rightedge_abs = packet_rightedge_abs;
1400  } else {
1401  /* adjust buffer to match chunk_size */
1402  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1403  if (mydata_len > chunk_size) {
1404  uint32_t excess = mydata_len - chunk_size;
1405  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1406 
1407  if (mydata_rightedge_abs == packet_rightedge_abs) {
1408  mydata += excess;
1409  mydata_len -= excess;
1410  mydata_offset += excess;
1411  SCLogDebug("cutting front of the buffer with %u", excess);
1412  } else if (mydata_offset == packet_leftedge_abs) {
1413  mydata_len -= excess;
1414  SCLogDebug("cutting tail of the buffer with %u", excess);
1415  } else {
1416  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1417  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1418  SCLogDebug("before %u after %u", before, after);
1419 
1420  if (after >= (chunk_size - p->payload_len) / 2) {
1421  // more trailing data than we need
1422 
1423  if (before >= (chunk_size - p->payload_len) / 2) {
1424  // also more heading data, divide evenly
1425  before = after = (chunk_size - p->payload_len) / 2;
1426  } else {
1427  // heading data is less than requested, give the
1428  // rest to the trailing data
1429  after = (chunk_size - p->payload_len) - before;
1430  }
1431  } else {
1432  // less trailing data than requested
1433 
1434  if (before >= (chunk_size - p->payload_len) / 2) {
1435  before = (chunk_size - p->payload_len) - after;
1436  } else {
1437  // both smaller than their requested size
1438  }
1439  }
1440 
1441  /* adjust the buffer */
1442  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1443  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1444  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1445  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1446  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1447  mydata += skip;
1448  mydata_len -= (skip + cut);
1449  mydata_offset += skip;
1450  }
1451  }
1452  }
1453 
1454  /* run the callback */
1455  r = Callback(cb_data, mydata, mydata_len);
1456  BUG_ON(r < 0);
1457 
1458  if (return_progress) {
1459  *progress_out = (mydata_offset + mydata_len);
1460  } else {
1461  /* several blocks of data, so we need to be a bit more careful:
1462  * - if last_ack is beyond last progress, move progress forward to last_ack
1463  * - if our block matches or starts before last ack, return right edge of
1464  * our block.
1465  */
1466  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1467  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1468  uint32_t delta = stream->last_ack - stream->base_seq;
1469  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1470  /* get max absolute offset */
1471  last_ack_abs += delta;
1472  }
1473  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1474 
1475  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1476  if (mydata_offset > last_ack_abs) {
1477  /* gap between us and last ack, set progress to last ack */
1478  *progress_out = last_ack_abs;
1479  } else {
1480  *progress_out = (mydata_offset + mydata_len);
1481  }
1482  } else {
1483  *progress_out = STREAM_RAW_PROGRESS(stream);
1484  }
1485  }
1486  return r;
1487 }
1488 
1489 /** \brief access 'raw' reassembly data.
1490  *
1491  * Access data as tracked by 'raw' tracker. Data is made available to
1492  * callback that is passed to this function.
1493  *
1494  * In the case of IDS the callback may be run multiple times if data
1495  * contains gaps. It will then be run for each block of data that is
1496  * continuous.
1497  *
1498  * The callback should give on of 2 return values:
1499  * - 0 ok
1500  * - 1 done
1501  * The value 1 will break the loop if there is a block list that is
1502  * inspected.
1503  *
1504  * This function will return the 'progress' value that has been
1505  * consumed until now.
1506  *
1507  * \param ssn tcp session
1508  * \param stream tcp stream
1509  * \param Callback the function pointer to the callback function
1510  * \param cb_data callback data
1511  * \param[in] progress_in progress to work from
1512  * \param[out] progress_out absolute progress value of the data this
1513  * call handled.
1514  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1515  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1516  *
1517  * `respect_inspect_depth` is used to avoid useless inspection of too
1518  * much data.
1519  */
1520 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1521  StreamReassembleRawFunc Callback, void *cb_data,
1522  const uint64_t progress_in,
1523  uint64_t *progress_out, bool eof,
1524  bool respect_inspect_depth)
1525 {
1526  SCEnter();
1527  int r = 0;
1528 
1529  StreamingBufferBlock *iter = NULL;
1530  uint64_t progress = progress_in;
1531  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1532 
1533  /* if the app layer triggered a flush, and we're supposed to
1534  * use a minimal inspect depth, we actually take the app progress
1535  * as that is the right edge of the data. Then we take the window
1536  * of 'min_inspect_depth' before that. */
1537 
1538  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1539  respect_inspect_depth ? "true" : "false",
1540  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1541  stream->min_inspect_depth);
1542 
1543  if (respect_inspect_depth &&
1545  && stream->min_inspect_depth)
1546  {
1547  progress = STREAM_APP_PROGRESS(stream);
1548  if (stream->min_inspect_depth >= progress) {
1549  progress = 0;
1550  } else {
1551  progress -= stream->min_inspect_depth;
1552  }
1553 
1554  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1555 
1556  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1557  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1558  }
1559 
1560  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1561 
1562  /* get window of data that is acked */
1563  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1564  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1565  uint32_t delta = stream->last_ack - stream->base_seq;
1566  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1567  /* get max absolute offset */
1568  last_ack_abs += delta;
1569  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1570  }
1571 
1572  /* loop through available buffers. On no packet loss we'll have a single
1573  * iteration. On missing data we'll walk the blocks */
1574  while (1) {
1575  const uint8_t *mydata;
1576  uint32_t mydata_len;
1577  uint64_t mydata_offset = 0;
1578 
1579  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1580  if (mydata_len == 0) {
1581  SCLogDebug("no data");
1582  break;
1583  }
1584  //PrintRawDataFp(stdout, mydata, mydata_len);
1585 
1586  SCLogDebug("raw progress %"PRIu64, progress);
1587  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1588  stream, &stream->sb, mydata_len, progress);
1589 
1590  if (eof) {
1591  // inspect all remaining data, ack'd or not
1592  } else {
1593  if (last_ack_abs < progress) {
1594  SCLogDebug("nothing to do");
1595  goto end;
1596  }
1597 
1598  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1599  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1600 
1601  /* see if the buffer contains unack'd data as well */
1602  if (progress + mydata_len > last_ack_abs) {
1603  uint32_t check = mydata_len;
1604  mydata_len = last_ack_abs - progress;
1605  BUG_ON(check < mydata_len);
1606  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1607  "data is considered", mydata_len);
1608  }
1609 
1610  }
1611  if (mydata_len == 0)
1612  break;
1613 
1614  SCLogDebug("data %p len %u", mydata, mydata_len);
1615 
1616  /* we have data. */
1617  r = Callback(cb_data, mydata, mydata_len);
1618  BUG_ON(r < 0);
1619 
1620  if (mydata_offset == progress) {
1621  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1622  progress, mydata_len, progress_in + mydata_len);
1623 
1624  progress += mydata_len;
1625  SCLogDebug("raw progress now %"PRIu64, progress);
1626 
1627  /* data is beyond the progress we'd like, and before last ack. Gap. */
1628  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1629  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1630  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1631  progress = mydata_offset;
1632  SCLogDebug("raw progress now %"PRIu64, progress);
1633 
1634  } else {
1635  SCLogDebug("not increasing progress, data gap => mydata_offset "
1636  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1637  }
1638 
1639  if (iter == NULL || r == 1)
1640  break;
1641  }
1642 end:
1643  *progress_out = progress;
1644  return r;
1645 }
1646 
1648  StreamReassembleRawFunc Callback, void *cb_data,
1649  uint64_t *progress_out, bool respect_inspect_depth)
1650 {
1651  /* handle inline separately as the logic is very different */
1652  if (StreamTcpInlineMode() == TRUE) {
1653  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1654  }
1655 
1656  TcpStream *stream;
1657  if (PKT_IS_TOSERVER(p)) {
1658  stream = &ssn->client;
1659  } else {
1660  stream = &ssn->server;
1661  }
1662 
1664  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1665  {
1666  *progress_out = STREAM_RAW_PROGRESS(stream);
1667  return 0;
1668  }
1669 
1670  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1671  STREAM_RAW_PROGRESS(stream), progress_out,
1672  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1673 }
1674 
1676  StreamReassembleRawFunc Callback, void *cb_data,
1677  uint64_t progress_in,
1678  uint64_t *progress_out, bool eof)
1679 {
1680  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1681  return 0;
1682 
1683  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1684  progress_in, progress_out, eof, false);
1685 }
1686 
1687 /** \internal
1688  * \brief update app layer based on received ACK
1689  *
1690  * \retval r 0 on success, -1 on error
1691  */
1692 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1693  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1694 {
1695  SCEnter();
1696 
1697  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1698  SCReturnInt(-1);
1699 
1700  SCReturnInt(0);
1701 }
1702 
1704  TcpSession *ssn, TcpStream *stream,
1705  Packet *p, PacketQueue *pq)
1706 {
1707  SCEnter();
1708 
1709  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1710 
1711  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1712  ssn, stream, p, p->payload_len);
1713 
1714  /* we need to update the opposing stream in
1715  * StreamTcpReassembleHandleSegmentUpdateACK */
1716  TcpStream *opposing_stream = NULL;
1717  if (stream == &ssn->client) {
1718  opposing_stream = &ssn->server;
1719  } else {
1720  opposing_stream = &ssn->client;
1721  }
1722 
1723  /* default IDS: update opposing side (triggered by ACK) */
1725  /* inline and stream end and flow timeout packets trigger same dir handling */
1726  if (StreamTcpInlineMode()) {
1727  dir = UPDATE_DIR_PACKET;
1728  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1729  dir = UPDATE_DIR_PACKET;
1730  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1731  dir = UPDATE_DIR_PACKET;
1732  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1733  dir = UPDATE_DIR_PACKET;
1734  } else if (ssn->state == TCP_CLOSED) {
1735  dir = UPDATE_DIR_BOTH;
1736  }
1737 
1738  /* handle ack received */
1739  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1740  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1741  {
1742  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1743  SCReturnInt(-1);
1744  }
1745 
1746  /* if this segment contains data, insert it */
1747  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1748  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1749 
1750  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1751  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1752  SCReturnInt(-1);
1753  }
1754 
1755  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1756  p->flags |= PKT_STREAM_ADD;
1757  } else {
1758  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1759  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1760  ssn, stream, p->payload_len,
1761  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1762 
1763  }
1764 
1765  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1766  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1767  * was *just* set. In this case we trigger the AppLayer Truncate
1768  * logic, to inform the applayer no more data in this direction is
1769  * to be expected. */
1770  if ((stream->flags &
1773  {
1774  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1775  if (dir != UPDATE_DIR_PACKET) {
1776  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1777  "can trigger Truncate");
1778  dir = UPDATE_DIR_PACKET;
1779  }
1780  }
1781 
1782  /* in stream inline mode even if we have no data we call the reassembly
1783  * functions to handle EOF */
1784  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1785  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1786  StreamTcpInlineMode()?"true":"false",
1787  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1788  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1789  SCReturnInt(-1);
1790  }
1791  }
1792 
1793  SCReturnInt(0);
1794 }
1795 
1796 /**
1797  * \brief get a segment from the pool
1798  *
1799  * \retval seg Segment from the pool or NULL
1800  */
1802 {
1803  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1804  SCLogDebug("seg we return is %p", seg);
1805  if (seg == NULL) {
1806  /* Increment the counter to show that we are not able to serve the
1807  segment request due to memcap limit */
1809  } else {
1810  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1811  }
1812 
1813  return seg;
1814 }
1815 
1816 /**
1817  * \brief Trigger RAW stream reassembly
1818  *
1819  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1820  * reassembly from the applayer, for example upon completion of a
1821  * HTTP request.
1822  *
1823  * It sets a flag in the stream so that the next Raw call will return
1824  * the data.
1825  *
1826  * \param ssn TcpSession
1827  */
1829 {
1830 #ifdef DEBUG
1831  BUG_ON(ssn == NULL);
1832 #endif
1833 
1834  if (ssn != NULL) {
1835  if (direction == STREAM_TOSERVER) {
1837  } else {
1839  }
1840 
1841  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1842  }
1843 }
1844 
1845 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1846 {
1847 #ifdef DEBUG
1848  BUG_ON(ssn == NULL);
1849 #endif
1850 
1851  if (ssn != NULL) {
1852  if (direction == STREAM_TOSERVER) {
1853  ssn->client.min_inspect_depth = depth;
1854  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1855  } else {
1856  ssn->server.min_inspect_depth = depth;
1857  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1858  }
1859  }
1860 }
1861 
1862 #ifdef UNITTESTS
1863 /** unit tests and it's support functions below */
1864 
1865 #define SET_ISN(stream, setseq) \
1866  (stream)->isn = (setseq); \
1867  (stream)->base_seq = (setseq) + 1
1868 
1869 /** \brief The Function to create the packet with given payload, which is used
1870  * to test the reassembly of the engine.
1871  *
1872  * \param payload The variable used to store the payload contents of the
1873  * current packet.
1874  * \param value The value which current payload will have for this packet
1875  * \param payload_len The length of the filed payload for current packet.
1876  * \param len Length of the payload array
1877  */
1878 
1879 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1880  uint8_t payload_len, uint8_t len)
1881 {
1882  uint8_t i;
1883  for (i = 0; i < payload_len; i++)
1884  payload[i] = value;
1885  for (; i < len; i++)
1886  payload = NULL;
1887 }
1888 
1889 /** \brief The Function Checks the reassembled stream contents against predefined
1890  * stream contents according to OS policy used.
1891  *
1892  * \param stream_policy Predefined value of stream for different OS policies
1893  * \param stream Reassembled stream returned from the reassembly functions
1894  */
1895 
1896 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
1897 {
1898  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
1899  {
1900  //PrintRawDataFp(stdout, stream_policy, sp_size);
1901  return 0;
1902  }
1903  return 1;
1904 }
1905 
1906 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
1907 {
1908  if (StreamingBufferCompareRawData(&stream->sb,
1909  data, data_len) == 0)
1910  {
1911  SCReturnInt(0);
1912  }
1913  SCLogInfo("OK");
1914  PrintRawDataFp(stdout, data, data_len);
1915  return 1;
1916 }
1917 
1918 #define MISSED_START(isn) \
1919  TcpReassemblyThreadCtx *ra_ctx = NULL; \
1920  TcpSession ssn; \
1921  ThreadVars tv; \
1922  memset(&tv, 0, sizeof(tv)); \
1923  \
1924  StreamTcpUTInit(&ra_ctx); \
1925  \
1926  StreamTcpUTSetupSession(&ssn); \
1927  StreamTcpUTSetupStream(&ssn.server, (isn)); \
1928  StreamTcpUTSetupStream(&ssn.client, (isn)); \
1929  \
1930  TcpStream *stream = &ssn.client;
1931 
1932 #define MISSED_END \
1933  StreamTcpUTClearSession(&ssn); \
1934  StreamTcpUTDeinit(ra_ctx); \
1935  PASS
1936 
1937 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
1938  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
1939  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
1940 
1941 /**
1942  * \test Test the handling of packets missed by both IDS and the end host.
1943  * The packet is missed in the starting of the stream.
1944  *
1945  * \retval On success it returns 1 and on failure 0.
1946  */
1947 
1948 static int StreamTcpReassembleTest25 (void)
1949 {
1950  MISSED_START(6);
1951  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
1952  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
1953  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
1954  MISSED_END;
1955 }
1956 
1957 /**
1958  * \test Test the handling of packets missed by both IDS and the end host.
1959  * The packet is missed in the middle of the stream.
1960  *
1961  * \retval On success it returns 1 and on failure 0.
1962  */
1963 
1964 static int StreamTcpReassembleTest26 (void)
1965 {
1966  MISSED_START(9);
1967  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1968  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
1969  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
1970  MISSED_END;
1971 }
1972 
1973 /**
1974  * \test Test the handling of packets missed by both IDS and the end host.
1975  * The packet is missed in the end of the stream.
1976  *
1977  * \retval On success it returns 1 and on failure 0.
1978  */
1979 
1980 static int StreamTcpReassembleTest27 (void)
1981 {
1982  MISSED_START(9);
1983  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1984  MISSED_STEP(13, "BB", 2, "AAABB", 5);
1985  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
1986  MISSED_END;
1987 }
1988 
1989 /**
1990  * \test Test the handling of packets missed by IDS, but the end host has
1991  * received it and send the acknowledgment of it. The packet is missed
1992  * in the starting of the stream.
1993  *
1994  * \retval On success it returns 1 and on failure 0.
1995  */
1996 
1997 static int StreamTcpReassembleTest28 (void)
1998 {
1999  MISSED_START(6);
2000  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2001  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2002  ssn.state = TCP_TIME_WAIT;
2003  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2004  MISSED_END;
2005 }
2006 
2007 /**
2008  * \test Test the handling of packets missed by IDS, but the end host has
2009  * received it and send the acknowledgment of it. The packet is missed
2010  * in the middle of the stream.
2011  *
2012  * \retval On success it returns 1 and on failure 0.
2013  */
2014 
2015 static int StreamTcpReassembleTest29 (void)
2016 {
2017  MISSED_START(9);
2018  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2019  ssn.state = TCP_TIME_WAIT;
2020  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2021  MISSED_END;
2022 }
2023 
2024 static int StreamTcpReassembleTest33(void)
2025 {
2026  TcpSession ssn;
2027  Packet *p = PacketGetFromAlloc();
2028  FAIL_IF(unlikely(p == NULL));
2029  Flow f;
2030  TCPHdr tcph;
2031  TcpReassemblyThreadCtx *ra_ctx = NULL;
2033  uint8_t packet[1460] = "";
2034 
2035  StreamTcpUTInit(&ra_ctx);
2037 
2038  PacketQueue pq;
2039  memset(&pq,0,sizeof(PacketQueue));
2040  memset(&f, 0, sizeof (Flow));
2041  memset(&tcph, 0, sizeof (TCPHdr));
2042  ThreadVars tv;
2043  memset(&tv, 0, sizeof (ThreadVars));
2044  FLOW_INITIALIZE(&f);
2045  f.protoctx = &ssn;
2046  f.proto = IPPROTO_TCP;
2047  p->src.family = AF_INET;
2048  p->dst.family = AF_INET;
2049  p->proto = IPPROTO_TCP;
2050  p->flow = &f;
2051  tcph.th_win = 5480;
2052  tcph.th_flags = TH_PUSH | TH_ACK;
2053  p->tcph = &tcph;
2055  p->payload = packet;
2056 
2057  p->tcph->th_seq = htonl(10);
2058  p->tcph->th_ack = htonl(31);
2059  p->payload_len = 10;
2060 
2061  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2062 
2063  p->tcph->th_seq = htonl(20);
2064  p->tcph->th_ack = htonl(31);
2065  p->payload_len = 10;
2066 
2067  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2068 
2069  p->tcph->th_seq = htonl(40);
2070  p->tcph->th_ack = htonl(31);
2071  p->payload_len = 10;
2072 
2073  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2074 
2075  p->tcph->th_seq = htonl(5);
2076  p->tcph->th_ack = htonl(31);
2077  p->payload_len = 30;
2078 
2079  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2080 
2082  StreamTcpUTDeinit(ra_ctx);
2083  SCFree(p);
2084  PASS;
2085 }
2086 
2087 static int StreamTcpReassembleTest34(void)
2088 {
2089  TcpSession ssn;
2090  Packet *p = PacketGetFromAlloc();
2091  FAIL_IF(unlikely(p == NULL));
2092  Flow f;
2093  TCPHdr tcph;
2094  TcpReassemblyThreadCtx *ra_ctx = NULL;
2096  uint8_t packet[1460] = "";
2097 
2098  StreamTcpUTInit(&ra_ctx);
2100  PacketQueue pq;
2101  memset(&pq,0,sizeof(PacketQueue));
2102  memset(&f, 0, sizeof (Flow));
2103  memset(&tcph, 0, sizeof (TCPHdr));
2104  ThreadVars tv;
2105  memset(&tv, 0, sizeof (ThreadVars));
2106  FLOW_INITIALIZE(&f);
2107  f.protoctx = &ssn;
2108  f.proto = IPPROTO_TCP;
2109  p->src.family = AF_INET;
2110  p->dst.family = AF_INET;
2111  p->proto = IPPROTO_TCP;
2112  p->flow = &f;
2113  tcph.th_win = 5480;
2114  tcph.th_flags = TH_PUSH | TH_ACK;
2115  p->tcph = &tcph;
2117  p->payload = packet;
2118  SET_ISN(&ssn.client, 857961230);
2119 
2120  p->tcph->th_seq = htonl(857961230);
2121  p->tcph->th_ack = htonl(31);
2122  p->payload_len = 304;
2123 
2124  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2125 
2126  p->tcph->th_seq = htonl(857961534);
2127  p->tcph->th_ack = htonl(31);
2128  p->payload_len = 1460;
2129 
2130  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2131 
2132  p->tcph->th_seq = htonl(857963582);
2133  p->tcph->th_ack = htonl(31);
2134  p->payload_len = 1460;
2135 
2136  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2137 
2138  p->tcph->th_seq = htonl(857960946);
2139  p->tcph->th_ack = htonl(31);
2140  p->payload_len = 1460;
2141 
2142  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2143 
2145  StreamTcpUTDeinit(ra_ctx);
2146  SCFree(p);
2147  PASS;
2148 }
2149 
2150 /** \test Test the bug 76 condition */
2151 static int StreamTcpReassembleTest37(void)
2152 {
2153  TcpSession ssn;
2154  Flow f;
2155  TCPHdr tcph;
2156  TcpReassemblyThreadCtx *ra_ctx = NULL;
2157  uint8_t packet[1460] = "";
2158  PacketQueue pq;
2159  ThreadVars tv;
2160  memset(&tv, 0, sizeof (ThreadVars));
2161 
2162  Packet *p = PacketGetFromAlloc();
2163  FAIL_IF(unlikely(p == NULL));
2164 
2165  StreamTcpUTInit(&ra_ctx);
2167  memset(&pq,0,sizeof(PacketQueue));
2168  memset(&f, 0, sizeof (Flow));
2169  memset(&tcph, 0, sizeof (TCPHdr));
2170  memset(&tv, 0, sizeof (ThreadVars));
2171 
2172  FLOW_INITIALIZE(&f);
2173  f.protoctx = &ssn;
2174  f.proto = IPPROTO_TCP;
2175  p->src.family = AF_INET;
2176  p->dst.family = AF_INET;
2177  p->proto = IPPROTO_TCP;
2178  p->flow = &f;
2179  tcph.th_win = 5480;
2180  tcph.th_flags = TH_PUSH | TH_ACK;
2181  p->tcph = &tcph;
2183  p->payload = packet;
2185 
2186  p->tcph->th_seq = htonl(3061088537UL);
2187  p->tcph->th_ack = htonl(1729548549UL);
2188  p->payload_len = 1391;
2189  ssn.client.last_ack = 3061091137UL;
2190  SET_ISN(&ssn.client, 3061091309UL);
2191 
2192  /* pre base_seq, so should be rejected */
2193  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2194 
2195  p->tcph->th_seq = htonl(3061089928UL);
2196  p->tcph->th_ack = htonl(1729548549UL);
2197  p->payload_len = 1391;
2198  ssn.client.last_ack = 3061091137UL;
2199 
2200  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2201 
2202  p->tcph->th_seq = htonl(3061091319UL);
2203  p->tcph->th_ack = htonl(1729548549UL);
2204  p->payload_len = 1391;
2205  ssn.client.last_ack = 3061091137UL;
2206 
2207  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2208 
2210  StreamTcpUTDeinit(ra_ctx);
2211  SCFree(p);
2212  PASS;
2213 }
2214 
2215 /**
2216  * \test Test to make sure that we don't return the segments until the app
2217  * layer proto has been detected and after that remove the processed
2218  * segments.
2219  *
2220  * \retval On success it returns 1 and on failure 0.
2221  */
2222 
2223 static int StreamTcpReassembleTest39 (void)
2224 {
2225  Packet *p = PacketGetFromAlloc();
2226  FAIL_IF(unlikely(p == NULL));
2227  Flow f;
2228  ThreadVars tv;
2229  StreamTcpThread stt;
2230  TCPHdr tcph;
2231  PacketQueue pq;
2232  memset(&pq,0,sizeof(PacketQueue));
2233  memset (&f, 0, sizeof(Flow));
2234  memset(&tv, 0, sizeof (ThreadVars));
2235  memset(&stt, 0, sizeof (stt));
2236  memset(&tcph, 0, sizeof (TCPHdr));
2237 
2238  FLOW_INITIALIZE(&f);
2239  f.flags = FLOW_IPV4;
2240  f.proto = IPPROTO_TCP;
2241  p->flow = &f;
2242  p->tcph = &tcph;
2243 
2244  StreamTcpUTInit(&stt.ra_ctx);
2245 
2246  /* handshake */
2247  tcph.th_win = htons(5480);
2248  tcph.th_flags = TH_SYN;
2250  p->payload_len = 0;
2251  p->payload = NULL;
2252  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2253 
2254  TcpSession *ssn = (TcpSession *)f.protoctx;
2255  FAIL_IF_NULL(ssn);
2256 
2267  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2268  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2269  FAIL_IF(ssn->data_first_seen_dir != 0);
2270 
2271  /* handshake */
2272  p->tcph->th_ack = htonl(1);
2273  p->tcph->th_flags = TH_SYN | TH_ACK;
2275  p->payload_len = 0;
2276  p->payload = NULL;
2277  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2288  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2289  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2290  FAIL_IF(ssn->data_first_seen_dir != 0);
2291 
2292  /* handshake */
2293  p->tcph->th_ack = htonl(1);
2294  p->tcph->th_seq = htonl(1);
2295  p->tcph->th_flags = TH_ACK;
2297  p->payload_len = 0;
2298  p->payload = NULL;
2299  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2310  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2311  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2312  FAIL_IF(ssn->data_first_seen_dir != 0);
2313 
2314  /* partial request */
2315  uint8_t request1[] = { 0x47, 0x45, };
2316  p->tcph->th_ack = htonl(1);
2317  p->tcph->th_seq = htonl(1);
2318  p->tcph->th_flags = TH_PUSH | TH_ACK;
2320  p->payload_len = sizeof(request1);
2321  p->payload = request1;
2322  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2333  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2334  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2335  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2337 
2338  /* response ack against partial request */
2339  p->tcph->th_ack = htonl(3);
2340  p->tcph->th_seq = htonl(1);
2341  p->tcph->th_flags = TH_ACK;
2343  p->payload_len = 0;
2344  p->payload = NULL;
2345  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2356  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2357  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2358  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2360 
2361  /* complete partial request */
2362  uint8_t request2[] = {
2363  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2364  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2365  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2366  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2367  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2368  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2369  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2370  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2371  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2372  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2373  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2374  p->tcph->th_ack = htonl(1);
2375  p->tcph->th_seq = htonl(3);
2376  p->tcph->th_flags = TH_PUSH | TH_ACK;
2378  p->payload_len = sizeof(request2);
2379  p->payload = request2;
2380  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2391  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2392  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2393  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2394  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2396 
2397  /* response - request ack */
2398  uint8_t response[] = {
2399  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2400  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2401  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2402  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2403  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2404  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2405  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2406  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2407  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2408  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2409  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2410  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2411  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2412  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2413  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2414  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2415  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2416  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2417  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2418  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2419  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2420  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2421  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2422  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2423  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2424  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2425  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2426  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2427  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2428  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2429  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2430  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2431  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2432  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2433  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2434  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2435  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2436  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2437  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2438  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2439  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2440  p->tcph->th_ack = htonl(88);
2441  p->tcph->th_seq = htonl(1);
2442  p->tcph->th_flags = TH_PUSH | TH_ACK;
2444  p->payload_len = sizeof(response);
2445  p->payload = response;
2446 
2447  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2459  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2460  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2461  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2462  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2463 
2464  /* response ack from request */
2465  p->tcph->th_ack = htonl(328);
2466  p->tcph->th_seq = htonl(88);
2467  p->tcph->th_flags = TH_ACK;
2469  p->payload_len = 0;
2470  p->payload = NULL;
2471  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2483  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2484  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2485  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2486  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2487  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2488 
2489  /* response - acking */
2490  p->tcph->th_ack = htonl(88);
2491  p->tcph->th_seq = htonl(328);
2492  p->tcph->th_flags = TH_PUSH | TH_ACK;
2494  p->payload_len = 0;
2495  p->payload = NULL;
2496  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2508  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2509  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2510  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2511  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2512  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2513 
2514  /* response ack from request */
2515  p->tcph->th_ack = htonl(328);
2516  p->tcph->th_seq = htonl(88);
2517  p->tcph->th_flags = TH_ACK;
2519  p->payload_len = 0;
2520  p->payload = NULL;
2521  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2533  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2534  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2535  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2536  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2537 
2538  /* response - acking the request again*/
2539  p->tcph->th_ack = htonl(88);
2540  p->tcph->th_seq = htonl(328);
2541  p->tcph->th_flags = TH_PUSH | TH_ACK;
2543  p->payload_len = 0;
2544  p->payload = NULL;
2545  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2557  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2558  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2559  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2560  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2561 
2562  /*** New Request ***/
2563 
2564  /* partial request */
2565  p->tcph->th_ack = htonl(328);
2566  p->tcph->th_seq = htonl(88);
2567  p->tcph->th_flags = TH_PUSH | TH_ACK;
2569  p->payload_len = sizeof(request1);
2570  p->payload = request1;
2571  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2583  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2584  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2585  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2586  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2587  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2588 
2589  /* response ack against partial request */
2590  p->tcph->th_ack = htonl(90);
2591  p->tcph->th_seq = htonl(328);
2592  p->tcph->th_flags = TH_ACK;
2594  p->payload_len = 0;
2595  p->payload = NULL;
2596  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2597  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2609  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2610  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2611  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2612  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2613  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2614 
2615  /* complete request */
2616  p->tcph->th_ack = htonl(328);
2617  p->tcph->th_seq = htonl(90);
2618  p->tcph->th_flags = TH_PUSH | TH_ACK;
2620  p->payload_len = sizeof(request2);
2621  p->payload = request2;
2622  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2634  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2635  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2636  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2637  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2638  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2639  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2640 
2641  /* response ack against second partial request */
2642  p->tcph->th_ack = htonl(175);
2643  p->tcph->th_seq = htonl(328);
2644  p->tcph->th_flags = TH_ACK;
2646  p->payload_len = 0;
2647  p->payload = NULL;
2648 
2649  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2661  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2662  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2663  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2664  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2665  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2666  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2667 
2668  /* response acking a request */
2669  p->tcph->th_ack = htonl(175);
2670  p->tcph->th_seq = htonl(328);
2671  p->tcph->th_flags = TH_ACK;
2673  p->payload_len = 0;
2674  p->payload = NULL;
2675  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2681 
2684 
2685  /* request acking a response */
2686  p->tcph->th_ack = htonl(328);
2687  p->tcph->th_seq = htonl(175);
2688  p->tcph->th_flags = TH_ACK;
2690  p->payload_len = 0;
2691  p->payload = NULL;
2692  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2693 
2694  StreamTcpSessionClear(ssn);
2696  SCFree(p);
2697  PASS;
2698 }
2699 
2700 /**
2701  * \test Test to make sure that we sent all the segments from the initial
2702  * segments to app layer until we have detected the app layer proto.
2703  *
2704  * \retval On success it returns 1 and on failure 0.
2705  */
2706 
2707 static int StreamTcpReassembleTest40 (void)
2708 {
2709  Packet *p = PacketGetFromAlloc();
2710  FAIL_IF_NULL(p);
2711  Flow *f = NULL;
2712  TCPHdr tcph;
2713  TcpSession ssn;
2714  PacketQueue pq;
2715  memset(&pq,0,sizeof(PacketQueue));
2716  memset(&tcph, 0, sizeof (TCPHdr));
2717  ThreadVars tv;
2718  memset(&tv, 0, sizeof (ThreadVars));
2719 
2722 
2724  FAIL_IF_NULL(ra_ctx);
2725 
2726  uint8_t httpbuf1[] = "P";
2727  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2728  uint8_t httpbuf3[] = "O";
2729  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2730  uint8_t httpbuf4[] = "S";
2731  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2732  uint8_t httpbuf5[] = "T \r\n";
2733  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2734 
2735  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2736  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2737 
2738  SET_ISN(&ssn.server, 9);
2739  ssn.server.last_ack = 10;
2740  SET_ISN(&ssn.client, 9);
2741  ssn.client.isn = 9;
2742 
2743  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2744  FAIL_IF_NULL(f);
2745  f->protoctx = &ssn;
2746  f->proto = IPPROTO_TCP;
2747  p->flow = f;
2748 
2749  tcph.th_win = htons(5480);
2750  tcph.th_seq = htonl(10);
2751  tcph.th_ack = htonl(10);
2752  tcph.th_flags = TH_ACK|TH_PUSH;
2753  p->tcph = &tcph;
2755  p->payload = httpbuf1;
2756  p->payload_len = httplen1;
2757  ssn.state = TCP_ESTABLISHED;
2758  TcpStream *s = &ssn.client;
2759  SCLogDebug("1 -- start");
2760  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2761 
2763  p->payload = httpbuf2;
2764  p->payload_len = httplen2;
2765  tcph.th_seq = htonl(10);
2766  tcph.th_ack = htonl(11);
2767  s = &ssn.server;
2768  ssn.server.last_ack = 11;
2769  SCLogDebug("2 -- start");
2770  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2771 
2773  p->payload = httpbuf3;
2774  p->payload_len = httplen3;
2775  tcph.th_seq = htonl(11);
2776  tcph.th_ack = htonl(55);
2777  s = &ssn.client;
2778  ssn.client.last_ack = 55;
2779  SCLogDebug("3 -- start");
2780  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2781 
2783  p->payload = httpbuf2;
2784  p->payload_len = httplen2;
2785  tcph.th_seq = htonl(55);
2786  tcph.th_ack = htonl(12);
2787  s = &ssn.server;
2788  ssn.server.last_ack = 12;
2789  SCLogDebug("4 -- start");
2790  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2791 
2792  /* check is have the segment in the list and flagged or not */
2793  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2794  FAIL_IF_NULL(seg);
2795  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2796 
2798  p->payload = httpbuf4;
2799  p->payload_len = httplen4;
2800  tcph.th_seq = htonl(12);
2801  tcph.th_ack = htonl(100);
2802  s = &ssn.client;
2803  ssn.client.last_ack = 100;
2804  SCLogDebug("5 -- start");
2805  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2806 
2808  p->payload = httpbuf2;
2809  p->payload_len = httplen2;
2810  tcph.th_seq = htonl(100);
2811  tcph.th_ack = htonl(13);
2812  s = &ssn.server;
2813  ssn.server.last_ack = 13;
2814  SCLogDebug("6 -- start");
2815  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2816 
2818  p->payload = httpbuf5;
2819  p->payload_len = httplen5;
2820  tcph.th_seq = htonl(13);
2821  tcph.th_ack = htonl(145);
2822  s = &ssn.client;
2823  ssn.client.last_ack = 145;
2824  SCLogDebug("7 -- start");
2825  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2826 
2828  p->payload = httpbuf2;
2829  p->payload_len = httplen2;
2830  tcph.th_seq = htonl(145);
2831  tcph.th_ack = htonl(16);
2832  s = &ssn.server;
2833  ssn.server.last_ack = 16;
2834  SCLogDebug("8 -- start");
2835  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2836  FAIL_IF(f->alproto != ALPROTO_HTTP);
2837 
2841  SCFree(p);
2842  UTHFreeFlow(f);
2843  PASS;
2844 }
2845 
2846 /** \test Test the memcap incrementing/decrementing and memcap check */
2847 static int StreamTcpReassembleTest44(void)
2848 {
2850  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2852  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2854  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2856  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2858  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2859  PASS;
2860 }
2861 
2862 /**
2863  * \test Test to make sure that reassembly_depth is enforced.
2864  *
2865  * \retval On success it returns 1 and on failure 0.
2866  */
2867 
2868 static int StreamTcpReassembleTest45 (void)
2869 {
2870  TcpReassemblyThreadCtx *ra_ctx = NULL;
2871  TcpSession ssn;
2872  ThreadVars tv;
2873  memset(&tv, 0, sizeof(tv));
2874  uint8_t payload[100] = {0};
2875  uint16_t payload_size = 100;
2876 
2877  StreamTcpUTInit(&ra_ctx);
2879 
2881  ssn.reassembly_depth = 100;
2882  StreamTcpUTSetupStream(&ssn.server, 100);
2883  StreamTcpUTSetupStream(&ssn.client, 100);
2884 
2885  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2886  FAIL_IF(r != 0);
2888 
2889  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2890  FAIL_IF(r != 0);
2892 
2896  StreamTcpUTDeinit(ra_ctx);
2897  PASS;
2898 }
2899 
2900 /**
2901  * \test Test the unlimited config value of reassembly depth.
2902  *
2903  * \retval On success it returns 1 and on failure 0.
2904  */
2905 
2906 static int StreamTcpReassembleTest46 (void)
2907 {
2908  int result = 0;
2909  TcpReassemblyThreadCtx *ra_ctx = NULL;
2910  TcpSession ssn;
2911  ThreadVars tv;
2912  memset(&tv, 0, sizeof(tv));
2913  uint8_t payload[100] = {0};
2914  uint16_t payload_size = 100;
2915 
2916  StreamTcpUTInit(&ra_ctx);
2918 
2920  StreamTcpUTSetupStream(&ssn.server, 100);
2921  StreamTcpUTSetupStream(&ssn.client, 100);
2922 
2923  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2924  if (r != 0)
2925  goto end;
2927  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2928  goto end;
2929  }
2930 
2931  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2932  if (r != 0)
2933  goto end;
2935  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2936  goto end;
2937  }
2938 
2939  result = 1;
2940 end:
2944  StreamTcpUTDeinit(ra_ctx);
2945  return result;
2946 }
2947 
2948 /**
2949  * \test Test to make sure we detect the sequence wrap around and continue
2950  * stream reassembly properly.
2951  *
2952  * \retval On success it returns 1 and on failure 0.
2953  */
2954 
2955 static int StreamTcpReassembleTest47 (void)
2956 {
2957  Packet *p = PacketGetFromAlloc();
2958  FAIL_IF(unlikely(p == NULL));
2959  Flow *f = NULL;
2960  TCPHdr tcph;
2961  TcpSession ssn;
2962  ThreadVars tv;
2963  PacketQueue pq;
2964  memset(&pq,0,sizeof(PacketQueue));
2965  memset(&tcph, 0, sizeof (TCPHdr));
2966  memset(&tv, 0, sizeof (ThreadVars));
2970 
2971  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
2972  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2973 
2974  SET_ISN(&ssn.server, 572799781UL);
2975  ssn.server.last_ack = 572799782UL;
2976 
2977  SET_ISN(&ssn.client, 4294967289UL);
2978  ssn.client.last_ack = 21;
2979 
2980  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2981  FAIL_IF(f == NULL);
2982  f->protoctx = &ssn;
2983  f->proto = IPPROTO_TCP;
2984  p->flow = f;
2985 
2986  tcph.th_win = htons(5480);
2987  ssn.state = TCP_ESTABLISHED;
2988  TcpStream *s = NULL;
2989  uint8_t cnt = 0;
2990 
2991  for (cnt=0; cnt < httplen1; cnt++) {
2992  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
2993  tcph.th_ack = htonl(572799782UL);
2994  tcph.th_flags = TH_ACK|TH_PUSH;
2995  p->tcph = &tcph;
2997  p->payload = &httpbuf1[cnt];
2998  p->payload_len = 1;
2999  s = &ssn.client;
3000 
3001  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3002 
3004  p->payload = NULL;
3005  p->payload_len = 0;
3006  tcph.th_seq = htonl(572799782UL);
3007  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3008  tcph.th_flags = TH_ACK;
3009  p->tcph = &tcph;
3010  s = &ssn.server;
3011 
3012  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3013  }
3014 
3015  FAIL_IF(f->alproto != ALPROTO_HTTP);
3016 
3020  SCFree(p);
3021  UTHFreeFlow(f);
3022  PASS;
3023 }
3024 
3025 /** \test 3 in order segments in inline reassembly */
3026 static int StreamTcpReassembleInlineTest01(void)
3027 {
3028  int ret = 0;
3029  TcpReassemblyThreadCtx *ra_ctx = NULL;
3030  ThreadVars tv;
3031  TcpSession ssn;
3032  Flow f;
3033 
3034  memset(&tv, 0x00, sizeof(tv));
3035 
3036  StreamTcpUTInit(&ra_ctx);
3039  StreamTcpUTSetupStream(&ssn.client, 1);
3040  FLOW_INITIALIZE(&f);
3041 
3042  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3043  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3044  if (p == NULL) {
3045  printf("couldn't get a packet: ");
3046  goto end;
3047  }
3048  p->tcph->th_seq = htonl(12);
3049  p->flow = &f;
3050 
3051  FLOWLOCK_WRLOCK(&f);
3052  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3053  printf("failed to add segment 1: ");
3054  goto end;
3055  }
3056  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3057  printf("failed to add segment 2: ");
3058  goto end;
3059  }
3060  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3061  printf("failed to add segment 3: ");
3062  goto end;
3063  }
3064  ssn.client.next_seq = 17;
3065  ret = 1;
3066 end:
3067  FLOWLOCK_UNLOCK(&f);
3068  FLOW_DESTROY(&f);
3069  UTHFreePacket(p);
3071  StreamTcpUTDeinit(ra_ctx);
3072  return ret;
3073 }
3074 
3075 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3076  * test the sliding window reassembly.
3077  */
3078 static int StreamTcpReassembleInlineTest02(void)
3079 {
3080  int ret = 0;
3081  TcpReassemblyThreadCtx *ra_ctx = NULL;
3082  ThreadVars tv;
3083  TcpSession ssn;
3084  Flow f;
3085 
3086  memset(&tv, 0x00, sizeof(tv));
3087 
3088  StreamTcpUTInit(&ra_ctx);
3091  StreamTcpUTSetupStream(&ssn.client, 1);
3092  FLOW_INITIALIZE(&f);
3093 
3094  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3095  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3096  if (p == NULL) {
3097  printf("couldn't get a packet: ");
3098  goto end;
3099  }
3100  p->tcph->th_seq = htonl(12);
3101  p->flow = &f;
3102 
3103  FLOWLOCK_WRLOCK(&f);
3104  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3105  printf("failed to add segment 1: ");
3106  goto end;
3107  }
3108  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3109  printf("failed to add segment 2: ");
3110  goto end;
3111  }
3112  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3113  printf("failed to add segment 3: ");
3114  goto end;
3115  }
3116  ssn.client.next_seq = 17;
3117  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3118  printf("failed to add segment 4: ");
3119  goto end;
3120  }
3121  ssn.client.next_seq = 22;
3122  ret = 1;
3123 end:
3124  FLOWLOCK_UNLOCK(&f);
3125  FLOW_DESTROY(&f);
3126  UTHFreePacket(p);
3128  StreamTcpUTDeinit(ra_ctx);
3129  return ret;
3130 }
3131 
3132 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3133  * test the sliding window reassembly with a small window size so that we
3134  * cutting off at the start (left edge)
3135  */
3136 static int StreamTcpReassembleInlineTest03(void)
3137 {
3138  int ret = 0;
3139  TcpReassemblyThreadCtx *ra_ctx = NULL;
3140  ThreadVars tv;
3141  TcpSession ssn;
3142  Flow f;
3143 
3144  memset(&tv, 0x00, sizeof(tv));
3145 
3146  StreamTcpUTInit(&ra_ctx);
3149  StreamTcpUTSetupStream(&ssn.client, 1);
3150  FLOW_INITIALIZE(&f);
3151 
3153 
3154  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3155  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3156  if (p == NULL) {
3157  printf("couldn't get a packet: ");
3158  goto end;
3159  }
3160  p->tcph->th_seq = htonl(12);
3161  p->flow = &f;
3163 
3164  FLOWLOCK_WRLOCK(&f);
3165  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3166  printf("failed to add segment 1: ");
3167  goto end;
3168  }
3169  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3170  printf("failed to add segment 2: ");
3171  goto end;
3172  }
3173  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3174  printf("failed to add segment 3: ");
3175  goto end;
3176  }
3177  ssn.client.next_seq = 17;
3178  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3179  printf("failed to add segment 4: ");
3180  goto end;
3181  }
3182  ssn.client.next_seq = 22;
3183 
3184  p->tcph->th_seq = htonl(17);
3185  ret = 1;
3186 end:
3187  FLOWLOCK_UNLOCK(&f);
3188  FLOW_DESTROY(&f);
3189  UTHFreePacket(p);
3191  StreamTcpUTDeinit(ra_ctx);
3192  return ret;
3193 }
3194 
3195 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3196  * test the sliding window reassembly with a small window size so that we
3197  * cutting off at the start (left edge) with small packet overlap.
3198  */
3199 static int StreamTcpReassembleInlineTest04(void)
3200 {
3201  int ret = 0;
3202  TcpReassemblyThreadCtx *ra_ctx = NULL;
3203  ThreadVars tv;
3204  TcpSession ssn;
3205  Flow f;
3206 
3207  memset(&tv, 0x00, sizeof(tv));
3208 
3209  StreamTcpUTInit(&ra_ctx);
3212  StreamTcpUTSetupStream(&ssn.client, 1);
3213  FLOW_INITIALIZE(&f);
3214 
3216 
3217  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3218  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3219  if (p == NULL) {
3220  printf("couldn't get a packet: ");
3221  goto end;
3222  }
3223  p->tcph->th_seq = htonl(12);
3224  p->flow = &f;
3226 
3227  FLOWLOCK_WRLOCK(&f);
3228  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3229  printf("failed to add segment 1: ");
3230  goto end;
3231  }
3232  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3233  printf("failed to add segment 2: ");
3234  goto end;
3235  }
3236  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3237  printf("failed to add segment 3: ");
3238  goto end;
3239  }
3240  ssn.client.next_seq = 17;
3241  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3242  printf("failed to add segment 4: ");
3243  goto end;
3244  }
3245  ssn.client.next_seq = 22;
3246 
3247  p->tcph->th_seq = htonl(17);
3248  ret = 1;
3249 end:
3250  FLOWLOCK_UNLOCK(&f);
3251  FLOW_DESTROY(&f);
3252  UTHFreePacket(p);
3254  StreamTcpUTDeinit(ra_ctx);
3255  return ret;
3256 }
3257 
3258 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3259  * test the sliding window reassembly with a small window size so that we
3260  * cutting off at the start (left edge). Test if the first segment is
3261  * removed from the list.
3262  */
3263 static int StreamTcpReassembleInlineTest08(void)
3264 {
3265  TcpReassemblyThreadCtx *ra_ctx = NULL;
3266  ThreadVars tv;
3267  memset(&tv, 0x00, sizeof(tv));
3268  TcpSession ssn;
3269  Flow f;
3270  StreamTcpUTInit(&ra_ctx);
3273  StreamTcpUTSetupStream(&ssn.client, 1);
3274  FLOW_INITIALIZE(&f);
3275 
3278  f.protoctx = &ssn;
3279 
3280  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3281  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3282  FAIL_IF(p == NULL);
3283  p->tcph->th_seq = htonl(12);
3284  p->flow = &f;
3286 
3287  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3288  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3289  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3290  ssn.client.next_seq = 17;
3291  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3292  ssn.client.next_seq = 22;
3293  p->tcph->th_seq = htonl(17);
3295 
3296  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3297  FAIL_IF_NULL(seg);
3298  FAIL_IF_NOT(seg->seq == 2);
3299 
3300  FLOW_DESTROY(&f);
3301  UTHFreePacket(p);
3303  StreamTcpUTDeinit(ra_ctx);
3304  PASS;
3305 }
3306 
3307 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3308  * test the sliding window reassembly with a small window size so that we
3309  * cutting off at the start (left edge). Test if the first segment is
3310  * removed from the list.
3311  */
3312 static int StreamTcpReassembleInlineTest09(void)
3313 {
3314  int ret = 0;
3315  TcpReassemblyThreadCtx *ra_ctx = NULL;
3316  ThreadVars tv;
3317  TcpSession ssn;
3318  Flow f;
3319 
3320  memset(&tv, 0x00, sizeof(tv));
3321 
3322  StreamTcpUTInit(&ra_ctx);
3325  StreamTcpUTSetupStream(&ssn.client, 1);
3326  FLOW_INITIALIZE(&f);
3327 
3330 
3331  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3332  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3333  if (p == NULL) {
3334  printf("couldn't get a packet: ");
3335  goto end;
3336  }
3337  p->tcph->th_seq = htonl(17);
3338  p->flow = &f;
3340 
3341  FLOWLOCK_WRLOCK(&f);
3342  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3343  printf("failed to add segment 1: ");
3344  goto end;
3345  }
3346  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3347  printf("failed to add segment 2: ");
3348  goto end;
3349  }
3350  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3351  printf("failed to add segment 3: ");
3352  goto end;
3353  }
3354  ssn.client.next_seq = 12;
3355  ssn.client.last_ack = 10;
3356 
3357  /* close the GAP and see if we properly reassemble and update base_seq */
3358  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3359  printf("failed to add segment 4: ");
3360  goto end;
3361  }
3362  ssn.client.next_seq = 22;
3363 
3364  p->tcph->th_seq = htonl(12);
3365 
3366  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3367  FAIL_IF_NULL(seg);
3368  FAIL_IF_NOT(seg->seq == 2);
3369 
3370  ret = 1;
3371 end:
3372  FLOWLOCK_UNLOCK(&f);
3373  FLOW_DESTROY(&f);
3374  UTHFreePacket(p);
3376  StreamTcpUTDeinit(ra_ctx);
3377  return ret;
3378 }
3379 
3380 /** \test App Layer reassembly.
3381  */
3382 static int StreamTcpReassembleInlineTest10(void)
3383 {
3384  int ret = 0;
3385  TcpReassemblyThreadCtx *ra_ctx = NULL;
3386  ThreadVars tv;
3387  TcpSession ssn;
3388  Flow *f = NULL;
3389  Packet *p = NULL;
3390 
3391  memset(&tv, 0x00, sizeof(tv));
3392 
3393  StreamTcpUTInit(&ra_ctx);
3396  StreamTcpUTSetupStream(&ssn.server, 1);
3397  ssn.server.last_ack = 2;
3398  StreamTcpUTSetupStream(&ssn.client, 1);
3399  ssn.client.last_ack = 2;
3401 
3402  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3403  if (f == NULL)
3404  goto end;
3405  f->protoctx = &ssn;
3406  f->proto = IPPROTO_TCP;
3407 
3408  uint8_t stream_payload1[] = "GE";
3409  uint8_t stream_payload2[] = "T /";
3410  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3411 
3412  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3413  if (p == NULL) {
3414  printf("couldn't get a packet: ");
3415  goto end;
3416  }
3417  p->tcph->th_seq = htonl(7);
3418  p->flow = f;
3420 
3421  FLOWLOCK_WRLOCK(f);
3422  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3423  printf("failed to add segment 1: ");
3424  goto end;
3425  }
3426  ssn.client.next_seq = 4;
3427 
3428  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3429  if (r < 0) {
3430  printf("StreamTcpReassembleAppLayer failed: ");
3431  goto end;
3432  }
3433 
3434  /* ssn.server.ra_app_base_seq should be isn here. */
3435  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3436  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3437  goto end;
3438  }
3439 
3440  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3441  printf("failed to add segment 2: ");
3442  goto end;
3443  }
3444  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3445  printf("failed to add segment 3: ");
3446  goto end;
3447  }
3448  ssn.client.next_seq = 19;
3449 
3450  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3451  if (r < 0) {
3452  printf("StreamTcpReassembleAppLayer failed: ");
3453  goto end;
3454  }
3455 
3456  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3457 
3458  ret = 1;
3459 end:
3460  UTHFreePacket(p);
3462  StreamTcpUTDeinit(ra_ctx);
3463  FLOWLOCK_UNLOCK(f);
3464  UTHFreeFlow(f);
3465  return ret;
3466 }
3467 
3468 /** \test test insert with overlap
3469  */
3470 static int StreamTcpReassembleInsertTest01(void)
3471 {
3472  TcpReassemblyThreadCtx *ra_ctx = NULL;
3473  ThreadVars tv;
3474  TcpSession ssn;
3475  Flow f;
3476 
3477  memset(&tv, 0x00, sizeof(tv));
3478 
3479  StreamTcpUTInit(&ra_ctx);
3481  StreamTcpUTSetupStream(&ssn.client, 1);
3483  FLOW_INITIALIZE(&f);
3484 
3485  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3486  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3487  FAIL_IF(p == NULL);
3488  p->tcph->th_seq = htonl(12);
3489  p->flow = &f;
3490 
3491  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3492  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3493  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3494  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3495  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3496  ssn.client.next_seq = 21;
3497 
3498  FLOW_DESTROY(&f);
3499  UTHFreePacket(p);
3501  StreamTcpUTDeinit(ra_ctx);
3502  PASS;
3503 }
3504 
3505 /** \test test insert with overlaps
3506  */
3507 static int StreamTcpReassembleInsertTest02(void)
3508 {
3509  int ret = 0;
3510  TcpReassemblyThreadCtx *ra_ctx = NULL;
3511  ThreadVars tv;
3512  TcpSession ssn;
3513 
3514  memset(&tv, 0x00, sizeof(tv));
3515 
3516  StreamTcpUTInit(&ra_ctx);
3518  StreamTcpUTSetupStream(&ssn.client, 1);
3519 
3520  int i;
3521  for (i = 2; i < 10; i++) {
3522  int len;
3523  len = i % 2;
3524  if (len == 0)
3525  len = 1;
3526  int seq;
3527  seq = i * 10;
3528  if (seq < 2)
3529  seq = 2;
3530 
3531  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3532  printf("failed to add segment 1: ");
3533  goto end;
3534  }
3535  }
3536  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3537  printf("failed to add segment 2: ");
3538  goto end;
3539  }
3540 
3541  ret = 1;
3542 end:
3544  StreamTcpUTDeinit(ra_ctx);
3545  return ret;
3546 }
3547 
3548 /** \test test insert with overlaps
3549  */
3550 static int StreamTcpReassembleInsertTest03(void)
3551 {
3552  int ret = 0;
3553  TcpReassemblyThreadCtx *ra_ctx = NULL;
3554  ThreadVars tv;
3555  TcpSession ssn;
3556 
3557  memset(&tv, 0x00, sizeof(tv));
3558 
3559  StreamTcpUTInit(&ra_ctx);
3561  StreamTcpUTSetupStream(&ssn.client, 1);
3562 
3563  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3564  printf("failed to add segment 2: ");
3565  goto end;
3566  }
3567 
3568  int i;
3569  for (i = 2; i < 10; i++) {
3570  int len;
3571  len = i % 2;
3572  if (len == 0)
3573  len = 1;
3574  int seq;
3575  seq = i * 10;
3576  if (seq < 2)
3577  seq = 2;
3578 
3579  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3580  printf("failed to add segment 2: ");
3581  goto end;
3582  }
3583  }
3584  ret = 1;
3585 end:
3587  StreamTcpUTDeinit(ra_ctx);
3588  return ret;
3589 }
3590 
3592 #endif /* UNITTESTS */
3593 
3594 /** \brief The Function Register the Unit tests to test the reassembly engine
3595  * for various OS policies.
3596  */
3597 
3599 {
3600 #ifdef UNITTESTS
3601  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3602  StreamTcpReassembleTest25);
3603  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3604  StreamTcpReassembleTest26);
3605  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3606  StreamTcpReassembleTest27);
3607  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3608  StreamTcpReassembleTest28);
3609  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3610  StreamTcpReassembleTest29);
3611  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3612  StreamTcpReassembleTest33);
3613  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3614  StreamTcpReassembleTest34);
3615  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3616  StreamTcpReassembleTest37);
3617  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3618  StreamTcpReassembleTest39);
3619  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3620  StreamTcpReassembleTest40);
3621  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3622  StreamTcpReassembleTest44);
3623  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3624  StreamTcpReassembleTest45);
3625  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3626  StreamTcpReassembleTest46);
3627  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3628  StreamTcpReassembleTest47);
3629 
3630  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3631  StreamTcpReassembleInlineTest01);
3632  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3633  StreamTcpReassembleInlineTest02);
3634  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3635  StreamTcpReassembleInlineTest03);
3636  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3637  StreamTcpReassembleInlineTest04);
3638  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3639  StreamTcpReassembleInlineTest08);
3640  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3641  StreamTcpReassembleInlineTest09);
3642 
3643  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3644  StreamTcpReassembleInlineTest10);
3645 
3646  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3647  StreamTcpReassembleInsertTest01);
3648  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3649  StreamTcpReassembleInsertTest02);
3650  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3651  StreamTcpReassembleInsertTest03);
3652 
3656  StreamTcpReassembleRawRegisterTests();
3657 #endif /* UNITTESTS */
3658 }
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
#define SCMutex
StreamingBufferSegment sbseg
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
#define SCLogDebug(...)
Definition: util-debug.h:335
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
void *(* Malloc)(size_t size)
int StreamTcpAppLayerIsDisabled(Flow *f)
struct Flow_ * flow
Definition: decode.h:445
AppProto alproto_tc
Definition: flow.h:411
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
#define STREAM_BASE_OFFSET(stream)
int StreamTcpReassembleInit(char quiet)
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:344
#define FALSE
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:243
StreamingBuffer sb
#define PASS
Pass the test.
#define unlikely(expr)
Definition: util-optimize.h:35
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
#define STREAMTCP_STREAM_FLAG_GAP
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void StreamTcpListRegisterTests(void)
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define TH_RST
Definition: decode-tcp.h:37
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet...
Definition: stream-tcp.c:770
void StreamTcpDisableAppLayer(Flow *f)
#define TH_FIN
Definition: decode-tcp.h:35
#define MIN(x, y)
#define STREAMING_BUFFER_NOFLAGS
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
uint64_t offset
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Address dst
Definition: decode.h:413
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
#define SCReturnUInt(x)
Definition: util-debug.h:343
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
int ByteExtractStringUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:244
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:669
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:240
uint64_t pcap_cnt
Definition: decode.h:561
#define STREAM_RAW_PROGRESS(stream)
void StreamTcpReassembleInitMemuse(void)
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:124
char * val
Definition: conf.h:34
TCPHdr * tcph
Definition: decode.h:522
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define TRUE
#define SCMutexLock(mut)
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
StreamingBufferBlock * head
#define STREAMTCP_FLAG_MIDSTREAM
void * protoctx
Definition: flow.h:400
void *(* Calloc)(size_t n, size_t size)
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
#define STREAM_DEPTH
Definition: stream.h:34
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
uint32_t min_inspect_depth
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueue *pq)
Definition: stream-tcp.c:4726
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define StreamTcpSetEvent(p, e)
AppProto alproto_ts
Definition: flow.h:410
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
#define SCCalloc(nm, a)
Definition: util-mem.h:253
#define SEQ_GEQ(a, b)
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:113
#define STREAM_GAP
Definition: stream.h:33
#define SCMutexUnlock(mut)
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:820
uint32_t seq
uint32_t raw_progress_rel
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:248
char family
Definition: decode.h:111
uint8_t proto
Definition: decode.h:430
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
#define SCMUTEX_INITIALIZER
#define TCP_SEG_LEN(seg)
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
#define STREAM_HAS_SEEN_DATA(stream)
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Data structures and function prototypes for keeping state for the detection engine.
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define FLOW_DESTROY(f)
Definition: flow-util.h:121
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies...
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
#define STREAM_EOF
Definition: stream.h:30
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define StreamTcpDisableAppLayerReassembly(ssn)
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
#define SCEnter(...)
Definition: util-debug.h:337
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
void StreamTcpReassembleFree(char quiet)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:439
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PKT_IS_TOSERVER(p)
Definition: decode.h:257
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
uint32_t reassembly_depth
#define TH_PUSH
Definition: decode-tcp.h:38
#define SEQ_GT(a, b)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:168
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1094
int RunmodeIsUnittests(void)
Definition: suricata.c:267
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1001
#define SCMutexInit(mut, mutattrs)
int8_t data_first_seen_dir
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define SCReturnInt(x)
Definition: util-debug.h:341
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueue *pq)
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
void StreamTcpUTInitInline(void)
#define SEQ_LEQ(a, b)
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
uint32_t reassembly_depth
Definition: stream-tcp.h:60
#define SCRealloc(x, a)
Definition: util-mem.h:238
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:563
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define RB_MIN(name, x)
Definition: tree.h:780
Definition: conf.h:32
#define SCMalloc(a)
Definition: util-mem.h:222
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:207
struct TCPSEG seg_tree
uint16_t payload_len
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define TH_SYN
Definition: decode-tcp.h:36
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
block of continues data
#define SCFree(a)
Definition: util-mem.h:322
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:249
#define RB_REMOVE(name, x, y)
Definition: tree.h:775
void StreamTcpUtilRegisterTests(void)
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:842
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
#define STREAM_START
Definition: stream.h:29
uint32_t prealloc_segments
Definition: stream-tcp.h:57
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
#define PKT_STREAM_ADD
Definition: decode.h:1090
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself...
void StreamTcpInlineRegisterTests(void)
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
void StreamTcpReassemblePseudoPacketCreate(TcpStream *, Packet *, PacketQueue *)
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
#define SCReturn
Definition: util-debug.h:339
void StreamTcpUTSetupSession(TcpSession *ssn)
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define MISSED_END
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of &#39;threads&#39;, so array elements)
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
uint32_t flags
Definition: decode.h:443
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
void StreamTcpReassembleConfigEnableOverlapCheck(void)
uint16_t payload_len
Definition: decode.h:541
#define STREAM_APP_PROGRESS(stream)
#define SEQ_LT(a, b)
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6320
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
#define STREAM_RIGHT_EDGE(stream)
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
void StreamTcpUTClearSession(TcpSession *ssn)
#define MISSED_START(isn)
Flow data structure.
Definition: flow.h:325
#define FLOW_IPV4
Definition: flow.h:94
uint8_t * payload
Definition: decode.h:540
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
uint32_t flags
Definition: flow.h:379
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
void StreamTcpUTClearStream(TcpStream *s)
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
void(* Free)(void *ptr, size_t size)
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:141
Address src
Definition: decode.h:412
void UTHFreeFlow(Flow *flow)
#define STREAM_MIDSTREAM
Definition: stream.h:35
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SET_ISN(stream, setseq)
#define SCMutexDestroy
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.