suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 
64 #include "detect-engine-state.h"
65 
66 #include "util-profiling.h"
67 #include "util-validate.h"
68 
69 #ifdef DEBUG
70 static SCMutex segment_pool_memuse_mutex;
71 static uint64_t segment_pool_memuse = 0;
72 static uint64_t segment_pool_memcnt = 0;
73 #endif
74 
75 static PoolThread *segment_thread_pool = NULL;
76 /* init only, protect initializing and growing pool */
77 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
78 
79 /* Memory use counter */
80 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
81 
82 /* prototypes */
84 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
85 
87 {
88  SC_ATOMIC_INIT(ra_memuse);
89 }
90 
91 /**
92  * \brief Function to Increment the memory usage counter for the TCP reassembly
93  * segments
94  *
95  * \param size Size of the TCP segment and its payload length memory allocated
96  */
97 void StreamTcpReassembleIncrMemuse(uint64_t size)
98 {
99  (void) SC_ATOMIC_ADD(ra_memuse, size);
100  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
101  return;
102 }
103 
104 /**
105  * \brief Function to Decrease the memory usage counter for the TCP reassembly
106  * segments
107  *
108  * \param size Size of the TCP segment and its payload length memory allocated
109  */
110 void StreamTcpReassembleDecrMemuse(uint64_t size)
111 {
112 #ifdef UNITTESTS
113  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
114  if (RunmodeIsUnittests()) {
115  BUG_ON(presize > UINT_MAX);
116  }
117 #endif
118 
119  (void) SC_ATOMIC_SUB(ra_memuse, size);
120 
121 #ifdef UNITTESTS
122  if (RunmodeIsUnittests()) {
123  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
124  BUG_ON(postsize > presize);
125  }
126 #endif
127  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
128  return;
129 }
130 
132 {
133  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
134  return smemuse;
135 }
136 
137 /**
138  * \brief Function to Check the reassembly memory usage counter against the
139  * allowed max memory usgae for TCP segments.
140  *
141  * \param size Size of the TCP segment and its payload length memory allocated
142  * \retval 1 if in bounds
143  * \retval 0 if not in bounds
144  */
146 {
147  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
148  if (memcapcopy == 0 ||
149  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
150  return 1;
151  return 0;
152 }
153 
154 /**
155  * \brief Update memcap value
156  *
157  * \param size new memcap value
158  */
160 {
161  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
162  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
163  return 1;
164  }
165 
166  return 0;
167 }
168 
169 /**
170  * \brief Return memcap value
171  *
172  * \return memcap memcap value
173  */
175 {
176  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
177  return memcapcopy;
178 }
179 
180 /* memory functions for the streaming buffer API */
181 
182 /*
183  void *(*Malloc)(size_t size);
184 */
185 static void *ReassembleMalloc(size_t size)
186 {
187  if (StreamTcpReassembleCheckMemcap(size) == 0)
188  return NULL;
189  void *ptr = SCMalloc(size);
190  if (ptr == NULL)
191  return NULL;
193  return ptr;
194 }
195 
196 /*
197  void *(*Calloc)(size_t n, size_t size);
198 */
199 static void *ReassembleCalloc(size_t n, size_t size)
200 {
201  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
202  return NULL;
203  void *ptr = SCCalloc(n, size);
204  if (ptr == NULL)
205  return NULL;
207  return ptr;
208 }
209 
210 /*
211  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
212 */
213 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
214 {
215  if (size > orig_size) {
216  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
217  return NULL;
218  }
219  void *nptr = SCRealloc(optr, size);
220  if (nptr == NULL)
221  return NULL;
222 
223  if (size > orig_size) {
224  StreamTcpReassembleIncrMemuse(size - orig_size);
225  } else {
226  StreamTcpReassembleDecrMemuse(orig_size - size);
227  }
228  return nptr;
229 }
230 
231 /*
232  void (*Free)(void *ptr, size_t size);
233 */
234 static void ReassembleFree(void *ptr, size_t size)
235 {
236  SCFree(ptr);
238 }
239 
240 /** \brief alloc a tcp segment pool entry */
241 static void *TcpSegmentPoolAlloc(void)
242 {
243  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
244  return NULL;
245  }
246 
247  TcpSegment *seg = NULL;
248 
249  seg = SCMalloc(sizeof (TcpSegment));
250  if (unlikely(seg == NULL))
251  return NULL;
252  return seg;
253 }
254 
255 static int TcpSegmentPoolInit(void *data, void *initdata)
256 {
257  TcpSegment *seg = (TcpSegment *) data;
258 
259  /* do this before the can bail, so TcpSegmentPoolCleanup
260  * won't have uninitialized memory to consider. */
261  memset(seg, 0, sizeof (TcpSegment));
262 
263  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
264  return 0;
265  }
266 
267 #ifdef DEBUG
268  SCMutexLock(&segment_pool_memuse_mutex);
269  segment_pool_memuse += sizeof(TcpSegment);
270  segment_pool_memcnt++;
271  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
272  SCMutexUnlock(&segment_pool_memuse_mutex);
273 #endif
274 
275  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
276  return 1;
277 }
278 
279 /** \brief clean up a tcp segment pool entry */
280 static void TcpSegmentPoolCleanup(void *ptr)
281 {
282  if (ptr == NULL)
283  return;
284 
285  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
286 
287 #ifdef DEBUG
288  SCMutexLock(&segment_pool_memuse_mutex);
289  segment_pool_memuse -= sizeof(TcpSegment);
290  segment_pool_memcnt--;
291  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
292  SCMutexUnlock(&segment_pool_memuse_mutex);
293 #endif
294 }
295 
296 /**
297  * \brief Function to return the segment back to the pool.
298  *
299  * \param seg Segment which will be returned back to the pool.
300  */
302 {
303  if (seg == NULL)
304  return;
305 
306  PoolThreadReturn(segment_thread_pool, seg);
307 }
308 
309 /**
310  * \brief return all segments in this stream into the pool(s)
311  *
312  * \param stream the stream to cleanup
313  */
315 {
316  TcpSegment *seg = NULL, *safe = NULL;
317  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
318  {
319  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
321  }
322 }
323 
324 #ifdef UNITTESTS
325 /** \internal
326  * \brief check if segments falls before stream 'offset' */
327 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
328 {
329  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
330  return 1;
331  return 0;
332 }
333 #endif
334 
335 /** \param f locked flow */
337 {
338  if (f->protoctx == NULL)
339  return;
340 
341  TcpSession *ssn = (TcpSession *)f->protoctx;
345 }
346 
347 /** \param f locked flow */
349 {
350  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
351  return 0;
352 
353  TcpSession *ssn = (TcpSession *)f->protoctx;
355 }
356 
357 static int StreamTcpReassemblyConfig(char quiet)
358 {
359  uint32_t segment_prealloc = 2048;
360  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
361  if (seg) {
362  uint32_t prealloc = 0;
363  if (ByteExtractStringUint32(&prealloc, 10, strlen(seg->val), seg->val) == -1)
364  {
365  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
366  "%s is invalid", seg->val);
367  return -1;
368  }
369  segment_prealloc = prealloc;
370  }
371  if (!quiet)
372  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
373  stream_config.prealloc_segments = segment_prealloc;
374 
375  int overlap_diff_data = 0;
376  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
377  if (overlap_diff_data) {
379  }
380  if (StreamTcpInlineMode() == TRUE) {
382  }
383 
386  stream_config.sbcnf.Malloc = ReassembleMalloc;
387  stream_config.sbcnf.Calloc = ReassembleCalloc;
388  stream_config.sbcnf.Realloc = ReassembleRealloc;
389  stream_config.sbcnf.Free = ReassembleFree;
390 
391  return 0;
392 }
393 
394 int StreamTcpReassembleInit(char quiet)
395 {
396  /* init the memcap/use tracker */
398 
399  if (StreamTcpReassemblyConfig(quiet) < 0)
400  return -1;
401 
402 #ifdef DEBUG
403  SCMutexInit(&segment_pool_memuse_mutex, NULL);
404 #endif
405  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
407  return 0;
408 }
409 
410 void StreamTcpReassembleFree(char quiet)
411 {
412  SCMutexLock(&segment_thread_pool_mutex);
413  if (segment_thread_pool != NULL) {
414  PoolThreadFree(segment_thread_pool);
415  segment_thread_pool = NULL;
416  }
417  SCMutexUnlock(&segment_thread_pool_mutex);
418  SCMutexDestroy(&segment_thread_pool_mutex);
419 
420 #ifdef DEBUG
421  if (segment_pool_memuse > 0)
422  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
423  if (segment_pool_memcnt > 0)
424  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
425  SCMutexDestroy(&segment_pool_memuse_mutex);
426 #endif
427 }
428 
430 {
431  SCEnter();
433  if (unlikely(ra_ctx == NULL))
434  return NULL;
435 
436  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
437 
438  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
439 
440  SCMutexLock(&segment_thread_pool_mutex);
441  if (segment_thread_pool == NULL) {
442  segment_thread_pool = PoolThreadInit(1, /* thread */
443  0, /* unlimited */
445  sizeof(TcpSegment),
446  TcpSegmentPoolAlloc,
447  TcpSegmentPoolInit, NULL,
448  TcpSegmentPoolCleanup, NULL);
449  ra_ctx->segment_thread_pool_id = 0;
450  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
451  PoolThreadSize(segment_thread_pool),
452  ra_ctx->segment_thread_pool_id);
453  } else {
454  /* grow segment_thread_pool until we have a element for our thread id */
455  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
456  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
457  PoolThreadSize(segment_thread_pool),
458  ra_ctx->segment_thread_pool_id);
459  }
460  SCMutexUnlock(&segment_thread_pool_mutex);
461  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
462  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
464  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
465  }
466 
467  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
468 }
469 
471 {
472  SCEnter();
474  SCFree(ra_ctx);
475  SCReturn;
476 }
477 
478 /**
479  * \brief check if stream in pkt direction has depth reached
480  *
481  * \param p packet with *LOCKED* flow
482  *
483  * \retval 1 stream has depth reached
484  * \retval 0 stream does not have depth reached
485  */
487 {
488  if (p->flow != NULL && p->flow->protoctx != NULL) {
489  TcpSession *ssn = p->flow->protoctx;
490  TcpStream *stream;
491  if (p->flowflags & FLOW_PKT_TOSERVER) {
492  stream = &ssn->client;
493  } else {
494  stream = &ssn->server;
495  }
496 
497  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
498  }
499 
500  return 0;
501 }
502 
503 /**
504  * \internal
505  * \brief Function to Check the reassembly depth valuer against the
506  * allowed max depth of the stream reassembly for TCP streams.
507  *
508  * \param stream stream direction
509  * \param seq sequence number where "size" starts
510  * \param size size of the segment that is added
511  *
512  * \retval size Part of the size that fits in the depth, 0 if none
513  */
514 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
515  uint32_t seq, uint32_t size)
516 {
517  SCEnter();
518 
519  /* if the configured depth value is 0, it means there is no limit on
520  reassembly depth. Otherwise carry on my boy ;) */
521  if (ssn->reassembly_depth == 0) {
522  SCReturnUInt(size);
523  }
524 
525  /* if the final flag is set, we're not accepting anymore */
527  SCReturnUInt(0);
528  }
529 
530  uint64_t seg_depth;
531  if (SEQ_GT(stream->base_seq, seq)) {
532  if (SEQ_LEQ(seq+size, stream->base_seq)) {
533  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
534  stream->base_seq, seq, seq+size);
535  SCReturnUInt(0);
536  }
537 
538  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
539  } else {
540  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
541  }
542 
543  /* if the base_seq has moved passed the depth window we stop
544  * checking and just reject the rest of the packets including
545  * retransmissions. Saves us the hassle of dealing with sequence
546  * wraps as well */
547  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
548  stream->base_seq, seg_depth,
549  ssn->reassembly_depth);
550 
551  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
552  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
554  SCReturnUInt(0);
555  }
556  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
557  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
558 #if 0
559  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
560  (stream->base_seq_offset + stream->base_seq + size),
561  (stream->isn + ssn->reassembly_depth));
562 #endif
563  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
564  /* packet (partly?) fits the depth window */
565 
566  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
567  /* complete fit */
568  SCReturnUInt(size);
569  } else {
571  /* partial fit, return only what fits */
572  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
573  DEBUG_VALIDATE_BUG_ON(part > size);
574  if (part > size)
575  part = size;
576  SCReturnUInt(part);
577  }
578  }
579 
580  SCReturnUInt(0);
581 }
582 
583 /**
584  * \brief Insert a packets TCP data into the stream reassembly engine.
585  *
586  * \retval 0 good segment, as far as we checked.
587  * \retval -1 badness, reason to drop in inline mode
588  *
589  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
590  * or it wasn't added because of reassembly depth.
591  *
592  */
594  TcpSession *ssn, TcpStream *stream, Packet *p)
595 {
596  SCEnter();
597 
598  if (ssn->data_first_seen_dir == 0) {
599  if (PKT_IS_TOSERVER(p)) {
601  } else {
603  }
604  }
605 
606  /* If the OS policy is not set then set the OS policy for this stream */
607  if (stream->os_policy == 0) {
608  StreamTcpSetOSPolicy(stream, p);
609  }
610 
613  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
614  SCReturnInt(0);
615  }
616 
617  /* If we have reached the defined depth for either of the stream, then stop
618  reassembling the TCP session */
619  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
620  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
621 
623  /* increment stream depth counter */
624  StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
625  }
626  if (size == 0) {
627  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
628  SCReturnInt(0);
629  }
630 
632  if (size > p->payload_len)
633  size = p->payload_len;
634 
635  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
636  if (seg == NULL) {
637  SCLogDebug("segment_pool is empty");
639  SCReturnInt(-1);
640  }
641 
642  TCP_SEG_LEN(seg) = size;
643  seg->seq = TCP_GET_SEQ(p);
644 
645  /* proto detection skipped, but now we do get data. Set event. */
646  if (RB_EMPTY(&stream->seg_tree) &&
648 
651  }
652 
653  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
654  SCLogDebug("StreamTcpReassembleInsertSegment failed");
655  SCReturnInt(-1);
656  }
657  SCReturnInt(0);
658 }
659 
660 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
661  Packet *p, enum StreamUpdateDir dir)
662 {
663  uint8_t flag = 0;
664 
666  flag |= STREAM_START;
667  }
668 
669  if (ssn->state == TCP_CLOSED) {
670  flag |= STREAM_EOF;
671  }
672 
673  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
674  flag |= STREAM_MIDSTREAM;
675  }
676 
677  if (p->flags & PKT_PSEUDO_STREAM_END) {
678  flag |= STREAM_EOF;
679  }
680 
681  if (dir == UPDATE_DIR_OPPOSING) {
682  if (p->flowflags & FLOW_PKT_TOSERVER) {
683  flag |= STREAM_TOCLIENT;
684  } else {
685  flag |= STREAM_TOSERVER;
686  }
687  } else {
688  if (p->flowflags & FLOW_PKT_TOSERVER) {
689  flag |= STREAM_TOSERVER;
690  } else {
691  flag |= STREAM_TOCLIENT;
692  }
693  }
694 
696  flag |= STREAM_DEPTH;
697  }
698  return flag;
699 }
700 
701 /**
702  * \brief Check the minimum size limits for reassembly.
703  *
704  * \retval 0 don't reassemble yet
705  * \retval 1 do reassemble
706  */
707 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
708  const TcpStream *stream, const Packet *p)
709 {
710  SCEnter();
711 
712  /* if any of these flags is set we always inspect immediately */
713 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
714  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
715  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
716  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
717 
720  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
721  "is set, so not expecting any new data segments");
722  }
724  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
725  }
727  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
728  "so no new segments will be considered");
729  }
730  SCReturnInt(1);
731  }
732 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
733 
734  /* some states mean we reassemble no matter how much data we have */
735  if (ssn->state > TCP_TIME_WAIT)
736  SCReturnInt(1);
737 
738  if (p->flags & PKT_PSEUDO_STREAM_END)
739  SCReturnInt(1);
740 
741  /* check if we have enough data to do raw reassembly */
742  if (PKT_IS_TOSERVER(p)) {
743  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
744  uint32_t delta = stream->last_ack - stream->base_seq;
745  /* get max absolute offset */
746  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
747 
748  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
749  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
750  SCReturnInt(1);
751  } else {
752  SCLogDebug("toserver min chunk len not yet reached: "
753  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
754  "%"PRIu32"", stream->last_ack, stream->base_seq,
755  (stream->last_ack - stream->base_seq),
757  SCReturnInt(0);
758  }
759  }
760  } else {
761  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
762  uint32_t delta = stream->last_ack - stream->base_seq;
763  /* get max absolute offset */
764  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
765 
766  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
767 
768  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
769  SCReturnInt(1);
770  } else {
771  SCLogDebug("toclient min chunk len not yet reached: "
772  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
773  "%"PRIu32"", stream->last_ack, stream->base_seq,
774  (stream->last_ack - stream->base_seq),
776  SCReturnInt(0);
777  }
778  }
779  }
780 
781  SCReturnInt(0);
782 }
783 
784 /**
785  * \brief see what if any work the TCP session still needs
786  */
787 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
788 {
789  const TcpStream *stream = NULL;
790 #ifdef DEBUG
791  const char *dirstr = NULL;
792 #endif
793  if (direction == STREAM_TOSERVER) {
794  stream = &ssn->client;
795 #ifdef DEBUG
796  dirstr = "client";
797 #endif
798  } else {
799  stream = &ssn->server;
800 #ifdef DEBUG
801  dirstr = "server";
802 #endif
803  }
804 
805  int use_app = 1;
806  int use_raw = 1;
807 
809  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
810  {
811  // app is dead
812  use_app = 0;
813  }
814 
816  // raw is dead
817  use_raw = 0;
818  }
819 
820  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
821 
822  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
823  dirstr,
824  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
825  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
826  right_edge);
827  if (use_raw) {
828  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
829  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
831  }
832  }
833  if (use_app) {
834  if (right_edge > STREAM_APP_PROGRESS(stream)) {
835  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
837  }
838  }
839 
840  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
842 }
843 
844 #ifdef DEBUG
845 static uint64_t GetStreamSize(TcpStream *stream)
846 {
847  if (stream) {
848  uint64_t size = 0;
849  uint32_t cnt = 0;
850 
851  TcpSegment *seg;
852  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
853  cnt++;
854  size += (uint64_t)TCP_SEG_LEN(seg);
855  }
856 
857  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
858  return size;
859  }
860  return (uint64_t)0;
861 }
862 
863 static void GetSessionSize(TcpSession *ssn, Packet *p)
864 {
865  uint64_t size = 0;
866  if (ssn) {
867  size = GetStreamSize(&ssn->client);
868  size += GetStreamSize(&ssn->server);
869 
870  //if (size > 900000)
871  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
872  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
873  }
874 }
875 #endif
876 
877 /** \internal
878  *
879  * Get buffer, or first part of the buffer if data gaps exist.
880  *
881  * \brief get stream data from offset
882  * \param offset stream offset */
883 static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset)
884 {
885  const uint8_t *mydata;
886  uint32_t mydata_len;
887 
888  if (RB_EMPTY(&stream->sb.sbb_tree)) {
889  SCLogDebug("getting one blob");
890 
891  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
892 
893  *data = mydata;
894  *data_len = mydata_len;
895  } else {
896  StreamingBufferBlock *blk = stream->sb.head;
897 
898  if (blk->offset > offset) {
899  SCLogDebug("gap, want data at offset %"PRIu64", "
900  "got data at %"PRIu64". GAP of size %"PRIu64,
901  offset, blk->offset, blk->offset - offset);
902  *data = NULL;
903  *data_len = blk->offset - offset;
904 
905  } else if (offset >= (blk->offset + blk->len)) {
906 
907  *data = NULL;
908  StreamingBufferBlock *nblk = SBB_RB_NEXT(blk);
909  *data_len = nblk ? nblk->offset - offset : 0;
910  if (nblk) {
911  SCLogDebug("gap, want data at offset %"PRIu64", "
912  "got data at %"PRIu64". GAP of size %"PRIu64,
913  offset, nblk->offset, nblk->offset - offset);
914  }
915 
916  } else if (offset > blk->offset && offset < (blk->offset + blk->len)) {
917  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
918  offset, blk->offset, blk->len);
919  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
920  SCLogDebug("data %p, data_len %u", *data, *data_len);
921  } else {
922  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
923  }
924  }
925 }
926 
927 /** \internal
928  * \brief check to see if we should declare a GAP
929  * Call this when the app layer didn't get data at the requested
930  * offset.
931  */
932 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
933 {
934  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
935  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
936 
937  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
938  /* get window of data that is acked */
939  const uint32_t delta = stream->last_ack - stream->base_seq;
940  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
941  /* get max absolute offset */
942  last_ack_abs += delta;
943 
944  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
945  last_ack_abs -= ackadded;
946 
947  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
948  SCLogDebug("next_seq %u", stream->next_seq);
949 
950  /* if last_ack_abs is beyond the app_progress data that we haven't seen
951  * has been ack'd. This looks like a GAP. */
952  if (last_ack_abs > app_progress) {
953  /* however, we can accept ACKs a bit too liberally. If last_ack
954  * is beyond next_seq, we only consider it a gap now if we do
955  * already have data beyond the gap. */
956  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
957  if (RB_EMPTY(&stream->sb.sbb_tree)) {
958  SCLogDebug("packet %"PRIu64": no GAP. "
959  "next_seq %u < last_ack %u, but no data in list",
960  p->pcap_cnt, stream->next_seq, stream->last_ack);
961  return false;
962  } else {
963  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
964  const StreamingBufferBlock *blk = stream->sb.head;
965  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
966  /* ack'd data after the gap */
967  SCLogDebug("packet %"PRIu64": GAP. "
968  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
969  p->pcap_cnt, stream->next_seq, stream->last_ack);
970  return true;
971  }
972  }
973  }
974 
975  SCLogDebug("packet %"PRIu64": GAP! "
976  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
977  "but we have no data.",
978  p->pcap_cnt, last_ack_abs, app_progress);
979  return true;
980  }
981  }
982  SCLogDebug("packet %"PRIu64": no GAP. "
983  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
984  p->pcap_cnt, last_ack_abs, app_progress);
985  return false;
986 }
987 
988 /** \internal
989  * \brief get stream buffer and update the app-layer
990  * \param stream pointer to pointer as app-layer can switch flow dir
991  * \retval 0 success
992  */
993 static int ReassembleUpdateAppLayer (ThreadVars *tv,
994  TcpReassemblyThreadCtx *ra_ctx,
995  TcpSession *ssn, TcpStream **stream,
996  Packet *p, enum StreamUpdateDir dir)
997 {
998  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
999 
1000  SCLogDebug("app progress %"PRIu64, app_progress);
1001  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1002 
1003  const uint8_t *mydata;
1004  uint32_t mydata_len;
1005 
1006  while (1) {
1007  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress);
1008  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1009  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1010 
1011  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1012  NULL, mydata_len,
1013  StreamGetAppLayerFlags(ssn, *stream, p, dir)|STREAM_GAP);
1014  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1015 
1017  StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
1018 
1019  /* AppLayerHandleTCPData has likely updated progress. */
1020  app_progress = STREAM_APP_PROGRESS(*stream);
1021 
1022  if (r < 0)
1023  return 0;
1024 
1025  continue;
1026  } else if (mydata == NULL || mydata_len == 0) {
1027  /* Possibly a gap, but no new data. */
1028  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1029  SCReturnInt(0);
1030 
1031  mydata = NULL;
1032  mydata_len = 0;
1033  }
1034  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1035  break;
1036  }
1037 
1038  //PrintRawDataFp(stdout, mydata, mydata_len);
1039 
1040  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1041  *stream, &(*stream)->sb, mydata_len, app_progress);
1042 
1043  /* get window of data that is acked */
1044  if (StreamTcpInlineMode() == FALSE) {
1045  if (p->flags & PKT_PSEUDO_STREAM_END) {
1046  // fall through, we use all available data
1047  } else {
1048  uint64_t last_ack_abs = app_progress; /* absolute right edge of ack'd data */
1049  if (STREAM_LASTACK_GT_BASESEQ(*stream)) {
1050  /* get window of data that is acked */
1051  uint32_t delta = (*stream)->last_ack - (*stream)->base_seq;
1052  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > (*stream)->window);
1053  /* get max absolute offset */
1054  last_ack_abs += delta;
1055  }
1056 
1057  /* see if the buffer contains unack'd data as well */
1058  if (app_progress + mydata_len > last_ack_abs) {
1059  uint32_t check = mydata_len;
1060  mydata_len = last_ack_abs - app_progress;
1061  BUG_ON(mydata_len > check);
1062  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1063  "data is considered", mydata_len);
1064  }
1065  }
1066  }
1067 
1068  /* update the app-layer */
1069  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1070  (uint8_t *)mydata, mydata_len,
1071  StreamGetAppLayerFlags(ssn, *stream, p, dir));
1072  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1073 
1074  SCReturnInt(0);
1075 }
1076 
1077 /**
1078  * \brief Update the stream reassembly upon receiving a packet.
1079  *
1080  * For IDS mode, the stream is in the opposite direction of the packet,
1081  * as the ACK-packet is ACK'ing the stream.
1082  *
1083  * One of the utilities call by this function AppLayerHandleTCPData(),
1084  * has a feature where it will call this very same function for the
1085  * stream opposing the stream it is called with. This shouldn't cause
1086  * any issues, since processing of each stream is independent of the
1087  * other stream.
1088  */
1090  TcpSession *ssn, TcpStream *stream,
1091  Packet *p, enum StreamUpdateDir dir)
1092 {
1093  SCEnter();
1094 
1095  /* this function can be directly called by app layer protocol
1096  * detection. */
1099  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1100  SCReturnInt(0);
1101  }
1102 
1103 #ifdef DEBUG
1104  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1105  GetSessionSize(ssn, p);
1106 #endif
1107  /* if no segments are in the list or all are already processed,
1108  * and state is beyond established, we send an empty msg */
1109  if (STREAM_HAS_SEEN_DATA(stream) && STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1110  {
1111  /* send an empty EOF msg if we have no segments but TCP state
1112  * is beyond ESTABLISHED */
1113  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1114  SCLogDebug("sending empty eof message");
1115  /* send EOF to app layer */
1116  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1117  NULL, 0,
1118  StreamGetAppLayerFlags(ssn, stream, p, dir));
1119  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1120 
1121  SCReturnInt(0);
1122  }
1123  }
1124 
1125  /* with all that out of the way, lets update the app-layer */
1126  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1127 }
1128 
1129 /** \internal
1130  * \brief get stream data from offset
1131  * \param offset stream offset */
1132 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1133  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1134 {
1135  const uint8_t *mydata;
1136  uint32_t mydata_len;
1137  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1138  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1139 
1140  uint64_t roffset = offset;
1141  if (offset)
1142  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1143  else {
1144  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1145  }
1146 
1147  *data = mydata;
1148  *data_len = mydata_len;
1149  *data_offset = roffset;
1150  } else {
1151  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1152  *iter == NULL ? "starting" : "continuing", offset);
1153  if (*iter == NULL) {
1154  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1155  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1156  SCLogDebug("*iter %p", *iter);
1157  }
1158  if (*iter == NULL) {
1159  SCLogDebug("no data");
1160  *data = NULL;
1161  *data_len = 0;
1162  *data_offset = 0;
1163  return 0;
1164  }
1165  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1166 
1167  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1168  SCLogDebug("mydata %p", mydata);
1169 
1170  if ((*iter)->offset < offset) {
1171  uint64_t delta = offset - (*iter)->offset;
1172  if (delta < mydata_len) {
1173  *data = mydata + delta;
1174  *data_len = mydata_len - delta;
1175  *data_offset = offset;
1176  } else {
1177  SCLogDebug("no data (yet)");
1178  *data = NULL;
1179  *data_len = 0;
1180  *data_offset = 0;
1181  }
1182 
1183  } else {
1184  *data = mydata;
1185  *data_len = mydata_len;
1186  *data_offset = (*iter)->offset;
1187  }
1188 
1189  *iter = SBB_RB_NEXT(*iter);
1190  SCLogDebug("*iter %p", *iter);
1191  }
1192  return 0;
1193 }
1194 
1195 /** \brief does the stream engine have data to inspect?
1196  *
1197  * Returns true if there is data to inspect. In IDS case this is
1198  * about ACK'd data in the packet's direction.
1199  *
1200  * In the IPS case this is about the packet itself.
1201  */
1203 {
1204  TcpStream *stream;
1205  if (PKT_IS_TOSERVER(p)) {
1206  stream = &ssn->client;
1207  } else {
1208  stream = &ssn->server;
1209  }
1210 
1211  if (RB_EMPTY(&stream->seg_tree)) {
1212  return false;
1213  }
1214 
1217  return false;
1218 
1219  if (StreamTcpInlineMode() == FALSE) {
1220  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1221  return false;
1222  }
1223  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1224  return true;
1225  }
1226  } else {
1227  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1228  return true;
1229  }
1230  }
1231  return false;
1232 }
1233 
1234 /** \brief update stream engine after detection
1235  *
1236  * Tasked with progressing the 'progress' for Raw reassembly.
1237  * 2 main scenario's:
1238  * 1. progress is != 0, so we use this
1239  * 2. progress is 0, meaning the detect engine didn't touch
1240  * raw at all. In this case we need to look into progressing
1241  * raw anyway.
1242  *
1243  * Additionally, this function is tasked with disabling raw
1244  * reassembly if the app-layer requested to disable it.
1245  */
1246 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1247 {
1248  TcpStream *stream;
1249  if (PKT_IS_TOSERVER(p)) {
1250  stream = &ssn->client;
1251  } else {
1252  stream = &ssn->server;
1253  }
1254 
1255  if (progress > STREAM_RAW_PROGRESS(stream)) {
1256  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1257  stream->raw_progress_rel += slide;
1259 
1260  /* if app is active and beyond raw, sync raw to app */
1261  } else if (progress == 0 &&
1262  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1264  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1265  {
1266  /* if trigger raw is set we sync the 2 trackers */
1268  {
1269  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1270  stream->raw_progress_rel += slide;
1272 
1273  /* otherwise mix in the tcp window */
1274  } else {
1275  uint64_t tcp_window = stream->window;
1276  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1277  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1278  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1279  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1280  stream->raw_progress_rel += slide;
1281  }
1282  }
1283  }
1284  /* app is dead */
1285  } else if (progress == 0) {
1286  uint64_t tcp_window = stream->window;
1287  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1288  if (tcp_window < stream_right_edge) {
1289  uint64_t new_raw = stream_right_edge - tcp_window;
1290  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1291  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1292  stream->raw_progress_rel += slide;
1293  }
1294  }
1296 
1297  } else {
1298  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1299  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1300  STREAM_RAW_PROGRESS(stream), stream->window);
1301  }
1302 
1303  /* if we were told to accept no more raw data, we can mark raw as
1304  * disabled now. */
1307  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1308  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1309  }
1310 
1311  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1312 }
1313 
1314 /** \internal
1315  * \brief get a buffer around the current packet and run the callback on it
1316  *
1317  * The inline/IPS scanning method takes the current payload and wraps it in
1318  * data from other segments.
1319  *
1320  * How much data is inspected is controlled by the available data, chunk_size
1321  * and the payload size of the packet.
1322  *
1323  * Large packets: if payload size is close to the chunk_size, where close is
1324  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1325  * used: payload_len + 33% of the chunk_size.
1326  * If the payload size if equal to or bigger than the chunk_size, we use
1327  * payload len + 33% of the chunk size.
1328  */
1329 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1330  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1331 {
1332  SCEnter();
1333  int r = 0;
1334 
1335  TcpStream *stream;
1336  if (PKT_IS_TOSERVER(p)) {
1337  stream = &ssn->client;
1338  } else {
1339  stream = &ssn->server;
1340  }
1341 
1342  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1344  {
1345  *progress_out = STREAM_RAW_PROGRESS(stream);
1346  return 0;
1347  }
1348 
1349  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1352  if (chunk_size <= p->payload_len) {
1353  chunk_size = p->payload_len + (chunk_size / 3);
1354  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1355  p->payload_len, chunk_size);
1356  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1357  chunk_size = p->payload_len + ((chunk_size / 3));
1358  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1359  p->payload_len, chunk_size);
1360  }
1361 
1362  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1363  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1364  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1365  packet_leftedge_abs, packet_rightedge_abs);
1366 
1367  const uint8_t *mydata = NULL;
1368  uint32_t mydata_len = 0;
1369  uint64_t mydata_offset = 0;
1370  /* simply return progress from the block we inspected. */
1371  bool return_progress = false;
1372 
1373  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1374  /* continues block */
1375  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1376  return_progress = true;
1377 
1378  } else {
1379  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1380  /* find our block */
1381  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1382  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1383  if (sbb) {
1384  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1385  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1386  mydata_offset = sbb->offset;
1387  }
1388  }
1389 
1390  /* this can only happen if the segment insert of our current 'p' failed */
1391  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1392  if ((mydata == NULL || mydata_len == 0) || /* no data */
1393  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1394  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1395  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1396  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1397  {
1398  /* no data, or data is incomplete or wrong: use packet data */
1399  mydata = p->payload;
1400  mydata_len = p->payload_len;
1401  mydata_offset = packet_leftedge_abs;
1402  //mydata_rightedge_abs = packet_rightedge_abs;
1403  } else {
1404  /* adjust buffer to match chunk_size */
1405  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1406  if (mydata_len > chunk_size) {
1407  uint32_t excess = mydata_len - chunk_size;
1408  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1409 
1410  if (mydata_rightedge_abs == packet_rightedge_abs) {
1411  mydata += excess;
1412  mydata_len -= excess;
1413  mydata_offset += excess;
1414  SCLogDebug("cutting front of the buffer with %u", excess);
1415  } else if (mydata_offset == packet_leftedge_abs) {
1416  mydata_len -= excess;
1417  SCLogDebug("cutting tail of the buffer with %u", excess);
1418  } else {
1419  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1420  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1421  SCLogDebug("before %u after %u", before, after);
1422 
1423  if (after >= (chunk_size - p->payload_len) / 2) {
1424  // more trailing data than we need
1425 
1426  if (before >= (chunk_size - p->payload_len) / 2) {
1427  // also more heading data, divide evenly
1428  before = after = (chunk_size - p->payload_len) / 2;
1429  } else {
1430  // heading data is less than requested, give the
1431  // rest to the trailing data
1432  after = (chunk_size - p->payload_len) - before;
1433  }
1434  } else {
1435  // less trailing data than requested
1436 
1437  if (before >= (chunk_size - p->payload_len) / 2) {
1438  before = (chunk_size - p->payload_len) - after;
1439  } else {
1440  // both smaller than their requested size
1441  }
1442  }
1443 
1444  /* adjust the buffer */
1445  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1446  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1447  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1448  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1449  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1450  mydata += skip;
1451  mydata_len -= (skip + cut);
1452  mydata_offset += skip;
1453  }
1454  }
1455  }
1456 
1457  /* run the callback */
1458  r = Callback(cb_data, mydata, mydata_len);
1459  BUG_ON(r < 0);
1460 
1461  if (return_progress) {
1462  *progress_out = (mydata_offset + mydata_len);
1463  } else {
1464  /* several blocks of data, so we need to be a bit more careful:
1465  * - if last_ack is beyond last progress, move progress forward to last_ack
1466  * - if our block matches or starts before last ack, return right edge of
1467  * our block.
1468  */
1469  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1470  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1471  uint32_t delta = stream->last_ack - stream->base_seq;
1472  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1473  /* get max absolute offset */
1474  last_ack_abs += delta;
1475  }
1476  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1477 
1478  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1479  if (mydata_offset > last_ack_abs) {
1480  /* gap between us and last ack, set progress to last ack */
1481  *progress_out = last_ack_abs;
1482  } else {
1483  *progress_out = (mydata_offset + mydata_len);
1484  }
1485  } else {
1486  *progress_out = STREAM_RAW_PROGRESS(stream);
1487  }
1488  }
1489  return r;
1490 }
1491 
1492 /** \brief access 'raw' reassembly data.
1493  *
1494  * Access data as tracked by 'raw' tracker. Data is made available to
1495  * callback that is passed to this function.
1496  *
1497  * In the case of IDS the callback may be run multiple times if data
1498  * contains gaps. It will then be run for each block of data that is
1499  * continuous.
1500  *
1501  * The callback should give on of 2 return values:
1502  * - 0 ok
1503  * - 1 done
1504  * The value 1 will break the loop if there is a block list that is
1505  * inspected.
1506  *
1507  * This function will return the 'progress' value that has been
1508  * consumed until now.
1509  *
1510  * \param ssn tcp session
1511  * \param stream tcp stream
1512  * \param Callback the function pointer to the callback function
1513  * \param cb_data callback data
1514  * \param[in] progress_in progress to work from
1515  * \param[out] progress_out absolute progress value of the data this
1516  * call handled.
1517  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1518  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1519  *
1520  * `respect_inspect_depth` is used to avoid useless inspection of too
1521  * much data.
1522  */
1523 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1524  StreamReassembleRawFunc Callback, void *cb_data,
1525  const uint64_t progress_in,
1526  uint64_t *progress_out, bool eof,
1527  bool respect_inspect_depth)
1528 {
1529  SCEnter();
1530  int r = 0;
1531 
1532  StreamingBufferBlock *iter = NULL;
1533  uint64_t progress = progress_in;
1534  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1535 
1536  /* if the app layer triggered a flush, and we're supposed to
1537  * use a minimal inspect depth, we actually take the app progress
1538  * as that is the right edge of the data. Then we take the window
1539  * of 'min_inspect_depth' before that. */
1540 
1541  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1542  respect_inspect_depth ? "true" : "false",
1543  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1544  stream->min_inspect_depth);
1545 
1546  if (respect_inspect_depth &&
1548  && stream->min_inspect_depth)
1549  {
1550  progress = STREAM_APP_PROGRESS(stream);
1551  if (stream->min_inspect_depth >= progress) {
1552  progress = 0;
1553  } else {
1554  progress -= stream->min_inspect_depth;
1555  }
1556 
1557  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1558 
1559  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1560  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1561  }
1562 
1563  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1564 
1565  /* get window of data that is acked */
1566  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1567  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1568  uint32_t delta = stream->last_ack - stream->base_seq;
1569  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1570  /* get max absolute offset */
1571  last_ack_abs += delta;
1572  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1573  }
1574 
1575  /* loop through available buffers. On no packet loss we'll have a single
1576  * iteration. On missing data we'll walk the blocks */
1577  while (1) {
1578  const uint8_t *mydata;
1579  uint32_t mydata_len;
1580  uint64_t mydata_offset = 0;
1581 
1582  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1583  if (mydata_len == 0) {
1584  SCLogDebug("no data");
1585  break;
1586  }
1587  //PrintRawDataFp(stdout, mydata, mydata_len);
1588 
1589  SCLogDebug("raw progress %"PRIu64, progress);
1590  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1591  stream, &stream->sb, mydata_len, progress);
1592 
1593  if (eof) {
1594  // inspect all remaining data, ack'd or not
1595  } else {
1596  if (last_ack_abs < progress) {
1597  SCLogDebug("nothing to do");
1598  goto end;
1599  }
1600 
1601  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1602  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1603 
1604  /* see if the buffer contains unack'd data as well */
1605  if (progress + mydata_len > last_ack_abs) {
1606  uint32_t check = mydata_len;
1607  mydata_len = last_ack_abs - progress;
1608  BUG_ON(check < mydata_len);
1609  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1610  "data is considered", mydata_len);
1611  }
1612 
1613  }
1614  if (mydata_len == 0)
1615  break;
1616 
1617  SCLogDebug("data %p len %u", mydata, mydata_len);
1618 
1619  /* we have data. */
1620  r = Callback(cb_data, mydata, mydata_len);
1621  BUG_ON(r < 0);
1622 
1623  if (mydata_offset == progress) {
1624  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1625  progress, mydata_len, progress_in + mydata_len);
1626 
1627  progress += mydata_len;
1628  SCLogDebug("raw progress now %"PRIu64, progress);
1629 
1630  /* data is beyond the progress we'd like, and before last ack. Gap. */
1631  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1632  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1633  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1634  progress = mydata_offset;
1635  SCLogDebug("raw progress now %"PRIu64, progress);
1636 
1637  } else {
1638  SCLogDebug("not increasing progress, data gap => mydata_offset "
1639  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1640  }
1641 
1642  if (iter == NULL || r == 1)
1643  break;
1644  }
1645 end:
1646  *progress_out = progress;
1647  return r;
1648 }
1649 
1651  StreamReassembleRawFunc Callback, void *cb_data,
1652  uint64_t *progress_out, bool respect_inspect_depth)
1653 {
1654  /* handle inline separately as the logic is very different */
1655  if (StreamTcpInlineMode() == TRUE) {
1656  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1657  }
1658 
1659  TcpStream *stream;
1660  if (PKT_IS_TOSERVER(p)) {
1661  stream = &ssn->client;
1662  } else {
1663  stream = &ssn->server;
1664  }
1665 
1667  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1668  {
1669  *progress_out = STREAM_RAW_PROGRESS(stream);
1670  return 0;
1671  }
1672 
1673  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1674  STREAM_RAW_PROGRESS(stream), progress_out,
1675  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1676 }
1677 
1679  StreamReassembleRawFunc Callback, void *cb_data,
1680  uint64_t progress_in,
1681  uint64_t *progress_out, bool eof)
1682 {
1683  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1684  return 0;
1685 
1686  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1687  progress_in, progress_out, eof, false);
1688 }
1689 
1690 /** \internal
1691  * \brief update app layer based on received ACK
1692  *
1693  * \retval r 0 on success, -1 on error
1694  */
1695 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1696  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1697 {
1698  SCEnter();
1699 
1700  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1701  SCReturnInt(-1);
1702 
1703  SCReturnInt(0);
1704 }
1705 
1707  TcpSession *ssn, TcpStream *stream,
1708  Packet *p, PacketQueue *pq)
1709 {
1710  SCEnter();
1711 
1712  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1713 
1714  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1715  ssn, stream, p, p->payload_len);
1716 
1717  /* we need to update the opposing stream in
1718  * StreamTcpReassembleHandleSegmentUpdateACK */
1719  TcpStream *opposing_stream = NULL;
1720  if (stream == &ssn->client) {
1721  opposing_stream = &ssn->server;
1722  } else {
1723  opposing_stream = &ssn->client;
1724  }
1725 
1726  /* default IDS: update opposing side (triggered by ACK) */
1728  /* inline and stream end and flow timeout packets trigger same dir handling */
1729  if (StreamTcpInlineMode()) {
1730  dir = UPDATE_DIR_PACKET;
1731  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1732  dir = UPDATE_DIR_PACKET;
1733  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1734  dir = UPDATE_DIR_PACKET;
1735  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1736  dir = UPDATE_DIR_PACKET;
1737  } else if (ssn->state == TCP_CLOSED) {
1738  dir = UPDATE_DIR_BOTH;
1739  }
1740 
1741  /* handle ack received */
1742  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1743  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1744  {
1745  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1746  SCReturnInt(-1);
1747  }
1748 
1749  /* if this segment contains data, insert it */
1750  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1751  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1752 
1753  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1754  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1755  SCReturnInt(-1);
1756  }
1757 
1758  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1759  p->flags |= PKT_STREAM_ADD;
1760  } else {
1761  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1762  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1763  ssn, stream, p->payload_len,
1764  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1765 
1766  }
1767 
1768  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1769  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1770  * was *just* set. In this case we trigger the AppLayer Truncate
1771  * logic, to inform the applayer no more data in this direction is
1772  * to be expected. */
1773  if ((stream->flags &
1776  {
1777  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1778  if (dir != UPDATE_DIR_PACKET) {
1779  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1780  "can trigger Truncate");
1781  dir = UPDATE_DIR_PACKET;
1782  }
1783  }
1784 
1785  /* in stream inline mode even if we have no data we call the reassembly
1786  * functions to handle EOF */
1787  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1788  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1789  StreamTcpInlineMode()?"true":"false",
1790  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1791  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1792  SCReturnInt(-1);
1793  }
1794  }
1795 
1796  SCReturnInt(0);
1797 }
1798 
1799 /**
1800  * \brief get a segment from the pool
1801  *
1802  * \retval seg Segment from the pool or NULL
1803  */
1805 {
1806  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1807  SCLogDebug("seg we return is %p", seg);
1808  if (seg == NULL) {
1809  /* Increment the counter to show that we are not able to serve the
1810  segment request due to memcap limit */
1812  } else {
1813  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1814  }
1815 
1816  return seg;
1817 }
1818 
1819 /**
1820  * \brief Trigger RAW stream reassembly
1821  *
1822  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1823  * reassembly from the applayer, for example upon completion of a
1824  * HTTP request.
1825  *
1826  * It sets a flag in the stream so that the next Raw call will return
1827  * the data.
1828  *
1829  * \param ssn TcpSession
1830  */
1832 {
1833 #ifdef DEBUG
1834  BUG_ON(ssn == NULL);
1835 #endif
1836 
1837  if (ssn != NULL) {
1838  if (direction == STREAM_TOSERVER) {
1840  } else {
1842  }
1843 
1844  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1845  }
1846 }
1847 
1848 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1849 {
1850 #ifdef DEBUG
1851  BUG_ON(ssn == NULL);
1852 #endif
1853 
1854  if (ssn != NULL) {
1855  if (direction == STREAM_TOSERVER) {
1856  ssn->client.min_inspect_depth = depth;
1857  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1858  } else {
1859  ssn->server.min_inspect_depth = depth;
1860  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1861  }
1862  }
1863 }
1864 
1865 #ifdef UNITTESTS
1866 /** unit tests and it's support functions below */
1867 
1868 #define SET_ISN(stream, setseq) \
1869  (stream)->isn = (setseq); \
1870  (stream)->base_seq = (setseq) + 1
1871 
1872 /** \brief The Function to create the packet with given payload, which is used
1873  * to test the reassembly of the engine.
1874  *
1875  * \param payload The variable used to store the payload contents of the
1876  * current packet.
1877  * \param value The value which current payload will have for this packet
1878  * \param payload_len The length of the filed payload for current packet.
1879  * \param len Length of the payload array
1880  */
1881 
1882 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1883  uint8_t payload_len, uint8_t len)
1884 {
1885  uint8_t i;
1886  for (i = 0; i < payload_len; i++)
1887  payload[i] = value;
1888  for (; i < len; i++)
1889  payload = NULL;
1890 }
1891 
1892 /** \brief The Function Checks the reassembled stream contents against predefined
1893  * stream contents according to OS policy used.
1894  *
1895  * \param stream_policy Predefined value of stream for different OS policies
1896  * \param stream Reassembled stream returned from the reassembly functions
1897  */
1898 
1899 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
1900 {
1901  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
1902  {
1903  //PrintRawDataFp(stdout, stream_policy, sp_size);
1904  return 0;
1905  }
1906  return 1;
1907 }
1908 
1909 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
1910 {
1911  if (StreamingBufferCompareRawData(&stream->sb,
1912  data, data_len) == 0)
1913  {
1914  SCReturnInt(0);
1915  }
1916  SCLogInfo("OK");
1917  PrintRawDataFp(stdout, data, data_len);
1918  return 1;
1919 }
1920 
1921 #define MISSED_START(isn) \
1922  TcpReassemblyThreadCtx *ra_ctx = NULL; \
1923  TcpSession ssn; \
1924  ThreadVars tv; \
1925  memset(&tv, 0, sizeof(tv)); \
1926  \
1927  StreamTcpUTInit(&ra_ctx); \
1928  \
1929  StreamTcpUTSetupSession(&ssn); \
1930  StreamTcpUTSetupStream(&ssn.server, (isn)); \
1931  StreamTcpUTSetupStream(&ssn.client, (isn)); \
1932  \
1933  TcpStream *stream = &ssn.client;
1934 
1935 #define MISSED_END \
1936  StreamTcpUTClearSession(&ssn); \
1937  StreamTcpUTDeinit(ra_ctx); \
1938  PASS
1939 
1940 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
1941  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
1942  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
1943 
1944 /**
1945  * \test Test the handling of packets missed by both IDS and the end host.
1946  * The packet is missed in the starting of the stream.
1947  *
1948  * \retval On success it returns 1 and on failure 0.
1949  */
1950 
1951 static int StreamTcpReassembleTest25 (void)
1952 {
1953  MISSED_START(6);
1954  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
1955  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
1956  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
1957  MISSED_END;
1958 }
1959 
1960 /**
1961  * \test Test the handling of packets missed by both IDS and the end host.
1962  * The packet is missed in the middle of the stream.
1963  *
1964  * \retval On success it returns 1 and on failure 0.
1965  */
1966 
1967 static int StreamTcpReassembleTest26 (void)
1968 {
1969  MISSED_START(9);
1970  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1971  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
1972  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
1973  MISSED_END;
1974 }
1975 
1976 /**
1977  * \test Test the handling of packets missed by both IDS and the end host.
1978  * The packet is missed in the end of the stream.
1979  *
1980  * \retval On success it returns 1 and on failure 0.
1981  */
1982 
1983 static int StreamTcpReassembleTest27 (void)
1984 {
1985  MISSED_START(9);
1986  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1987  MISSED_STEP(13, "BB", 2, "AAABB", 5);
1988  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
1989  MISSED_END;
1990 }
1991 
1992 /**
1993  * \test Test the handling of packets missed by IDS, but the end host has
1994  * received it and send the acknowledgment of it. The packet is missed
1995  * in the starting of the stream.
1996  *
1997  * \retval On success it returns 1 and on failure 0.
1998  */
1999 
2000 static int StreamTcpReassembleTest28 (void)
2001 {
2002  MISSED_START(6);
2003  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2004  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2005  ssn.state = TCP_TIME_WAIT;
2006  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2007  MISSED_END;
2008 }
2009 
2010 /**
2011  * \test Test the handling of packets missed by IDS, but the end host has
2012  * received it and send the acknowledgment of it. The packet is missed
2013  * in the middle of the stream.
2014  *
2015  * \retval On success it returns 1 and on failure 0.
2016  */
2017 
2018 static int StreamTcpReassembleTest29 (void)
2019 {
2020  MISSED_START(9);
2021  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2022  ssn.state = TCP_TIME_WAIT;
2023  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2024  MISSED_END;
2025 }
2026 
2027 static int StreamTcpReassembleTest33(void)
2028 {
2029  TcpSession ssn;
2030  Packet *p = PacketGetFromAlloc();
2031  FAIL_IF(unlikely(p == NULL));
2032  Flow f;
2033  TCPHdr tcph;
2034  TcpReassemblyThreadCtx *ra_ctx = NULL;
2036  uint8_t packet[1460] = "";
2037 
2038  StreamTcpUTInit(&ra_ctx);
2040 
2041  PacketQueue pq;
2042  memset(&pq,0,sizeof(PacketQueue));
2043  memset(&f, 0, sizeof (Flow));
2044  memset(&tcph, 0, sizeof (TCPHdr));
2045  ThreadVars tv;
2046  memset(&tv, 0, sizeof (ThreadVars));
2047  FLOW_INITIALIZE(&f);
2048  f.protoctx = &ssn;
2049  f.proto = IPPROTO_TCP;
2050  p->src.family = AF_INET;
2051  p->dst.family = AF_INET;
2052  p->proto = IPPROTO_TCP;
2053  p->flow = &f;
2054  tcph.th_win = 5480;
2055  tcph.th_flags = TH_PUSH | TH_ACK;
2056  p->tcph = &tcph;
2058  p->payload = packet;
2059 
2060  p->tcph->th_seq = htonl(10);
2061  p->tcph->th_ack = htonl(31);
2062  p->payload_len = 10;
2063 
2064  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2065 
2066  p->tcph->th_seq = htonl(20);
2067  p->tcph->th_ack = htonl(31);
2068  p->payload_len = 10;
2069 
2070  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2071 
2072  p->tcph->th_seq = htonl(40);
2073  p->tcph->th_ack = htonl(31);
2074  p->payload_len = 10;
2075 
2076  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2077 
2078  p->tcph->th_seq = htonl(5);
2079  p->tcph->th_ack = htonl(31);
2080  p->payload_len = 30;
2081 
2082  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2083 
2085  StreamTcpUTDeinit(ra_ctx);
2086  SCFree(p);
2087  PASS;
2088 }
2089 
2090 static int StreamTcpReassembleTest34(void)
2091 {
2092  TcpSession ssn;
2093  Packet *p = PacketGetFromAlloc();
2094  FAIL_IF(unlikely(p == NULL));
2095  Flow f;
2096  TCPHdr tcph;
2097  TcpReassemblyThreadCtx *ra_ctx = NULL;
2099  uint8_t packet[1460] = "";
2100 
2101  StreamTcpUTInit(&ra_ctx);
2103  PacketQueue pq;
2104  memset(&pq,0,sizeof(PacketQueue));
2105  memset(&f, 0, sizeof (Flow));
2106  memset(&tcph, 0, sizeof (TCPHdr));
2107  ThreadVars tv;
2108  memset(&tv, 0, sizeof (ThreadVars));
2109  FLOW_INITIALIZE(&f);
2110  f.protoctx = &ssn;
2111  f.proto = IPPROTO_TCP;
2112  p->src.family = AF_INET;
2113  p->dst.family = AF_INET;
2114  p->proto = IPPROTO_TCP;
2115  p->flow = &f;
2116  tcph.th_win = 5480;
2117  tcph.th_flags = TH_PUSH | TH_ACK;
2118  p->tcph = &tcph;
2120  p->payload = packet;
2121  SET_ISN(&ssn.client, 857961230);
2122 
2123  p->tcph->th_seq = htonl(857961230);
2124  p->tcph->th_ack = htonl(31);
2125  p->payload_len = 304;
2126 
2127  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2128 
2129  p->tcph->th_seq = htonl(857961534);
2130  p->tcph->th_ack = htonl(31);
2131  p->payload_len = 1460;
2132 
2133  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2134 
2135  p->tcph->th_seq = htonl(857963582);
2136  p->tcph->th_ack = htonl(31);
2137  p->payload_len = 1460;
2138 
2139  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2140 
2141  p->tcph->th_seq = htonl(857960946);
2142  p->tcph->th_ack = htonl(31);
2143  p->payload_len = 1460;
2144 
2145  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2146 
2148  StreamTcpUTDeinit(ra_ctx);
2149  SCFree(p);
2150  PASS;
2151 }
2152 
2153 /** \test Test the bug 76 condition */
2154 static int StreamTcpReassembleTest37(void)
2155 {
2156  TcpSession ssn;
2157  Flow f;
2158  TCPHdr tcph;
2159  TcpReassemblyThreadCtx *ra_ctx = NULL;
2160  uint8_t packet[1460] = "";
2161  PacketQueue pq;
2162  ThreadVars tv;
2163  memset(&tv, 0, sizeof (ThreadVars));
2164 
2165  Packet *p = PacketGetFromAlloc();
2166  FAIL_IF(unlikely(p == NULL));
2167 
2168  StreamTcpUTInit(&ra_ctx);
2170  memset(&pq,0,sizeof(PacketQueue));
2171  memset(&f, 0, sizeof (Flow));
2172  memset(&tcph, 0, sizeof (TCPHdr));
2173  memset(&tv, 0, sizeof (ThreadVars));
2174 
2175  FLOW_INITIALIZE(&f);
2176  f.protoctx = &ssn;
2177  f.proto = IPPROTO_TCP;
2178  p->src.family = AF_INET;
2179  p->dst.family = AF_INET;
2180  p->proto = IPPROTO_TCP;
2181  p->flow = &f;
2182  tcph.th_win = 5480;
2183  tcph.th_flags = TH_PUSH | TH_ACK;
2184  p->tcph = &tcph;
2186  p->payload = packet;
2188 
2189  p->tcph->th_seq = htonl(3061088537UL);
2190  p->tcph->th_ack = htonl(1729548549UL);
2191  p->payload_len = 1391;
2192  ssn.client.last_ack = 3061091137UL;
2193  SET_ISN(&ssn.client, 3061091309UL);
2194 
2195  /* pre base_seq, so should be rejected */
2196  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2197 
2198  p->tcph->th_seq = htonl(3061089928UL);
2199  p->tcph->th_ack = htonl(1729548549UL);
2200  p->payload_len = 1391;
2201  ssn.client.last_ack = 3061091137UL;
2202 
2203  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2204 
2205  p->tcph->th_seq = htonl(3061091319UL);
2206  p->tcph->th_ack = htonl(1729548549UL);
2207  p->payload_len = 1391;
2208  ssn.client.last_ack = 3061091137UL;
2209 
2210  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2211 
2213  StreamTcpUTDeinit(ra_ctx);
2214  SCFree(p);
2215  PASS;
2216 }
2217 
2218 /**
2219  * \test Test to make sure that we don't return the segments until the app
2220  * layer proto has been detected and after that remove the processed
2221  * segments.
2222  *
2223  * \retval On success it returns 1 and on failure 0.
2224  */
2225 
2226 static int StreamTcpReassembleTest39 (void)
2227 {
2228  Packet *p = PacketGetFromAlloc();
2229  FAIL_IF(unlikely(p == NULL));
2230  Flow f;
2231  ThreadVars tv;
2232  StreamTcpThread stt;
2233  TCPHdr tcph;
2234  PacketQueue pq;
2235  memset(&pq,0,sizeof(PacketQueue));
2236  memset (&f, 0, sizeof(Flow));
2237  memset(&tv, 0, sizeof (ThreadVars));
2238  memset(&stt, 0, sizeof (stt));
2239  memset(&tcph, 0, sizeof (TCPHdr));
2240 
2241  FLOW_INITIALIZE(&f);
2242  f.flags = FLOW_IPV4;
2243  f.proto = IPPROTO_TCP;
2244  p->flow = &f;
2245  p->tcph = &tcph;
2246 
2247  StreamTcpUTInit(&stt.ra_ctx);
2248 
2249  /* handshake */
2250  tcph.th_win = htons(5480);
2251  tcph.th_flags = TH_SYN;
2253  p->payload_len = 0;
2254  p->payload = NULL;
2255  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2256 
2257  TcpSession *ssn = (TcpSession *)f.protoctx;
2258  FAIL_IF_NULL(ssn);
2259 
2270  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2271  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2272  FAIL_IF(ssn->data_first_seen_dir != 0);
2273 
2274  /* handshake */
2275  p->tcph->th_ack = htonl(1);
2276  p->tcph->th_flags = TH_SYN | TH_ACK;
2278  p->payload_len = 0;
2279  p->payload = NULL;
2280  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2291  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2292  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2293  FAIL_IF(ssn->data_first_seen_dir != 0);
2294 
2295  /* handshake */
2296  p->tcph->th_ack = htonl(1);
2297  p->tcph->th_seq = htonl(1);
2298  p->tcph->th_flags = TH_ACK;
2300  p->payload_len = 0;
2301  p->payload = NULL;
2302  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2313  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2314  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2315  FAIL_IF(ssn->data_first_seen_dir != 0);
2316 
2317  /* partial request */
2318  uint8_t request1[] = { 0x47, 0x45, };
2319  p->tcph->th_ack = htonl(1);
2320  p->tcph->th_seq = htonl(1);
2321  p->tcph->th_flags = TH_PUSH | TH_ACK;
2323  p->payload_len = sizeof(request1);
2324  p->payload = request1;
2325  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2336  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2337  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2338  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2340 
2341  /* response ack against partial request */
2342  p->tcph->th_ack = htonl(3);
2343  p->tcph->th_seq = htonl(1);
2344  p->tcph->th_flags = TH_ACK;
2346  p->payload_len = 0;
2347  p->payload = NULL;
2348  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2359  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2360  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2361  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2363 
2364  /* complete partial request */
2365  uint8_t request2[] = {
2366  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2367  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2368  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2369  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2370  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2371  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2372  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2373  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2374  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2375  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2376  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2377  p->tcph->th_ack = htonl(1);
2378  p->tcph->th_seq = htonl(3);
2379  p->tcph->th_flags = TH_PUSH | TH_ACK;
2381  p->payload_len = sizeof(request2);
2382  p->payload = request2;
2383  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2394  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2395  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2396  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2397  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2399 
2400  /* response - request ack */
2401  uint8_t response[] = {
2402  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2403  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2404  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2405  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2406  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2407  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2408  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2409  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2410  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2411  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2412  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2413  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2414  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2415  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2416  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2417  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2418  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2419  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2420  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2421  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2422  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2423  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2424  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2425  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2426  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2427  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2428  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2429  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2430  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2431  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2432  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2433  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2434  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2435  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2436  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2437  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2438  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2439  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2440  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2441  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2442  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2443  p->tcph->th_ack = htonl(88);
2444  p->tcph->th_seq = htonl(1);
2445  p->tcph->th_flags = TH_PUSH | TH_ACK;
2447  p->payload_len = sizeof(response);
2448  p->payload = response;
2449 
2450  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2462  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2463  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2464  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2465  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2466 
2467  /* response ack from request */
2468  p->tcph->th_ack = htonl(328);
2469  p->tcph->th_seq = htonl(88);
2470  p->tcph->th_flags = TH_ACK;
2472  p->payload_len = 0;
2473  p->payload = NULL;
2474  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2486  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2487  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2488  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2489  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2490  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2491 
2492  /* response - acking */
2493  p->tcph->th_ack = htonl(88);
2494  p->tcph->th_seq = htonl(328);
2495  p->tcph->th_flags = TH_PUSH | TH_ACK;
2497  p->payload_len = 0;
2498  p->payload = NULL;
2499  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2511  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2512  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2513  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2514  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2515  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2516 
2517  /* response ack from request */
2518  p->tcph->th_ack = htonl(328);
2519  p->tcph->th_seq = htonl(88);
2520  p->tcph->th_flags = TH_ACK;
2522  p->payload_len = 0;
2523  p->payload = NULL;
2524  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2536  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2537  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2538  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2539  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2540 
2541  /* response - acking the request again*/
2542  p->tcph->th_ack = htonl(88);
2543  p->tcph->th_seq = htonl(328);
2544  p->tcph->th_flags = TH_PUSH | TH_ACK;
2546  p->payload_len = 0;
2547  p->payload = NULL;
2548  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2560  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2561  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2562  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2563  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2564 
2565  /*** New Request ***/
2566 
2567  /* partial request */
2568  p->tcph->th_ack = htonl(328);
2569  p->tcph->th_seq = htonl(88);
2570  p->tcph->th_flags = TH_PUSH | TH_ACK;
2572  p->payload_len = sizeof(request1);
2573  p->payload = request1;
2574  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2586  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2587  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2588  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2589  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2590  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2591 
2592  /* response ack against partial request */
2593  p->tcph->th_ack = htonl(90);
2594  p->tcph->th_seq = htonl(328);
2595  p->tcph->th_flags = TH_ACK;
2597  p->payload_len = 0;
2598  p->payload = NULL;
2599  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2600  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2612  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2613  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2614  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2615  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2616  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2617 
2618  /* complete request */
2619  p->tcph->th_ack = htonl(328);
2620  p->tcph->th_seq = htonl(90);
2621  p->tcph->th_flags = TH_PUSH | TH_ACK;
2623  p->payload_len = sizeof(request2);
2624  p->payload = request2;
2625  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2637  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2638  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2639  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2640  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2641  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2642  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2643 
2644  /* response ack against second partial request */
2645  p->tcph->th_ack = htonl(175);
2646  p->tcph->th_seq = htonl(328);
2647  p->tcph->th_flags = TH_ACK;
2649  p->payload_len = 0;
2650  p->payload = NULL;
2651 
2652  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2664  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2665  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2666  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2667  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2668  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2669  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2670 
2671  /* response acking a request */
2672  p->tcph->th_ack = htonl(175);
2673  p->tcph->th_seq = htonl(328);
2674  p->tcph->th_flags = TH_ACK;
2676  p->payload_len = 0;
2677  p->payload = NULL;
2678  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2684 
2687 
2688  /* request acking a response */
2689  p->tcph->th_ack = htonl(328);
2690  p->tcph->th_seq = htonl(175);
2691  p->tcph->th_flags = TH_ACK;
2693  p->payload_len = 0;
2694  p->payload = NULL;
2695  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2696 
2697  StreamTcpSessionClear(ssn);
2699  SCFree(p);
2700  PASS;
2701 }
2702 
2703 /**
2704  * \test Test to make sure that we sent all the segments from the initial
2705  * segments to app layer until we have detected the app layer proto.
2706  *
2707  * \retval On success it returns 1 and on failure 0.
2708  */
2709 
2710 static int StreamTcpReassembleTest40 (void)
2711 {
2712  Packet *p = PacketGetFromAlloc();
2713  FAIL_IF_NULL(p);
2714  Flow *f = NULL;
2715  TCPHdr tcph;
2716  TcpSession ssn;
2717  PacketQueue pq;
2718  memset(&pq,0,sizeof(PacketQueue));
2719  memset(&tcph, 0, sizeof (TCPHdr));
2720  ThreadVars tv;
2721  memset(&tv, 0, sizeof (ThreadVars));
2722 
2725 
2727  FAIL_IF_NULL(ra_ctx);
2728 
2729  uint8_t httpbuf1[] = "P";
2730  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2731  uint8_t httpbuf3[] = "O";
2732  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2733  uint8_t httpbuf4[] = "S";
2734  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2735  uint8_t httpbuf5[] = "T \r\n";
2736  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2737 
2738  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2739  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2740 
2741  SET_ISN(&ssn.server, 9);
2742  ssn.server.last_ack = 10;
2743  SET_ISN(&ssn.client, 9);
2744  ssn.client.isn = 9;
2745 
2746  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2747  FAIL_IF_NULL(f);
2748  f->protoctx = &ssn;
2749  f->proto = IPPROTO_TCP;
2750  p->flow = f;
2751 
2752  tcph.th_win = htons(5480);
2753  tcph.th_seq = htonl(10);
2754  tcph.th_ack = htonl(10);
2755  tcph.th_flags = TH_ACK|TH_PUSH;
2756  p->tcph = &tcph;
2758  p->payload = httpbuf1;
2759  p->payload_len = httplen1;
2760  ssn.state = TCP_ESTABLISHED;
2761  TcpStream *s = &ssn.client;
2762  SCLogDebug("1 -- start");
2763  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2764 
2766  p->payload = httpbuf2;
2767  p->payload_len = httplen2;
2768  tcph.th_seq = htonl(10);
2769  tcph.th_ack = htonl(11);
2770  s = &ssn.server;
2771  ssn.server.last_ack = 11;
2772  SCLogDebug("2 -- start");
2773  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2774 
2776  p->payload = httpbuf3;
2777  p->payload_len = httplen3;
2778  tcph.th_seq = htonl(11);
2779  tcph.th_ack = htonl(55);
2780  s = &ssn.client;
2781  ssn.client.last_ack = 55;
2782  SCLogDebug("3 -- start");
2783  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2784 
2786  p->payload = httpbuf2;
2787  p->payload_len = httplen2;
2788  tcph.th_seq = htonl(55);
2789  tcph.th_ack = htonl(12);
2790  s = &ssn.server;
2791  ssn.server.last_ack = 12;
2792  SCLogDebug("4 -- start");
2793  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2794 
2795  /* check is have the segment in the list and flagged or not */
2796  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2797  FAIL_IF_NULL(seg);
2798  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2799 
2801  p->payload = httpbuf4;
2802  p->payload_len = httplen4;
2803  tcph.th_seq = htonl(12);
2804  tcph.th_ack = htonl(100);
2805  s = &ssn.client;
2806  ssn.client.last_ack = 100;
2807  SCLogDebug("5 -- start");
2808  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2809 
2811  p->payload = httpbuf2;
2812  p->payload_len = httplen2;
2813  tcph.th_seq = htonl(100);
2814  tcph.th_ack = htonl(13);
2815  s = &ssn.server;
2816  ssn.server.last_ack = 13;
2817  SCLogDebug("6 -- start");
2818  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2819 
2821  p->payload = httpbuf5;
2822  p->payload_len = httplen5;
2823  tcph.th_seq = htonl(13);
2824  tcph.th_ack = htonl(145);
2825  s = &ssn.client;
2826  ssn.client.last_ack = 145;
2827  SCLogDebug("7 -- start");
2828  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2829 
2831  p->payload = httpbuf2;
2832  p->payload_len = httplen2;
2833  tcph.th_seq = htonl(145);
2834  tcph.th_ack = htonl(16);
2835  s = &ssn.server;
2836  ssn.server.last_ack = 16;
2837  SCLogDebug("8 -- start");
2838  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2839  FAIL_IF(f->alproto != ALPROTO_HTTP);
2840 
2844  SCFree(p);
2845  UTHFreeFlow(f);
2846  PASS;
2847 }
2848 
2849 /** \test Test the memcap incrementing/decrementing and memcap check */
2850 static int StreamTcpReassembleTest44(void)
2851 {
2853  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2855  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2857  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2859  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2861  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2862  PASS;
2863 }
2864 
2865 /**
2866  * \test Test to make sure that reassembly_depth is enforced.
2867  *
2868  * \retval On success it returns 1 and on failure 0.
2869  */
2870 
2871 static int StreamTcpReassembleTest45 (void)
2872 {
2873  TcpReassemblyThreadCtx *ra_ctx = NULL;
2874  TcpSession ssn;
2875  ThreadVars tv;
2876  memset(&tv, 0, sizeof(tv));
2877  uint8_t payload[100] = {0};
2878  uint16_t payload_size = 100;
2879 
2880  StreamTcpUTInit(&ra_ctx);
2882 
2884  ssn.reassembly_depth = 100;
2885  StreamTcpUTSetupStream(&ssn.server, 100);
2886  StreamTcpUTSetupStream(&ssn.client, 100);
2887 
2888  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2889  FAIL_IF(r != 0);
2891 
2892  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2893  FAIL_IF(r != 0);
2895 
2899  StreamTcpUTDeinit(ra_ctx);
2900  PASS;
2901 }
2902 
2903 /**
2904  * \test Test the unlimited config value of reassembly depth.
2905  *
2906  * \retval On success it returns 1 and on failure 0.
2907  */
2908 
2909 static int StreamTcpReassembleTest46 (void)
2910 {
2911  int result = 0;
2912  TcpReassemblyThreadCtx *ra_ctx = NULL;
2913  TcpSession ssn;
2914  ThreadVars tv;
2915  memset(&tv, 0, sizeof(tv));
2916  uint8_t payload[100] = {0};
2917  uint16_t payload_size = 100;
2918 
2919  StreamTcpUTInit(&ra_ctx);
2921 
2923  StreamTcpUTSetupStream(&ssn.server, 100);
2924  StreamTcpUTSetupStream(&ssn.client, 100);
2925 
2926  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2927  if (r != 0)
2928  goto end;
2930  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2931  goto end;
2932  }
2933 
2934  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2935  if (r != 0)
2936  goto end;
2938  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2939  goto end;
2940  }
2941 
2942  result = 1;
2943 end:
2947  StreamTcpUTDeinit(ra_ctx);
2948  return result;
2949 }
2950 
2951 /**
2952  * \test Test to make sure we detect the sequence wrap around and continue
2953  * stream reassembly properly.
2954  *
2955  * \retval On success it returns 1 and on failure 0.
2956  */
2957 
2958 static int StreamTcpReassembleTest47 (void)
2959 {
2960  Packet *p = PacketGetFromAlloc();
2961  FAIL_IF(unlikely(p == NULL));
2962  Flow *f = NULL;
2963  TCPHdr tcph;
2964  TcpSession ssn;
2965  ThreadVars tv;
2966  PacketQueue pq;
2967  memset(&pq,0,sizeof(PacketQueue));
2968  memset(&tcph, 0, sizeof (TCPHdr));
2969  memset(&tv, 0, sizeof (ThreadVars));
2973 
2974  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
2975  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2976 
2977  SET_ISN(&ssn.server, 572799781UL);
2978  ssn.server.last_ack = 572799782UL;
2979 
2980  SET_ISN(&ssn.client, 4294967289UL);
2981  ssn.client.last_ack = 21;
2982 
2983  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2984  FAIL_IF(f == NULL);
2985  f->protoctx = &ssn;
2986  f->proto = IPPROTO_TCP;
2987  p->flow = f;
2988 
2989  tcph.th_win = htons(5480);
2990  ssn.state = TCP_ESTABLISHED;
2991  TcpStream *s = NULL;
2992  uint8_t cnt = 0;
2993 
2994  for (cnt=0; cnt < httplen1; cnt++) {
2995  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
2996  tcph.th_ack = htonl(572799782UL);
2997  tcph.th_flags = TH_ACK|TH_PUSH;
2998  p->tcph = &tcph;
3000  p->payload = &httpbuf1[cnt];
3001  p->payload_len = 1;
3002  s = &ssn.client;
3003 
3004  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3005 
3007  p->payload = NULL;
3008  p->payload_len = 0;
3009  tcph.th_seq = htonl(572799782UL);
3010  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3011  tcph.th_flags = TH_ACK;
3012  p->tcph = &tcph;
3013  s = &ssn.server;
3014 
3015  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3016  }
3017 
3018  FAIL_IF(f->alproto != ALPROTO_HTTP);
3019 
3023  SCFree(p);
3024  UTHFreeFlow(f);
3025  PASS;
3026 }
3027 
3028 /** \test 3 in order segments in inline reassembly */
3029 static int StreamTcpReassembleInlineTest01(void)
3030 {
3031  int ret = 0;
3032  TcpReassemblyThreadCtx *ra_ctx = NULL;
3033  ThreadVars tv;
3034  TcpSession ssn;
3035  Flow f;
3036 
3037  memset(&tv, 0x00, sizeof(tv));
3038 
3039  StreamTcpUTInit(&ra_ctx);
3042  StreamTcpUTSetupStream(&ssn.client, 1);
3043  FLOW_INITIALIZE(&f);
3044 
3045  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3046  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3047  if (p == NULL) {
3048  printf("couldn't get a packet: ");
3049  goto end;
3050  }
3051  p->tcph->th_seq = htonl(12);
3052  p->flow = &f;
3053 
3054  FLOWLOCK_WRLOCK(&f);
3055  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3056  printf("failed to add segment 1: ");
3057  goto end;
3058  }
3059  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3060  printf("failed to add segment 2: ");
3061  goto end;
3062  }
3063  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3064  printf("failed to add segment 3: ");
3065  goto end;
3066  }
3067  ssn.client.next_seq = 17;
3068  ret = 1;
3069 end:
3070  FLOWLOCK_UNLOCK(&f);
3071  FLOW_DESTROY(&f);
3072  UTHFreePacket(p);
3074  StreamTcpUTDeinit(ra_ctx);
3075  return ret;
3076 }
3077 
3078 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3079  * test the sliding window reassembly.
3080  */
3081 static int StreamTcpReassembleInlineTest02(void)
3082 {
3083  int ret = 0;
3084  TcpReassemblyThreadCtx *ra_ctx = NULL;
3085  ThreadVars tv;
3086  TcpSession ssn;
3087  Flow f;
3088 
3089  memset(&tv, 0x00, sizeof(tv));
3090 
3091  StreamTcpUTInit(&ra_ctx);
3094  StreamTcpUTSetupStream(&ssn.client, 1);
3095  FLOW_INITIALIZE(&f);
3096 
3097  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3098  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3099  if (p == NULL) {
3100  printf("couldn't get a packet: ");
3101  goto end;
3102  }
3103  p->tcph->th_seq = htonl(12);
3104  p->flow = &f;
3105 
3106  FLOWLOCK_WRLOCK(&f);
3107  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3108  printf("failed to add segment 1: ");
3109  goto end;
3110  }
3111  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3112  printf("failed to add segment 2: ");
3113  goto end;
3114  }
3115  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3116  printf("failed to add segment 3: ");
3117  goto end;
3118  }
3119  ssn.client.next_seq = 17;
3120  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3121  printf("failed to add segment 4: ");
3122  goto end;
3123  }
3124  ssn.client.next_seq = 22;
3125  ret = 1;
3126 end:
3127  FLOWLOCK_UNLOCK(&f);
3128  FLOW_DESTROY(&f);
3129  UTHFreePacket(p);
3131  StreamTcpUTDeinit(ra_ctx);
3132  return ret;
3133 }
3134 
3135 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3136  * test the sliding window reassembly with a small window size so that we
3137  * cutting off at the start (left edge)
3138  */
3139 static int StreamTcpReassembleInlineTest03(void)
3140 {
3141  int ret = 0;
3142  TcpReassemblyThreadCtx *ra_ctx = NULL;
3143  ThreadVars tv;
3144  TcpSession ssn;
3145  Flow f;
3146 
3147  memset(&tv, 0x00, sizeof(tv));
3148 
3149  StreamTcpUTInit(&ra_ctx);
3152  StreamTcpUTSetupStream(&ssn.client, 1);
3153  FLOW_INITIALIZE(&f);
3154 
3156 
3157  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3158  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3159  if (p == NULL) {
3160  printf("couldn't get a packet: ");
3161  goto end;
3162  }
3163  p->tcph->th_seq = htonl(12);
3164  p->flow = &f;
3166 
3167  FLOWLOCK_WRLOCK(&f);
3168  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3169  printf("failed to add segment 1: ");
3170  goto end;
3171  }
3172  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3173  printf("failed to add segment 2: ");
3174  goto end;
3175  }
3176  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3177  printf("failed to add segment 3: ");
3178  goto end;
3179  }
3180  ssn.client.next_seq = 17;
3181  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3182  printf("failed to add segment 4: ");
3183  goto end;
3184  }
3185  ssn.client.next_seq = 22;
3186 
3187  p->tcph->th_seq = htonl(17);
3188  ret = 1;
3189 end:
3190  FLOWLOCK_UNLOCK(&f);
3191  FLOW_DESTROY(&f);
3192  UTHFreePacket(p);
3194  StreamTcpUTDeinit(ra_ctx);
3195  return ret;
3196 }
3197 
3198 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3199  * test the sliding window reassembly with a small window size so that we
3200  * cutting off at the start (left edge) with small packet overlap.
3201  */
3202 static int StreamTcpReassembleInlineTest04(void)
3203 {
3204  int ret = 0;
3205  TcpReassemblyThreadCtx *ra_ctx = NULL;
3206  ThreadVars tv;
3207  TcpSession ssn;
3208  Flow f;
3209 
3210  memset(&tv, 0x00, sizeof(tv));
3211 
3212  StreamTcpUTInit(&ra_ctx);
3215  StreamTcpUTSetupStream(&ssn.client, 1);
3216  FLOW_INITIALIZE(&f);
3217 
3219 
3220  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3221  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3222  if (p == NULL) {
3223  printf("couldn't get a packet: ");
3224  goto end;
3225  }
3226  p->tcph->th_seq = htonl(12);
3227  p->flow = &f;
3229 
3230  FLOWLOCK_WRLOCK(&f);
3231  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3232  printf("failed to add segment 1: ");
3233  goto end;
3234  }
3235  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3236  printf("failed to add segment 2: ");
3237  goto end;
3238  }
3239  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3240  printf("failed to add segment 3: ");
3241  goto end;
3242  }
3243  ssn.client.next_seq = 17;
3244  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3245  printf("failed to add segment 4: ");
3246  goto end;
3247  }
3248  ssn.client.next_seq = 22;
3249 
3250  p->tcph->th_seq = htonl(17);
3251  ret = 1;
3252 end:
3253  FLOWLOCK_UNLOCK(&f);
3254  FLOW_DESTROY(&f);
3255  UTHFreePacket(p);
3257  StreamTcpUTDeinit(ra_ctx);
3258  return ret;
3259 }
3260 
3261 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3262  * test the sliding window reassembly with a small window size so that we
3263  * cutting off at the start (left edge). Test if the first segment is
3264  * removed from the list.
3265  */
3266 static int StreamTcpReassembleInlineTest08(void)
3267 {
3268  TcpReassemblyThreadCtx *ra_ctx = NULL;
3269  ThreadVars tv;
3270  memset(&tv, 0x00, sizeof(tv));
3271  TcpSession ssn;
3272  Flow f;
3273  StreamTcpUTInit(&ra_ctx);
3276  StreamTcpUTSetupStream(&ssn.client, 1);
3277  FLOW_INITIALIZE(&f);
3278 
3281  f.protoctx = &ssn;
3282 
3283  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3284  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3285  FAIL_IF(p == NULL);
3286  p->tcph->th_seq = htonl(12);
3287  p->flow = &f;
3289 
3290  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3291  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3292  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3293  ssn.client.next_seq = 17;
3294  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3295  ssn.client.next_seq = 22;
3296  p->tcph->th_seq = htonl(17);
3298 
3299  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3300  FAIL_IF_NULL(seg);
3301  FAIL_IF_NOT(seg->seq == 2);
3302 
3303  FLOW_DESTROY(&f);
3304  UTHFreePacket(p);
3306  StreamTcpUTDeinit(ra_ctx);
3307  PASS;
3308 }
3309 
3310 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3311  * test the sliding window reassembly with a small window size so that we
3312  * cutting off at the start (left edge). Test if the first segment is
3313  * removed from the list.
3314  */
3315 static int StreamTcpReassembleInlineTest09(void)
3316 {
3317  int ret = 0;
3318  TcpReassemblyThreadCtx *ra_ctx = NULL;
3319  ThreadVars tv;
3320  TcpSession ssn;
3321  Flow f;
3322 
3323  memset(&tv, 0x00, sizeof(tv));
3324 
3325  StreamTcpUTInit(&ra_ctx);
3328  StreamTcpUTSetupStream(&ssn.client, 1);
3329  FLOW_INITIALIZE(&f);
3330 
3333 
3334  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3335  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3336  if (p == NULL) {
3337  printf("couldn't get a packet: ");
3338  goto end;
3339  }
3340  p->tcph->th_seq = htonl(17);
3341  p->flow = &f;
3343 
3344  FLOWLOCK_WRLOCK(&f);
3345  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3346  printf("failed to add segment 1: ");
3347  goto end;
3348  }
3349  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3350  printf("failed to add segment 2: ");
3351  goto end;
3352  }
3353  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3354  printf("failed to add segment 3: ");
3355  goto end;
3356  }
3357  ssn.client.next_seq = 12;
3358  ssn.client.last_ack = 10;
3359 
3360  /* close the GAP and see if we properly reassemble and update base_seq */
3361  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3362  printf("failed to add segment 4: ");
3363  goto end;
3364  }
3365  ssn.client.next_seq = 22;
3366 
3367  p->tcph->th_seq = htonl(12);
3368 
3369  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3370  FAIL_IF_NULL(seg);
3371  FAIL_IF_NOT(seg->seq == 2);
3372 
3373  ret = 1;
3374 end:
3375  FLOWLOCK_UNLOCK(&f);
3376  FLOW_DESTROY(&f);
3377  UTHFreePacket(p);
3379  StreamTcpUTDeinit(ra_ctx);
3380  return ret;
3381 }
3382 
3383 /** \test App Layer reassembly.
3384  */
3385 static int StreamTcpReassembleInlineTest10(void)
3386 {
3387  int ret = 0;
3388  TcpReassemblyThreadCtx *ra_ctx = NULL;
3389  ThreadVars tv;
3390  TcpSession ssn;
3391  Flow *f = NULL;
3392  Packet *p = NULL;
3393 
3394  memset(&tv, 0x00, sizeof(tv));
3395 
3396  StreamTcpUTInit(&ra_ctx);
3399  StreamTcpUTSetupStream(&ssn.server, 1);
3400  ssn.server.last_ack = 2;
3401  StreamTcpUTSetupStream(&ssn.client, 1);
3402  ssn.client.last_ack = 2;
3404 
3405  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3406  if (f == NULL)
3407  goto end;
3408  f->protoctx = &ssn;
3409  f->proto = IPPROTO_TCP;
3410 
3411  uint8_t stream_payload1[] = "GE";
3412  uint8_t stream_payload2[] = "T /";
3413  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3414 
3415  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3416  if (p == NULL) {
3417  printf("couldn't get a packet: ");
3418  goto end;
3419  }
3420  p->tcph->th_seq = htonl(7);
3421  p->flow = f;
3423 
3424  FLOWLOCK_WRLOCK(f);
3425  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3426  printf("failed to add segment 1: ");
3427  goto end;
3428  }
3429  ssn.client.next_seq = 4;
3430 
3431  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3432  if (r < 0) {
3433  printf("StreamTcpReassembleAppLayer failed: ");
3434  goto end;
3435  }
3436 
3437  /* ssn.server.ra_app_base_seq should be isn here. */
3438  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3439  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3440  goto end;
3441  }
3442 
3443  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3444  printf("failed to add segment 2: ");
3445  goto end;
3446  }
3447  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3448  printf("failed to add segment 3: ");
3449  goto end;
3450  }
3451  ssn.client.next_seq = 19;
3452 
3453  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3454  if (r < 0) {
3455  printf("StreamTcpReassembleAppLayer failed: ");
3456  goto end;
3457  }
3458 
3459  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3460 
3461  ret = 1;
3462 end:
3463  UTHFreePacket(p);
3465  StreamTcpUTDeinit(ra_ctx);
3466  FLOWLOCK_UNLOCK(f);
3467  UTHFreeFlow(f);
3468  return ret;
3469 }
3470 
3471 /** \test test insert with overlap
3472  */
3473 static int StreamTcpReassembleInsertTest01(void)
3474 {
3475  TcpReassemblyThreadCtx *ra_ctx = NULL;
3476  ThreadVars tv;
3477  TcpSession ssn;
3478  Flow f;
3479 
3480  memset(&tv, 0x00, sizeof(tv));
3481 
3482  StreamTcpUTInit(&ra_ctx);
3484  StreamTcpUTSetupStream(&ssn.client, 1);
3486  FLOW_INITIALIZE(&f);
3487 
3488  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3489  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3490  FAIL_IF(p == NULL);
3491  p->tcph->th_seq = htonl(12);
3492  p->flow = &f;
3493 
3494  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3495  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3496  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3497  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3498  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3499  ssn.client.next_seq = 21;
3500 
3501  FLOW_DESTROY(&f);
3502  UTHFreePacket(p);
3504  StreamTcpUTDeinit(ra_ctx);
3505  PASS;
3506 }
3507 
3508 /** \test test insert with overlaps
3509  */
3510 static int StreamTcpReassembleInsertTest02(void)
3511 {
3512  int ret = 0;
3513  TcpReassemblyThreadCtx *ra_ctx = NULL;
3514  ThreadVars tv;
3515  TcpSession ssn;
3516 
3517  memset(&tv, 0x00, sizeof(tv));
3518 
3519  StreamTcpUTInit(&ra_ctx);
3521  StreamTcpUTSetupStream(&ssn.client, 1);
3522 
3523  int i;
3524  for (i = 2; i < 10; i++) {
3525  int len;
3526  len = i % 2;
3527  if (len == 0)
3528  len = 1;
3529  int seq;
3530  seq = i * 10;
3531  if (seq < 2)
3532  seq = 2;
3533 
3534  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3535  printf("failed to add segment 1: ");
3536  goto end;
3537  }
3538  }
3539  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3540  printf("failed to add segment 2: ");
3541  goto end;
3542  }
3543 
3544  ret = 1;
3545 end:
3547  StreamTcpUTDeinit(ra_ctx);
3548  return ret;
3549 }
3550 
3551 /** \test test insert with overlaps
3552  */
3553 static int StreamTcpReassembleInsertTest03(void)
3554 {
3555  int ret = 0;
3556  TcpReassemblyThreadCtx *ra_ctx = NULL;
3557  ThreadVars tv;
3558  TcpSession ssn;
3559 
3560  memset(&tv, 0x00, sizeof(tv));
3561 
3562  StreamTcpUTInit(&ra_ctx);
3564  StreamTcpUTSetupStream(&ssn.client, 1);
3565 
3566  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3567  printf("failed to add segment 2: ");
3568  goto end;
3569  }
3570 
3571  int i;
3572  for (i = 2; i < 10; i++) {
3573  int len;
3574  len = i % 2;
3575  if (len == 0)
3576  len = 1;
3577  int seq;
3578  seq = i * 10;
3579  if (seq < 2)
3580  seq = 2;
3581 
3582  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3583  printf("failed to add segment 2: ");
3584  goto end;
3585  }
3586  }
3587  ret = 1;
3588 end:
3590  StreamTcpUTDeinit(ra_ctx);
3591  return ret;
3592 }
3593 
3595 #endif /* UNITTESTS */
3596 
3597 /** \brief The Function Register the Unit tests to test the reassembly engine
3598  * for various OS policies.
3599  */
3600 
3602 {
3603 #ifdef UNITTESTS
3604  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3605  StreamTcpReassembleTest25);
3606  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3607  StreamTcpReassembleTest26);
3608  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3609  StreamTcpReassembleTest27);
3610  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3611  StreamTcpReassembleTest28);
3612  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3613  StreamTcpReassembleTest29);
3614  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3615  StreamTcpReassembleTest33);
3616  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3617  StreamTcpReassembleTest34);
3618  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3619  StreamTcpReassembleTest37);
3620  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3621  StreamTcpReassembleTest39);
3622  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3623  StreamTcpReassembleTest40);
3624  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3625  StreamTcpReassembleTest44);
3626  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3627  StreamTcpReassembleTest45);
3628  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3629  StreamTcpReassembleTest46);
3630  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3631  StreamTcpReassembleTest47);
3632 
3633  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3634  StreamTcpReassembleInlineTest01);
3635  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3636  StreamTcpReassembleInlineTest02);
3637  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3638  StreamTcpReassembleInlineTest03);
3639  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3640  StreamTcpReassembleInlineTest04);
3641  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3642  StreamTcpReassembleInlineTest08);
3643  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3644  StreamTcpReassembleInlineTest09);
3645 
3646  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3647  StreamTcpReassembleInlineTest10);
3648 
3649  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3650  StreamTcpReassembleInsertTest01);
3651  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3652  StreamTcpReassembleInsertTest02);
3653  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3654  StreamTcpReassembleInsertTest03);
3655 
3659  StreamTcpReassembleRawRegisterTests();
3660 #endif /* UNITTESTS */
3661 }
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
#define SCMutex
StreamingBufferSegment sbseg
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
#define SCLogDebug(...)
Definition: util-debug.h:335
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
void *(* Malloc)(size_t size)
int StreamTcpAppLayerIsDisabled(Flow *f)
struct Flow_ * flow
Definition: decode.h:446
AppProto alproto_tc
Definition: flow.h:411
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:568
#define STREAM_BASE_OFFSET(stream)
int StreamTcpReassembleInit(char quiet)
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:344
#define FALSE
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:243
StreamingBuffer sb
#define PASS
Pass the test.
#define unlikely(expr)
Definition: util-optimize.h:35
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
#define STREAMTCP_STREAM_FLAG_GAP
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void StreamTcpListRegisterTests(void)
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define TH_RST
Definition: decode-tcp.h:37
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet...
Definition: stream-tcp.c:769
void StreamTcpDisableAppLayer(Flow *f)
#define TH_FIN
Definition: decode-tcp.h:35
#define MIN(x, y)
#define STREAMING_BUFFER_NOFLAGS
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
uint64_t offset
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Address dst
Definition: decode.h:414
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
#define SCReturnUInt(x)
Definition: util-debug.h:343
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
int ByteExtractStringUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:244
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:668
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:240
uint64_t pcap_cnt
Definition: decode.h:562
#define STREAM_RAW_PROGRESS(stream)
void StreamTcpReassembleInitMemuse(void)
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:124
char * val
Definition: conf.h:34
TCPHdr * tcph
Definition: decode.h:523
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define TRUE
#define SCMutexLock(mut)
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
StreamingBufferBlock * head
#define STREAMTCP_FLAG_MIDSTREAM
void * protoctx
Definition: flow.h:400
void *(* Calloc)(size_t n, size_t size)
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
#define STREAM_DEPTH
Definition: stream.h:34
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
uint32_t min_inspect_depth
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueue *pq)
Definition: stream-tcp.c:4762
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define StreamTcpSetEvent(p, e)
AppProto alproto_ts
Definition: flow.h:410
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
#define SCCalloc(nm, a)
Definition: util-mem.h:253
#define SEQ_GEQ(a, b)
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:113
#define STREAM_GAP
Definition: stream.h:33
#define SCMutexUnlock(mut)
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:820
uint32_t seq
uint32_t raw_progress_rel
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:248
char family
Definition: decode.h:112
uint8_t proto
Definition: decode.h:431
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
#define SCMUTEX_INITIALIZER
#define TCP_SEG_LEN(seg)
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
#define STREAM_HAS_SEEN_DATA(stream)
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Data structures and function prototypes for keeping state for the detection engine.
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define FLOW_DESTROY(f)
Definition: flow-util.h:121
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies...
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
#define STREAM_EOF
Definition: stream.h:30
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define StreamTcpDisableAppLayerReassembly(ssn)
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:248
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
#define SCEnter(...)
Definition: util-debug.h:337
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:364
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
void StreamTcpReassembleFree(char quiet)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:440
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PKT_IS_TOSERVER(p)
Definition: decode.h:258
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
uint32_t reassembly_depth
#define TH_PUSH
Definition: decode-tcp.h:38
#define SEQ_GT(a, b)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:168
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1095
int RunmodeIsUnittests(void)
Definition: suricata.c:267
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1001
#define SCMutexInit(mut, mutattrs)
int8_t data_first_seen_dir
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define SCReturnInt(x)
Definition: util-debug.h:341
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueue *pq)
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
void StreamTcpUTInitInline(void)
#define SEQ_LEQ(a, b)
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
uint32_t reassembly_depth
Definition: stream-tcp.h:60
#define SCRealloc(x, a)
Definition: util-mem.h:238
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:563
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define RB_MIN(name, x)
Definition: tree.h:780
Definition: conf.h:32
#define SCMalloc(a)
Definition: util-mem.h:222
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:207
struct TCPSEG seg_tree
uint16_t payload_len
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define TH_SYN
Definition: decode-tcp.h:36
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
block of continues data
#define SCFree(a)
Definition: util-mem.h:322
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:249
#define RB_REMOVE(name, x, y)
Definition: tree.h:775
void StreamTcpUtilRegisterTests(void)
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:842
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
#define STREAM_START
Definition: stream.h:29
uint32_t prealloc_segments
Definition: stream-tcp.h:57
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
#define PKT_STREAM_ADD
Definition: decode.h:1091
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself...
void StreamTcpInlineRegisterTests(void)
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
#define SCReturn
Definition: util-debug.h:339
void StreamTcpUTSetupSession(TcpSession *ssn)
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define MISSED_END
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of &#39;threads&#39;, so array elements)
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
uint32_t flags
Definition: decode.h:444
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
void StreamTcpReassembleConfigEnableOverlapCheck(void)
uint16_t payload_len
Definition: decode.h:542
#define STREAM_APP_PROGRESS(stream)
#define SEQ_LT(a, b)
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6242
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
#define STREAM_RIGHT_EDGE(stream)
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
void StreamTcpUTClearSession(TcpSession *ssn)
#define MISSED_START(isn)
Flow data structure.
Definition: flow.h:325
#define FLOW_IPV4
Definition: flow.h:94
uint8_t * payload
Definition: decode.h:541
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
uint32_t flags
Definition: flow.h:379
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
void StreamTcpUTClearStream(TcpStream *s)
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
void(* Free)(void *ptr, size_t size)
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:141
Address src
Definition: decode.h:413
void UTHFreeFlow(Flow *flow)
#define STREAM_MIDSTREAM
Definition: stream.h:35
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SET_ISN(stream, setseq)
#define SCMutexDestroy
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.