suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 #include "app-layer-parser.h"
64 
65 #include "detect-engine-state.h"
66 
67 #include "util-profiling.h"
68 #include "util-validate.h"
69 
70 #ifdef DEBUG
71 static SCMutex segment_pool_memuse_mutex;
72 static uint64_t segment_pool_memuse = 0;
73 static uint64_t segment_pool_memcnt = 0;
74 #endif
75 
76 static PoolThread *segment_thread_pool = NULL;
77 /* init only, protect initializing and growing pool */
78 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
79 
80 /* Memory use counter */
81 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
82 
83 /* prototypes */
85 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346  if (f->alparser) {
349  }
350 }
351 
352 /** \param f locked flow */
354 {
355  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
356  return 0;
357 
358  TcpSession *ssn = (TcpSession *)f->protoctx;
360 }
361 
362 static int StreamTcpReassemblyConfig(char quiet)
363 {
364  uint32_t segment_prealloc = 2048;
365  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
366  if (seg) {
367  uint32_t prealloc = 0;
368  if (StringParseUint32(&prealloc, 10, strlen(seg->val), seg->val) < 0)
369  {
370  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
371  "%s is invalid", seg->val);
372  return -1;
373  }
374  segment_prealloc = prealloc;
375  }
376  if (!quiet)
377  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
378  stream_config.prealloc_segments = segment_prealloc;
379 
380  int overlap_diff_data = 0;
381  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
382  if (overlap_diff_data) {
384  }
385  if (StreamTcpInlineMode() == TRUE) {
387  }
388 
391  stream_config.sbcnf.Malloc = ReassembleMalloc;
392  stream_config.sbcnf.Calloc = ReassembleCalloc;
393  stream_config.sbcnf.Realloc = ReassembleRealloc;
394  stream_config.sbcnf.Free = ReassembleFree;
395 
396  return 0;
397 }
398 
399 int StreamTcpReassembleInit(char quiet)
400 {
401  /* init the memcap/use tracker */
403 
404  if (StreamTcpReassemblyConfig(quiet) < 0)
405  return -1;
406 
407 #ifdef DEBUG
408  SCMutexInit(&segment_pool_memuse_mutex, NULL);
409 #endif
410  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
412  return 0;
413 }
414 
415 void StreamTcpReassembleFree(char quiet)
416 {
417  SCMutexLock(&segment_thread_pool_mutex);
418  if (segment_thread_pool != NULL) {
419  PoolThreadFree(segment_thread_pool);
420  segment_thread_pool = NULL;
421  }
422  SCMutexUnlock(&segment_thread_pool_mutex);
423  SCMutexDestroy(&segment_thread_pool_mutex);
424 
425 #ifdef DEBUG
426  if (segment_pool_memuse > 0)
427  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
428  if (segment_pool_memcnt > 0)
429  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
430  SCMutexDestroy(&segment_pool_memuse_mutex);
431 #endif
432 }
433 
435 {
436  SCEnter();
438  if (unlikely(ra_ctx == NULL))
439  return NULL;
440 
441  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
442 
443  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
444 
445  SCMutexLock(&segment_thread_pool_mutex);
446  if (segment_thread_pool == NULL) {
447  segment_thread_pool = PoolThreadInit(1, /* thread */
448  0, /* unlimited */
450  sizeof(TcpSegment),
451  TcpSegmentPoolAlloc,
452  TcpSegmentPoolInit, NULL,
453  TcpSegmentPoolCleanup, NULL);
454  ra_ctx->segment_thread_pool_id = 0;
455  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
456  PoolThreadSize(segment_thread_pool),
457  ra_ctx->segment_thread_pool_id);
458  } else {
459  /* grow segment_thread_pool until we have a element for our thread id */
460  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
461  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
462  PoolThreadSize(segment_thread_pool),
463  ra_ctx->segment_thread_pool_id);
464  }
465  SCMutexUnlock(&segment_thread_pool_mutex);
466  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
467  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
469  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
470  }
471 
472  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
473 }
474 
476 {
477  SCEnter();
478  if (ra_ctx) {
480  SCFree(ra_ctx);
481  }
482  SCReturn;
483 }
484 
485 /**
486  * \brief check if stream in pkt direction has depth reached
487  *
488  * \param p packet with *LOCKED* flow
489  *
490  * \retval 1 stream has depth reached
491  * \retval 0 stream does not have depth reached
492  */
494 {
495  if (p->flow != NULL && p->flow->protoctx != NULL) {
496  TcpSession *ssn = p->flow->protoctx;
497  TcpStream *stream;
498  if (p->flowflags & FLOW_PKT_TOSERVER) {
499  stream = &ssn->client;
500  } else {
501  stream = &ssn->server;
502  }
503 
504  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
505  }
506 
507  return 0;
508 }
509 
510 /**
511  * \internal
512  * \brief Function to Check the reassembly depth valuer against the
513  * allowed max depth of the stream reassembly for TCP streams.
514  *
515  * \param stream stream direction
516  * \param seq sequence number where "size" starts
517  * \param size size of the segment that is added
518  *
519  * \retval size Part of the size that fits in the depth, 0 if none
520  */
521 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
522  uint32_t seq, uint32_t size)
523 {
524  SCEnter();
525 
526  /* if the configured depth value is 0, it means there is no limit on
527  reassembly depth. Otherwise carry on my boy ;) */
528  if (ssn->reassembly_depth == 0) {
529  SCReturnUInt(size);
530  }
531 
532  /* if the final flag is set, we're not accepting anymore */
534  SCReturnUInt(0);
535  }
536 
537  uint64_t seg_depth;
538  if (SEQ_GT(stream->base_seq, seq)) {
539  if (SEQ_LEQ(seq+size, stream->base_seq)) {
540  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
541  stream->base_seq, seq, seq+size);
542  SCReturnUInt(0);
543  }
544 
545  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
546  } else {
547  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
548  }
549 
550  /* if the base_seq has moved passed the depth window we stop
551  * checking and just reject the rest of the packets including
552  * retransmissions. Saves us the hassle of dealing with sequence
553  * wraps as well */
554  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
555  stream->base_seq, seg_depth,
556  ssn->reassembly_depth);
557 
558  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
559  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
561  SCReturnUInt(0);
562  }
563  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
564  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
565 #if 0
566  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
567  (stream->base_seq_offset + stream->base_seq + size),
568  (stream->isn + ssn->reassembly_depth));
569 #endif
570  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
571  /* packet (partly?) fits the depth window */
572 
573  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
574  /* complete fit */
575  SCReturnUInt(size);
576  } else {
578  /* partial fit, return only what fits */
579  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
580  DEBUG_VALIDATE_BUG_ON(part > size);
581  if (part > size)
582  part = size;
583  SCReturnUInt(part);
584  }
585  }
586 
587  SCReturnUInt(0);
588 }
589 
590 /**
591  * \brief Insert a packets TCP data into the stream reassembly engine.
592  *
593  * \retval 0 good segment, as far as we checked.
594  * \retval -1 badness, reason to drop in inline mode
595  *
596  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
597  * or it wasn't added because of reassembly depth.
598  *
599  */
601  TcpSession *ssn, TcpStream *stream, Packet *p)
602 {
603  SCEnter();
604 
605  if (ssn->data_first_seen_dir == 0) {
606  if (PKT_IS_TOSERVER(p)) {
608  } else {
610  }
611  }
612 
613  /* If the OS policy is not set then set the OS policy for this stream */
614  if (stream->os_policy == 0) {
615  StreamTcpSetOSPolicy(stream, p);
616  }
617 
620  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
621  SCReturnInt(0);
622  }
623 
624  /* If we have reached the defined depth for either of the stream, then stop
625  reassembling the TCP session */
626  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
627  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
628 
630  /* increment stream depth counter */
632  }
633  if (size == 0) {
634  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
635  SCReturnInt(0);
636  }
637 
639  if (size > p->payload_len)
640  size = p->payload_len;
641 
642  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
643  if (seg == NULL) {
644  SCLogDebug("segment_pool is empty");
646  SCReturnInt(-1);
647  }
648 
649  TCP_SEG_LEN(seg) = size;
650  seg->seq = TCP_GET_SEQ(p);
651 
652  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
653  if (TCP_HAS_TFO(p) && p->payload_len && p->tcph->th_flags == TH_SYN)
654  seg->seq += 1;
655 
656  /* proto detection skipped, but now we do get data. Set event. */
657  if (RB_EMPTY(&stream->seg_tree) &&
659 
662  }
663 
664  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
665  SCLogDebug("StreamTcpReassembleInsertSegment failed");
666  SCReturnInt(-1);
667  }
668  SCReturnInt(0);
669 }
670 
671 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
672  Packet *p)
673 {
674  uint8_t flag = 0;
675 
677  flag |= STREAM_START;
678  }
679 
680  if (ssn->state == TCP_CLOSED) {
681  flag |= STREAM_EOF;
682  }
683 
684  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
685  flag |= STREAM_MIDSTREAM;
686  }
687 
688  if (p->flags & PKT_PSEUDO_STREAM_END) {
689  flag |= STREAM_EOF;
690  }
691 
692  if (&ssn->client == stream) {
693  flag |= STREAM_TOSERVER;
694  } else {
695  flag |= STREAM_TOCLIENT;
696  }
698  flag |= STREAM_DEPTH;
699  }
700  return flag;
701 }
702 
703 /**
704  * \brief Check the minimum size limits for reassembly.
705  *
706  * \retval 0 don't reassemble yet
707  * \retval 1 do reassemble
708  */
709 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
710  const TcpStream *stream, const Packet *p)
711 {
712  SCEnter();
713 
714  /* if any of these flags is set we always inspect immediately */
715 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
716  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
717  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
718  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
719 
722  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
723  "is set, so not expecting any new data segments");
724  }
726  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
727  }
729  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
730  "so no new segments will be considered");
731  }
732  SCReturnInt(1);
733  }
734 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
735 
736  /* some states mean we reassemble no matter how much data we have */
737  if (ssn->state > TCP_TIME_WAIT)
738  SCReturnInt(1);
739 
740  if (p->flags & PKT_PSEUDO_STREAM_END)
741  SCReturnInt(1);
742 
743  /* check if we have enough data to do raw reassembly */
744  if (PKT_IS_TOSERVER(p)) {
745  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
746  uint32_t delta = stream->last_ack - stream->base_seq;
747  /* get max absolute offset */
748  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
749 
750  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
751  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
752  SCReturnInt(1);
753  } else {
754  SCLogDebug("toserver min chunk len not yet reached: "
755  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
756  "%"PRIu32"", stream->last_ack, stream->base_seq,
757  (stream->last_ack - stream->base_seq),
759  SCReturnInt(0);
760  }
761  }
762  } else {
763  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
764  uint32_t delta = stream->last_ack - stream->base_seq;
765  /* get max absolute offset */
766  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
767 
768  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
769 
770  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
771  SCReturnInt(1);
772  } else {
773  SCLogDebug("toclient min chunk len not yet reached: "
774  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
775  "%"PRIu32"", stream->last_ack, stream->base_seq,
776  (stream->last_ack - stream->base_seq),
778  SCReturnInt(0);
779  }
780  }
781  }
782 
783  SCReturnInt(0);
784 }
785 
786 /**
787  * \brief see what if any work the TCP session still needs
788  */
789 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
790 {
791  const TcpStream *stream = NULL;
792 #ifdef DEBUG
793  const char *dirstr = NULL;
794 #endif
795  if (direction == STREAM_TOSERVER) {
796  stream = &ssn->client;
797 #ifdef DEBUG
798  dirstr = "client";
799 #endif
800  } else {
801  stream = &ssn->server;
802 #ifdef DEBUG
803  dirstr = "server";
804 #endif
805  }
806 
807  int use_app = 1;
808  int use_raw = 1;
809 
811  // app is dead
812  use_app = 0;
813  }
814 
816  // raw is dead
817  use_raw = 0;
818  }
819 
820  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
821 
822  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
823  dirstr,
824  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
825  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
826  right_edge);
827  if (use_raw) {
828  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
829  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
831  }
832  }
833  if (use_app) {
834  if (right_edge > STREAM_APP_PROGRESS(stream)) {
835  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
837  }
838  }
839 
840  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
842 }
843 
844 #ifdef DEBUG
845 static uint64_t GetStreamSize(TcpStream *stream)
846 {
847  if (stream) {
848  uint64_t size = 0;
849  uint32_t cnt = 0;
850 
851  TcpSegment *seg;
852  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
853  cnt++;
854  size += (uint64_t)TCP_SEG_LEN(seg);
855  }
856 
857  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
858  return size;
859  }
860  return (uint64_t)0;
861 }
862 
863 static void GetSessionSize(TcpSession *ssn, Packet *p)
864 {
865  uint64_t size = 0;
866  if (ssn) {
867  size = GetStreamSize(&ssn->client);
868  size += GetStreamSize(&ssn->server);
869 
870  //if (size > 900000)
871  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
872  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
873  }
874 }
875 #endif
876 
877 static StreamingBufferBlock *GetBlock(StreamingBuffer *sb, const uint64_t offset)
878 {
879  StreamingBufferBlock *blk = sb->head;
880  if (blk == NULL)
881  return NULL;
882 
883  for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
884  if (blk->offset >= offset)
885  return blk;
886  else if ((blk->offset + blk->len) > offset) {
887  return blk;
888  }
889  }
890  return NULL;
891 }
892 
893 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
894 {
895  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
896  return STREAM_BASE_OFFSET(stream) +
897  (stream->last_ack - stream->base_seq);
898  } else {
899  return STREAM_BASE_OFFSET(stream);
900  }
901 }
902 
903 static inline bool GapAhead(TcpStream *stream, StreamingBufferBlock *cur_blk)
904 {
905  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
906  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
907  GetAbsLastAck(stream) >= (cur_blk->offset + cur_blk->len)) {
908  return true;
909  }
910  return false;
911 }
912 
913 /** \internal
914  *
915  * Get buffer, or first part of the buffer if data gaps exist.
916  *
917  * \brief get stream data from offset
918  * \param offset stream offset
919  * \param check_for_gap check if there is a gap ahead. Optional as it is only
920  * needed for app-layer incomplete support.
921  * \retval bool pkt loss ahead */
922 static bool GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
923  uint64_t offset, const bool check_for_gap)
924 {
925  const uint8_t *mydata;
926  uint32_t mydata_len;
927  bool gap_ahead = false;
928 
929  if (RB_EMPTY(&stream->sb.sbb_tree)) {
930  SCLogDebug("getting one blob");
931 
932  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
933 
934  *data = mydata;
935  *data_len = mydata_len;
936  } else {
937  StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
938  if (blk == NULL) {
939  *data = NULL;
940  *data_len = 0;
941  return false;
942  }
943 
944  /* block at expected offset */
945  if (blk->offset == offset) {
946 
947  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
948 
949  gap_ahead = check_for_gap && GapAhead(stream, blk);
950 
951  /* block past out offset */
952  } else if (blk->offset > offset) {
953  SCLogDebug("gap, want data at offset %"PRIu64", "
954  "got data at %"PRIu64". GAP of size %"PRIu64,
955  offset, blk->offset, blk->offset - offset);
956  *data = NULL;
957  *data_len = blk->offset - offset;
958 
959  /* block starts before offset, but ends after */
960  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
961  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
962  offset, blk->offset, blk->len);
963  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
964  SCLogDebug("data %p, data_len %u", *data, *data_len);
965 
966  gap_ahead = check_for_gap && GapAhead(stream, blk);
967 
968  } else {
969  *data = NULL;
970  *data_len = 0;
971  }
972  }
973  return gap_ahead;
974 }
975 
976 /** \internal
977  * \brief check to see if we should declare a GAP
978  * Call this when the app layer didn't get data at the requested
979  * offset.
980  */
981 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
982 {
983  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
984  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
985 
986  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
987  /* get window of data that is acked */
988  const uint32_t delta = stream->last_ack - stream->base_seq;
989  /* get max absolute offset */
990  last_ack_abs += delta;
991 
992  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
993  last_ack_abs -= ackadded;
994 
995  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
996  SCLogDebug("next_seq %u", stream->next_seq);
997 
998  /* if last_ack_abs is beyond the app_progress data that we haven't seen
999  * has been ack'd. This looks like a GAP. */
1000  if (last_ack_abs > app_progress) {
1001  /* however, we can accept ACKs a bit too liberally. If last_ack
1002  * is beyond next_seq, we only consider it a gap now if we do
1003  * already have data beyond the gap. */
1004  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1005  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1006  SCLogDebug("packet %"PRIu64": no GAP. "
1007  "next_seq %u < last_ack %u, but no data in list",
1008  p->pcap_cnt, stream->next_seq, stream->last_ack);
1009  return false;
1010  } else {
1011  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1012  const StreamingBufferBlock *blk = stream->sb.head;
1013  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1014  /* ack'd data after the gap */
1015  SCLogDebug("packet %"PRIu64": GAP. "
1016  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1017  p->pcap_cnt, stream->next_seq, stream->last_ack);
1018  return true;
1019  }
1020  }
1021  }
1022 
1023  SCLogDebug("packet %"PRIu64": GAP! "
1024  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
1025  "but we have no data.",
1026  p->pcap_cnt, last_ack_abs, app_progress);
1027  return true;
1028  }
1029  }
1030  SCLogDebug("packet %"PRIu64": no GAP. "
1031  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1032  p->pcap_cnt, last_ack_abs, app_progress);
1033  return false;
1034 }
1035 
1036 static inline uint32_t AdjustToAcked(const Packet *p,
1037  const TcpSession *ssn, const TcpStream *stream,
1038  const uint64_t app_progress, const uint32_t data_len)
1039 {
1040  uint32_t adjusted = data_len;
1041 
1042  /* get window of data that is acked */
1043  if (StreamTcpInlineMode() == FALSE) {
1044  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1045  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1046  (ssn->state == TCP_CLOSED &&
1047  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1048  (p->flags & PKT_PSEUDO_STREAM_END))) {
1049  // fall through, we use all available data
1050  } else {
1051  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1052  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1053  /* get window of data that is acked */
1054  uint32_t delta = stream->last_ack - stream->base_seq;
1055  /* get max absolute offset */
1056  last_ack_abs += delta;
1057  }
1058  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1059 
1060  /* see if the buffer contains unack'd data as well */
1061  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1062  uint32_t check = data_len;
1063  adjusted = last_ack_abs - app_progress;
1064  BUG_ON(adjusted > check);
1065  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1066  "data is considered", adjusted);
1067  }
1068  }
1069  }
1070  return adjusted;
1071 }
1072 
1073 /** \internal
1074  * \brief get stream buffer and update the app-layer
1075  * \param stream pointer to pointer as app-layer can switch flow dir
1076  * \retval 0 success
1077  */
1078 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1079  TcpReassemblyThreadCtx *ra_ctx,
1080  TcpSession *ssn, TcpStream **stream,
1081  Packet *p, enum StreamUpdateDir dir)
1082 {
1083  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1084 
1085  SCLogDebug("app progress %"PRIu64, app_progress);
1086  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1087 
1088  const uint8_t *mydata;
1089  uint32_t mydata_len;
1090  bool gap_ahead = false;
1091  bool last_was_gap = false;
1092 
1093  while (1) {
1094  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1095  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1096  gap_ahead = GetAppBuffer(*stream, &mydata, &mydata_len,
1097  app_progress, check_for_gap_ahead);
1098  if (last_was_gap && mydata_len == 0) {
1099  break;
1100  }
1101  last_was_gap = false;
1102 
1103  /* make sure to only deal with ACK'd data */
1104  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1105  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1106  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1107  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1108 
1109  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1110  NULL, mydata_len,
1111  StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP);
1112  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1113 
1116 
1117  /* AppLayerHandleTCPData has likely updated progress. */
1118  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1119  app_progress = STREAM_APP_PROGRESS(*stream);
1120 
1121  /* a GAP also consumes 'data required'. TODO perhaps we can use
1122  * this to skip post GAP data until the start of a next record. */
1123  if ((*stream)->data_required > 0) {
1124  if ((*stream)->data_required > mydata_len) {
1125  (*stream)->data_required -= mydata_len;
1126  } else {
1127  (*stream)->data_required = 0;
1128  }
1129  }
1130  if (r < 0)
1131  return 0;
1132  if (no_progress_update)
1133  break;
1134  last_was_gap = true;
1135  continue;
1136 
1137  } else if (flags & STREAM_DEPTH) {
1138  // we're just called once with this flag, so make sure we pass it on
1139 
1140  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1141  /* Possibly a gap, but no new data. */
1142  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1143  SCReturnInt(0);
1144 
1145  mydata = NULL;
1146  mydata_len = 0;
1147  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1148  break;
1149  }
1150 
1151  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1152  *stream, &(*stream)->sb, mydata_len, app_progress);
1153 
1154  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1155  if (mydata_len < (*stream)->data_required) {
1156  if (gap_ahead) {
1157  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1158  (*stream)->data_required, mydata_len);
1159  (*stream)->app_progress_rel += mydata_len;
1160  (*stream)->data_required -= mydata_len;
1161  // TODO send incomplete data to app-layer with special flag
1162  // indicating its all there is for this rec?
1163  } else {
1164  SCReturnInt(0);
1165  }
1166  app_progress = STREAM_APP_PROGRESS(*stream);
1167  continue;
1168  }
1169  }
1170  (*stream)->data_required = 0;
1171 
1172  /* update the app-layer */
1173  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1174  (uint8_t *)mydata, mydata_len, flags);
1175  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1176  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1177  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1178  break;
1179  app_progress = new_app_progress;
1180  if (flags & STREAM_DEPTH)
1181  break;
1182  }
1183 
1184  SCReturnInt(0);
1185 }
1186 
1187 /**
1188  * \brief Update the stream reassembly upon receiving a packet.
1189  *
1190  * For IDS mode, the stream is in the opposite direction of the packet,
1191  * as the ACK-packet is ACK'ing the stream.
1192  *
1193  * One of the utilities call by this function AppLayerHandleTCPData(),
1194  * has a feature where it will call this very same function for the
1195  * stream opposing the stream it is called with. This shouldn't cause
1196  * any issues, since processing of each stream is independent of the
1197  * other stream.
1198  */
1200  TcpSession *ssn, TcpStream *stream,
1201  Packet *p, enum StreamUpdateDir dir)
1202 {
1203  SCEnter();
1204 
1205  /* this function can be directly called by app layer protocol
1206  * detection. */
1209  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1210  SCReturnInt(0);
1211  }
1212 
1213 #ifdef DEBUG
1214  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1215  GetSessionSize(ssn, p);
1216 #endif
1217  /* if no segments are in the list or all are already processed,
1218  * and state is beyond established, we send an empty msg */
1219  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1220  {
1221  /* send an empty EOF msg if we have no segments but TCP state
1222  * is beyond ESTABLISHED */
1223  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1224  SCLogDebug("sending empty eof message");
1225  /* send EOF to app layer */
1226  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1227  NULL, 0,
1228  StreamGetAppLayerFlags(ssn, stream, p));
1229  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1230 
1231  SCReturnInt(0);
1232  }
1233  }
1234 
1235  /* with all that out of the way, lets update the app-layer */
1236  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1237 }
1238 
1239 /** \internal
1240  * \brief get stream data from offset
1241  * \param offset stream offset */
1242 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1243  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1244 {
1245  const uint8_t *mydata;
1246  uint32_t mydata_len;
1247  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1248  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1249 
1250  uint64_t roffset = offset;
1251  if (offset)
1252  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1253  else {
1254  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1255  }
1256 
1257  *data = mydata;
1258  *data_len = mydata_len;
1259  *data_offset = roffset;
1260  } else {
1261  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1262  *iter == NULL ? "starting" : "continuing", offset);
1263  if (*iter == NULL) {
1264  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1265  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1266  SCLogDebug("*iter %p", *iter);
1267  }
1268  if (*iter == NULL) {
1269  SCLogDebug("no data");
1270  *data = NULL;
1271  *data_len = 0;
1272  *data_offset = 0;
1273  return 0;
1274  }
1275  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1276 
1277  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1278  SCLogDebug("mydata %p", mydata);
1279 
1280  if ((*iter)->offset < offset) {
1281  uint64_t delta = offset - (*iter)->offset;
1282  if (delta < mydata_len) {
1283  *data = mydata + delta;
1284  *data_len = mydata_len - delta;
1285  *data_offset = offset;
1286  } else {
1287  SCLogDebug("no data (yet)");
1288  *data = NULL;
1289  *data_len = 0;
1290  *data_offset = 0;
1291  }
1292 
1293  } else {
1294  *data = mydata;
1295  *data_len = mydata_len;
1296  *data_offset = (*iter)->offset;
1297  }
1298 
1299  *iter = SBB_RB_NEXT(*iter);
1300  SCLogDebug("*iter %p", *iter);
1301  }
1302  return 0;
1303 }
1304 
1305 /** \brief does the stream engine have data to inspect?
1306  *
1307  * Returns true if there is data to inspect. In IDS case this is
1308  * about ACK'd data in the packet's direction.
1309  *
1310  * In the IPS case this is about the packet itself.
1311  */
1313 {
1314  TcpStream *stream;
1315  if (PKT_IS_TOSERVER(p)) {
1316  stream = &ssn->client;
1317  } else {
1318  stream = &ssn->server;
1319  }
1320 
1321  if (RB_EMPTY(&stream->seg_tree)) {
1322  return false;
1323  }
1324 
1327  return false;
1328 
1329  if (StreamTcpInlineMode() == FALSE) {
1330  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1331  return false;
1332  }
1333  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1334  return true;
1335  }
1336  } else {
1337  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1338  return true;
1339  }
1340  }
1341  return false;
1342 }
1343 
1344 /** \brief update stream engine after detection
1345  *
1346  * Tasked with progressing the 'progress' for Raw reassembly.
1347  * 2 main scenario's:
1348  * 1. progress is != 0, so we use this
1349  * 2. progress is 0, meaning the detect engine didn't touch
1350  * raw at all. In this case we need to look into progressing
1351  * raw anyway.
1352  *
1353  * Additionally, this function is tasked with disabling raw
1354  * reassembly if the app-layer requested to disable it.
1355  */
1356 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1357 {
1358  TcpStream *stream;
1359  if (PKT_IS_TOSERVER(p)) {
1360  stream = &ssn->client;
1361  } else {
1362  stream = &ssn->server;
1363  }
1364 
1365  if (progress > STREAM_RAW_PROGRESS(stream)) {
1366  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1367  stream->raw_progress_rel += slide;
1369 
1370  /* if app is active and beyond raw, sync raw to app */
1371  } else if (progress == 0 && STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1373  /* if trigger raw is set we sync the 2 trackers */
1375  {
1376  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1377  stream->raw_progress_rel += slide;
1379 
1380  /* otherwise mix in the tcp window */
1381  } else {
1382  uint64_t tcp_window = stream->window;
1383  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1384  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1385  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1386  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1387  stream->raw_progress_rel += slide;
1388  }
1389  }
1390  }
1391  /* app is dead */
1392  } else if (progress == 0) {
1393  uint64_t tcp_window = stream->window;
1394  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1395  if (tcp_window < stream_right_edge) {
1396  uint64_t new_raw = stream_right_edge - tcp_window;
1397  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1398  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1399  stream->raw_progress_rel += slide;
1400  }
1401  }
1403 
1404  } else {
1405  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1406  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1407  STREAM_RAW_PROGRESS(stream), stream->window);
1408  }
1409 
1410  /* if we were told to accept no more raw data, we can mark raw as
1411  * disabled now. */
1414  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1415  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1416  }
1417 
1418  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1419 }
1420 
1421 /** \internal
1422  * \brief get a buffer around the current packet and run the callback on it
1423  *
1424  * The inline/IPS scanning method takes the current payload and wraps it in
1425  * data from other segments.
1426  *
1427  * How much data is inspected is controlled by the available data, chunk_size
1428  * and the payload size of the packet.
1429  *
1430  * Large packets: if payload size is close to the chunk_size, where close is
1431  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1432  * used: payload_len + 33% of the chunk_size.
1433  * If the payload size if equal to or bigger than the chunk_size, we use
1434  * payload len + 33% of the chunk size.
1435  */
1436 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1437  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1438 {
1439  SCEnter();
1440  int r = 0;
1441 
1442  TcpStream *stream;
1443  if (PKT_IS_TOSERVER(p)) {
1444  stream = &ssn->client;
1445  } else {
1446  stream = &ssn->server;
1447  }
1448 
1449  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1451  {
1452  *progress_out = STREAM_RAW_PROGRESS(stream);
1453  return 0;
1454  }
1455 
1456  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1459  if (chunk_size <= p->payload_len) {
1460  chunk_size = p->payload_len + (chunk_size / 3);
1461  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1462  p->payload_len, chunk_size);
1463  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1464  chunk_size = p->payload_len + ((chunk_size / 3));
1465  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1466  p->payload_len, chunk_size);
1467  }
1468 
1469  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1470  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1471  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1472  packet_leftedge_abs, packet_rightedge_abs);
1473 
1474  const uint8_t *mydata = NULL;
1475  uint32_t mydata_len = 0;
1476  uint64_t mydata_offset = 0;
1477  /* simply return progress from the block we inspected. */
1478  bool return_progress = false;
1479 
1480  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1481  /* continues block */
1482  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1483  return_progress = true;
1484 
1485  } else {
1486  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1487  /* find our block */
1488  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1489  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1490  if (sbb) {
1491  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1492  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1493  mydata_offset = sbb->offset;
1494  }
1495  }
1496 
1497  /* this can only happen if the segment insert of our current 'p' failed */
1498  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1499  if ((mydata == NULL || mydata_len == 0) || /* no data */
1500  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1501  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1502  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1503  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1504  {
1505  /* no data, or data is incomplete or wrong: use packet data */
1506  mydata = p->payload;
1507  mydata_len = p->payload_len;
1508  mydata_offset = packet_leftedge_abs;
1509  //mydata_rightedge_abs = packet_rightedge_abs;
1510  } else {
1511  /* adjust buffer to match chunk_size */
1512  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1513  if (mydata_len > chunk_size) {
1514  uint32_t excess = mydata_len - chunk_size;
1515  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1516 
1517  if (mydata_rightedge_abs == packet_rightedge_abs) {
1518  mydata += excess;
1519  mydata_len -= excess;
1520  mydata_offset += excess;
1521  SCLogDebug("cutting front of the buffer with %u", excess);
1522  } else if (mydata_offset == packet_leftedge_abs) {
1523  mydata_len -= excess;
1524  SCLogDebug("cutting tail of the buffer with %u", excess);
1525  } else {
1526  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1527  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1528  SCLogDebug("before %u after %u", before, after);
1529 
1530  if (after >= (chunk_size - p->payload_len) / 2) {
1531  // more trailing data than we need
1532 
1533  if (before >= (chunk_size - p->payload_len) / 2) {
1534  // also more heading data, divide evenly
1535  before = after = (chunk_size - p->payload_len) / 2;
1536  } else {
1537  // heading data is less than requested, give the
1538  // rest to the trailing data
1539  after = (chunk_size - p->payload_len) - before;
1540  }
1541  } else {
1542  // less trailing data than requested
1543 
1544  if (before >= (chunk_size - p->payload_len) / 2) {
1545  before = (chunk_size - p->payload_len) - after;
1546  } else {
1547  // both smaller than their requested size
1548  }
1549  }
1550 
1551  /* adjust the buffer */
1552  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1553  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1554  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1555  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1556  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1557  mydata += skip;
1558  mydata_len -= (skip + cut);
1559  mydata_offset += skip;
1560  }
1561  }
1562  }
1563 
1564  /* run the callback */
1565  r = Callback(cb_data, mydata, mydata_len);
1566  BUG_ON(r < 0);
1567 
1568  if (return_progress) {
1569  *progress_out = (mydata_offset + mydata_len);
1570  } else {
1571  /* several blocks of data, so we need to be a bit more careful:
1572  * - if last_ack is beyond last progress, move progress forward to last_ack
1573  * - if our block matches or starts before last ack, return right edge of
1574  * our block.
1575  */
1576  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1577  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1578  uint32_t delta = stream->last_ack - stream->base_seq;
1579  /* get max absolute offset */
1580  last_ack_abs += delta;
1581  }
1582  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1583 
1584  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1585  if (mydata_offset > last_ack_abs) {
1586  /* gap between us and last ack, set progress to last ack */
1587  *progress_out = last_ack_abs;
1588  } else {
1589  *progress_out = (mydata_offset + mydata_len);
1590  }
1591  } else {
1592  *progress_out = STREAM_RAW_PROGRESS(stream);
1593  }
1594  }
1595  return r;
1596 }
1597 
1598 /** \brief access 'raw' reassembly data.
1599  *
1600  * Access data as tracked by 'raw' tracker. Data is made available to
1601  * callback that is passed to this function.
1602  *
1603  * In the case of IDS the callback may be run multiple times if data
1604  * contains gaps. It will then be run for each block of data that is
1605  * continuous.
1606  *
1607  * The callback should give on of 2 return values:
1608  * - 0 ok
1609  * - 1 done
1610  * The value 1 will break the loop if there is a block list that is
1611  * inspected.
1612  *
1613  * This function will return the 'progress' value that has been
1614  * consumed until now.
1615  *
1616  * \param ssn tcp session
1617  * \param stream tcp stream
1618  * \param Callback the function pointer to the callback function
1619  * \param cb_data callback data
1620  * \param[in] progress_in progress to work from
1621  * \param[out] progress_out absolute progress value of the data this
1622  * call handled.
1623  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1624  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1625  *
1626  * `respect_inspect_depth` is used to avoid useless inspection of too
1627  * much data.
1628  */
1629 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1630  StreamReassembleRawFunc Callback, void *cb_data,
1631  const uint64_t progress_in,
1632  uint64_t *progress_out, bool eof,
1633  bool respect_inspect_depth)
1634 {
1635  SCEnter();
1636  int r = 0;
1637 
1638  StreamingBufferBlock *iter = NULL;
1639  uint64_t progress = progress_in;
1640  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1641 
1642  /* if the app layer triggered a flush, and we're supposed to
1643  * use a minimal inspect depth, we actually take the app progress
1644  * as that is the right edge of the data. Then we take the window
1645  * of 'min_inspect_depth' before that. */
1646 
1647  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1648  respect_inspect_depth ? "true" : "false",
1649  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1650  stream->min_inspect_depth);
1651 
1652  if (respect_inspect_depth &&
1654  && stream->min_inspect_depth)
1655  {
1656  progress = STREAM_APP_PROGRESS(stream);
1657  if (stream->min_inspect_depth >= progress) {
1658  progress = 0;
1659  } else {
1660  progress -= stream->min_inspect_depth;
1661  }
1662 
1663  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1664 
1665  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1666  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1667  }
1668 
1669  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1670 
1671  /* get window of data that is acked */
1672  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1673  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1674  uint32_t delta = stream->last_ack - stream->base_seq;
1675  /* get max absolute offset */
1676  last_ack_abs += delta;
1677  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1678  }
1679 
1680  /* loop through available buffers. On no packet loss we'll have a single
1681  * iteration. On missing data we'll walk the blocks */
1682  while (1) {
1683  const uint8_t *mydata;
1684  uint32_t mydata_len;
1685  uint64_t mydata_offset = 0;
1686 
1687  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1688  if (mydata_len == 0) {
1689  SCLogDebug("no data");
1690  break;
1691  }
1692  //PrintRawDataFp(stdout, mydata, mydata_len);
1693 
1694  SCLogDebug("raw progress %"PRIu64, progress);
1695  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1696  stream, &stream->sb, mydata_len, progress);
1697 
1698  if (eof) {
1699  // inspect all remaining data, ack'd or not
1700  } else {
1701  if (last_ack_abs < progress) {
1702  SCLogDebug("nothing to do");
1703  goto end;
1704  }
1705 
1706  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1707  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1708 
1709  /* see if the buffer contains unack'd data as well */
1710  if (progress + mydata_len > last_ack_abs) {
1711  uint32_t check = mydata_len;
1712  mydata_len = last_ack_abs - progress;
1713  BUG_ON(check < mydata_len);
1714  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1715  "data is considered", mydata_len);
1716  }
1717 
1718  }
1719  if (mydata_len == 0)
1720  break;
1721 
1722  SCLogDebug("data %p len %u", mydata, mydata_len);
1723 
1724  /* we have data. */
1725  r = Callback(cb_data, mydata, mydata_len);
1726  BUG_ON(r < 0);
1727 
1728  if (mydata_offset == progress) {
1729  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1730  progress, mydata_len, progress_in + mydata_len);
1731 
1732  progress += mydata_len;
1733  SCLogDebug("raw progress now %"PRIu64, progress);
1734 
1735  /* data is beyond the progress we'd like, and before last ack. Gap. */
1736  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1737  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1738  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1739  progress = mydata_offset;
1740  SCLogDebug("raw progress now %"PRIu64, progress);
1741 
1742  } else {
1743  SCLogDebug("not increasing progress, data gap => mydata_offset "
1744  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1745  }
1746 
1747  if (iter == NULL || r == 1)
1748  break;
1749  }
1750 end:
1751  *progress_out = progress;
1752  return r;
1753 }
1754 
1756  StreamReassembleRawFunc Callback, void *cb_data,
1757  uint64_t *progress_out, bool respect_inspect_depth)
1758 {
1759  /* handle inline separately as the logic is very different */
1760  if (StreamTcpInlineMode() == TRUE) {
1761  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1762  }
1763 
1764  TcpStream *stream;
1765  if (PKT_IS_TOSERVER(p)) {
1766  stream = &ssn->client;
1767  } else {
1768  stream = &ssn->server;
1769  }
1770 
1772  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1773  {
1774  *progress_out = STREAM_RAW_PROGRESS(stream);
1775  return 0;
1776  }
1777 
1778  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1779  STREAM_RAW_PROGRESS(stream), progress_out,
1780  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1781 }
1782 
1784  StreamReassembleRawFunc Callback, void *cb_data,
1785  uint64_t progress_in,
1786  uint64_t *progress_out, bool eof)
1787 {
1788  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1789  return 0;
1790 
1791  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1792  progress_in, progress_out, eof, false);
1793 }
1794 
1795 /** \internal
1796  * \brief update app layer based on received ACK
1797  *
1798  * \retval r 0 on success, -1 on error
1799  */
1800 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1801  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1802 {
1803  SCEnter();
1804 
1805  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1806  SCReturnInt(-1);
1807 
1808  SCReturnInt(0);
1809 }
1810 
1812  TcpSession *ssn, TcpStream *stream,
1813  Packet *p, PacketQueueNoLock *pq)
1814 {
1815  SCEnter();
1816 
1817  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1818 
1819  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1820  ssn, stream, p, p->payload_len);
1821 
1822  /* default IDS: update opposing side (triggered by ACK) */
1824  /* inline and stream end and flow timeout packets trigger same dir handling */
1825  if (StreamTcpInlineMode()) {
1826  dir = UPDATE_DIR_PACKET;
1827  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1828  dir = UPDATE_DIR_PACKET;
1829  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1830  dir = UPDATE_DIR_PACKET;
1831  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1832  dir = UPDATE_DIR_PACKET;
1833  } else if (ssn->state == TCP_CLOSED) {
1834  dir = UPDATE_DIR_BOTH;
1835  }
1836 
1837  /* handle ack received */
1838  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
1839  /* we need to update the opposing stream in
1840  * StreamTcpReassembleHandleSegmentUpdateACK */
1841  TcpStream *opposing_stream = NULL;
1842  if (stream == &ssn->client) {
1843  opposing_stream = &ssn->server;
1844  } else {
1845  opposing_stream = &ssn->client;
1846  }
1847 
1848  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
1849 
1850  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
1851  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1852  SCReturnInt(-1);
1853  }
1854 
1855  /* StreamTcpReassembleHandleSegmentUpdateACK
1856  * may swap content of ssn->server and ssn->client structures.
1857  * We have to continue with initial content of the stream in such case */
1858  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
1859  if (reversed_before_ack_handling != reversed_after_ack_handling) {
1860  SCLogDebug("TCP streams were swapped");
1861  stream = opposing_stream;
1862  }
1863  }
1864  /* if this segment contains data, insert it */
1865  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1866  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1867 
1868  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1869  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1870  SCReturnInt(-1);
1871  }
1872 
1873  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1874  p->flags |= PKT_STREAM_ADD;
1875  } else {
1876  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1877  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1878  ssn, stream, p->payload_len,
1879  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1880 
1881  }
1882 
1883  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1884  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1885  * was *just* set. In this case we trigger the AppLayer Truncate
1886  * logic, to inform the applayer no more data in this direction is
1887  * to be expected. */
1888  if ((stream->flags &
1891  {
1892  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1893  if (dir != UPDATE_DIR_PACKET) {
1894  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1895  "can trigger Truncate");
1896  dir = UPDATE_DIR_PACKET;
1897  }
1898  }
1899 
1900  /* in stream inline mode even if we have no data we call the reassembly
1901  * functions to handle EOF */
1902  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1903  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1904  StreamTcpInlineMode()?"true":"false",
1905  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1906  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1907  SCReturnInt(-1);
1908  }
1909  }
1910 
1911  SCReturnInt(0);
1912 }
1913 
1914 /**
1915  * \brief get a segment from the pool
1916  *
1917  * \retval seg Segment from the pool or NULL
1918  */
1920 {
1921  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1922  SCLogDebug("seg we return is %p", seg);
1923  if (seg == NULL) {
1924  /* Increment the counter to show that we are not able to serve the
1925  segment request due to memcap limit */
1927  } else {
1928  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1929  }
1930 
1931  return seg;
1932 }
1933 
1934 /**
1935  * \brief Trigger RAW stream reassembly
1936  *
1937  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1938  * reassembly from the applayer, for example upon completion of a
1939  * HTTP request.
1940  *
1941  * It sets a flag in the stream so that the next Raw call will return
1942  * the data.
1943  *
1944  * \param ssn TcpSession
1945  */
1947 {
1948 #ifdef DEBUG
1949  BUG_ON(ssn == NULL);
1950 #endif
1951 
1952  if (ssn != NULL) {
1953  if (direction == STREAM_TOSERVER) {
1955  } else {
1957  }
1958 
1959  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1960  }
1961 }
1962 
1963 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1964 {
1965 #ifdef DEBUG
1966  BUG_ON(ssn == NULL);
1967 #endif
1968 
1969  if (ssn != NULL) {
1970  if (direction == STREAM_TOSERVER) {
1971  ssn->client.min_inspect_depth = depth;
1972  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1973  } else {
1974  ssn->server.min_inspect_depth = depth;
1975  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1976  }
1977  }
1978 }
1979 
1980 #ifdef UNITTESTS
1981 /** unit tests and it's support functions below */
1982 
1983 #define SET_ISN(stream, setseq) \
1984  (stream)->isn = (setseq); \
1985  (stream)->base_seq = (setseq) + 1
1986 
1987 /** \brief The Function to create the packet with given payload, which is used
1988  * to test the reassembly of the engine.
1989  *
1990  * \param payload The variable used to store the payload contents of the
1991  * current packet.
1992  * \param value The value which current payload will have for this packet
1993  * \param payload_len The length of the filed payload for current packet.
1994  * \param len Length of the payload array
1995  */
1996 
1997 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1998  uint8_t payload_len, uint8_t len)
1999 {
2000  uint8_t i;
2001  for (i = 0; i < payload_len; i++)
2002  payload[i] = value;
2003  for (; i < len; i++)
2004  payload = NULL;
2005 }
2006 
2007 /** \brief The Function Checks the reassembled stream contents against predefined
2008  * stream contents according to OS policy used.
2009  *
2010  * \param stream_policy Predefined value of stream for different OS policies
2011  * \param stream Reassembled stream returned from the reassembly functions
2012  */
2013 
2014 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
2015 {
2016  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
2017  {
2018  //PrintRawDataFp(stdout, stream_policy, sp_size);
2019  return 0;
2020  }
2021  return 1;
2022 }
2023 
2024 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2025 {
2026  if (StreamingBufferCompareRawData(&stream->sb,
2027  data, data_len) == 0)
2028  {
2029  SCReturnInt(0);
2030  }
2031  SCLogInfo("OK");
2032  PrintRawDataFp(stdout, data, data_len);
2033  return 1;
2034 }
2035 
2036 #define MISSED_START(isn) \
2037  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2038  TcpSession ssn; \
2039  ThreadVars tv; \
2040  memset(&tv, 0, sizeof(tv)); \
2041  \
2042  StreamTcpUTInit(&ra_ctx); \
2043  \
2044  StreamTcpUTSetupSession(&ssn); \
2045  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2046  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2047  \
2048  TcpStream *stream = &ssn.client;
2049 
2050 #define MISSED_END \
2051  StreamTcpUTClearSession(&ssn); \
2052  StreamTcpUTDeinit(ra_ctx); \
2053  PASS
2054 
2055 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2056  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2057  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2058 
2059 /**
2060  * \test Test the handling of packets missed by both IDS and the end host.
2061  * The packet is missed in the starting of the stream.
2062  *
2063  * \retval On success it returns 1 and on failure 0.
2064  */
2065 
2066 static int StreamTcpReassembleTest25 (void)
2067 {
2068  MISSED_START(6);
2069  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
2070  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
2071  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2072  MISSED_END;
2073 }
2074 
2075 /**
2076  * \test Test the handling of packets missed by both IDS and the end host.
2077  * The packet is missed in the middle of the stream.
2078  *
2079  * \retval On success it returns 1 and on failure 0.
2080  */
2081 
2082 static int StreamTcpReassembleTest26 (void)
2083 {
2084  MISSED_START(9);
2085  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2086  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2087  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2088  MISSED_END;
2089 }
2090 
2091 /**
2092  * \test Test the handling of packets missed by both IDS and the end host.
2093  * The packet is missed in the end of the stream.
2094  *
2095  * \retval On success it returns 1 and on failure 0.
2096  */
2097 
2098 static int StreamTcpReassembleTest27 (void)
2099 {
2100  MISSED_START(9);
2101  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2102  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2103  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2104  MISSED_END;
2105 }
2106 
2107 /**
2108  * \test Test the handling of packets missed by IDS, but the end host has
2109  * received it and send the acknowledgment of it. The packet is missed
2110  * in the starting of the stream.
2111  *
2112  * \retval On success it returns 1 and on failure 0.
2113  */
2114 
2115 static int StreamTcpReassembleTest28 (void)
2116 {
2117  MISSED_START(6);
2118  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2119  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2120  ssn.state = TCP_TIME_WAIT;
2121  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2122  MISSED_END;
2123 }
2124 
2125 /**
2126  * \test Test the handling of packets missed by IDS, but the end host has
2127  * received it and send the acknowledgment of it. The packet is missed
2128  * in the middle of the stream.
2129  *
2130  * \retval On success it returns 1 and on failure 0.
2131  */
2132 
2133 static int StreamTcpReassembleTest29 (void)
2134 {
2135  MISSED_START(9);
2136  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2137  ssn.state = TCP_TIME_WAIT;
2138  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2139  MISSED_END;
2140 }
2141 
2142 static int StreamTcpReassembleTest33(void)
2143 {
2144  TcpSession ssn;
2145  Packet *p = PacketGetFromAlloc();
2146  FAIL_IF(unlikely(p == NULL));
2147  Flow f;
2148  TCPHdr tcph;
2149  TcpReassemblyThreadCtx *ra_ctx = NULL;
2151  uint8_t packet[1460] = "";
2152 
2153  StreamTcpUTInit(&ra_ctx);
2155 
2156  PacketQueueNoLock pq;
2157  memset(&pq,0,sizeof(PacketQueueNoLock));
2158  memset(&f, 0, sizeof (Flow));
2159  memset(&tcph, 0, sizeof (TCPHdr));
2160  ThreadVars tv;
2161  memset(&tv, 0, sizeof (ThreadVars));
2162  FLOW_INITIALIZE(&f);
2163  f.protoctx = &ssn;
2164  f.proto = IPPROTO_TCP;
2165  p->src.family = AF_INET;
2166  p->dst.family = AF_INET;
2167  p->proto = IPPROTO_TCP;
2168  p->flow = &f;
2169  tcph.th_win = 5480;
2170  tcph.th_flags = TH_PUSH | TH_ACK;
2171  p->tcph = &tcph;
2173  p->payload = packet;
2174 
2175  p->tcph->th_seq = htonl(10);
2176  p->tcph->th_ack = htonl(31);
2177  p->payload_len = 10;
2178 
2179  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2180 
2181  p->tcph->th_seq = htonl(20);
2182  p->tcph->th_ack = htonl(31);
2183  p->payload_len = 10;
2184 
2185  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2186 
2187  p->tcph->th_seq = htonl(40);
2188  p->tcph->th_ack = htonl(31);
2189  p->payload_len = 10;
2190 
2191  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2192 
2193  p->tcph->th_seq = htonl(5);
2194  p->tcph->th_ack = htonl(31);
2195  p->payload_len = 30;
2196 
2197  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2198 
2200  StreamTcpUTDeinit(ra_ctx);
2201  SCFree(p);
2202  PASS;
2203 }
2204 
2205 static int StreamTcpReassembleTest34(void)
2206 {
2207  TcpSession ssn;
2208  Packet *p = PacketGetFromAlloc();
2209  FAIL_IF(unlikely(p == NULL));
2210  Flow f;
2211  TCPHdr tcph;
2212  TcpReassemblyThreadCtx *ra_ctx = NULL;
2214  uint8_t packet[1460] = "";
2215 
2216  StreamTcpUTInit(&ra_ctx);
2218  PacketQueueNoLock pq;
2219  memset(&pq,0,sizeof(PacketQueueNoLock));
2220  memset(&f, 0, sizeof (Flow));
2221  memset(&tcph, 0, sizeof (TCPHdr));
2222  ThreadVars tv;
2223  memset(&tv, 0, sizeof (ThreadVars));
2224  FLOW_INITIALIZE(&f);
2225  f.protoctx = &ssn;
2226  f.proto = IPPROTO_TCP;
2227  p->src.family = AF_INET;
2228  p->dst.family = AF_INET;
2229  p->proto = IPPROTO_TCP;
2230  p->flow = &f;
2231  tcph.th_win = 5480;
2232  tcph.th_flags = TH_PUSH | TH_ACK;
2233  p->tcph = &tcph;
2235  p->payload = packet;
2236  SET_ISN(&ssn.client, 857961230);
2237 
2238  p->tcph->th_seq = htonl(857961230);
2239  p->tcph->th_ack = htonl(31);
2240  p->payload_len = 304;
2241 
2242  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2243 
2244  p->tcph->th_seq = htonl(857961534);
2245  p->tcph->th_ack = htonl(31);
2246  p->payload_len = 1460;
2247 
2248  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2249 
2250  p->tcph->th_seq = htonl(857963582);
2251  p->tcph->th_ack = htonl(31);
2252  p->payload_len = 1460;
2253 
2254  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2255 
2256  p->tcph->th_seq = htonl(857960946);
2257  p->tcph->th_ack = htonl(31);
2258  p->payload_len = 1460;
2259 
2260  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2261 
2263  StreamTcpUTDeinit(ra_ctx);
2264  SCFree(p);
2265  PASS;
2266 }
2267 
2268 /** \test Test the bug 76 condition */
2269 static int StreamTcpReassembleTest37(void)
2270 {
2271  TcpSession ssn;
2272  Flow f;
2273  TCPHdr tcph;
2274  TcpReassemblyThreadCtx *ra_ctx = NULL;
2275  uint8_t packet[1460] = "";
2276  PacketQueueNoLock pq;
2277  ThreadVars tv;
2278  memset(&tv, 0, sizeof (ThreadVars));
2279 
2280  Packet *p = PacketGetFromAlloc();
2281  FAIL_IF(unlikely(p == NULL));
2282 
2283  StreamTcpUTInit(&ra_ctx);
2285  memset(&pq,0,sizeof(PacketQueueNoLock));
2286  memset(&f, 0, sizeof (Flow));
2287  memset(&tcph, 0, sizeof (TCPHdr));
2288  memset(&tv, 0, sizeof (ThreadVars));
2289 
2290  FLOW_INITIALIZE(&f);
2291  f.protoctx = &ssn;
2292  f.proto = IPPROTO_TCP;
2293  p->src.family = AF_INET;
2294  p->dst.family = AF_INET;
2295  p->proto = IPPROTO_TCP;
2296  p->flow = &f;
2297  tcph.th_win = 5480;
2298  tcph.th_flags = TH_PUSH | TH_ACK;
2299  p->tcph = &tcph;
2301  p->payload = packet;
2303 
2304  p->tcph->th_seq = htonl(3061088537UL);
2305  p->tcph->th_ack = htonl(1729548549UL);
2306  p->payload_len = 1391;
2307  ssn.client.last_ack = 3061091137UL;
2308  SET_ISN(&ssn.client, 3061091309UL);
2309 
2310  /* pre base_seq, so should be rejected */
2311  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2312 
2313  p->tcph->th_seq = htonl(3061089928UL);
2314  p->tcph->th_ack = htonl(1729548549UL);
2315  p->payload_len = 1391;
2316  ssn.client.last_ack = 3061091137UL;
2317 
2318  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2319 
2320  p->tcph->th_seq = htonl(3061091319UL);
2321  p->tcph->th_ack = htonl(1729548549UL);
2322  p->payload_len = 1391;
2323  ssn.client.last_ack = 3061091137UL;
2324 
2325  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2326 
2328  StreamTcpUTDeinit(ra_ctx);
2329  SCFree(p);
2330  PASS;
2331 }
2332 
2333 /**
2334  * \test Test to make sure that we don't return the segments until the app
2335  * layer proto has been detected and after that remove the processed
2336  * segments.
2337  *
2338  * \retval On success it returns 1 and on failure 0.
2339  */
2340 
2341 static int StreamTcpReassembleTest39 (void)
2342 {
2343  Packet *p = PacketGetFromAlloc();
2344  FAIL_IF(unlikely(p == NULL));
2345  Flow f;
2346  ThreadVars tv;
2347  StreamTcpThread stt;
2348  TCPHdr tcph;
2349  PacketQueueNoLock pq;
2350  memset(&pq,0,sizeof(PacketQueueNoLock));
2351  memset (&f, 0, sizeof(Flow));
2352  memset(&tv, 0, sizeof (ThreadVars));
2353  memset(&stt, 0, sizeof (stt));
2354  memset(&tcph, 0, sizeof (TCPHdr));
2355 
2356  FLOW_INITIALIZE(&f);
2357  f.flags = FLOW_IPV4;
2358  f.proto = IPPROTO_TCP;
2359  p->flow = &f;
2360  p->tcph = &tcph;
2361 
2362  StreamTcpUTInit(&stt.ra_ctx);
2363 
2364  /* handshake */
2365  tcph.th_win = htons(5480);
2366  tcph.th_flags = TH_SYN;
2368  p->payload_len = 0;
2369  p->payload = NULL;
2370  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2371 
2372  TcpSession *ssn = (TcpSession *)f.protoctx;
2373  FAIL_IF_NULL(ssn);
2374 
2385  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2386  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2387  FAIL_IF(ssn->data_first_seen_dir != 0);
2388 
2389  /* handshake */
2390  p->tcph->th_ack = htonl(1);
2391  p->tcph->th_flags = TH_SYN | TH_ACK;
2393  p->payload_len = 0;
2394  p->payload = NULL;
2395  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2406  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2407  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2408  FAIL_IF(ssn->data_first_seen_dir != 0);
2409 
2410  /* handshake */
2411  p->tcph->th_ack = htonl(1);
2412  p->tcph->th_seq = htonl(1);
2413  p->tcph->th_flags = TH_ACK;
2415  p->payload_len = 0;
2416  p->payload = NULL;
2417  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2428  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2429  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2430  FAIL_IF(ssn->data_first_seen_dir != 0);
2431 
2432  /* partial request */
2433  uint8_t request1[] = { 0x47, 0x45, };
2434  p->tcph->th_ack = htonl(1);
2435  p->tcph->th_seq = htonl(1);
2436  p->tcph->th_flags = TH_PUSH | TH_ACK;
2438  p->payload_len = sizeof(request1);
2439  p->payload = request1;
2440  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2451  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2452  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2453  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2455 
2456  /* response ack against partial request */
2457  p->tcph->th_ack = htonl(3);
2458  p->tcph->th_seq = htonl(1);
2459  p->tcph->th_flags = TH_ACK;
2461  p->payload_len = 0;
2462  p->payload = NULL;
2463  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2474  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2475  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2476  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2478 
2479  /* complete partial request */
2480  uint8_t request2[] = {
2481  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2482  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2483  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2484  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2485  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2486  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2487  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2488  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2489  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2490  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2491  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2492  p->tcph->th_ack = htonl(1);
2493  p->tcph->th_seq = htonl(3);
2494  p->tcph->th_flags = TH_PUSH | TH_ACK;
2496  p->payload_len = sizeof(request2);
2497  p->payload = request2;
2498  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2509  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2510  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2511  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2512  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2514 
2515  /* response - request ack */
2516  uint8_t response[] = {
2517  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2518  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2519  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2520  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2521  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2522  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2523  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2524  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2525  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2526  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2527  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2528  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2529  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2530  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2531  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2532  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2533  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2534  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2535  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2536  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2537  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2538  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2539  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2540  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2541  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2542  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2543  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2544  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2545  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2546  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2547  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2548  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2549  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2550  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2551  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2552  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2553  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2554  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2555  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2556  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2557  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2558  p->tcph->th_ack = htonl(88);
2559  p->tcph->th_seq = htonl(1);
2560  p->tcph->th_flags = TH_PUSH | TH_ACK;
2562  p->payload_len = sizeof(response);
2563  p->payload = response;
2564 
2565  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2577  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2578  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2579  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2580  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2581 
2582  /* response ack from request */
2583  p->tcph->th_ack = htonl(328);
2584  p->tcph->th_seq = htonl(88);
2585  p->tcph->th_flags = TH_ACK;
2587  p->payload_len = 0;
2588  p->payload = NULL;
2589  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2601  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2602  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2603  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2604  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2605  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2606 
2607  /* response - acking */
2608  p->tcph->th_ack = htonl(88);
2609  p->tcph->th_seq = htonl(328);
2610  p->tcph->th_flags = TH_PUSH | TH_ACK;
2612  p->payload_len = 0;
2613  p->payload = NULL;
2614  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2626  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2627  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2628  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2629  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2630  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2631 
2632  /* response ack from request */
2633  p->tcph->th_ack = htonl(328);
2634  p->tcph->th_seq = htonl(88);
2635  p->tcph->th_flags = TH_ACK;
2637  p->payload_len = 0;
2638  p->payload = NULL;
2639  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2651  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2652  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2653  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2654  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2655 
2656  /* response - acking the request again*/
2657  p->tcph->th_ack = htonl(88);
2658  p->tcph->th_seq = htonl(328);
2659  p->tcph->th_flags = TH_PUSH | TH_ACK;
2661  p->payload_len = 0;
2662  p->payload = NULL;
2663  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2675  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2676  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2677  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2678  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2679 
2680  /*** New Request ***/
2681 
2682  /* partial request */
2683  p->tcph->th_ack = htonl(328);
2684  p->tcph->th_seq = htonl(88);
2685  p->tcph->th_flags = TH_PUSH | TH_ACK;
2687  p->payload_len = sizeof(request1);
2688  p->payload = request1;
2689  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2701  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2702  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2703  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2704  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2705  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2706 
2707  /* response ack against partial request */
2708  p->tcph->th_ack = htonl(90);
2709  p->tcph->th_seq = htonl(328);
2710  p->tcph->th_flags = TH_ACK;
2712  p->payload_len = 0;
2713  p->payload = NULL;
2714  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2715  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2727  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2728  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2729  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2730  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2731  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2732 
2733  /* complete request */
2734  p->tcph->th_ack = htonl(328);
2735  p->tcph->th_seq = htonl(90);
2736  p->tcph->th_flags = TH_PUSH | TH_ACK;
2738  p->payload_len = sizeof(request2);
2739  p->payload = request2;
2740  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2752  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2753  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2754  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2755  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2756  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2757  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2758 
2759  /* response ack against second partial request */
2760  p->tcph->th_ack = htonl(175);
2761  p->tcph->th_seq = htonl(328);
2762  p->tcph->th_flags = TH_ACK;
2764  p->payload_len = 0;
2765  p->payload = NULL;
2766 
2767  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2779  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2780  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2781  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2782  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2783  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2784  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2785 
2786  /* response acking a request */
2787  p->tcph->th_ack = htonl(175);
2788  p->tcph->th_seq = htonl(328);
2789  p->tcph->th_flags = TH_ACK;
2791  p->payload_len = 0;
2792  p->payload = NULL;
2793  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2799 
2802 
2803  /* request acking a response */
2804  p->tcph->th_ack = htonl(328);
2805  p->tcph->th_seq = htonl(175);
2806  p->tcph->th_flags = TH_ACK;
2808  p->payload_len = 0;
2809  p->payload = NULL;
2810  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2811 
2812  StreamTcpSessionClear(ssn);
2814  SCFree(p);
2815  PASS;
2816 }
2817 
2818 /**
2819  * \test Test to make sure that we sent all the segments from the initial
2820  * segments to app layer until we have detected the app layer proto.
2821  *
2822  * \retval On success it returns 1 and on failure 0.
2823  */
2824 
2825 static int StreamTcpReassembleTest40 (void)
2826 {
2827  Packet *p = PacketGetFromAlloc();
2828  FAIL_IF_NULL(p);
2829  Flow *f = NULL;
2830  TCPHdr tcph;
2831  TcpSession ssn;
2832  PacketQueueNoLock pq;
2833  memset(&pq,0,sizeof(PacketQueueNoLock));
2834  memset(&tcph, 0, sizeof (TCPHdr));
2835  ThreadVars tv;
2836  memset(&tv, 0, sizeof (ThreadVars));
2837 
2840 
2842  FAIL_IF_NULL(ra_ctx);
2843 
2844  uint8_t httpbuf1[] = "P";
2845  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2846  uint8_t httpbuf3[] = "O";
2847  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2848  uint8_t httpbuf4[] = "S";
2849  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2850  uint8_t httpbuf5[] = "T \r\n";
2851  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2852 
2853  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2854  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2855 
2856  SET_ISN(&ssn.server, 9);
2857  ssn.server.last_ack = 10;
2858  SET_ISN(&ssn.client, 9);
2859  ssn.client.isn = 9;
2860 
2861  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2862  FAIL_IF_NULL(f);
2863  f->protoctx = &ssn;
2864  f->proto = IPPROTO_TCP;
2865  p->flow = f;
2866 
2867  tcph.th_win = htons(5480);
2868  tcph.th_seq = htonl(10);
2869  tcph.th_ack = htonl(10);
2870  tcph.th_flags = TH_ACK|TH_PUSH;
2871  p->tcph = &tcph;
2873  p->payload = httpbuf1;
2874  p->payload_len = httplen1;
2875  ssn.state = TCP_ESTABLISHED;
2876  TcpStream *s = &ssn.client;
2877  SCLogDebug("1 -- start");
2878  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2879 
2881  p->payload = httpbuf2;
2882  p->payload_len = httplen2;
2883  tcph.th_seq = htonl(10);
2884  tcph.th_ack = htonl(11);
2885  s = &ssn.server;
2886  ssn.server.last_ack = 11;
2887  SCLogDebug("2 -- start");
2888  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2889 
2891  p->payload = httpbuf3;
2892  p->payload_len = httplen3;
2893  tcph.th_seq = htonl(11);
2894  tcph.th_ack = htonl(55);
2895  s = &ssn.client;
2896  ssn.client.last_ack = 55;
2897  SCLogDebug("3 -- start");
2898  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2899 
2901  p->payload = httpbuf2;
2902  p->payload_len = httplen2;
2903  tcph.th_seq = htonl(55);
2904  tcph.th_ack = htonl(12);
2905  s = &ssn.server;
2906  ssn.server.last_ack = 12;
2907  SCLogDebug("4 -- start");
2908  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2909 
2910  /* check is have the segment in the list and flagged or not */
2911  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2912  FAIL_IF_NULL(seg);
2913  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2914 
2916  p->payload = httpbuf4;
2917  p->payload_len = httplen4;
2918  tcph.th_seq = htonl(12);
2919  tcph.th_ack = htonl(100);
2920  s = &ssn.client;
2921  ssn.client.last_ack = 100;
2922  SCLogDebug("5 -- start");
2923  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2924 
2926  p->payload = httpbuf2;
2927  p->payload_len = httplen2;
2928  tcph.th_seq = htonl(100);
2929  tcph.th_ack = htonl(13);
2930  s = &ssn.server;
2931  ssn.server.last_ack = 13;
2932  SCLogDebug("6 -- start");
2933  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2934 
2936  p->payload = httpbuf5;
2937  p->payload_len = httplen5;
2938  tcph.th_seq = htonl(13);
2939  tcph.th_ack = htonl(145);
2940  s = &ssn.client;
2941  ssn.client.last_ack = 145;
2942  SCLogDebug("7 -- start");
2943  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2944 
2946  p->payload = httpbuf2;
2947  p->payload_len = httplen2;
2948  tcph.th_seq = htonl(145);
2949  tcph.th_ack = htonl(16);
2950  s = &ssn.server;
2951  ssn.server.last_ack = 16;
2952  SCLogDebug("8 -- start");
2953  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2954  FAIL_IF(f->alproto != ALPROTO_HTTP1);
2955 
2959  SCFree(p);
2960  UTHFreeFlow(f);
2961  PASS;
2962 }
2963 
2964 /** \test Test the memcap incrementing/decrementing and memcap check */
2965 static int StreamTcpReassembleTest44(void)
2966 {
2968  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2970  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2972  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2974  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2976  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2977  PASS;
2978 }
2979 
2980 /**
2981  * \test Test to make sure that reassembly_depth is enforced.
2982  *
2983  * \retval On success it returns 1 and on failure 0.
2984  */
2985 
2986 static int StreamTcpReassembleTest45 (void)
2987 {
2988  TcpReassemblyThreadCtx *ra_ctx = NULL;
2989  TcpSession ssn;
2990  ThreadVars tv;
2991  memset(&tv, 0, sizeof(tv));
2992  uint8_t payload[100] = {0};
2993  uint16_t payload_size = 100;
2994 
2995  StreamTcpUTInit(&ra_ctx);
2997 
2999  ssn.reassembly_depth = 100;
3000  StreamTcpUTSetupStream(&ssn.server, 100);
3001  StreamTcpUTSetupStream(&ssn.client, 100);
3002 
3003  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3004  FAIL_IF(r != 0);
3006 
3007  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3008  FAIL_IF(r != 0);
3010 
3014  StreamTcpUTDeinit(ra_ctx);
3015  PASS;
3016 }
3017 
3018 /**
3019  * \test Test the unlimited config value of reassembly depth.
3020  *
3021  * \retval On success it returns 1 and on failure 0.
3022  */
3023 
3024 static int StreamTcpReassembleTest46 (void)
3025 {
3026  int result = 0;
3027  TcpReassemblyThreadCtx *ra_ctx = NULL;
3028  TcpSession ssn;
3029  ThreadVars tv;
3030  memset(&tv, 0, sizeof(tv));
3031  uint8_t payload[100] = {0};
3032  uint16_t payload_size = 100;
3033 
3034  StreamTcpUTInit(&ra_ctx);
3036 
3038  StreamTcpUTSetupStream(&ssn.server, 100);
3039  StreamTcpUTSetupStream(&ssn.client, 100);
3040 
3041  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3042  if (r != 0)
3043  goto end;
3045  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3046  goto end;
3047  }
3048 
3049  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3050  if (r != 0)
3051  goto end;
3053  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3054  goto end;
3055  }
3056 
3057  result = 1;
3058 end:
3062  StreamTcpUTDeinit(ra_ctx);
3063  return result;
3064 }
3065 
3066 /**
3067  * \test Test to make sure we detect the sequence wrap around and continue
3068  * stream reassembly properly.
3069  *
3070  * \retval On success it returns 1 and on failure 0.
3071  */
3072 
3073 static int StreamTcpReassembleTest47 (void)
3074 {
3075  Packet *p = PacketGetFromAlloc();
3076  FAIL_IF(unlikely(p == NULL));
3077  Flow *f = NULL;
3078  TCPHdr tcph;
3079  TcpSession ssn;
3080  ThreadVars tv;
3081  PacketQueueNoLock pq;
3082  memset(&pq,0,sizeof(PacketQueueNoLock));
3083  memset(&tcph, 0, sizeof (TCPHdr));
3084  memset(&tv, 0, sizeof (ThreadVars));
3088 
3089  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3090  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3091 
3092  SET_ISN(&ssn.server, 572799781UL);
3093  ssn.server.last_ack = 572799782UL;
3094 
3095  SET_ISN(&ssn.client, 4294967289UL);
3096  ssn.client.last_ack = 21;
3097 
3098  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3099  FAIL_IF(f == NULL);
3100  f->protoctx = &ssn;
3101  f->proto = IPPROTO_TCP;
3102  p->flow = f;
3103 
3104  tcph.th_win = htons(5480);
3105  ssn.state = TCP_ESTABLISHED;
3106  TcpStream *s = NULL;
3107  uint8_t cnt = 0;
3108 
3109  for (cnt=0; cnt < httplen1; cnt++) {
3110  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3111  tcph.th_ack = htonl(572799782UL);
3112  tcph.th_flags = TH_ACK|TH_PUSH;
3113  p->tcph = &tcph;
3115  p->payload = &httpbuf1[cnt];
3116  p->payload_len = 1;
3117  s = &ssn.client;
3118 
3119  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3120 
3122  p->payload = NULL;
3123  p->payload_len = 0;
3124  tcph.th_seq = htonl(572799782UL);
3125  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3126  tcph.th_flags = TH_ACK;
3127  p->tcph = &tcph;
3128  s = &ssn.server;
3129 
3130  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3131  }
3132 
3133  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3134 
3138  SCFree(p);
3139  UTHFreeFlow(f);
3140  PASS;
3141 }
3142 
3143 /** \test 3 in order segments in inline reassembly */
3144 static int StreamTcpReassembleInlineTest01(void)
3145 {
3146  int ret = 0;
3147  TcpReassemblyThreadCtx *ra_ctx = NULL;
3148  ThreadVars tv;
3149  TcpSession ssn;
3150  Flow f;
3151 
3152  memset(&tv, 0x00, sizeof(tv));
3153 
3154  StreamTcpUTInit(&ra_ctx);
3157  StreamTcpUTSetupStream(&ssn.client, 1);
3158  FLOW_INITIALIZE(&f);
3159 
3160  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3161  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3162  if (p == NULL) {
3163  printf("couldn't get a packet: ");
3164  goto end;
3165  }
3166  p->tcph->th_seq = htonl(12);
3167  p->flow = &f;
3168 
3169  FLOWLOCK_WRLOCK(&f);
3170  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3171  printf("failed to add segment 1: ");
3172  goto end;
3173  }
3174  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3175  printf("failed to add segment 2: ");
3176  goto end;
3177  }
3178  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3179  printf("failed to add segment 3: ");
3180  goto end;
3181  }
3182  ssn.client.next_seq = 17;
3183  ret = 1;
3184 end:
3185  FLOWLOCK_UNLOCK(&f);
3186  FLOW_DESTROY(&f);
3187  UTHFreePacket(p);
3189  StreamTcpUTDeinit(ra_ctx);
3190  return ret;
3191 }
3192 
3193 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3194  * test the sliding window reassembly.
3195  */
3196 static int StreamTcpReassembleInlineTest02(void)
3197 {
3198  int ret = 0;
3199  TcpReassemblyThreadCtx *ra_ctx = NULL;
3200  ThreadVars tv;
3201  TcpSession ssn;
3202  Flow f;
3203 
3204  memset(&tv, 0x00, sizeof(tv));
3205 
3206  StreamTcpUTInit(&ra_ctx);
3209  StreamTcpUTSetupStream(&ssn.client, 1);
3210  FLOW_INITIALIZE(&f);
3211 
3212  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3213  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3214  if (p == NULL) {
3215  printf("couldn't get a packet: ");
3216  goto end;
3217  }
3218  p->tcph->th_seq = htonl(12);
3219  p->flow = &f;
3220 
3221  FLOWLOCK_WRLOCK(&f);
3222  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3223  printf("failed to add segment 1: ");
3224  goto end;
3225  }
3226  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3227  printf("failed to add segment 2: ");
3228  goto end;
3229  }
3230  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3231  printf("failed to add segment 3: ");
3232  goto end;
3233  }
3234  ssn.client.next_seq = 17;
3235  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3236  printf("failed to add segment 4: ");
3237  goto end;
3238  }
3239  ssn.client.next_seq = 22;
3240  ret = 1;
3241 end:
3242  FLOWLOCK_UNLOCK(&f);
3243  FLOW_DESTROY(&f);
3244  UTHFreePacket(p);
3246  StreamTcpUTDeinit(ra_ctx);
3247  return ret;
3248 }
3249 
3250 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3251  * test the sliding window reassembly with a small window size so that we
3252  * cutting off at the start (left edge)
3253  */
3254 static int StreamTcpReassembleInlineTest03(void)
3255 {
3256  int ret = 0;
3257  TcpReassemblyThreadCtx *ra_ctx = NULL;
3258  ThreadVars tv;
3259  TcpSession ssn;
3260  Flow f;
3261 
3262  memset(&tv, 0x00, sizeof(tv));
3263 
3264  StreamTcpUTInit(&ra_ctx);
3267  StreamTcpUTSetupStream(&ssn.client, 1);
3268  FLOW_INITIALIZE(&f);
3269 
3271 
3272  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3273  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3274  if (p == NULL) {
3275  printf("couldn't get a packet: ");
3276  goto end;
3277  }
3278  p->tcph->th_seq = htonl(12);
3279  p->flow = &f;
3281 
3282  FLOWLOCK_WRLOCK(&f);
3283  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3284  printf("failed to add segment 1: ");
3285  goto end;
3286  }
3287  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3288  printf("failed to add segment 2: ");
3289  goto end;
3290  }
3291  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3292  printf("failed to add segment 3: ");
3293  goto end;
3294  }
3295  ssn.client.next_seq = 17;
3296  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3297  printf("failed to add segment 4: ");
3298  goto end;
3299  }
3300  ssn.client.next_seq = 22;
3301 
3302  p->tcph->th_seq = htonl(17);
3303  ret = 1;
3304 end:
3305  FLOWLOCK_UNLOCK(&f);
3306  FLOW_DESTROY(&f);
3307  UTHFreePacket(p);
3309  StreamTcpUTDeinit(ra_ctx);
3310  return ret;
3311 }
3312 
3313 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3314  * test the sliding window reassembly with a small window size so that we
3315  * cutting off at the start (left edge) with small packet overlap.
3316  */
3317 static int StreamTcpReassembleInlineTest04(void)
3318 {
3319  int ret = 0;
3320  TcpReassemblyThreadCtx *ra_ctx = NULL;
3321  ThreadVars tv;
3322  TcpSession ssn;
3323  Flow f;
3324 
3325  memset(&tv, 0x00, sizeof(tv));
3326 
3327  StreamTcpUTInit(&ra_ctx);
3330  StreamTcpUTSetupStream(&ssn.client, 1);
3331  FLOW_INITIALIZE(&f);
3332 
3334 
3335  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3336  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3337  if (p == NULL) {
3338  printf("couldn't get a packet: ");
3339  goto end;
3340  }
3341  p->tcph->th_seq = htonl(12);
3342  p->flow = &f;
3344 
3345  FLOWLOCK_WRLOCK(&f);
3346  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3347  printf("failed to add segment 1: ");
3348  goto end;
3349  }
3350  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3351  printf("failed to add segment 2: ");
3352  goto end;
3353  }
3354  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3355  printf("failed to add segment 3: ");
3356  goto end;
3357  }
3358  ssn.client.next_seq = 17;
3359  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3360  printf("failed to add segment 4: ");
3361  goto end;
3362  }
3363  ssn.client.next_seq = 22;
3364 
3365  p->tcph->th_seq = htonl(17);
3366  ret = 1;
3367 end:
3368  FLOWLOCK_UNLOCK(&f);
3369  FLOW_DESTROY(&f);
3370  UTHFreePacket(p);
3372  StreamTcpUTDeinit(ra_ctx);
3373  return ret;
3374 }
3375 
3376 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3377  * test the sliding window reassembly with a small window size so that we
3378  * cutting off at the start (left edge). Test if the first segment is
3379  * removed from the list.
3380  */
3381 static int StreamTcpReassembleInlineTest08(void)
3382 {
3383  TcpReassemblyThreadCtx *ra_ctx = NULL;
3384  ThreadVars tv;
3385  memset(&tv, 0x00, sizeof(tv));
3386  TcpSession ssn;
3387  Flow f;
3388  StreamTcpUTInit(&ra_ctx);
3391  StreamTcpUTSetupStream(&ssn.client, 1);
3392  FLOW_INITIALIZE(&f);
3393 
3395  f.protoctx = &ssn;
3396 
3397  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3398  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3399  FAIL_IF(p == NULL);
3400  p->tcph->th_seq = htonl(12);
3401  p->flow = &f;
3403 
3404  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3405  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3406  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3407  ssn.client.next_seq = 17;
3408  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3409  ssn.client.next_seq = 22;
3410  p->tcph->th_seq = htonl(17);
3412 
3413  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3414  FAIL_IF_NULL(seg);
3415  FAIL_IF_NOT(seg->seq == 2);
3416 
3417  FLOW_DESTROY(&f);
3418  UTHFreePacket(p);
3420  StreamTcpUTDeinit(ra_ctx);
3421  PASS;
3422 }
3423 
3424 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3425  * test the sliding window reassembly with a small window size so that we
3426  * cutting off at the start (left edge). Test if the first segment is
3427  * removed from the list.
3428  */
3429 static int StreamTcpReassembleInlineTest09(void)
3430 {
3431  int ret = 0;
3432  TcpReassemblyThreadCtx *ra_ctx = NULL;
3433  ThreadVars tv;
3434  TcpSession ssn;
3435  Flow f;
3436 
3437  memset(&tv, 0x00, sizeof(tv));
3438 
3439  StreamTcpUTInit(&ra_ctx);
3442  StreamTcpUTSetupStream(&ssn.client, 1);
3443  FLOW_INITIALIZE(&f);
3444 
3446 
3447  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3448  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3449  if (p == NULL) {
3450  printf("couldn't get a packet: ");
3451  goto end;
3452  }
3453  p->tcph->th_seq = htonl(17);
3454  p->flow = &f;
3456 
3457  FLOWLOCK_WRLOCK(&f);
3458  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3459  printf("failed to add segment 1: ");
3460  goto end;
3461  }
3462  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3463  printf("failed to add segment 2: ");
3464  goto end;
3465  }
3466  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3467  printf("failed to add segment 3: ");
3468  goto end;
3469  }
3470  ssn.client.next_seq = 12;
3471  ssn.client.last_ack = 10;
3472 
3473  /* close the GAP and see if we properly reassemble and update base_seq */
3474  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3475  printf("failed to add segment 4: ");
3476  goto end;
3477  }
3478  ssn.client.next_seq = 22;
3479 
3480  p->tcph->th_seq = htonl(12);
3481 
3482  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3483  FAIL_IF_NULL(seg);
3484  FAIL_IF_NOT(seg->seq == 2);
3485 
3486  ret = 1;
3487 end:
3488  FLOWLOCK_UNLOCK(&f);
3489  FLOW_DESTROY(&f);
3490  UTHFreePacket(p);
3492  StreamTcpUTDeinit(ra_ctx);
3493  return ret;
3494 }
3495 
3496 /** \test App Layer reassembly.
3497  */
3498 static int StreamTcpReassembleInlineTest10(void)
3499 {
3500  int ret = 0;
3501  TcpReassemblyThreadCtx *ra_ctx = NULL;
3502  ThreadVars tv;
3503  TcpSession ssn;
3504  Flow *f = NULL;
3505  Packet *p = NULL;
3506 
3507  memset(&tv, 0x00, sizeof(tv));
3508 
3509  StreamTcpUTInit(&ra_ctx);
3512  StreamTcpUTSetupStream(&ssn.server, 1);
3513  ssn.server.last_ack = 2;
3514  StreamTcpUTSetupStream(&ssn.client, 1);
3515  ssn.client.last_ack = 2;
3517 
3518  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3519  if (f == NULL)
3520  goto end;
3521  f->protoctx = &ssn;
3522  f->proto = IPPROTO_TCP;
3523 
3524  uint8_t stream_payload1[] = "GE";
3525  uint8_t stream_payload2[] = "T /";
3526  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3527 
3528  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3529  if (p == NULL) {
3530  printf("couldn't get a packet: ");
3531  goto end;
3532  }
3533  p->tcph->th_seq = htonl(7);
3534  p->flow = f;
3536 
3537  FLOWLOCK_WRLOCK(f);
3538  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3539  printf("failed to add segment 1: ");
3540  goto end;
3541  }
3542  ssn.client.next_seq = 4;
3543 
3544  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3545  if (r < 0) {
3546  printf("StreamTcpReassembleAppLayer failed: ");
3547  goto end;
3548  }
3549 
3550  /* ssn.server.ra_app_base_seq should be isn here. */
3551  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3552  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3553  goto end;
3554  }
3555 
3556  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3557  printf("failed to add segment 2: ");
3558  goto end;
3559  }
3560  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3561  printf("failed to add segment 3: ");
3562  goto end;
3563  }
3564  ssn.client.next_seq = 19;
3565 
3566  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3567  if (r < 0) {
3568  printf("StreamTcpReassembleAppLayer failed: ");
3569  goto end;
3570  }
3571 
3572  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3573 
3574  ret = 1;
3575 end:
3576  UTHFreePacket(p);
3578  StreamTcpUTDeinit(ra_ctx);
3579  FLOWLOCK_UNLOCK(f);
3580  UTHFreeFlow(f);
3581  return ret;
3582 }
3583 
3584 /** \test test insert with overlap
3585  */
3586 static int StreamTcpReassembleInsertTest01(void)
3587 {
3588  TcpReassemblyThreadCtx *ra_ctx = NULL;
3589  ThreadVars tv;
3590  TcpSession ssn;
3591  Flow f;
3592 
3593  memset(&tv, 0x00, sizeof(tv));
3594 
3595  StreamTcpUTInit(&ra_ctx);
3597  StreamTcpUTSetupStream(&ssn.client, 1);
3599  FLOW_INITIALIZE(&f);
3600 
3601  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3602  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3603  FAIL_IF(p == NULL);
3604  p->tcph->th_seq = htonl(12);
3605  p->flow = &f;
3606 
3607  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3608  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3609  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3610  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3611  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3612  ssn.client.next_seq = 21;
3613 
3614  FLOW_DESTROY(&f);
3615  UTHFreePacket(p);
3617  StreamTcpUTDeinit(ra_ctx);
3618  PASS;
3619 }
3620 
3621 /** \test test insert with overlaps
3622  */
3623 static int StreamTcpReassembleInsertTest02(void)
3624 {
3625  int ret = 0;
3626  TcpReassemblyThreadCtx *ra_ctx = NULL;
3627  ThreadVars tv;
3628  TcpSession ssn;
3629 
3630  memset(&tv, 0x00, sizeof(tv));
3631 
3632  StreamTcpUTInit(&ra_ctx);
3634  StreamTcpUTSetupStream(&ssn.client, 1);
3635 
3636  int i;
3637  for (i = 2; i < 10; i++) {
3638  int len;
3639  len = i % 2;
3640  if (len == 0)
3641  len = 1;
3642  int seq;
3643  seq = i * 10;
3644  if (seq < 2)
3645  seq = 2;
3646 
3647  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3648  printf("failed to add segment 1: ");
3649  goto end;
3650  }
3651  }
3652  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3653  printf("failed to add segment 2: ");
3654  goto end;
3655  }
3656 
3657  ret = 1;
3658 end:
3660  StreamTcpUTDeinit(ra_ctx);
3661  return ret;
3662 }
3663 
3664 /** \test test insert with overlaps
3665  */
3666 static int StreamTcpReassembleInsertTest03(void)
3667 {
3668  int ret = 0;
3669  TcpReassemblyThreadCtx *ra_ctx = NULL;
3670  ThreadVars tv;
3671  TcpSession ssn;
3672 
3673  memset(&tv, 0x00, sizeof(tv));
3674 
3675  StreamTcpUTInit(&ra_ctx);
3677  StreamTcpUTSetupStream(&ssn.client, 1);
3678 
3679  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3680  printf("failed to add segment 2: ");
3681  goto end;
3682  }
3683 
3684  int i;
3685  for (i = 2; i < 10; i++) {
3686  int len;
3687  len = i % 2;
3688  if (len == 0)
3689  len = 1;
3690  int seq;
3691  seq = i * 10;
3692  if (seq < 2)
3693  seq = 2;
3694 
3695  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3696  printf("failed to add segment 2: ");
3697  goto end;
3698  }
3699  }
3700  ret = 1;
3701 end:
3703  StreamTcpUTDeinit(ra_ctx);
3704  return ret;
3705 }
3706 
3708 #endif /* UNITTESTS */
3709 
3710 /** \brief The Function Register the Unit tests to test the reassembly engine
3711  * for various OS policies.
3712  */
3713 
3715 {
3716 #ifdef UNITTESTS
3717  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3718  StreamTcpReassembleTest25);
3719  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3720  StreamTcpReassembleTest26);
3721  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3722  StreamTcpReassembleTest27);
3723  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3724  StreamTcpReassembleTest28);
3725  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3726  StreamTcpReassembleTest29);
3727  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3728  StreamTcpReassembleTest33);
3729  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3730  StreamTcpReassembleTest34);
3731  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3732  StreamTcpReassembleTest37);
3733  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3734  StreamTcpReassembleTest39);
3735  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3736  StreamTcpReassembleTest40);
3737  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3738  StreamTcpReassembleTest44);
3739  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3740  StreamTcpReassembleTest45);
3741  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3742  StreamTcpReassembleTest46);
3743  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3744  StreamTcpReassembleTest47);
3745 
3746  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3747  StreamTcpReassembleInlineTest01);
3748  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3749  StreamTcpReassembleInlineTest02);
3750  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3751  StreamTcpReassembleInlineTest03);
3752  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3753  StreamTcpReassembleInlineTest04);
3754  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3755  StreamTcpReassembleInlineTest08);
3756  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3757  StreamTcpReassembleInlineTest09);
3758 
3759  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3760  StreamTcpReassembleInlineTest10);
3761 
3762  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3763  StreamTcpReassembleInsertTest01);
3764  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3765  StreamTcpReassembleInsertTest02);
3766  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3767  StreamTcpReassembleInsertTest03);
3768 
3772  StreamTcpReassembleRawRegisterTests();
3773 #endif /* UNITTESTS */
3774 }
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:249
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:57
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:55
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:437
TcpStream_
Definition: stream-tcp-private.h:94
len
uint8_t len
Definition: app-layer-dnp3.h:2
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:149
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:101
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:101
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:1919
StreamingBufferConfig_::Malloc
void *(* Malloc)(size_t size)
Definition: util-streaming-buffer.h:71
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:169
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:302
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:92
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:70
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1755
STREAM_MIDSTREAM
#define STREAM_MIDSTREAM
Definition: stream.h:35
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:95
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:82
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:72
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:104
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:516
stream-tcp.h
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:493
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
Definition: stream-tcp-reassemble.c:175
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:557
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:123
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:1983
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:573
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:997
StreamTcpThread_
Definition: stream-tcp.h:70
Flow_::proto
uint8_t proto
Definition: flow.h:369
Packet_::payload
uint8_t * payload
Definition: decode.h:552
StreamTcpInlineMode
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6278
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:1963
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:61
Packet_::flags
uint32_t flags
Definition: decode.h:450
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:175
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:98
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:60
Flow_
Flow data structure.
Definition: flow.h:347
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:66
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:68
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:1946
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:144
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:38
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1356
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:849
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:219
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:143
StreamNeedsReassembly
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:789
MIN
#define MIN(x, y)
Definition: suricata-common.h:372
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:64
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:95
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
StreamingBufferConfig_::flags
uint32_t flags
Definition: util-streaming-buffer.h:68
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:150
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:280
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:866
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:446
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:119
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:41
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:242
Flow_::protoctx
void * protoctx
Definition: flow.h:445
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:711
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:95
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:553
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:32
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:204
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:579
util-unittest.h
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2055
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
TcpSession_::flags
uint16_t flags
Definition: stream-tcp-private.h:269
FLOWLOCK_UNLOCK
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:264
STREAM_START
#define STREAM_START
Definition: stream.h:29
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:158
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:269
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:103
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:479
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:434
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:521
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1199
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:270
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:353
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:581
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:264
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:92
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:270
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1123
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1127
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:131
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
FLOWLOCK_WRLOCK
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:261
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:118
StreamReassembleLog
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
Definition: stream-tcp-reassemble.c:1783
util-print.h
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:50
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:866
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:174
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:281
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:4780
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1312
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:262
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:87
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:1997
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:798
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
Definition: stream-tcp-reassemble.c:146
TRUE
#define TRUE
Definition: suricata-common.h:33
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueueNoLock *pq)
Definition: stream-tcp-reassemble.c:1811
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:277
TCP_GET_SEQ
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:112
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
FALSE
#define FALSE
Definition: suricata-common.h:34
SCReturn
#define SCReturn
Definition: util-debug.h:302
stream.h
TcpSegment
Definition: stream-tcp-private.h:61
StreamTcpReassembleFree
void StreamTcpReassembleFree(char quiet)
Definition: stream-tcp-reassemble.c:415
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:278
Packet_
Definition: decode.h:415
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:215
STREAM_DEPTH
#define STREAM_DEPTH
Definition: stream.h:34
StreamTcpFreeConfig
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:670
stream-tcp-private.h
PoolThreadReturn
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
Definition: util-pool-thread.c:187
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:42
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:306
conf.h
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:276
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:889
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:105
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:475
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
STREAM_GAP
#define STREAM_GAP
Definition: stream.h:33
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:315
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:220
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:266
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
StreamTcpReassembleInit
int StreamTcpReassembleInit(char quiet)
Definition: stream-tcp-reassemble.c:399
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:70
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:115
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:79
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:526
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:56
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:103
StreamingBuffer_
Definition: util-streaming-buffer.h:95
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:600
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:63
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:150
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:769
StringParseUint32
int StringParseUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:313
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:168
Packet_::flow
struct Flow_ * flow
Definition: decode.h:452
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:241
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:337
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:240
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:532
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:112
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:30
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:89
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2036
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:122
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:217
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:149
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:102
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:150
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
StreamTcpReassembleIncrMemuse
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:98
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:455
ConfNode_
Definition: conf.h:32
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:485
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:888
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:238
Flow_::flags
uint32_t flags
Definition: flow.h:425
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:768
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:74
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:50
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:132
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:282
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2050
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:146
StreamTcpInitConfig
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
STREAM_EOF
#define STREAM_EOF
Definition: stream.h:30
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:60
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:57
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:168
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:188
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:200
suricata.h
Address_::family
char family
Definition: decode.h:119
Packet_::dst
Address dst
Definition: decode.h:420
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:6293
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:239
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint8_t flag)
Definition: app-layer-parser.c:1664
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:219
STREAMING_BUFFER_NOFLAGS
#define STREAMING_BUFFER_NOFLAGS
Definition: util-streaming-buffer.h:64
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3714
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:90
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:171
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:132
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:212
StreamTcpReassembleDecrMemuse
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:111
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
TcpSession_
Definition: stream-tcp-private.h:260
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:265
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:160
flow.h
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:456
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:454
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:107
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:206
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:143
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:85
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:83
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:66
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:743
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:82
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:130
Packet_::src
Address src
Definition: decode.h:419
debug.h
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:73
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:213
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:133
app-layer.h
StreamTcpCheckStreamContents
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
Definition: stream-tcp-reassemble.c:2014