suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 #include "app-layer-parser.h"
64 
65 #include "detect-engine-state.h"
66 
67 #include "util-profiling.h"
68 #include "util-validate.h"
69 
70 #ifdef DEBUG
71 static SCMutex segment_pool_memuse_mutex;
72 static uint64_t segment_pool_memuse = 0;
73 static uint64_t segment_pool_memcnt = 0;
74 #endif
75 
76 static PoolThread *segment_thread_pool = NULL;
77 /* init only, protect initializing and growing pool */
78 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
79 
80 /* Memory use counter */
81 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
82 
83 /* prototypes */
85 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346  if (f->alparser) {
349  }
350 }
351 
352 /** \param f locked flow */
354 {
355  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
356  return 0;
357 
358  TcpSession *ssn = (TcpSession *)f->protoctx;
360 }
361 
362 static int StreamTcpReassemblyConfig(char quiet)
363 {
364  uint32_t segment_prealloc = 2048;
365  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
366  if (seg) {
367  uint32_t prealloc = 0;
368  if (StringParseUint32(&prealloc, 10, strlen(seg->val), seg->val) < 0)
369  {
370  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
371  "%s is invalid", seg->val);
372  return -1;
373  }
374  segment_prealloc = prealloc;
375  }
376  if (!quiet)
377  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
378  stream_config.prealloc_segments = segment_prealloc;
379 
380  int overlap_diff_data = 0;
381  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
382  if (overlap_diff_data) {
384  }
385  if (StreamTcpInlineMode() == TRUE) {
387  }
388 
391  stream_config.sbcnf.Malloc = ReassembleMalloc;
392  stream_config.sbcnf.Calloc = ReassembleCalloc;
393  stream_config.sbcnf.Realloc = ReassembleRealloc;
394  stream_config.sbcnf.Free = ReassembleFree;
395 
396  return 0;
397 }
398 
399 int StreamTcpReassembleInit(char quiet)
400 {
401  /* init the memcap/use tracker */
403 
404  if (StreamTcpReassemblyConfig(quiet) < 0)
405  return -1;
406 
407 #ifdef DEBUG
408  SCMutexInit(&segment_pool_memuse_mutex, NULL);
409 #endif
410  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
412  return 0;
413 }
414 
415 void StreamTcpReassembleFree(char quiet)
416 {
417  SCMutexLock(&segment_thread_pool_mutex);
418  if (segment_thread_pool != NULL) {
419  PoolThreadFree(segment_thread_pool);
420  segment_thread_pool = NULL;
421  }
422  SCMutexUnlock(&segment_thread_pool_mutex);
423  SCMutexDestroy(&segment_thread_pool_mutex);
424 
425 #ifdef DEBUG
426  if (segment_pool_memuse > 0)
427  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
428  if (segment_pool_memcnt > 0)
429  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
430  SCMutexDestroy(&segment_pool_memuse_mutex);
431 #endif
432 }
433 
435 {
436  SCEnter();
438  if (unlikely(ra_ctx == NULL))
439  return NULL;
440 
441  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
442 
443  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
444 
445  SCMutexLock(&segment_thread_pool_mutex);
446  if (segment_thread_pool == NULL) {
447  segment_thread_pool = PoolThreadInit(1, /* thread */
448  0, /* unlimited */
450  sizeof(TcpSegment),
451  TcpSegmentPoolAlloc,
452  TcpSegmentPoolInit, NULL,
453  TcpSegmentPoolCleanup, NULL);
454  ra_ctx->segment_thread_pool_id = 0;
455  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
456  PoolThreadSize(segment_thread_pool),
457  ra_ctx->segment_thread_pool_id);
458  } else {
459  /* grow segment_thread_pool until we have a element for our thread id */
460  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
461  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
462  PoolThreadSize(segment_thread_pool),
463  ra_ctx->segment_thread_pool_id);
464  }
465  SCMutexUnlock(&segment_thread_pool_mutex);
466  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
467  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
469  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
470  }
471 
472  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
473 }
474 
476 {
477  SCEnter();
478  if (ra_ctx) {
480  SCFree(ra_ctx);
481  }
482  SCReturn;
483 }
484 
485 /**
486  * \brief check if stream in pkt direction has depth reached
487  *
488  * \param p packet with *LOCKED* flow
489  *
490  * \retval 1 stream has depth reached
491  * \retval 0 stream does not have depth reached
492  */
494 {
495  if (p->flow != NULL && p->flow->protoctx != NULL) {
496  TcpSession *ssn = p->flow->protoctx;
497  TcpStream *stream;
498  if (p->flowflags & FLOW_PKT_TOSERVER) {
499  stream = &ssn->client;
500  } else {
501  stream = &ssn->server;
502  }
503 
504  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
505  }
506 
507  return 0;
508 }
509 
510 /**
511  * \internal
512  * \brief Function to Check the reassembly depth valuer against the
513  * allowed max depth of the stream reassembly for TCP streams.
514  *
515  * \param stream stream direction
516  * \param seq sequence number where "size" starts
517  * \param size size of the segment that is added
518  *
519  * \retval size Part of the size that fits in the depth, 0 if none
520  */
521 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
522  uint32_t seq, uint32_t size)
523 {
524  SCEnter();
525 
526  /* if the configured depth value is 0, it means there is no limit on
527  reassembly depth. Otherwise carry on my boy ;) */
528  if (ssn->reassembly_depth == 0) {
529  SCReturnUInt(size);
530  }
531 
532  /* if the final flag is set, we're not accepting anymore */
534  SCReturnUInt(0);
535  }
536 
537  uint64_t seg_depth;
538  if (SEQ_GT(stream->base_seq, seq)) {
539  if (SEQ_LEQ(seq+size, stream->base_seq)) {
540  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
541  stream->base_seq, seq, seq+size);
542  SCReturnUInt(0);
543  }
544 
545  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
546  } else {
547  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
548  }
549 
550  /* if the base_seq has moved passed the depth window we stop
551  * checking and just reject the rest of the packets including
552  * retransmissions. Saves us the hassle of dealing with sequence
553  * wraps as well */
554  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
555  stream->base_seq, seg_depth,
556  ssn->reassembly_depth);
557 
558  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
559  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
561  SCReturnUInt(0);
562  }
563  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
564  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
565 #if 0
566  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
567  (stream->base_seq_offset + stream->base_seq + size),
568  (stream->isn + ssn->reassembly_depth));
569 #endif
570  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
571  /* packet (partly?) fits the depth window */
572 
573  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
574  /* complete fit */
575  SCReturnUInt(size);
576  } else {
578  /* partial fit, return only what fits */
579  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
580  DEBUG_VALIDATE_BUG_ON(part > size);
581  if (part > size)
582  part = size;
583  SCReturnUInt(part);
584  }
585  }
586 
587  SCReturnUInt(0);
588 }
589 
590 /**
591  * \brief Insert a packets TCP data into the stream reassembly engine.
592  *
593  * \retval 0 good segment, as far as we checked.
594  * \retval -1 badness, reason to drop in inline mode
595  *
596  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
597  * or it wasn't added because of reassembly depth.
598  *
599  */
601  TcpSession *ssn, TcpStream *stream, Packet *p)
602 {
603  SCEnter();
604 
605  if (ssn->data_first_seen_dir == 0) {
606  if (PKT_IS_TOSERVER(p)) {
608  } else {
610  }
611  }
612 
613  /* If the OS policy is not set then set the OS policy for this stream */
614  if (stream->os_policy == 0) {
615  StreamTcpSetOSPolicy(stream, p);
616  }
617 
620  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
621  SCReturnInt(0);
622  }
623 
624  /* If we have reached the defined depth for either of the stream, then stop
625  reassembling the TCP session */
626  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
627  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
628 
630  /* increment stream depth counter */
632  }
633  if (size == 0) {
634  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
635  SCReturnInt(0);
636  }
637 
639  if (size > p->payload_len)
640  size = p->payload_len;
641 
642  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
643  if (seg == NULL) {
644  SCLogDebug("segment_pool is empty");
646  SCReturnInt(-1);
647  }
648 
649  TCP_SEG_LEN(seg) = size;
650  seg->seq = TCP_GET_SEQ(p);
651 
652  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
653  if (TCP_HAS_TFO(p) && p->payload_len && p->tcph->th_flags == TH_SYN)
654  seg->seq += 1;
655 
656  /* proto detection skipped, but now we do get data. Set event. */
657  if (RB_EMPTY(&stream->seg_tree) &&
659 
662  }
663 
664  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
665  SCLogDebug("StreamTcpReassembleInsertSegment failed");
666  SCReturnInt(-1);
667  }
668  SCReturnInt(0);
669 }
670 
671 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
672  Packet *p)
673 {
674  uint8_t flag = 0;
675 
677  flag |= STREAM_START;
678  }
679 
680  if (ssn->state == TCP_CLOSED) {
681  flag |= STREAM_EOF;
682  }
683 
684  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
685  flag |= STREAM_MIDSTREAM;
686  }
687 
688  if (p->flags & PKT_PSEUDO_STREAM_END) {
689  flag |= STREAM_EOF;
690  }
691 
692  if (&ssn->client == stream) {
693  flag |= STREAM_TOSERVER;
694  } else {
695  flag |= STREAM_TOCLIENT;
696  }
698  flag |= STREAM_DEPTH;
699  }
700  return flag;
701 }
702 
703 /**
704  * \brief Check the minimum size limits for reassembly.
705  *
706  * \retval 0 don't reassemble yet
707  * \retval 1 do reassemble
708  */
709 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
710  const TcpStream *stream, const Packet *p)
711 {
712  SCEnter();
713 
714  /* if any of these flags is set we always inspect immediately */
715 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
716  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
717  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
718  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
719 
722  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
723  "is set, so not expecting any new data segments");
724  }
726  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
727  }
729  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
730  "so no new segments will be considered");
731  }
732  SCReturnInt(1);
733  }
734 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
735 
736  /* some states mean we reassemble no matter how much data we have */
737  if (ssn->state > TCP_TIME_WAIT)
738  SCReturnInt(1);
739 
740  if (p->flags & PKT_PSEUDO_STREAM_END)
741  SCReturnInt(1);
742 
743  /* check if we have enough data to do raw reassembly */
744  if (PKT_IS_TOSERVER(p)) {
745  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
746  uint32_t delta = stream->last_ack - stream->base_seq;
747  /* get max absolute offset */
748  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
749 
750  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
751  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
752  SCReturnInt(1);
753  } else {
754  SCLogDebug("toserver min chunk len not yet reached: "
755  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
756  "%"PRIu32"", stream->last_ack, stream->base_seq,
757  (stream->last_ack - stream->base_seq),
759  SCReturnInt(0);
760  }
761  }
762  } else {
763  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
764  uint32_t delta = stream->last_ack - stream->base_seq;
765  /* get max absolute offset */
766  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
767 
768  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
769 
770  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
771  SCReturnInt(1);
772  } else {
773  SCLogDebug("toclient min chunk len not yet reached: "
774  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
775  "%"PRIu32"", stream->last_ack, stream->base_seq,
776  (stream->last_ack - stream->base_seq),
778  SCReturnInt(0);
779  }
780  }
781  }
782 
783  SCReturnInt(0);
784 }
785 
786 /**
787  * \brief see what if any work the TCP session still needs
788  */
789 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
790 {
791  const TcpStream *stream = NULL;
792 #ifdef DEBUG
793  const char *dirstr = NULL;
794 #endif
795  if (direction == STREAM_TOSERVER) {
796  stream = &ssn->client;
797 #ifdef DEBUG
798  dirstr = "client";
799 #endif
800  } else {
801  stream = &ssn->server;
802 #ifdef DEBUG
803  dirstr = "server";
804 #endif
805  }
806 
807  int use_app = 1;
808  int use_raw = 1;
809 
811  // app is dead
812  use_app = 0;
813  }
814 
816  // raw is dead
817  use_raw = 0;
818  }
819 
820  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
821 
822  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
823  dirstr,
824  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
825  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
826  right_edge);
827  if (use_raw) {
828  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
829  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
831  }
832  }
833  if (use_app) {
834  if (right_edge > STREAM_APP_PROGRESS(stream)) {
835  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
837  }
838  }
839 
840  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
842 }
843 
844 #ifdef DEBUG
845 static uint64_t GetStreamSize(TcpStream *stream)
846 {
847  if (stream) {
848  uint64_t size = 0;
849  uint32_t cnt = 0;
850 
851  TcpSegment *seg;
852  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
853  cnt++;
854  size += (uint64_t)TCP_SEG_LEN(seg);
855  }
856 
857  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
858  return size;
859  }
860  return (uint64_t)0;
861 }
862 
863 static void GetSessionSize(TcpSession *ssn, Packet *p)
864 {
865  uint64_t size = 0;
866  if (ssn) {
867  size = GetStreamSize(&ssn->client);
868  size += GetStreamSize(&ssn->server);
869 
870  //if (size > 900000)
871  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
872  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
873  }
874 }
875 #endif
876 
877 static StreamingBufferBlock *GetBlock(StreamingBuffer *sb, const uint64_t offset)
878 {
879  StreamingBufferBlock *blk = sb->head;
880  if (blk == NULL)
881  return NULL;
882 
883  for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
884  if (blk->offset >= offset)
885  return blk;
886  else if ((blk->offset + blk->len) > offset) {
887  return blk;
888  }
889  }
890  return NULL;
891 }
892 
893 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
894 {
895  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
896  return STREAM_BASE_OFFSET(stream) +
897  (stream->last_ack - stream->base_seq);
898  } else {
899  return STREAM_BASE_OFFSET(stream);
900  }
901 }
902 
903 static inline bool GapAhead(TcpStream *stream, StreamingBufferBlock *cur_blk)
904 {
905  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
906  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
907  GetAbsLastAck(stream) >= (cur_blk->offset + cur_blk->len)) {
908  return true;
909  }
910  return false;
911 }
912 
913 /** \internal
914  *
915  * Get buffer, or first part of the buffer if data gaps exist.
916  *
917  * \brief get stream data from offset
918  * \param offset stream offset
919  * \param check_for_gap check if there is a gap ahead. Optional as it is only
920  * needed for app-layer incomplete support.
921  * \retval bool pkt loss ahead */
922 static bool GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
923  uint64_t offset, const bool check_for_gap)
924 {
925  const uint8_t *mydata;
926  uint32_t mydata_len;
927  bool gap_ahead = false;
928 
929  if (RB_EMPTY(&stream->sb.sbb_tree)) {
930  SCLogDebug("getting one blob");
931 
932  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
933 
934  *data = mydata;
935  *data_len = mydata_len;
936  } else {
937  StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
938  if (blk == NULL) {
939  *data = NULL;
940  *data_len = 0;
941  return false;
942  }
943 
944  /* block at expected offset */
945  if (blk->offset == offset) {
946 
947  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
948 
949  gap_ahead = check_for_gap && GapAhead(stream, blk);
950 
951  /* block past out offset */
952  } else if (blk->offset > offset) {
953  SCLogDebug("gap, want data at offset %"PRIu64", "
954  "got data at %"PRIu64". GAP of size %"PRIu64,
955  offset, blk->offset, blk->offset - offset);
956  *data = NULL;
957  *data_len = blk->offset - offset;
958 
959  /* block starts before offset, but ends after */
960  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
961  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
962  offset, blk->offset, blk->len);
963  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
964  SCLogDebug("data %p, data_len %u", *data, *data_len);
965 
966  gap_ahead = check_for_gap && GapAhead(stream, blk);
967 
968  } else {
969  *data = NULL;
970  *data_len = 0;
971  }
972  }
973  return gap_ahead;
974 }
975 
976 /** \internal
977  * \brief check to see if we should declare a GAP
978  * Call this when the app layer didn't get data at the requested
979  * offset.
980  */
981 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
982 {
983  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
984  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
985 
986  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
987  /* get window of data that is acked */
988  const uint32_t delta = stream->last_ack - stream->base_seq;
989  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
990  /* get max absolute offset */
991  last_ack_abs += delta;
992 
993  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
994  last_ack_abs -= ackadded;
995 
996  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
997  SCLogDebug("next_seq %u", stream->next_seq);
998 
999  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1000  * has been ack'd. This looks like a GAP. */
1001  if (last_ack_abs > app_progress) {
1002  /* however, we can accept ACKs a bit too liberally. If last_ack
1003  * is beyond next_seq, we only consider it a gap now if we do
1004  * already have data beyond the gap. */
1005  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1006  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1007  SCLogDebug("packet %"PRIu64": no GAP. "
1008  "next_seq %u < last_ack %u, but no data in list",
1009  p->pcap_cnt, stream->next_seq, stream->last_ack);
1010  return false;
1011  } else {
1012  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1013  const StreamingBufferBlock *blk = stream->sb.head;
1014  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1015  /* ack'd data after the gap */
1016  SCLogDebug("packet %"PRIu64": GAP. "
1017  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1018  p->pcap_cnt, stream->next_seq, stream->last_ack);
1019  return true;
1020  }
1021  }
1022  }
1023 
1024  SCLogDebug("packet %"PRIu64": GAP! "
1025  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
1026  "but we have no data.",
1027  p->pcap_cnt, last_ack_abs, app_progress);
1028  return true;
1029  }
1030  }
1031  SCLogDebug("packet %"PRIu64": no GAP. "
1032  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1033  p->pcap_cnt, last_ack_abs, app_progress);
1034  return false;
1035 }
1036 
1037 static inline uint32_t AdjustToAcked(const Packet *p,
1038  const TcpSession *ssn, const TcpStream *stream,
1039  const uint64_t app_progress, const uint32_t data_len)
1040 {
1041  uint32_t adjusted = data_len;
1042 
1043  /* get window of data that is acked */
1044  if (StreamTcpInlineMode() == FALSE) {
1045  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1046  if ((ssn->state < TCP_CLOSED ||
1047  (ssn->state == TCP_CLOSED &&
1048  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1050  {
1051  // fall through, we use all available data
1052  } else {
1053  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1054  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1055  /* get window of data that is acked */
1056  uint32_t delta = stream->last_ack - stream->base_seq;
1057  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1058  /* get max absolute offset */
1059  last_ack_abs += delta;
1060  }
1061 
1062  /* see if the buffer contains unack'd data as well */
1063  if (app_progress + data_len > last_ack_abs) {
1064  uint32_t check = data_len;
1065  adjusted = last_ack_abs - app_progress;
1066  BUG_ON(adjusted > check);
1067  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1068  "data is considered", adjusted);
1069  }
1070  }
1071  }
1072  return adjusted;
1073 }
1074 
1075 /** \internal
1076  * \brief get stream buffer and update the app-layer
1077  * \param stream pointer to pointer as app-layer can switch flow dir
1078  * \retval 0 success
1079  */
1080 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1081  TcpReassemblyThreadCtx *ra_ctx,
1082  TcpSession *ssn, TcpStream **stream,
1083  Packet *p, enum StreamUpdateDir dir)
1084 {
1085  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1086 
1087  SCLogDebug("app progress %"PRIu64, app_progress);
1088  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1089 
1090  const uint8_t *mydata;
1091  uint32_t mydata_len;
1092  bool gap_ahead = false;
1093  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1094 
1095  while (1) {
1096  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1097  gap_ahead = GetAppBuffer(*stream, &mydata, &mydata_len,
1098  app_progress, check_for_gap_ahead);
1099  /* make sure to only deal with ACK'd data */
1100  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1101  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1102  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1103  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1104 
1105  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1106  NULL, mydata_len,
1107  StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP);
1108  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1109 
1112 
1113  /* AppLayerHandleTCPData has likely updated progress. */
1114  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1115  app_progress = STREAM_APP_PROGRESS(*stream);
1116 
1117  /* a GAP also consumes 'data required'. TODO perhaps we can use
1118  * this to skip post GAP data until the start of a next record. */
1119  if ((*stream)->data_required > 0) {
1120  if ((*stream)->data_required > mydata_len) {
1121  (*stream)->data_required -= mydata_len;
1122  } else {
1123  (*stream)->data_required = 0;
1124  }
1125  }
1126  if (r < 0)
1127  return 0;
1128  if (no_progress_update)
1129  break;
1130  continue;
1131 
1132  } else if (flags & STREAM_DEPTH) {
1133  // we're just called once with this flag, so make sure we pass it on
1134 
1135  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1136  /* Possibly a gap, but no new data. */
1137  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1138  SCReturnInt(0);
1139 
1140  mydata = NULL;
1141  mydata_len = 0;
1142  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1143  break;
1144  }
1145 
1146  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1147  *stream, &(*stream)->sb, mydata_len, app_progress);
1148 
1149  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1150  if (mydata_len < (*stream)->data_required) {
1151  if (gap_ahead) {
1152  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1153  (*stream)->data_required, mydata_len);
1154  (*stream)->app_progress_rel += mydata_len;
1155  (*stream)->data_required -= mydata_len;
1156  // TODO send incomplete data to app-layer with special flag
1157  // indicating its all there is for this rec?
1158  } else {
1159  SCReturnInt(0);
1160  }
1161  app_progress = STREAM_APP_PROGRESS(*stream);
1162  continue;
1163  }
1164  }
1165  (*stream)->data_required = 0;
1166 
1167  /* update the app-layer */
1168  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1169  (uint8_t *)mydata, mydata_len, flags);
1170  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1171  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1172  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1173  break;
1174  app_progress = new_app_progress;
1175  if (flags & STREAM_DEPTH)
1176  break;
1177  }
1178 
1179  SCReturnInt(0);
1180 }
1181 
1182 /**
1183  * \brief Update the stream reassembly upon receiving a packet.
1184  *
1185  * For IDS mode, the stream is in the opposite direction of the packet,
1186  * as the ACK-packet is ACK'ing the stream.
1187  *
1188  * One of the utilities call by this function AppLayerHandleTCPData(),
1189  * has a feature where it will call this very same function for the
1190  * stream opposing the stream it is called with. This shouldn't cause
1191  * any issues, since processing of each stream is independent of the
1192  * other stream.
1193  */
1195  TcpSession *ssn, TcpStream *stream,
1196  Packet *p, enum StreamUpdateDir dir)
1197 {
1198  SCEnter();
1199 
1200  /* this function can be directly called by app layer protocol
1201  * detection. */
1204  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1205  SCReturnInt(0);
1206  }
1207 
1208 #ifdef DEBUG
1209  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1210  GetSessionSize(ssn, p);
1211 #endif
1212  /* if no segments are in the list or all are already processed,
1213  * and state is beyond established, we send an empty msg */
1214  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1215  {
1216  /* send an empty EOF msg if we have no segments but TCP state
1217  * is beyond ESTABLISHED */
1218  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1219  SCLogDebug("sending empty eof message");
1220  /* send EOF to app layer */
1221  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1222  NULL, 0,
1223  StreamGetAppLayerFlags(ssn, stream, p));
1224  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1225 
1226  SCReturnInt(0);
1227  }
1228  }
1229 
1230  /* with all that out of the way, lets update the app-layer */
1231  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1232 }
1233 
1234 /** \internal
1235  * \brief get stream data from offset
1236  * \param offset stream offset */
1237 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1238  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1239 {
1240  const uint8_t *mydata;
1241  uint32_t mydata_len;
1242  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1243  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1244 
1245  uint64_t roffset = offset;
1246  if (offset)
1247  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1248  else {
1249  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1250  }
1251 
1252  *data = mydata;
1253  *data_len = mydata_len;
1254  *data_offset = roffset;
1255  } else {
1256  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1257  *iter == NULL ? "starting" : "continuing", offset);
1258  if (*iter == NULL) {
1259  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1260  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1261  SCLogDebug("*iter %p", *iter);
1262  }
1263  if (*iter == NULL) {
1264  SCLogDebug("no data");
1265  *data = NULL;
1266  *data_len = 0;
1267  *data_offset = 0;
1268  return 0;
1269  }
1270  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1271 
1272  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1273  SCLogDebug("mydata %p", mydata);
1274 
1275  if ((*iter)->offset < offset) {
1276  uint64_t delta = offset - (*iter)->offset;
1277  if (delta < mydata_len) {
1278  *data = mydata + delta;
1279  *data_len = mydata_len - delta;
1280  *data_offset = offset;
1281  } else {
1282  SCLogDebug("no data (yet)");
1283  *data = NULL;
1284  *data_len = 0;
1285  *data_offset = 0;
1286  }
1287 
1288  } else {
1289  *data = mydata;
1290  *data_len = mydata_len;
1291  *data_offset = (*iter)->offset;
1292  }
1293 
1294  *iter = SBB_RB_NEXT(*iter);
1295  SCLogDebug("*iter %p", *iter);
1296  }
1297  return 0;
1298 }
1299 
1300 /** \brief does the stream engine have data to inspect?
1301  *
1302  * Returns true if there is data to inspect. In IDS case this is
1303  * about ACK'd data in the packet's direction.
1304  *
1305  * In the IPS case this is about the packet itself.
1306  */
1308 {
1309  TcpStream *stream;
1310  if (PKT_IS_TOSERVER(p)) {
1311  stream = &ssn->client;
1312  } else {
1313  stream = &ssn->server;
1314  }
1315 
1316  if (RB_EMPTY(&stream->seg_tree)) {
1317  return false;
1318  }
1319 
1322  return false;
1323 
1324  if (StreamTcpInlineMode() == FALSE) {
1325  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1326  return false;
1327  }
1328  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1329  return true;
1330  }
1331  } else {
1332  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1333  return true;
1334  }
1335  }
1336  return false;
1337 }
1338 
1339 /** \brief update stream engine after detection
1340  *
1341  * Tasked with progressing the 'progress' for Raw reassembly.
1342  * 2 main scenario's:
1343  * 1. progress is != 0, so we use this
1344  * 2. progress is 0, meaning the detect engine didn't touch
1345  * raw at all. In this case we need to look into progressing
1346  * raw anyway.
1347  *
1348  * Additionally, this function is tasked with disabling raw
1349  * reassembly if the app-layer requested to disable it.
1350  */
1351 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1352 {
1353  TcpStream *stream;
1354  if (PKT_IS_TOSERVER(p)) {
1355  stream = &ssn->client;
1356  } else {
1357  stream = &ssn->server;
1358  }
1359 
1360  if (progress > STREAM_RAW_PROGRESS(stream)) {
1361  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1362  stream->raw_progress_rel += slide;
1364 
1365  /* if app is active and beyond raw, sync raw to app */
1366  } else if (progress == 0 && STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1368  /* if trigger raw is set we sync the 2 trackers */
1370  {
1371  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1372  stream->raw_progress_rel += slide;
1374 
1375  /* otherwise mix in the tcp window */
1376  } else {
1377  uint64_t tcp_window = stream->window;
1378  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1379  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1380  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1381  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1382  stream->raw_progress_rel += slide;
1383  }
1384  }
1385  }
1386  /* app is dead */
1387  } else if (progress == 0) {
1388  uint64_t tcp_window = stream->window;
1389  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1390  if (tcp_window < stream_right_edge) {
1391  uint64_t new_raw = stream_right_edge - tcp_window;
1392  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1393  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1394  stream->raw_progress_rel += slide;
1395  }
1396  }
1398 
1399  } else {
1400  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1401  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1402  STREAM_RAW_PROGRESS(stream), stream->window);
1403  }
1404 
1405  /* if we were told to accept no more raw data, we can mark raw as
1406  * disabled now. */
1409  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1410  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1411  }
1412 
1413  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1414 }
1415 
1416 /** \internal
1417  * \brief get a buffer around the current packet and run the callback on it
1418  *
1419  * The inline/IPS scanning method takes the current payload and wraps it in
1420  * data from other segments.
1421  *
1422  * How much data is inspected is controlled by the available data, chunk_size
1423  * and the payload size of the packet.
1424  *
1425  * Large packets: if payload size is close to the chunk_size, where close is
1426  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1427  * used: payload_len + 33% of the chunk_size.
1428  * If the payload size if equal to or bigger than the chunk_size, we use
1429  * payload len + 33% of the chunk size.
1430  */
1431 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1432  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1433 {
1434  SCEnter();
1435  int r = 0;
1436 
1437  TcpStream *stream;
1438  if (PKT_IS_TOSERVER(p)) {
1439  stream = &ssn->client;
1440  } else {
1441  stream = &ssn->server;
1442  }
1443 
1444  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1446  {
1447  *progress_out = STREAM_RAW_PROGRESS(stream);
1448  return 0;
1449  }
1450 
1451  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1454  if (chunk_size <= p->payload_len) {
1455  chunk_size = p->payload_len + (chunk_size / 3);
1456  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1457  p->payload_len, chunk_size);
1458  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1459  chunk_size = p->payload_len + ((chunk_size / 3));
1460  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1461  p->payload_len, chunk_size);
1462  }
1463 
1464  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1465  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1466  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1467  packet_leftedge_abs, packet_rightedge_abs);
1468 
1469  const uint8_t *mydata = NULL;
1470  uint32_t mydata_len = 0;
1471  uint64_t mydata_offset = 0;
1472  /* simply return progress from the block we inspected. */
1473  bool return_progress = false;
1474 
1475  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1476  /* continues block */
1477  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1478  return_progress = true;
1479 
1480  } else {
1481  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1482  /* find our block */
1483  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1484  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1485  if (sbb) {
1486  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1487  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1488  mydata_offset = sbb->offset;
1489  }
1490  }
1491 
1492  /* this can only happen if the segment insert of our current 'p' failed */
1493  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1494  if ((mydata == NULL || mydata_len == 0) || /* no data */
1495  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1496  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1497  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1498  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1499  {
1500  /* no data, or data is incomplete or wrong: use packet data */
1501  mydata = p->payload;
1502  mydata_len = p->payload_len;
1503  mydata_offset = packet_leftedge_abs;
1504  //mydata_rightedge_abs = packet_rightedge_abs;
1505  } else {
1506  /* adjust buffer to match chunk_size */
1507  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1508  if (mydata_len > chunk_size) {
1509  uint32_t excess = mydata_len - chunk_size;
1510  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1511 
1512  if (mydata_rightedge_abs == packet_rightedge_abs) {
1513  mydata += excess;
1514  mydata_len -= excess;
1515  mydata_offset += excess;
1516  SCLogDebug("cutting front of the buffer with %u", excess);
1517  } else if (mydata_offset == packet_leftedge_abs) {
1518  mydata_len -= excess;
1519  SCLogDebug("cutting tail of the buffer with %u", excess);
1520  } else {
1521  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1522  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1523  SCLogDebug("before %u after %u", before, after);
1524 
1525  if (after >= (chunk_size - p->payload_len) / 2) {
1526  // more trailing data than we need
1527 
1528  if (before >= (chunk_size - p->payload_len) / 2) {
1529  // also more heading data, divide evenly
1530  before = after = (chunk_size - p->payload_len) / 2;
1531  } else {
1532  // heading data is less than requested, give the
1533  // rest to the trailing data
1534  after = (chunk_size - p->payload_len) - before;
1535  }
1536  } else {
1537  // less trailing data than requested
1538 
1539  if (before >= (chunk_size - p->payload_len) / 2) {
1540  before = (chunk_size - p->payload_len) - after;
1541  } else {
1542  // both smaller than their requested size
1543  }
1544  }
1545 
1546  /* adjust the buffer */
1547  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1548  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1549  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1550  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1551  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1552  mydata += skip;
1553  mydata_len -= (skip + cut);
1554  mydata_offset += skip;
1555  }
1556  }
1557  }
1558 
1559  /* run the callback */
1560  r = Callback(cb_data, mydata, mydata_len);
1561  BUG_ON(r < 0);
1562 
1563  if (return_progress) {
1564  *progress_out = (mydata_offset + mydata_len);
1565  } else {
1566  /* several blocks of data, so we need to be a bit more careful:
1567  * - if last_ack is beyond last progress, move progress forward to last_ack
1568  * - if our block matches or starts before last ack, return right edge of
1569  * our block.
1570  */
1571  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1572  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1573  uint32_t delta = stream->last_ack - stream->base_seq;
1574  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1575  /* get max absolute offset */
1576  last_ack_abs += delta;
1577  }
1578  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1579 
1580  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1581  if (mydata_offset > last_ack_abs) {
1582  /* gap between us and last ack, set progress to last ack */
1583  *progress_out = last_ack_abs;
1584  } else {
1585  *progress_out = (mydata_offset + mydata_len);
1586  }
1587  } else {
1588  *progress_out = STREAM_RAW_PROGRESS(stream);
1589  }
1590  }
1591  return r;
1592 }
1593 
1594 /** \brief access 'raw' reassembly data.
1595  *
1596  * Access data as tracked by 'raw' tracker. Data is made available to
1597  * callback that is passed to this function.
1598  *
1599  * In the case of IDS the callback may be run multiple times if data
1600  * contains gaps. It will then be run for each block of data that is
1601  * continuous.
1602  *
1603  * The callback should give on of 2 return values:
1604  * - 0 ok
1605  * - 1 done
1606  * The value 1 will break the loop if there is a block list that is
1607  * inspected.
1608  *
1609  * This function will return the 'progress' value that has been
1610  * consumed until now.
1611  *
1612  * \param ssn tcp session
1613  * \param stream tcp stream
1614  * \param Callback the function pointer to the callback function
1615  * \param cb_data callback data
1616  * \param[in] progress_in progress to work from
1617  * \param[out] progress_out absolute progress value of the data this
1618  * call handled.
1619  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1620  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1621  *
1622  * `respect_inspect_depth` is used to avoid useless inspection of too
1623  * much data.
1624  */
1625 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1626  StreamReassembleRawFunc Callback, void *cb_data,
1627  const uint64_t progress_in,
1628  uint64_t *progress_out, bool eof,
1629  bool respect_inspect_depth)
1630 {
1631  SCEnter();
1632  int r = 0;
1633 
1634  StreamingBufferBlock *iter = NULL;
1635  uint64_t progress = progress_in;
1636  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1637 
1638  /* if the app layer triggered a flush, and we're supposed to
1639  * use a minimal inspect depth, we actually take the app progress
1640  * as that is the right edge of the data. Then we take the window
1641  * of 'min_inspect_depth' before that. */
1642 
1643  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1644  respect_inspect_depth ? "true" : "false",
1645  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1646  stream->min_inspect_depth);
1647 
1648  if (respect_inspect_depth &&
1650  && stream->min_inspect_depth)
1651  {
1652  progress = STREAM_APP_PROGRESS(stream);
1653  if (stream->min_inspect_depth >= progress) {
1654  progress = 0;
1655  } else {
1656  progress -= stream->min_inspect_depth;
1657  }
1658 
1659  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1660 
1661  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1662  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1663  }
1664 
1665  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1666 
1667  /* get window of data that is acked */
1668  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1669  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1670  uint32_t delta = stream->last_ack - stream->base_seq;
1671  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1672  /* get max absolute offset */
1673  last_ack_abs += delta;
1674  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1675  }
1676 
1677  /* loop through available buffers. On no packet loss we'll have a single
1678  * iteration. On missing data we'll walk the blocks */
1679  while (1) {
1680  const uint8_t *mydata;
1681  uint32_t mydata_len;
1682  uint64_t mydata_offset = 0;
1683 
1684  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1685  if (mydata_len == 0) {
1686  SCLogDebug("no data");
1687  break;
1688  }
1689  //PrintRawDataFp(stdout, mydata, mydata_len);
1690 
1691  SCLogDebug("raw progress %"PRIu64, progress);
1692  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1693  stream, &stream->sb, mydata_len, progress);
1694 
1695  if (eof) {
1696  // inspect all remaining data, ack'd or not
1697  } else {
1698  if (last_ack_abs < progress) {
1699  SCLogDebug("nothing to do");
1700  goto end;
1701  }
1702 
1703  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1704  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1705 
1706  /* see if the buffer contains unack'd data as well */
1707  if (progress + mydata_len > last_ack_abs) {
1708  uint32_t check = mydata_len;
1709  mydata_len = last_ack_abs - progress;
1710  BUG_ON(check < mydata_len);
1711  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1712  "data is considered", mydata_len);
1713  }
1714 
1715  }
1716  if (mydata_len == 0)
1717  break;
1718 
1719  SCLogDebug("data %p len %u", mydata, mydata_len);
1720 
1721  /* we have data. */
1722  r = Callback(cb_data, mydata, mydata_len);
1723  BUG_ON(r < 0);
1724 
1725  if (mydata_offset == progress) {
1726  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1727  progress, mydata_len, progress_in + mydata_len);
1728 
1729  progress += mydata_len;
1730  SCLogDebug("raw progress now %"PRIu64, progress);
1731 
1732  /* data is beyond the progress we'd like, and before last ack. Gap. */
1733  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1734  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1735  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1736  progress = mydata_offset;
1737  SCLogDebug("raw progress now %"PRIu64, progress);
1738 
1739  } else {
1740  SCLogDebug("not increasing progress, data gap => mydata_offset "
1741  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1742  }
1743 
1744  if (iter == NULL || r == 1)
1745  break;
1746  }
1747 end:
1748  *progress_out = progress;
1749  return r;
1750 }
1751 
1753  StreamReassembleRawFunc Callback, void *cb_data,
1754  uint64_t *progress_out, bool respect_inspect_depth)
1755 {
1756  /* handle inline separately as the logic is very different */
1757  if (StreamTcpInlineMode() == TRUE) {
1758  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1759  }
1760 
1761  TcpStream *stream;
1762  if (PKT_IS_TOSERVER(p)) {
1763  stream = &ssn->client;
1764  } else {
1765  stream = &ssn->server;
1766  }
1767 
1769  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1770  {
1771  *progress_out = STREAM_RAW_PROGRESS(stream);
1772  return 0;
1773  }
1774 
1775  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1776  STREAM_RAW_PROGRESS(stream), progress_out,
1777  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1778 }
1779 
1781  StreamReassembleRawFunc Callback, void *cb_data,
1782  uint64_t progress_in,
1783  uint64_t *progress_out, bool eof)
1784 {
1785  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1786  return 0;
1787 
1788  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1789  progress_in, progress_out, eof, false);
1790 }
1791 
1792 /** \internal
1793  * \brief update app layer based on received ACK
1794  *
1795  * \retval r 0 on success, -1 on error
1796  */
1797 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1798  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1799 {
1800  SCEnter();
1801 
1802  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1803  SCReturnInt(-1);
1804 
1805  SCReturnInt(0);
1806 }
1807 
1809  TcpSession *ssn, TcpStream *stream,
1810  Packet *p, PacketQueueNoLock *pq)
1811 {
1812  SCEnter();
1813 
1814  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1815 
1816  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1817  ssn, stream, p, p->payload_len);
1818 
1819  /* we need to update the opposing stream in
1820  * StreamTcpReassembleHandleSegmentUpdateACK */
1821  TcpStream *opposing_stream = NULL;
1822  if (stream == &ssn->client) {
1823  opposing_stream = &ssn->server;
1824  } else {
1825  opposing_stream = &ssn->client;
1826  }
1827 
1828  /* default IDS: update opposing side (triggered by ACK) */
1830  /* inline and stream end and flow timeout packets trigger same dir handling */
1831  if (StreamTcpInlineMode()) {
1832  dir = UPDATE_DIR_PACKET;
1833  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1834  dir = UPDATE_DIR_PACKET;
1835  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1836  dir = UPDATE_DIR_PACKET;
1837  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1838  dir = UPDATE_DIR_PACKET;
1839  } else if (ssn->state == TCP_CLOSED) {
1840  dir = UPDATE_DIR_BOTH;
1841  }
1842 
1843  /* handle ack received */
1844  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1845  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1846  {
1847  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1848  SCReturnInt(-1);
1849  }
1850 
1851  /* if this segment contains data, insert it */
1852  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1853  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1854 
1855  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1856  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1857  SCReturnInt(-1);
1858  }
1859 
1860  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1861  p->flags |= PKT_STREAM_ADD;
1862  } else {
1863  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1864  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1865  ssn, stream, p->payload_len,
1866  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1867 
1868  }
1869 
1870  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1871  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1872  * was *just* set. In this case we trigger the AppLayer Truncate
1873  * logic, to inform the applayer no more data in this direction is
1874  * to be expected. */
1875  if ((stream->flags &
1878  {
1879  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1880  if (dir != UPDATE_DIR_PACKET) {
1881  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1882  "can trigger Truncate");
1883  dir = UPDATE_DIR_PACKET;
1884  }
1885  }
1886 
1887  /* in stream inline mode even if we have no data we call the reassembly
1888  * functions to handle EOF */
1889  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1890  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1891  StreamTcpInlineMode()?"true":"false",
1892  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1893  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1894  SCReturnInt(-1);
1895  }
1896  }
1897 
1898  SCReturnInt(0);
1899 }
1900 
1901 /**
1902  * \brief get a segment from the pool
1903  *
1904  * \retval seg Segment from the pool or NULL
1905  */
1907 {
1908  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1909  SCLogDebug("seg we return is %p", seg);
1910  if (seg == NULL) {
1911  /* Increment the counter to show that we are not able to serve the
1912  segment request due to memcap limit */
1914  } else {
1915  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1916  }
1917 
1918  return seg;
1919 }
1920 
1921 /**
1922  * \brief Trigger RAW stream reassembly
1923  *
1924  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1925  * reassembly from the applayer, for example upon completion of a
1926  * HTTP request.
1927  *
1928  * It sets a flag in the stream so that the next Raw call will return
1929  * the data.
1930  *
1931  * \param ssn TcpSession
1932  */
1934 {
1935 #ifdef DEBUG
1936  BUG_ON(ssn == NULL);
1937 #endif
1938 
1939  if (ssn != NULL) {
1940  if (direction == STREAM_TOSERVER) {
1942  } else {
1944  }
1945 
1946  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1947  }
1948 }
1949 
1950 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1951 {
1952 #ifdef DEBUG
1953  BUG_ON(ssn == NULL);
1954 #endif
1955 
1956  if (ssn != NULL) {
1957  if (direction == STREAM_TOSERVER) {
1958  ssn->client.min_inspect_depth = depth;
1959  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1960  } else {
1961  ssn->server.min_inspect_depth = depth;
1962  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1963  }
1964  }
1965 }
1966 
1967 #ifdef UNITTESTS
1968 /** unit tests and it's support functions below */
1969 
1970 #define SET_ISN(stream, setseq) \
1971  (stream)->isn = (setseq); \
1972  (stream)->base_seq = (setseq) + 1
1973 
1974 /** \brief The Function to create the packet with given payload, which is used
1975  * to test the reassembly of the engine.
1976  *
1977  * \param payload The variable used to store the payload contents of the
1978  * current packet.
1979  * \param value The value which current payload will have for this packet
1980  * \param payload_len The length of the filed payload for current packet.
1981  * \param len Length of the payload array
1982  */
1983 
1984 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1985  uint8_t payload_len, uint8_t len)
1986 {
1987  uint8_t i;
1988  for (i = 0; i < payload_len; i++)
1989  payload[i] = value;
1990  for (; i < len; i++)
1991  payload = NULL;
1992 }
1993 
1994 /** \brief The Function Checks the reassembled stream contents against predefined
1995  * stream contents according to OS policy used.
1996  *
1997  * \param stream_policy Predefined value of stream for different OS policies
1998  * \param stream Reassembled stream returned from the reassembly functions
1999  */
2000 
2001 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
2002 {
2003  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
2004  {
2005  //PrintRawDataFp(stdout, stream_policy, sp_size);
2006  return 0;
2007  }
2008  return 1;
2009 }
2010 
2011 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2012 {
2013  if (StreamingBufferCompareRawData(&stream->sb,
2014  data, data_len) == 0)
2015  {
2016  SCReturnInt(0);
2017  }
2018  SCLogInfo("OK");
2019  PrintRawDataFp(stdout, data, data_len);
2020  return 1;
2021 }
2022 
2023 #define MISSED_START(isn) \
2024  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2025  TcpSession ssn; \
2026  ThreadVars tv; \
2027  memset(&tv, 0, sizeof(tv)); \
2028  \
2029  StreamTcpUTInit(&ra_ctx); \
2030  \
2031  StreamTcpUTSetupSession(&ssn); \
2032  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2033  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2034  \
2035  TcpStream *stream = &ssn.client;
2036 
2037 #define MISSED_END \
2038  StreamTcpUTClearSession(&ssn); \
2039  StreamTcpUTDeinit(ra_ctx); \
2040  PASS
2041 
2042 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2043  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2044  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2045 
2046 /**
2047  * \test Test the handling of packets missed by both IDS and the end host.
2048  * The packet is missed in the starting of the stream.
2049  *
2050  * \retval On success it returns 1 and on failure 0.
2051  */
2052 
2053 static int StreamTcpReassembleTest25 (void)
2054 {
2055  MISSED_START(6);
2056  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
2057  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
2058  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2059  MISSED_END;
2060 }
2061 
2062 /**
2063  * \test Test the handling of packets missed by both IDS and the end host.
2064  * The packet is missed in the middle of the stream.
2065  *
2066  * \retval On success it returns 1 and on failure 0.
2067  */
2068 
2069 static int StreamTcpReassembleTest26 (void)
2070 {
2071  MISSED_START(9);
2072  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2073  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2074  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2075  MISSED_END;
2076 }
2077 
2078 /**
2079  * \test Test the handling of packets missed by both IDS and the end host.
2080  * The packet is missed in the end of the stream.
2081  *
2082  * \retval On success it returns 1 and on failure 0.
2083  */
2084 
2085 static int StreamTcpReassembleTest27 (void)
2086 {
2087  MISSED_START(9);
2088  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2089  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2090  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2091  MISSED_END;
2092 }
2093 
2094 /**
2095  * \test Test the handling of packets missed by IDS, but the end host has
2096  * received it and send the acknowledgment of it. The packet is missed
2097  * in the starting of the stream.
2098  *
2099  * \retval On success it returns 1 and on failure 0.
2100  */
2101 
2102 static int StreamTcpReassembleTest28 (void)
2103 {
2104  MISSED_START(6);
2105  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2106  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2107  ssn.state = TCP_TIME_WAIT;
2108  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2109  MISSED_END;
2110 }
2111 
2112 /**
2113  * \test Test the handling of packets missed by IDS, but the end host has
2114  * received it and send the acknowledgment of it. The packet is missed
2115  * in the middle of the stream.
2116  *
2117  * \retval On success it returns 1 and on failure 0.
2118  */
2119 
2120 static int StreamTcpReassembleTest29 (void)
2121 {
2122  MISSED_START(9);
2123  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2124  ssn.state = TCP_TIME_WAIT;
2125  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2126  MISSED_END;
2127 }
2128 
2129 static int StreamTcpReassembleTest33(void)
2130 {
2131  TcpSession ssn;
2132  Packet *p = PacketGetFromAlloc();
2133  FAIL_IF(unlikely(p == NULL));
2134  Flow f;
2135  TCPHdr tcph;
2136  TcpReassemblyThreadCtx *ra_ctx = NULL;
2138  uint8_t packet[1460] = "";
2139 
2140  StreamTcpUTInit(&ra_ctx);
2142 
2143  PacketQueueNoLock pq;
2144  memset(&pq,0,sizeof(PacketQueueNoLock));
2145  memset(&f, 0, sizeof (Flow));
2146  memset(&tcph, 0, sizeof (TCPHdr));
2147  ThreadVars tv;
2148  memset(&tv, 0, sizeof (ThreadVars));
2149  FLOW_INITIALIZE(&f);
2150  f.protoctx = &ssn;
2151  f.proto = IPPROTO_TCP;
2152  p->src.family = AF_INET;
2153  p->dst.family = AF_INET;
2154  p->proto = IPPROTO_TCP;
2155  p->flow = &f;
2156  tcph.th_win = 5480;
2157  tcph.th_flags = TH_PUSH | TH_ACK;
2158  p->tcph = &tcph;
2160  p->payload = packet;
2161 
2162  p->tcph->th_seq = htonl(10);
2163  p->tcph->th_ack = htonl(31);
2164  p->payload_len = 10;
2165 
2166  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2167 
2168  p->tcph->th_seq = htonl(20);
2169  p->tcph->th_ack = htonl(31);
2170  p->payload_len = 10;
2171 
2172  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2173 
2174  p->tcph->th_seq = htonl(40);
2175  p->tcph->th_ack = htonl(31);
2176  p->payload_len = 10;
2177 
2178  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2179 
2180  p->tcph->th_seq = htonl(5);
2181  p->tcph->th_ack = htonl(31);
2182  p->payload_len = 30;
2183 
2184  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2185 
2187  StreamTcpUTDeinit(ra_ctx);
2188  SCFree(p);
2189  PASS;
2190 }
2191 
2192 static int StreamTcpReassembleTest34(void)
2193 {
2194  TcpSession ssn;
2195  Packet *p = PacketGetFromAlloc();
2196  FAIL_IF(unlikely(p == NULL));
2197  Flow f;
2198  TCPHdr tcph;
2199  TcpReassemblyThreadCtx *ra_ctx = NULL;
2201  uint8_t packet[1460] = "";
2202 
2203  StreamTcpUTInit(&ra_ctx);
2205  PacketQueueNoLock pq;
2206  memset(&pq,0,sizeof(PacketQueueNoLock));
2207  memset(&f, 0, sizeof (Flow));
2208  memset(&tcph, 0, sizeof (TCPHdr));
2209  ThreadVars tv;
2210  memset(&tv, 0, sizeof (ThreadVars));
2211  FLOW_INITIALIZE(&f);
2212  f.protoctx = &ssn;
2213  f.proto = IPPROTO_TCP;
2214  p->src.family = AF_INET;
2215  p->dst.family = AF_INET;
2216  p->proto = IPPROTO_TCP;
2217  p->flow = &f;
2218  tcph.th_win = 5480;
2219  tcph.th_flags = TH_PUSH | TH_ACK;
2220  p->tcph = &tcph;
2222  p->payload = packet;
2223  SET_ISN(&ssn.client, 857961230);
2224 
2225  p->tcph->th_seq = htonl(857961230);
2226  p->tcph->th_ack = htonl(31);
2227  p->payload_len = 304;
2228 
2229  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2230 
2231  p->tcph->th_seq = htonl(857961534);
2232  p->tcph->th_ack = htonl(31);
2233  p->payload_len = 1460;
2234 
2235  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2236 
2237  p->tcph->th_seq = htonl(857963582);
2238  p->tcph->th_ack = htonl(31);
2239  p->payload_len = 1460;
2240 
2241  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2242 
2243  p->tcph->th_seq = htonl(857960946);
2244  p->tcph->th_ack = htonl(31);
2245  p->payload_len = 1460;
2246 
2247  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2248 
2250  StreamTcpUTDeinit(ra_ctx);
2251  SCFree(p);
2252  PASS;
2253 }
2254 
2255 /** \test Test the bug 76 condition */
2256 static int StreamTcpReassembleTest37(void)
2257 {
2258  TcpSession ssn;
2259  Flow f;
2260  TCPHdr tcph;
2261  TcpReassemblyThreadCtx *ra_ctx = NULL;
2262  uint8_t packet[1460] = "";
2263  PacketQueueNoLock pq;
2264  ThreadVars tv;
2265  memset(&tv, 0, sizeof (ThreadVars));
2266 
2267  Packet *p = PacketGetFromAlloc();
2268  FAIL_IF(unlikely(p == NULL));
2269 
2270  StreamTcpUTInit(&ra_ctx);
2272  memset(&pq,0,sizeof(PacketQueueNoLock));
2273  memset(&f, 0, sizeof (Flow));
2274  memset(&tcph, 0, sizeof (TCPHdr));
2275  memset(&tv, 0, sizeof (ThreadVars));
2276 
2277  FLOW_INITIALIZE(&f);
2278  f.protoctx = &ssn;
2279  f.proto = IPPROTO_TCP;
2280  p->src.family = AF_INET;
2281  p->dst.family = AF_INET;
2282  p->proto = IPPROTO_TCP;
2283  p->flow = &f;
2284  tcph.th_win = 5480;
2285  tcph.th_flags = TH_PUSH | TH_ACK;
2286  p->tcph = &tcph;
2288  p->payload = packet;
2290 
2291  p->tcph->th_seq = htonl(3061088537UL);
2292  p->tcph->th_ack = htonl(1729548549UL);
2293  p->payload_len = 1391;
2294  ssn.client.last_ack = 3061091137UL;
2295  SET_ISN(&ssn.client, 3061091309UL);
2296 
2297  /* pre base_seq, so should be rejected */
2298  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2299 
2300  p->tcph->th_seq = htonl(3061089928UL);
2301  p->tcph->th_ack = htonl(1729548549UL);
2302  p->payload_len = 1391;
2303  ssn.client.last_ack = 3061091137UL;
2304 
2305  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2306 
2307  p->tcph->th_seq = htonl(3061091319UL);
2308  p->tcph->th_ack = htonl(1729548549UL);
2309  p->payload_len = 1391;
2310  ssn.client.last_ack = 3061091137UL;
2311 
2312  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2313 
2315  StreamTcpUTDeinit(ra_ctx);
2316  SCFree(p);
2317  PASS;
2318 }
2319 
2320 /**
2321  * \test Test to make sure that we don't return the segments until the app
2322  * layer proto has been detected and after that remove the processed
2323  * segments.
2324  *
2325  * \retval On success it returns 1 and on failure 0.
2326  */
2327 
2328 static int StreamTcpReassembleTest39 (void)
2329 {
2330  Packet *p = PacketGetFromAlloc();
2331  FAIL_IF(unlikely(p == NULL));
2332  Flow f;
2333  ThreadVars tv;
2334  StreamTcpThread stt;
2335  TCPHdr tcph;
2336  PacketQueueNoLock pq;
2337  memset(&pq,0,sizeof(PacketQueueNoLock));
2338  memset (&f, 0, sizeof(Flow));
2339  memset(&tv, 0, sizeof (ThreadVars));
2340  memset(&stt, 0, sizeof (stt));
2341  memset(&tcph, 0, sizeof (TCPHdr));
2342 
2343  FLOW_INITIALIZE(&f);
2344  f.flags = FLOW_IPV4;
2345  f.proto = IPPROTO_TCP;
2346  p->flow = &f;
2347  p->tcph = &tcph;
2348 
2349  StreamTcpUTInit(&stt.ra_ctx);
2350 
2351  /* handshake */
2352  tcph.th_win = htons(5480);
2353  tcph.th_flags = TH_SYN;
2355  p->payload_len = 0;
2356  p->payload = NULL;
2357  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2358 
2359  TcpSession *ssn = (TcpSession *)f.protoctx;
2360  FAIL_IF_NULL(ssn);
2361 
2372  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2373  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2374  FAIL_IF(ssn->data_first_seen_dir != 0);
2375 
2376  /* handshake */
2377  p->tcph->th_ack = htonl(1);
2378  p->tcph->th_flags = TH_SYN | TH_ACK;
2380  p->payload_len = 0;
2381  p->payload = NULL;
2382  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2393  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2394  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2395  FAIL_IF(ssn->data_first_seen_dir != 0);
2396 
2397  /* handshake */
2398  p->tcph->th_ack = htonl(1);
2399  p->tcph->th_seq = htonl(1);
2400  p->tcph->th_flags = TH_ACK;
2402  p->payload_len = 0;
2403  p->payload = NULL;
2404  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2415  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2416  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2417  FAIL_IF(ssn->data_first_seen_dir != 0);
2418 
2419  /* partial request */
2420  uint8_t request1[] = { 0x47, 0x45, };
2421  p->tcph->th_ack = htonl(1);
2422  p->tcph->th_seq = htonl(1);
2423  p->tcph->th_flags = TH_PUSH | TH_ACK;
2425  p->payload_len = sizeof(request1);
2426  p->payload = request1;
2427  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2438  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2439  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2440  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2442 
2443  /* response ack against partial request */
2444  p->tcph->th_ack = htonl(3);
2445  p->tcph->th_seq = htonl(1);
2446  p->tcph->th_flags = TH_ACK;
2448  p->payload_len = 0;
2449  p->payload = NULL;
2450  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2461  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2462  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2463  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2465 
2466  /* complete partial request */
2467  uint8_t request2[] = {
2468  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2469  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2470  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2471  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2472  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2473  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2474  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2475  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2476  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2477  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2478  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2479  p->tcph->th_ack = htonl(1);
2480  p->tcph->th_seq = htonl(3);
2481  p->tcph->th_flags = TH_PUSH | TH_ACK;
2483  p->payload_len = sizeof(request2);
2484  p->payload = request2;
2485  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2496  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2497  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2498  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2499  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2501 
2502  /* response - request ack */
2503  uint8_t response[] = {
2504  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2505  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2506  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2507  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2508  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2509  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2510  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2511  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2512  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2513  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2514  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2515  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2516  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2517  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2518  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2519  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2520  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2521  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2522  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2523  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2524  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2525  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2526  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2527  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2528  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2529  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2530  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2531  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2532  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2533  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2534  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2535  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2536  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2537  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2538  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2539  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2540  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2541  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2542  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2543  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2544  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2545  p->tcph->th_ack = htonl(88);
2546  p->tcph->th_seq = htonl(1);
2547  p->tcph->th_flags = TH_PUSH | TH_ACK;
2549  p->payload_len = sizeof(response);
2550  p->payload = response;
2551 
2552  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2564  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2565  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2566  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2567  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2568 
2569  /* response ack from request */
2570  p->tcph->th_ack = htonl(328);
2571  p->tcph->th_seq = htonl(88);
2572  p->tcph->th_flags = TH_ACK;
2574  p->payload_len = 0;
2575  p->payload = NULL;
2576  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2588  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2589  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2590  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2591  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2592  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2593 
2594  /* response - acking */
2595  p->tcph->th_ack = htonl(88);
2596  p->tcph->th_seq = htonl(328);
2597  p->tcph->th_flags = TH_PUSH | TH_ACK;
2599  p->payload_len = 0;
2600  p->payload = NULL;
2601  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2613  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2614  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2615  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2616  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2617  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2618 
2619  /* response ack from request */
2620  p->tcph->th_ack = htonl(328);
2621  p->tcph->th_seq = htonl(88);
2622  p->tcph->th_flags = TH_ACK;
2624  p->payload_len = 0;
2625  p->payload = NULL;
2626  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2638  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2639  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2640  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2641  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2642 
2643  /* response - acking the request again*/
2644  p->tcph->th_ack = htonl(88);
2645  p->tcph->th_seq = htonl(328);
2646  p->tcph->th_flags = TH_PUSH | TH_ACK;
2648  p->payload_len = 0;
2649  p->payload = NULL;
2650  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2662  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2663  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2664  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2665  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2666 
2667  /*** New Request ***/
2668 
2669  /* partial request */
2670  p->tcph->th_ack = htonl(328);
2671  p->tcph->th_seq = htonl(88);
2672  p->tcph->th_flags = TH_PUSH | TH_ACK;
2674  p->payload_len = sizeof(request1);
2675  p->payload = request1;
2676  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2688  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2689  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2690  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2691  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2692  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2693 
2694  /* response ack against partial request */
2695  p->tcph->th_ack = htonl(90);
2696  p->tcph->th_seq = htonl(328);
2697  p->tcph->th_flags = TH_ACK;
2699  p->payload_len = 0;
2700  p->payload = NULL;
2701  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2702  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2714  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2715  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2716  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2717  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2718  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2719 
2720  /* complete request */
2721  p->tcph->th_ack = htonl(328);
2722  p->tcph->th_seq = htonl(90);
2723  p->tcph->th_flags = TH_PUSH | TH_ACK;
2725  p->payload_len = sizeof(request2);
2726  p->payload = request2;
2727  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2739  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2740  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2741  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2742  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2743  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2744  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2745 
2746  /* response ack against second partial request */
2747  p->tcph->th_ack = htonl(175);
2748  p->tcph->th_seq = htonl(328);
2749  p->tcph->th_flags = TH_ACK;
2751  p->payload_len = 0;
2752  p->payload = NULL;
2753 
2754  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2766  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2767  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2768  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2769  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2770  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2771  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2772 
2773  /* response acking a request */
2774  p->tcph->th_ack = htonl(175);
2775  p->tcph->th_seq = htonl(328);
2776  p->tcph->th_flags = TH_ACK;
2778  p->payload_len = 0;
2779  p->payload = NULL;
2780  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2786 
2789 
2790  /* request acking a response */
2791  p->tcph->th_ack = htonl(328);
2792  p->tcph->th_seq = htonl(175);
2793  p->tcph->th_flags = TH_ACK;
2795  p->payload_len = 0;
2796  p->payload = NULL;
2797  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2798 
2799  StreamTcpSessionClear(ssn);
2801  SCFree(p);
2802  PASS;
2803 }
2804 
2805 /**
2806  * \test Test to make sure that we sent all the segments from the initial
2807  * segments to app layer until we have detected the app layer proto.
2808  *
2809  * \retval On success it returns 1 and on failure 0.
2810  */
2811 
2812 static int StreamTcpReassembleTest40 (void)
2813 {
2814  Packet *p = PacketGetFromAlloc();
2815  FAIL_IF_NULL(p);
2816  Flow *f = NULL;
2817  TCPHdr tcph;
2818  TcpSession ssn;
2819  PacketQueueNoLock pq;
2820  memset(&pq,0,sizeof(PacketQueueNoLock));
2821  memset(&tcph, 0, sizeof (TCPHdr));
2822  ThreadVars tv;
2823  memset(&tv, 0, sizeof (ThreadVars));
2824 
2827 
2829  FAIL_IF_NULL(ra_ctx);
2830 
2831  uint8_t httpbuf1[] = "P";
2832  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2833  uint8_t httpbuf3[] = "O";
2834  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2835  uint8_t httpbuf4[] = "S";
2836  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2837  uint8_t httpbuf5[] = "T \r\n";
2838  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2839 
2840  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2841  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2842 
2843  SET_ISN(&ssn.server, 9);
2844  ssn.server.last_ack = 10;
2845  SET_ISN(&ssn.client, 9);
2846  ssn.client.isn = 9;
2847 
2848  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2849  FAIL_IF_NULL(f);
2850  f->protoctx = &ssn;
2851  f->proto = IPPROTO_TCP;
2852  p->flow = f;
2853 
2854  tcph.th_win = htons(5480);
2855  tcph.th_seq = htonl(10);
2856  tcph.th_ack = htonl(10);
2857  tcph.th_flags = TH_ACK|TH_PUSH;
2858  p->tcph = &tcph;
2860  p->payload = httpbuf1;
2861  p->payload_len = httplen1;
2862  ssn.state = TCP_ESTABLISHED;
2863  TcpStream *s = &ssn.client;
2864  SCLogDebug("1 -- start");
2865  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2866 
2868  p->payload = httpbuf2;
2869  p->payload_len = httplen2;
2870  tcph.th_seq = htonl(10);
2871  tcph.th_ack = htonl(11);
2872  s = &ssn.server;
2873  ssn.server.last_ack = 11;
2874  SCLogDebug("2 -- start");
2875  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2876 
2878  p->payload = httpbuf3;
2879  p->payload_len = httplen3;
2880  tcph.th_seq = htonl(11);
2881  tcph.th_ack = htonl(55);
2882  s = &ssn.client;
2883  ssn.client.last_ack = 55;
2884  SCLogDebug("3 -- start");
2885  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2886 
2888  p->payload = httpbuf2;
2889  p->payload_len = httplen2;
2890  tcph.th_seq = htonl(55);
2891  tcph.th_ack = htonl(12);
2892  s = &ssn.server;
2893  ssn.server.last_ack = 12;
2894  SCLogDebug("4 -- start");
2895  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2896 
2897  /* check is have the segment in the list and flagged or not */
2898  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2899  FAIL_IF_NULL(seg);
2900  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2901 
2903  p->payload = httpbuf4;
2904  p->payload_len = httplen4;
2905  tcph.th_seq = htonl(12);
2906  tcph.th_ack = htonl(100);
2907  s = &ssn.client;
2908  ssn.client.last_ack = 100;
2909  SCLogDebug("5 -- start");
2910  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2911 
2913  p->payload = httpbuf2;
2914  p->payload_len = httplen2;
2915  tcph.th_seq = htonl(100);
2916  tcph.th_ack = htonl(13);
2917  s = &ssn.server;
2918  ssn.server.last_ack = 13;
2919  SCLogDebug("6 -- start");
2920  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2921 
2923  p->payload = httpbuf5;
2924  p->payload_len = httplen5;
2925  tcph.th_seq = htonl(13);
2926  tcph.th_ack = htonl(145);
2927  s = &ssn.client;
2928  ssn.client.last_ack = 145;
2929  SCLogDebug("7 -- start");
2930  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2931 
2933  p->payload = httpbuf2;
2934  p->payload_len = httplen2;
2935  tcph.th_seq = htonl(145);
2936  tcph.th_ack = htonl(16);
2937  s = &ssn.server;
2938  ssn.server.last_ack = 16;
2939  SCLogDebug("8 -- start");
2940  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2941  FAIL_IF(f->alproto != ALPROTO_HTTP);
2942 
2946  SCFree(p);
2947  UTHFreeFlow(f);
2948  PASS;
2949 }
2950 
2951 /** \test Test the memcap incrementing/decrementing and memcap check */
2952 static int StreamTcpReassembleTest44(void)
2953 {
2955  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2957  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2959  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2961  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2963  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2964  PASS;
2965 }
2966 
2967 /**
2968  * \test Test to make sure that reassembly_depth is enforced.
2969  *
2970  * \retval On success it returns 1 and on failure 0.
2971  */
2972 
2973 static int StreamTcpReassembleTest45 (void)
2974 {
2975  TcpReassemblyThreadCtx *ra_ctx = NULL;
2976  TcpSession ssn;
2977  ThreadVars tv;
2978  memset(&tv, 0, sizeof(tv));
2979  uint8_t payload[100] = {0};
2980  uint16_t payload_size = 100;
2981 
2982  StreamTcpUTInit(&ra_ctx);
2984 
2986  ssn.reassembly_depth = 100;
2987  StreamTcpUTSetupStream(&ssn.server, 100);
2988  StreamTcpUTSetupStream(&ssn.client, 100);
2989 
2990  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2991  FAIL_IF(r != 0);
2993 
2994  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2995  FAIL_IF(r != 0);
2997 
3001  StreamTcpUTDeinit(ra_ctx);
3002  PASS;
3003 }
3004 
3005 /**
3006  * \test Test the unlimited config value of reassembly depth.
3007  *
3008  * \retval On success it returns 1 and on failure 0.
3009  */
3010 
3011 static int StreamTcpReassembleTest46 (void)
3012 {
3013  int result = 0;
3014  TcpReassemblyThreadCtx *ra_ctx = NULL;
3015  TcpSession ssn;
3016  ThreadVars tv;
3017  memset(&tv, 0, sizeof(tv));
3018  uint8_t payload[100] = {0};
3019  uint16_t payload_size = 100;
3020 
3021  StreamTcpUTInit(&ra_ctx);
3023 
3025  StreamTcpUTSetupStream(&ssn.server, 100);
3026  StreamTcpUTSetupStream(&ssn.client, 100);
3027 
3028  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3029  if (r != 0)
3030  goto end;
3032  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3033  goto end;
3034  }
3035 
3036  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3037  if (r != 0)
3038  goto end;
3040  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3041  goto end;
3042  }
3043 
3044  result = 1;
3045 end:
3049  StreamTcpUTDeinit(ra_ctx);
3050  return result;
3051 }
3052 
3053 /**
3054  * \test Test to make sure we detect the sequence wrap around and continue
3055  * stream reassembly properly.
3056  *
3057  * \retval On success it returns 1 and on failure 0.
3058  */
3059 
3060 static int StreamTcpReassembleTest47 (void)
3061 {
3062  Packet *p = PacketGetFromAlloc();
3063  FAIL_IF(unlikely(p == NULL));
3064  Flow *f = NULL;
3065  TCPHdr tcph;
3066  TcpSession ssn;
3067  ThreadVars tv;
3068  PacketQueueNoLock pq;
3069  memset(&pq,0,sizeof(PacketQueueNoLock));
3070  memset(&tcph, 0, sizeof (TCPHdr));
3071  memset(&tv, 0, sizeof (ThreadVars));
3075 
3076  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3077  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3078 
3079  SET_ISN(&ssn.server, 572799781UL);
3080  ssn.server.last_ack = 572799782UL;
3081 
3082  SET_ISN(&ssn.client, 4294967289UL);
3083  ssn.client.last_ack = 21;
3084 
3085  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3086  FAIL_IF(f == NULL);
3087  f->protoctx = &ssn;
3088  f->proto = IPPROTO_TCP;
3089  p->flow = f;
3090 
3091  tcph.th_win = htons(5480);
3092  ssn.state = TCP_ESTABLISHED;
3093  TcpStream *s = NULL;
3094  uint8_t cnt = 0;
3095 
3096  for (cnt=0; cnt < httplen1; cnt++) {
3097  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3098  tcph.th_ack = htonl(572799782UL);
3099  tcph.th_flags = TH_ACK|TH_PUSH;
3100  p->tcph = &tcph;
3102  p->payload = &httpbuf1[cnt];
3103  p->payload_len = 1;
3104  s = &ssn.client;
3105 
3106  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3107 
3109  p->payload = NULL;
3110  p->payload_len = 0;
3111  tcph.th_seq = htonl(572799782UL);
3112  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3113  tcph.th_flags = TH_ACK;
3114  p->tcph = &tcph;
3115  s = &ssn.server;
3116 
3117  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3118  }
3119 
3120  FAIL_IF(f->alproto != ALPROTO_HTTP);
3121 
3125  SCFree(p);
3126  UTHFreeFlow(f);
3127  PASS;
3128 }
3129 
3130 /** \test 3 in order segments in inline reassembly */
3131 static int StreamTcpReassembleInlineTest01(void)
3132 {
3133  int ret = 0;
3134  TcpReassemblyThreadCtx *ra_ctx = NULL;
3135  ThreadVars tv;
3136  TcpSession ssn;
3137  Flow f;
3138 
3139  memset(&tv, 0x00, sizeof(tv));
3140 
3141  StreamTcpUTInit(&ra_ctx);
3144  StreamTcpUTSetupStream(&ssn.client, 1);
3145  FLOW_INITIALIZE(&f);
3146 
3147  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3148  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3149  if (p == NULL) {
3150  printf("couldn't get a packet: ");
3151  goto end;
3152  }
3153  p->tcph->th_seq = htonl(12);
3154  p->flow = &f;
3155 
3156  FLOWLOCK_WRLOCK(&f);
3157  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3158  printf("failed to add segment 1: ");
3159  goto end;
3160  }
3161  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3162  printf("failed to add segment 2: ");
3163  goto end;
3164  }
3165  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3166  printf("failed to add segment 3: ");
3167  goto end;
3168  }
3169  ssn.client.next_seq = 17;
3170  ret = 1;
3171 end:
3172  FLOWLOCK_UNLOCK(&f);
3173  FLOW_DESTROY(&f);
3174  UTHFreePacket(p);
3176  StreamTcpUTDeinit(ra_ctx);
3177  return ret;
3178 }
3179 
3180 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3181  * test the sliding window reassembly.
3182  */
3183 static int StreamTcpReassembleInlineTest02(void)
3184 {
3185  int ret = 0;
3186  TcpReassemblyThreadCtx *ra_ctx = NULL;
3187  ThreadVars tv;
3188  TcpSession ssn;
3189  Flow f;
3190 
3191  memset(&tv, 0x00, sizeof(tv));
3192 
3193  StreamTcpUTInit(&ra_ctx);
3196  StreamTcpUTSetupStream(&ssn.client, 1);
3197  FLOW_INITIALIZE(&f);
3198 
3199  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3200  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3201  if (p == NULL) {
3202  printf("couldn't get a packet: ");
3203  goto end;
3204  }
3205  p->tcph->th_seq = htonl(12);
3206  p->flow = &f;
3207 
3208  FLOWLOCK_WRLOCK(&f);
3209  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3210  printf("failed to add segment 1: ");
3211  goto end;
3212  }
3213  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3214  printf("failed to add segment 2: ");
3215  goto end;
3216  }
3217  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3218  printf("failed to add segment 3: ");
3219  goto end;
3220  }
3221  ssn.client.next_seq = 17;
3222  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3223  printf("failed to add segment 4: ");
3224  goto end;
3225  }
3226  ssn.client.next_seq = 22;
3227  ret = 1;
3228 end:
3229  FLOWLOCK_UNLOCK(&f);
3230  FLOW_DESTROY(&f);
3231  UTHFreePacket(p);
3233  StreamTcpUTDeinit(ra_ctx);
3234  return ret;
3235 }
3236 
3237 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3238  * test the sliding window reassembly with a small window size so that we
3239  * cutting off at the start (left edge)
3240  */
3241 static int StreamTcpReassembleInlineTest03(void)
3242 {
3243  int ret = 0;
3244  TcpReassemblyThreadCtx *ra_ctx = NULL;
3245  ThreadVars tv;
3246  TcpSession ssn;
3247  Flow f;
3248 
3249  memset(&tv, 0x00, sizeof(tv));
3250 
3251  StreamTcpUTInit(&ra_ctx);
3254  StreamTcpUTSetupStream(&ssn.client, 1);
3255  FLOW_INITIALIZE(&f);
3256 
3258 
3259  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3260  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3261  if (p == NULL) {
3262  printf("couldn't get a packet: ");
3263  goto end;
3264  }
3265  p->tcph->th_seq = htonl(12);
3266  p->flow = &f;
3268 
3269  FLOWLOCK_WRLOCK(&f);
3270  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3271  printf("failed to add segment 1: ");
3272  goto end;
3273  }
3274  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3275  printf("failed to add segment 2: ");
3276  goto end;
3277  }
3278  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3279  printf("failed to add segment 3: ");
3280  goto end;
3281  }
3282  ssn.client.next_seq = 17;
3283  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3284  printf("failed to add segment 4: ");
3285  goto end;
3286  }
3287  ssn.client.next_seq = 22;
3288 
3289  p->tcph->th_seq = htonl(17);
3290  ret = 1;
3291 end:
3292  FLOWLOCK_UNLOCK(&f);
3293  FLOW_DESTROY(&f);
3294  UTHFreePacket(p);
3296  StreamTcpUTDeinit(ra_ctx);
3297  return ret;
3298 }
3299 
3300 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3301  * test the sliding window reassembly with a small window size so that we
3302  * cutting off at the start (left edge) with small packet overlap.
3303  */
3304 static int StreamTcpReassembleInlineTest04(void)
3305 {
3306  int ret = 0;
3307  TcpReassemblyThreadCtx *ra_ctx = NULL;
3308  ThreadVars tv;
3309  TcpSession ssn;
3310  Flow f;
3311 
3312  memset(&tv, 0x00, sizeof(tv));
3313 
3314  StreamTcpUTInit(&ra_ctx);
3317  StreamTcpUTSetupStream(&ssn.client, 1);
3318  FLOW_INITIALIZE(&f);
3319 
3321 
3322  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3323  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3324  if (p == NULL) {
3325  printf("couldn't get a packet: ");
3326  goto end;
3327  }
3328  p->tcph->th_seq = htonl(12);
3329  p->flow = &f;
3331 
3332  FLOWLOCK_WRLOCK(&f);
3333  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3334  printf("failed to add segment 1: ");
3335  goto end;
3336  }
3337  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3338  printf("failed to add segment 2: ");
3339  goto end;
3340  }
3341  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3342  printf("failed to add segment 3: ");
3343  goto end;
3344  }
3345  ssn.client.next_seq = 17;
3346  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3347  printf("failed to add segment 4: ");
3348  goto end;
3349  }
3350  ssn.client.next_seq = 22;
3351 
3352  p->tcph->th_seq = htonl(17);
3353  ret = 1;
3354 end:
3355  FLOWLOCK_UNLOCK(&f);
3356  FLOW_DESTROY(&f);
3357  UTHFreePacket(p);
3359  StreamTcpUTDeinit(ra_ctx);
3360  return ret;
3361 }
3362 
3363 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3364  * test the sliding window reassembly with a small window size so that we
3365  * cutting off at the start (left edge). Test if the first segment is
3366  * removed from the list.
3367  */
3368 static int StreamTcpReassembleInlineTest08(void)
3369 {
3370  TcpReassemblyThreadCtx *ra_ctx = NULL;
3371  ThreadVars tv;
3372  memset(&tv, 0x00, sizeof(tv));
3373  TcpSession ssn;
3374  Flow f;
3375  StreamTcpUTInit(&ra_ctx);
3378  StreamTcpUTSetupStream(&ssn.client, 1);
3379  FLOW_INITIALIZE(&f);
3380 
3382  f.protoctx = &ssn;
3383 
3384  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3385  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3386  FAIL_IF(p == NULL);
3387  p->tcph->th_seq = htonl(12);
3388  p->flow = &f;
3390 
3391  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3392  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3393  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3394  ssn.client.next_seq = 17;
3395  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3396  ssn.client.next_seq = 22;
3397  p->tcph->th_seq = htonl(17);
3399 
3400  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3401  FAIL_IF_NULL(seg);
3402  FAIL_IF_NOT(seg->seq == 2);
3403 
3404  FLOW_DESTROY(&f);
3405  UTHFreePacket(p);
3407  StreamTcpUTDeinit(ra_ctx);
3408  PASS;
3409 }
3410 
3411 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3412  * test the sliding window reassembly with a small window size so that we
3413  * cutting off at the start (left edge). Test if the first segment is
3414  * removed from the list.
3415  */
3416 static int StreamTcpReassembleInlineTest09(void)
3417 {
3418  int ret = 0;
3419  TcpReassemblyThreadCtx *ra_ctx = NULL;
3420  ThreadVars tv;
3421  TcpSession ssn;
3422  Flow f;
3423 
3424  memset(&tv, 0x00, sizeof(tv));
3425 
3426  StreamTcpUTInit(&ra_ctx);
3429  StreamTcpUTSetupStream(&ssn.client, 1);
3430  FLOW_INITIALIZE(&f);
3431 
3433 
3434  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3435  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3436  if (p == NULL) {
3437  printf("couldn't get a packet: ");
3438  goto end;
3439  }
3440  p->tcph->th_seq = htonl(17);
3441  p->flow = &f;
3443 
3444  FLOWLOCK_WRLOCK(&f);
3445  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3446  printf("failed to add segment 1: ");
3447  goto end;
3448  }
3449  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3450  printf("failed to add segment 2: ");
3451  goto end;
3452  }
3453  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3454  printf("failed to add segment 3: ");
3455  goto end;
3456  }
3457  ssn.client.next_seq = 12;
3458  ssn.client.last_ack = 10;
3459 
3460  /* close the GAP and see if we properly reassemble and update base_seq */
3461  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3462  printf("failed to add segment 4: ");
3463  goto end;
3464  }
3465  ssn.client.next_seq = 22;
3466 
3467  p->tcph->th_seq = htonl(12);
3468 
3469  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3470  FAIL_IF_NULL(seg);
3471  FAIL_IF_NOT(seg->seq == 2);
3472 
3473  ret = 1;
3474 end:
3475  FLOWLOCK_UNLOCK(&f);
3476  FLOW_DESTROY(&f);
3477  UTHFreePacket(p);
3479  StreamTcpUTDeinit(ra_ctx);
3480  return ret;
3481 }
3482 
3483 /** \test App Layer reassembly.
3484  */
3485 static int StreamTcpReassembleInlineTest10(void)
3486 {
3487  int ret = 0;
3488  TcpReassemblyThreadCtx *ra_ctx = NULL;
3489  ThreadVars tv;
3490  TcpSession ssn;
3491  Flow *f = NULL;
3492  Packet *p = NULL;
3493 
3494  memset(&tv, 0x00, sizeof(tv));
3495 
3496  StreamTcpUTInit(&ra_ctx);
3499  StreamTcpUTSetupStream(&ssn.server, 1);
3500  ssn.server.last_ack = 2;
3501  StreamTcpUTSetupStream(&ssn.client, 1);
3502  ssn.client.last_ack = 2;
3504 
3505  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3506  if (f == NULL)
3507  goto end;
3508  f->protoctx = &ssn;
3509  f->proto = IPPROTO_TCP;
3510 
3511  uint8_t stream_payload1[] = "GE";
3512  uint8_t stream_payload2[] = "T /";
3513  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3514 
3515  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3516  if (p == NULL) {
3517  printf("couldn't get a packet: ");
3518  goto end;
3519  }
3520  p->tcph->th_seq = htonl(7);
3521  p->flow = f;
3523 
3524  FLOWLOCK_WRLOCK(f);
3525  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3526  printf("failed to add segment 1: ");
3527  goto end;
3528  }
3529  ssn.client.next_seq = 4;
3530 
3531  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3532  if (r < 0) {
3533  printf("StreamTcpReassembleAppLayer failed: ");
3534  goto end;
3535  }
3536 
3537  /* ssn.server.ra_app_base_seq should be isn here. */
3538  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3539  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3540  goto end;
3541  }
3542 
3543  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3544  printf("failed to add segment 2: ");
3545  goto end;
3546  }
3547  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3548  printf("failed to add segment 3: ");
3549  goto end;
3550  }
3551  ssn.client.next_seq = 19;
3552 
3553  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3554  if (r < 0) {
3555  printf("StreamTcpReassembleAppLayer failed: ");
3556  goto end;
3557  }
3558 
3559  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3560 
3561  ret = 1;
3562 end:
3563  UTHFreePacket(p);
3565  StreamTcpUTDeinit(ra_ctx);
3566  FLOWLOCK_UNLOCK(f);
3567  UTHFreeFlow(f);
3568  return ret;
3569 }
3570 
3571 /** \test test insert with overlap
3572  */
3573 static int StreamTcpReassembleInsertTest01(void)
3574 {
3575  TcpReassemblyThreadCtx *ra_ctx = NULL;
3576  ThreadVars tv;
3577  TcpSession ssn;
3578  Flow f;
3579 
3580  memset(&tv, 0x00, sizeof(tv));
3581 
3582  StreamTcpUTInit(&ra_ctx);
3584  StreamTcpUTSetupStream(&ssn.client, 1);
3586  FLOW_INITIALIZE(&f);
3587 
3588  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3589  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3590  FAIL_IF(p == NULL);
3591  p->tcph->th_seq = htonl(12);
3592  p->flow = &f;
3593 
3594  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3595  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3596  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3597  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3598  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3599  ssn.client.next_seq = 21;
3600 
3601  FLOW_DESTROY(&f);
3602  UTHFreePacket(p);
3604  StreamTcpUTDeinit(ra_ctx);
3605  PASS;
3606 }
3607 
3608 /** \test test insert with overlaps
3609  */
3610 static int StreamTcpReassembleInsertTest02(void)
3611 {
3612  int ret = 0;
3613  TcpReassemblyThreadCtx *ra_ctx = NULL;
3614  ThreadVars tv;
3615  TcpSession ssn;
3616 
3617  memset(&tv, 0x00, sizeof(tv));
3618 
3619  StreamTcpUTInit(&ra_ctx);
3621  StreamTcpUTSetupStream(&ssn.client, 1);
3622 
3623  int i;
3624  for (i = 2; i < 10; i++) {
3625  int len;
3626  len = i % 2;
3627  if (len == 0)
3628  len = 1;
3629  int seq;
3630  seq = i * 10;
3631  if (seq < 2)
3632  seq = 2;
3633 
3634  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3635  printf("failed to add segment 1: ");
3636  goto end;
3637  }
3638  }
3639  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3640  printf("failed to add segment 2: ");
3641  goto end;
3642  }
3643 
3644  ret = 1;
3645 end:
3647  StreamTcpUTDeinit(ra_ctx);
3648  return ret;
3649 }
3650 
3651 /** \test test insert with overlaps
3652  */
3653 static int StreamTcpReassembleInsertTest03(void)
3654 {
3655  int ret = 0;
3656  TcpReassemblyThreadCtx *ra_ctx = NULL;
3657  ThreadVars tv;
3658  TcpSession ssn;
3659 
3660  memset(&tv, 0x00, sizeof(tv));
3661 
3662  StreamTcpUTInit(&ra_ctx);
3664  StreamTcpUTSetupStream(&ssn.client, 1);
3665 
3666  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3667  printf("failed to add segment 2: ");
3668  goto end;
3669  }
3670 
3671  int i;
3672  for (i = 2; i < 10; i++) {
3673  int len;
3674  len = i % 2;
3675  if (len == 0)
3676  len = 1;
3677  int seq;
3678  seq = i * 10;
3679  if (seq < 2)
3680  seq = 2;
3681 
3682  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3683  printf("failed to add segment 2: ");
3684  goto end;
3685  }
3686  }
3687  ret = 1;
3688 end:
3690  StreamTcpUTDeinit(ra_ctx);
3691  return ret;
3692 }
3693 
3695 #endif /* UNITTESTS */
3696 
3697 /** \brief The Function Register the Unit tests to test the reassembly engine
3698  * for various OS policies.
3699  */
3700 
3702 {
3703 #ifdef UNITTESTS
3704  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3705  StreamTcpReassembleTest25);
3706  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3707  StreamTcpReassembleTest26);
3708  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3709  StreamTcpReassembleTest27);
3710  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3711  StreamTcpReassembleTest28);
3712  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3713  StreamTcpReassembleTest29);
3714  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3715  StreamTcpReassembleTest33);
3716  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3717  StreamTcpReassembleTest34);
3718  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3719  StreamTcpReassembleTest37);
3720  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3721  StreamTcpReassembleTest39);
3722  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3723  StreamTcpReassembleTest40);
3724  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3725  StreamTcpReassembleTest44);
3726  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3727  StreamTcpReassembleTest45);
3728  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3729  StreamTcpReassembleTest46);
3730  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3731  StreamTcpReassembleTest47);
3732 
3733  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3734  StreamTcpReassembleInlineTest01);
3735  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3736  StreamTcpReassembleInlineTest02);
3737  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3738  StreamTcpReassembleInlineTest03);
3739  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3740  StreamTcpReassembleInlineTest04);
3741  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3742  StreamTcpReassembleInlineTest08);
3743  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3744  StreamTcpReassembleInlineTest09);
3745 
3746  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3747  StreamTcpReassembleInlineTest10);
3748 
3749  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3750  StreamTcpReassembleInsertTest01);
3751  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3752  StreamTcpReassembleInsertTest02);
3753  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3754  StreamTcpReassembleInsertTest03);
3755 
3759  StreamTcpReassembleRawRegisterTests();
3760 #endif /* UNITTESTS */
3761 }
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:249
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:57
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:55
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:436
TcpStream_
Definition: stream-tcp-private.h:94
len
uint8_t len
Definition: app-layer-dnp3.h:2
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:149
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:101
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:101
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:1906
StreamingBufferConfig_::Malloc
void *(* Malloc)(size_t size)
Definition: util-streaming-buffer.h:71
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:169
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:302
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:92
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:70
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1752
STREAM_MIDSTREAM
#define STREAM_MIDSTREAM
Definition: stream.h:35
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:95
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:82
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:72
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:104
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:516
stream-tcp.h
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:493
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
Definition: stream-tcp-reassemble.c:175
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:557
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:123
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:1970
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:570
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:997
StreamTcpThread_
Definition: stream-tcp.h:70
Flow_::proto
uint8_t proto
Definition: flow.h:365
Packet_::payload
uint8_t * payload
Definition: decode.h:549
StreamTcpInlineMode
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6241
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:1950
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:61
Packet_::flags
uint32_t flags
Definition: decode.h:449
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:175
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:98
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:60
Flow_
Flow data structure.
Definition: flow.h:347
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:66
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:68
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:1933
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:144
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:38
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1351
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:849
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:219
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:143
StreamNeedsReassembly
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:789
MIN
#define MIN(x, y)
Definition: suricata-common.h:377
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:64
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:95
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
StreamingBufferConfig_::flags
uint32_t flags
Definition: util-streaming-buffer.h:68
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:150
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:866
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:445
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:119
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:41
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:242
Flow_::protoctx
void * protoctx
Definition: flow.h:441
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:711
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:95
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:550
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:32
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:204
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:576
util-unittest.h
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2042
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
TcpSession_::flags
uint16_t flags
Definition: stream-tcp-private.h:269
FLOWLOCK_UNLOCK
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:264
STREAM_START
#define STREAM_START
Definition: stream.h:29
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:158
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:269
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:103
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:475
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:434
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:521
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1194
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:270
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:353
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:568
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:263
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:92
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:270
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1106
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1110
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:131
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
FLOWLOCK_WRLOCK
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:261
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:118
StreamReassembleLog
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
Definition: stream-tcp-reassemble.c:1780
util-print.h
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:50
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:848
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:174
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:4748
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1307
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:262
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:87
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:1984
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:799
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
Definition: stream-tcp-reassemble.c:146
TRUE
#define TRUE
Definition: suricata-common.h:33
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueueNoLock *pq)
Definition: stream-tcp-reassemble.c:1808
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:282
TCP_GET_SEQ
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:112
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
FALSE
#define FALSE
Definition: suricata-common.h:34
SCReturn
#define SCReturn
Definition: util-debug.h:302
stream.h
TcpSegment
Definition: stream-tcp-private.h:61
StreamTcpReassembleFree
void StreamTcpReassembleFree(char quiet)
Definition: stream-tcp-reassemble.c:415
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:278
Packet_
Definition: decode.h:414
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:215
STREAM_DEPTH
#define STREAM_DEPTH
Definition: stream.h:34
StreamTcpFreeConfig
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:668
stream-tcp-private.h
PoolThreadReturn
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
Definition: util-pool-thread.c:187
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:42
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:306
conf.h
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:276
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:889
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:105
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:475
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
STREAM_GAP
#define STREAM_GAP
Definition: stream.h:33
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:315
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:220
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:264
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
StreamTcpReassembleInit
int StreamTcpReassembleInit(char quiet)
Definition: stream-tcp-reassemble.c:399
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:70
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:115
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:79
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:526
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:56
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:103
StreamingBuffer_
Definition: util-streaming-buffer.h:95
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:600
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:63
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:150
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:769
StringParseUint32
int StringParseUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:313
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:168
Packet_::flow
struct Flow_ * flow
Definition: decode.h:451
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:241
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:337
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:240
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:531
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:112
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:89
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2023
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:122
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:217
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:149
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:102
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:144
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
StreamTcpReassembleIncrMemuse
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:98
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:451
ConfNode_
Definition: conf.h:32
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:485
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:870
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:238
Flow_::flags
uint32_t flags
Definition: flow.h:421
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:766
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:74
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:50
ALPROTO_HTTP
@ ALPROTO_HTTP
Definition: app-layer-protos.h:30
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:132
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:282
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2037
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:146
StreamTcpInitConfig
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
STREAM_EOF
#define STREAM_EOF
Definition: stream.h:30
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:60
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:57
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:168
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:188
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:200
suricata.h
Address_::family
char family
Definition: decode.h:117
Packet_::dst
Address dst
Definition: decode.h:419
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:6256
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:267
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:239
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint8_t flag)
Definition: app-layer-parser.c:1640
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:219
STREAMING_BUFFER_NOFLAGS
#define STREAMING_BUFFER_NOFLAGS
Definition: util-streaming-buffer.h:64
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3701
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:90
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:171
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:132
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:212
StreamTcpReassembleDecrMemuse
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:111
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
TcpSession_
Definition: stream-tcp-private.h:260
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:265
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:160
flow.h
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:452
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:206
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:143
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:266
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:85
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:83
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:66
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:743
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:82
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:130
Packet_::src
Address src
Definition: decode.h:418
debug.h
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:73
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:213
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:133
app-layer.h
StreamTcpCheckStreamContents
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
Definition: stream-tcp-reassemble.c:2001