suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 #include "app-layer-parser.h"
64 
65 #include "detect-engine-state.h"
66 
67 #include "util-profiling.h"
68 #include "util-validate.h"
69 
70 #ifdef DEBUG
71 static SCMutex segment_pool_memuse_mutex;
72 static uint64_t segment_pool_memuse = 0;
73 static uint64_t segment_pool_memcnt = 0;
74 #endif
75 
76 static PoolThread *segment_thread_pool = NULL;
77 /* init only, protect initializing and growing pool */
78 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
79 
80 /* Memory use counter */
81 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
82 
83 /* prototypes */
85 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usage for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346  if (f->alparser) {
349  }
350 }
351 
352 /** \param f locked flow */
354 {
355  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
356  return 0;
357 
358  TcpSession *ssn = (TcpSession *)f->protoctx;
360 }
361 
362 static int StreamTcpReassemblyConfig(bool quiet)
363 {
364  uint32_t segment_prealloc = 2048;
365  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
366  if (seg) {
367  uint32_t prealloc = 0;
368  if (StringParseUint32(&prealloc, 10, strlen(seg->val), seg->val) < 0)
369  {
370  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
371  "%s is invalid", seg->val);
372  return -1;
373  }
374  segment_prealloc = prealloc;
375  }
376  if (!quiet)
377  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
378  stream_config.prealloc_segments = segment_prealloc;
379 
380  int overlap_diff_data = 0;
381  (void)ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
382  if (overlap_diff_data) {
384  }
385  if (StreamTcpInlineMode() == TRUE) {
387  }
388 
391  stream_config.sbcnf.Malloc = ReassembleMalloc;
392  stream_config.sbcnf.Calloc = ReassembleCalloc;
393  stream_config.sbcnf.Realloc = ReassembleRealloc;
394  stream_config.sbcnf.Free = ReassembleFree;
395 
396  return 0;
397 }
398 
399 int StreamTcpReassembleInit(bool quiet)
400 {
401  /* init the memcap/use tracker */
403 
404  if (StreamTcpReassemblyConfig(quiet) < 0)
405  return -1;
406 
407 #ifdef DEBUG
408  SCMutexInit(&segment_pool_memuse_mutex, NULL);
409 #endif
410  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
412  return 0;
413 }
414 
415 void StreamTcpReassembleFree(bool quiet)
416 {
417  SCMutexLock(&segment_thread_pool_mutex);
418  if (segment_thread_pool != NULL) {
419  PoolThreadFree(segment_thread_pool);
420  segment_thread_pool = NULL;
421  }
422  SCMutexUnlock(&segment_thread_pool_mutex);
423  SCMutexDestroy(&segment_thread_pool_mutex);
424 
425 #ifdef DEBUG
426  if (segment_pool_memuse > 0)
427  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
428  if (segment_pool_memcnt > 0)
429  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
430  SCMutexDestroy(&segment_pool_memuse_mutex);
431 #endif
432 }
433 
435 {
436  SCEnter();
438  if (unlikely(ra_ctx == NULL))
439  return NULL;
440 
441  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
442 
443  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
444 
445  SCMutexLock(&segment_thread_pool_mutex);
446  if (segment_thread_pool == NULL) {
447  segment_thread_pool = PoolThreadInit(1, /* thread */
448  0, /* unlimited */
450  sizeof(TcpSegment),
451  TcpSegmentPoolAlloc,
452  TcpSegmentPoolInit, NULL,
453  TcpSegmentPoolCleanup, NULL);
454  ra_ctx->segment_thread_pool_id = 0;
455  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
456  PoolThreadSize(segment_thread_pool),
457  ra_ctx->segment_thread_pool_id);
458  } else {
459  /* grow segment_thread_pool until we have an element for our thread id */
460  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
461  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
462  PoolThreadSize(segment_thread_pool),
463  ra_ctx->segment_thread_pool_id);
464  }
465  SCMutexUnlock(&segment_thread_pool_mutex);
466  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
467  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
469  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
470  }
471 
472  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
473 }
474 
476 {
477  SCEnter();
478  if (ra_ctx) {
480  SCFree(ra_ctx);
481  }
482  SCReturn;
483 }
484 
485 /**
486  * \brief check if stream in pkt direction has depth reached
487  *
488  * \param p packet with *LOCKED* flow
489  *
490  * \retval 1 stream has depth reached
491  * \retval 0 stream does not have depth reached
492  */
494 {
495  if (p->flow != NULL && p->flow->protoctx != NULL) {
496  TcpSession *ssn = p->flow->protoctx;
497  TcpStream *stream;
498  if (p->flowflags & FLOW_PKT_TOSERVER) {
499  stream = &ssn->client;
500  } else {
501  stream = &ssn->server;
502  }
503 
504  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
505  }
506 
507  return 0;
508 }
509 
510 /**
511  * \internal
512  * \brief Function to Check the reassembly depth valuer against the
513  * allowed max depth of the stream reassembly for TCP streams.
514  *
515  * \param stream stream direction
516  * \param seq sequence number where "size" starts
517  * \param size size of the segment that is added
518  *
519  * \retval size Part of the size that fits in the depth, 0 if none
520  */
521 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
522  uint32_t seq, uint32_t size)
523 {
524  SCEnter();
525 
526  /* if the configured depth value is 0, it means there is no limit on
527  reassembly depth. Otherwise carry on my boy ;) */
528  if (ssn->reassembly_depth == 0) {
529  SCReturnUInt(size);
530  }
531 
532  /* if the final flag is set, we're not accepting anymore */
534  SCReturnUInt(0);
535  }
536 
537  uint64_t seg_depth;
538  if (SEQ_GT(stream->base_seq, seq)) {
539  if (SEQ_LEQ(seq+size, stream->base_seq)) {
540  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
541  stream->base_seq, seq, seq+size);
542  SCReturnUInt(0);
543  }
544 
545  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
546  } else {
547  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
548  }
549 
550  /* if the base_seq has moved passed the depth window we stop
551  * checking and just reject the rest of the packets including
552  * retransmissions. Saves us the hassle of dealing with sequence
553  * wraps as well */
554  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
555  stream->base_seq, seg_depth,
556  ssn->reassembly_depth);
557 
558  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
559  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
561  SCReturnUInt(0);
562  }
563  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
564  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
565 #if 0
566  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
567  (stream->base_seq_offset + stream->base_seq + size),
568  (stream->isn + ssn->reassembly_depth));
569 #endif
570  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
571  /* packet (partly?) fits the depth window */
572 
573  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
574  /* complete fit */
575  SCReturnUInt(size);
576  } else {
578  /* partial fit, return only what fits */
579  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
580  DEBUG_VALIDATE_BUG_ON(part > size);
581  if (part > size)
582  part = size;
583  SCReturnUInt(part);
584  }
585  }
586 
587  SCReturnUInt(0);
588 }
589 
591 {
592  if (RB_EMPTY(&stream->sb.sbb_tree)) {
593  if (stream->sb.stream_offset != 0)
594  return 0;
595 
596  return stream->sb.buf_offset;
597  } else {
598  DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
599  DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
600  return stream->sb.sbb_size;
601  }
602 }
603 
604 /**
605  * \brief Insert a packets TCP data into the stream reassembly engine.
606  *
607  * \retval 0 good segment, as far as we checked.
608  * \retval -1 badness, reason to drop in inline mode
609  *
610  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
611  * or it wasn't added because of reassembly depth.
612  *
613  */
615  TcpSession *ssn, TcpStream *stream, Packet *p)
616 {
617  SCEnter();
618 
619  if (ssn->data_first_seen_dir == 0) {
620  if (PKT_IS_TOSERVER(p)) {
622  } else {
624  }
625  }
626 
627  /* If the OS policy is not set then set the OS policy for this stream */
628  if (stream->os_policy == 0) {
629  StreamTcpSetOSPolicy(stream, p);
630  }
631 
634  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
635  SCReturnInt(0);
636  }
637 
638  /* If we have reached the defined depth for either of the stream, then stop
639  reassembling the TCP session */
640  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
641  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
642 
644  /* increment stream depth counter */
646  }
647  if (size == 0) {
648  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
649  SCReturnInt(0);
650  }
651 
653  if (size > p->payload_len)
654  size = p->payload_len;
655 
656  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
657  if (seg == NULL) {
658  SCLogDebug("segment_pool is empty");
660  SCReturnInt(-1);
661  }
662 
663  TCP_SEG_LEN(seg) = size;
664  seg->seq = TCP_GET_SEQ(p);
665 
666  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
667  if (TCP_HAS_TFO(p) && p->payload_len && p->tcph->th_flags == TH_SYN)
668  seg->seq += 1;
669 
670  /* proto detection skipped, but now we do get data. Set event. */
671  if (RB_EMPTY(&stream->seg_tree) &&
673 
676  }
677 
678  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
679  SCLogDebug("StreamTcpReassembleInsertSegment failed");
680  SCReturnInt(-1);
681  }
682  SCReturnInt(0);
683 }
684 
685 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
686  Packet *p)
687 {
688  uint8_t flag = 0;
689 
691  flag |= STREAM_START;
692  }
693 
694  if (ssn->state == TCP_CLOSED) {
695  flag |= STREAM_EOF;
696  }
697 
698  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
699  flag |= STREAM_MIDSTREAM;
700  }
701 
702  if (p->flags & PKT_PSEUDO_STREAM_END) {
703  flag |= STREAM_EOF;
704  }
705 
706  if (&ssn->client == stream) {
707  flag |= STREAM_TOSERVER;
708  } else {
709  flag |= STREAM_TOCLIENT;
710  }
712  flag |= STREAM_DEPTH;
713  }
714  return flag;
715 }
716 
717 /**
718  * \brief Check the minimum size limits for reassembly.
719  *
720  * \retval 0 don't reassemble yet
721  * \retval 1 do reassemble
722  */
723 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
724  const TcpStream *stream, const Packet *p)
725 {
726  SCEnter();
727 
728  /* if any of these flags is set we always inspect immediately */
729 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
730  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
731  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
732  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
733 
736  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
737  "is set, so not expecting any new data segments");
738  }
740  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
741  }
743  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
744  "so no new segments will be considered");
745  }
746  SCReturnInt(1);
747  }
748 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
749 
750  /* some states mean we reassemble no matter how much data we have */
751  if (ssn->state > TCP_TIME_WAIT)
752  SCReturnInt(1);
753 
754  if (p->flags & PKT_PSEUDO_STREAM_END)
755  SCReturnInt(1);
756 
757  /* check if we have enough data to do raw reassembly */
758  if (PKT_IS_TOSERVER(p)) {
759  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
760  uint32_t delta = stream->last_ack - stream->base_seq;
761  /* get max absolute offset */
762  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
763 
764  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
765  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
766  SCReturnInt(1);
767  } else {
768  SCLogDebug("toserver min chunk len not yet reached: "
769  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
770  "%"PRIu32"", stream->last_ack, stream->base_seq,
771  (stream->last_ack - stream->base_seq),
773  SCReturnInt(0);
774  }
775  }
776  } else {
777  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
778  uint32_t delta = stream->last_ack - stream->base_seq;
779  /* get max absolute offset */
780  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
781 
782  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
783 
784  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
785  SCReturnInt(1);
786  } else {
787  SCLogDebug("toclient min chunk len not yet reached: "
788  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
789  "%"PRIu32"", stream->last_ack, stream->base_seq,
790  (stream->last_ack - stream->base_seq),
792  SCReturnInt(0);
793  }
794  }
795  }
796 
797  SCReturnInt(0);
798 }
799 
800 /**
801  * \brief see what if any work the TCP session still needs
802  */
803 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
804 {
805  const TcpStream *stream = NULL;
806 #ifdef DEBUG
807  const char *dirstr = NULL;
808 #endif
809  if (direction == STREAM_TOSERVER) {
810  stream = &ssn->client;
811 #ifdef DEBUG
812  dirstr = "client";
813 #endif
814  } else {
815  stream = &ssn->server;
816 #ifdef DEBUG
817  dirstr = "server";
818 #endif
819  }
820 
821  int use_app = 1;
822  int use_raw = 1;
823 
825  // app is dead
826  use_app = 0;
827  }
828 
830  // raw is dead
831  use_raw = 0;
832  }
833 
834  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
835 
836  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
837  dirstr,
838  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
839  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
840  right_edge);
841  if (use_raw) {
842  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
843  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
845  }
846  }
847  if (use_app) {
848  if (right_edge > STREAM_APP_PROGRESS(stream)) {
849  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
851  }
852  }
853 
854  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
856 }
857 
858 #ifdef DEBUG
859 static uint64_t GetStreamSize(TcpStream *stream)
860 {
861  if (stream) {
862  uint64_t size = 0;
863  uint32_t cnt = 0;
864 
865  TcpSegment *seg;
866  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
867  cnt++;
868  size += (uint64_t)TCP_SEG_LEN(seg);
869  }
870 
871  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
872  return size;
873  }
874  return (uint64_t)0;
875 }
876 
877 static void GetSessionSize(TcpSession *ssn, Packet *p)
878 {
879  uint64_t size = 0;
880  if (ssn) {
881  size = GetStreamSize(&ssn->client);
882  size += GetStreamSize(&ssn->server);
883 
884  //if (size > 900000)
885  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
886  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
887  }
888 }
889 #endif
890 
891 static StreamingBufferBlock *GetBlock(StreamingBuffer *sb, const uint64_t offset)
892 {
893  StreamingBufferBlock *blk = sb->head;
894  if (blk == NULL)
895  return NULL;
896 
897  for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
898  if (blk->offset >= offset)
899  return blk;
900  else if ((blk->offset + blk->len) > offset) {
901  return blk;
902  }
903  }
904  return NULL;
905 }
906 
907 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
908 {
909  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
910  return STREAM_BASE_OFFSET(stream) +
911  (stream->last_ack - stream->base_seq);
912  } else {
913  return STREAM_BASE_OFFSET(stream);
914  }
915 }
916 
917 static inline bool GapAhead(TcpStream *stream, StreamingBufferBlock *cur_blk)
918 {
919  StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
920  if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
921  GetAbsLastAck(stream) >= (cur_blk->offset + cur_blk->len)) {
922  return true;
923  }
924  return false;
925 }
926 
927 /** \internal
928  *
929  * Get buffer, or first part of the buffer if data gaps exist.
930  *
931  * \brief get stream data from offset
932  * \param offset stream offset
933  * \param check_for_gap check if there is a gap ahead. Optional as it is only
934  * needed for app-layer incomplete support.
935  * \retval bool pkt loss ahead */
936 static bool GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
937  uint64_t offset, const bool check_for_gap)
938 {
939  const uint8_t *mydata;
940  uint32_t mydata_len;
941  bool gap_ahead = false;
942 
943  if (RB_EMPTY(&stream->sb.sbb_tree)) {
944  SCLogDebug("getting one blob");
945 
946  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
947 
948  *data = mydata;
949  *data_len = mydata_len;
950  } else {
951  StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
952  if (blk == NULL) {
953  *data = NULL;
954  *data_len = 0;
955  return false;
956  }
957 
958  /* block at expected offset */
959  if (blk->offset == offset) {
960 
961  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
962 
963  gap_ahead = check_for_gap && GapAhead(stream, blk);
964 
965  /* block past out offset */
966  } else if (blk->offset > offset) {
967  SCLogDebug("gap, want data at offset %"PRIu64", "
968  "got data at %"PRIu64". GAP of size %"PRIu64,
969  offset, blk->offset, blk->offset - offset);
970  *data = NULL;
971  *data_len = blk->offset - offset;
972 
973  /* block starts before offset, but ends after */
974  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
975  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
976  offset, blk->offset, blk->len);
977  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
978  SCLogDebug("data %p, data_len %u", *data, *data_len);
979 
980  gap_ahead = check_for_gap && GapAhead(stream, blk);
981 
982  } else {
983  *data = NULL;
984  *data_len = 0;
985  }
986  }
987  return gap_ahead;
988 }
989 
990 /** \internal
991  * \brief check to see if we should declare a GAP
992  * Call this when the app layer didn't get data at the requested
993  * offset.
994  */
995 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
996 {
997  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
998  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
999 
1000  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1001  /* get window of data that is acked */
1002  const uint32_t delta = stream->last_ack - stream->base_seq;
1003  /* get max absolute offset */
1004  last_ack_abs += delta;
1005 
1006  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1007  last_ack_abs -= ackadded;
1008 
1009  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
1010  SCLogDebug("next_seq %u", stream->next_seq);
1011 
1012  /* if last_ack_abs is beyond the app_progress data that we haven't seen
1013  * has been ack'd. This looks like a GAP. */
1014  if (last_ack_abs > app_progress) {
1015  /* however, we can accept ACKs a bit too liberally. If last_ack
1016  * is beyond next_seq, we only consider it a gap now if we do
1017  * already have data beyond the gap. */
1018  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1019  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1020  SCLogDebug("packet %"PRIu64": no GAP. "
1021  "next_seq %u < last_ack %u, but no data in list",
1022  p->pcap_cnt, stream->next_seq, stream->last_ack);
1023  return false;
1024  } else {
1025  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1026  const StreamingBufferBlock *blk = stream->sb.head;
1027  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1028  /* ack'd data after the gap */
1029  SCLogDebug("packet %"PRIu64": GAP. "
1030  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1031  p->pcap_cnt, stream->next_seq, stream->last_ack);
1032  return true;
1033  }
1034  }
1035  }
1036 
1037  SCLogDebug("packet %"PRIu64": GAP! "
1038  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
1039  "but we have no data.",
1040  p->pcap_cnt, last_ack_abs, app_progress);
1041  return true;
1042  }
1043  }
1044  SCLogDebug("packet %"PRIu64": no GAP. "
1045  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1046  p->pcap_cnt, last_ack_abs, app_progress);
1047  return false;
1048 }
1049 
1050 static inline uint32_t AdjustToAcked(const Packet *p,
1051  const TcpSession *ssn, const TcpStream *stream,
1052  const uint64_t app_progress, const uint32_t data_len)
1053 {
1054  uint32_t adjusted = data_len;
1055 
1056  /* get window of data that is acked */
1057  if (StreamTcpInlineMode() == FALSE) {
1058  SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1059  if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1060  (ssn->state == TCP_CLOSED &&
1061  (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1062  (p->flags & PKT_PSEUDO_STREAM_END))) {
1063  // fall through, we use all available data
1064  } else {
1065  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1066  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1067  /* get window of data that is acked */
1068  uint32_t delta = stream->last_ack - stream->base_seq;
1069  /* get max absolute offset */
1070  last_ack_abs += delta;
1071  }
1072  DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1073 
1074  /* see if the buffer contains unack'd data as well */
1075  if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1076  uint32_t check = data_len;
1077  adjusted = last_ack_abs - app_progress;
1078  BUG_ON(adjusted > check);
1079  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1080  "data is considered", adjusted);
1081  }
1082  }
1083  }
1084  return adjusted;
1085 }
1086 
1087 /** \internal
1088  * \brief get stream buffer and update the app-layer
1089  * \param stream pointer to pointer as app-layer can switch flow dir
1090  * \retval 0 success
1091  */
1092 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1093  TcpReassemblyThreadCtx *ra_ctx,
1094  TcpSession *ssn, TcpStream **stream,
1095  Packet *p, enum StreamUpdateDir dir)
1096 {
1097  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1098 
1099  SCLogDebug("app progress %"PRIu64, app_progress);
1100  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1101 
1102  const uint8_t *mydata;
1103  uint32_t mydata_len;
1104  bool gap_ahead = false;
1105  bool last_was_gap = false;
1106 
1107  while (1) {
1108  const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1109  bool check_for_gap_ahead = ((*stream)->data_required > 0);
1110  gap_ahead = GetAppBuffer(*stream, &mydata, &mydata_len,
1111  app_progress, check_for_gap_ahead);
1112  if (last_was_gap && mydata_len == 0) {
1113  break;
1114  }
1115  last_was_gap = false;
1116 
1117  /* make sure to only deal with ACK'd data */
1118  mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1119  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1120  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1121  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1122 
1123  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1124  NULL, mydata_len,
1125  StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP);
1126  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1127 
1130 
1131  /* AppLayerHandleTCPData has likely updated progress. */
1132  const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1133  app_progress = STREAM_APP_PROGRESS(*stream);
1134 
1135  /* a GAP also consumes 'data required'. TODO perhaps we can use
1136  * this to skip post GAP data until the start of a next record. */
1137  if ((*stream)->data_required > 0) {
1138  if ((*stream)->data_required > mydata_len) {
1139  (*stream)->data_required -= mydata_len;
1140  } else {
1141  (*stream)->data_required = 0;
1142  }
1143  }
1144  if (r < 0)
1145  return 0;
1146  if (no_progress_update)
1147  break;
1148  last_was_gap = true;
1149  continue;
1150 
1151  } else if (flags & STREAM_DEPTH) {
1152  // we're just called once with this flag, so make sure we pass it on
1153 
1154  } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1155  /* Possibly a gap, but no new data. */
1156  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1157  SCReturnInt(0);
1158 
1159  mydata = NULL;
1160  mydata_len = 0;
1161  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1162  break;
1163  }
1164 
1165  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1166  *stream, &(*stream)->sb, mydata_len, app_progress);
1167 
1168  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1169  if (mydata_len < (*stream)->data_required) {
1170  if (gap_ahead) {
1171  SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1172  (*stream)->data_required, mydata_len);
1173  (*stream)->app_progress_rel += mydata_len;
1174  (*stream)->data_required -= mydata_len;
1175  // TODO send incomplete data to app-layer with special flag
1176  // indicating its all there is for this rec?
1177  } else {
1178  SCReturnInt(0);
1179  }
1180  app_progress = STREAM_APP_PROGRESS(*stream);
1181  continue;
1182  }
1183  }
1184  (*stream)->data_required = 0;
1185 
1186  /* update the app-layer */
1187  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1188  (uint8_t *)mydata, mydata_len, flags);
1189  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1190  uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1191  if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1192  break;
1193  app_progress = new_app_progress;
1194  if (flags & STREAM_DEPTH)
1195  break;
1196  }
1197 
1198  SCReturnInt(0);
1199 }
1200 
1201 /**
1202  * \brief Update the stream reassembly upon receiving a packet.
1203  *
1204  * For IDS mode, the stream is in the opposite direction of the packet,
1205  * as the ACK-packet is ACK'ing the stream.
1206  *
1207  * One of the utilities call by this function AppLayerHandleTCPData(),
1208  * has a feature where it will call this very same function for the
1209  * stream opposing the stream it is called with. This shouldn't cause
1210  * any issues, since processing of each stream is independent of the
1211  * other stream.
1212  */
1214  TcpSession *ssn, TcpStream *stream,
1215  Packet *p, enum StreamUpdateDir dir)
1216 {
1217  SCEnter();
1218 
1219  /* this function can be directly called by app layer protocol
1220  * detection. */
1223  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1224  SCReturnInt(0);
1225  }
1226 
1227 #ifdef DEBUG
1228  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1229  GetSessionSize(ssn, p);
1230 #endif
1231  /* if no segments are in the list or all are already processed,
1232  * and state is beyond established, we send an empty msg */
1233  if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1234  {
1235  /* send an empty EOF msg if we have no segments but TCP state
1236  * is beyond ESTABLISHED */
1237  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1238  SCLogDebug("sending empty eof message");
1239  /* send EOF to app layer */
1240  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1241  NULL, 0,
1242  StreamGetAppLayerFlags(ssn, stream, p));
1243  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1244 
1245  SCReturnInt(0);
1246  }
1247  }
1248 
1249  /* with all that out of the way, lets update the app-layer */
1250  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1251 }
1252 
1253 /** \internal
1254  * \brief get stream data from offset
1255  * \param offset stream offset */
1256 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1257  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1258 {
1259  const uint8_t *mydata;
1260  uint32_t mydata_len;
1261  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1262  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1263 
1264  uint64_t roffset = offset;
1265  if (offset)
1266  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1267  else {
1268  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1269  }
1270 
1271  *data = mydata;
1272  *data_len = mydata_len;
1273  *data_offset = roffset;
1274  } else {
1275  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1276  *iter == NULL ? "starting" : "continuing", offset);
1277  if (*iter == NULL) {
1278  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1279  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1280  SCLogDebug("*iter %p", *iter);
1281  }
1282  if (*iter == NULL) {
1283  SCLogDebug("no data");
1284  *data = NULL;
1285  *data_len = 0;
1286  *data_offset = 0;
1287  return 0;
1288  }
1289  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1290 
1291  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1292  SCLogDebug("mydata %p", mydata);
1293 
1294  if ((*iter)->offset < offset) {
1295  uint64_t delta = offset - (*iter)->offset;
1296  if (delta < mydata_len) {
1297  *data = mydata + delta;
1298  *data_len = mydata_len - delta;
1299  *data_offset = offset;
1300  } else {
1301  SCLogDebug("no data (yet)");
1302  *data = NULL;
1303  *data_len = 0;
1304  *data_offset = 0;
1305  }
1306 
1307  } else {
1308  *data = mydata;
1309  *data_len = mydata_len;
1310  *data_offset = (*iter)->offset;
1311  }
1312 
1313  *iter = SBB_RB_NEXT(*iter);
1314  SCLogDebug("*iter %p", *iter);
1315  }
1316  return 0;
1317 }
1318 
1319 /** \brief does the stream engine have data to inspect?
1320  *
1321  * Returns true if there is data to inspect. In IDS case this is
1322  * about ACK'd data in the packet's direction.
1323  *
1324  * In the IPS case this is about the packet itself.
1325  */
1327 {
1328  TcpStream *stream;
1329  if (PKT_IS_TOSERVER(p)) {
1330  stream = &ssn->client;
1331  } else {
1332  stream = &ssn->server;
1333  }
1334 
1335  if (RB_EMPTY(&stream->seg_tree)) {
1336  return false;
1337  }
1338 
1341  return false;
1342 
1343  if (StreamTcpInlineMode() == FALSE) {
1344  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1345  return false;
1346  }
1347  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1348  return true;
1349  }
1350  } else {
1351  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1352  return true;
1353  }
1354  }
1355  return false;
1356 }
1357 
1358 /** \brief update stream engine after detection
1359  *
1360  * Tasked with progressing the 'progress' for Raw reassembly.
1361  * 2 main scenario's:
1362  * 1. progress is != 0, so we use this
1363  * 2. progress is 0, meaning the detect engine didn't touch
1364  * raw at all. In this case we need to look into progressing
1365  * raw anyway.
1366  *
1367  * Additionally, this function is tasked with disabling raw
1368  * reassembly if the app-layer requested to disable it.
1369  */
1370 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1371 {
1372  TcpStream *stream;
1373  if (PKT_IS_TOSERVER(p)) {
1374  stream = &ssn->client;
1375  } else {
1376  stream = &ssn->server;
1377  }
1378 
1379  if (progress > STREAM_RAW_PROGRESS(stream)) {
1380  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1381  stream->raw_progress_rel += slide;
1383 
1384  /* if app is active and beyond raw, sync raw to app */
1385  } else if (progress == 0 && STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1387  /* if trigger raw is set we sync the 2 trackers */
1389  {
1390  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1391  stream->raw_progress_rel += slide;
1393 
1394  /* otherwise mix in the tcp window */
1395  } else {
1396  uint64_t tcp_window = stream->window;
1397  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1398  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1399  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1400  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1401  stream->raw_progress_rel += slide;
1402  }
1403  }
1404  }
1405  /* app is dead */
1406  } else if (progress == 0) {
1407  uint64_t tcp_window = stream->window;
1408  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1409  if (tcp_window < stream_right_edge) {
1410  uint64_t new_raw = stream_right_edge - tcp_window;
1411  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1412  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1413  stream->raw_progress_rel += slide;
1414  }
1415  }
1417 
1418  } else {
1419  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1420  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1421  STREAM_RAW_PROGRESS(stream), stream->window);
1422  }
1423 
1424  /* if we were told to accept no more raw data, we can mark raw as
1425  * disabled now. */
1428  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1429  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1430  }
1431 
1432  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1433 }
1434 
1435 /** \internal
1436  * \brief get a buffer around the current packet and run the callback on it
1437  *
1438  * The inline/IPS scanning method takes the current payload and wraps it in
1439  * data from other segments.
1440  *
1441  * How much data is inspected is controlled by the available data, chunk_size
1442  * and the payload size of the packet.
1443  *
1444  * Large packets: if payload size is close to the chunk_size, where close is
1445  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1446  * used: payload_len + 33% of the chunk_size.
1447  * If the payload size if equal to or bigger than the chunk_size, we use
1448  * payload len + 33% of the chunk size.
1449  */
1450 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1451  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1452 {
1453  SCEnter();
1454  int r = 0;
1455 
1456  TcpStream *stream;
1457  if (PKT_IS_TOSERVER(p)) {
1458  stream = &ssn->client;
1459  } else {
1460  stream = &ssn->server;
1461  }
1462 
1463  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1465  {
1466  *progress_out = STREAM_RAW_PROGRESS(stream);
1467  return 0;
1468  }
1469 
1470  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1473  if (chunk_size <= p->payload_len) {
1474  chunk_size = p->payload_len + (chunk_size / 3);
1475  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1476  p->payload_len, chunk_size);
1477  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1478  chunk_size = p->payload_len + ((chunk_size / 3));
1479  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1480  p->payload_len, chunk_size);
1481  }
1482 
1483  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1484  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1485  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1486  packet_leftedge_abs, packet_rightedge_abs);
1487 
1488  const uint8_t *mydata = NULL;
1489  uint32_t mydata_len = 0;
1490  uint64_t mydata_offset = 0;
1491  /* simply return progress from the block we inspected. */
1492  bool return_progress = false;
1493 
1494  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1495  /* continues block */
1496  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1497  return_progress = true;
1498 
1499  } else {
1500  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1501  /* find our block */
1502  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1503  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1504  if (sbb) {
1505  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1506  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1507  mydata_offset = sbb->offset;
1508  }
1509  }
1510 
1511  /* this can only happen if the segment insert of our current 'p' failed */
1512  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1513  if ((mydata == NULL || mydata_len == 0) || /* no data */
1514  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1515  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1516  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1517  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1518  {
1519  /* no data, or data is incomplete or wrong: use packet data */
1520  mydata = p->payload;
1521  mydata_len = p->payload_len;
1522  mydata_offset = packet_leftedge_abs;
1523  //mydata_rightedge_abs = packet_rightedge_abs;
1524  } else {
1525  /* adjust buffer to match chunk_size */
1526  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1527  if (mydata_len > chunk_size) {
1528  uint32_t excess = mydata_len - chunk_size;
1529  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1530 
1531  if (mydata_rightedge_abs == packet_rightedge_abs) {
1532  mydata += excess;
1533  mydata_len -= excess;
1534  mydata_offset += excess;
1535  SCLogDebug("cutting front of the buffer with %u", excess);
1536  } else if (mydata_offset == packet_leftedge_abs) {
1537  mydata_len -= excess;
1538  SCLogDebug("cutting tail of the buffer with %u", excess);
1539  } else {
1540  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1541  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1542  SCLogDebug("before %u after %u", before, after);
1543 
1544  if (after >= (chunk_size - p->payload_len) / 2) {
1545  // more trailing data than we need
1546 
1547  if (before >= (chunk_size - p->payload_len) / 2) {
1548  // also more heading data, divide evenly
1549  before = after = (chunk_size - p->payload_len) / 2;
1550  } else {
1551  // heading data is less than requested, give the
1552  // rest to the trailing data
1553  after = (chunk_size - p->payload_len) - before;
1554  }
1555  } else {
1556  // less trailing data than requested
1557 
1558  if (before >= (chunk_size - p->payload_len) / 2) {
1559  before = (chunk_size - p->payload_len) - after;
1560  } else {
1561  // both smaller than their requested size
1562  }
1563  }
1564 
1565  /* adjust the buffer */
1566  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1567  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1568  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1569  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1570  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1571  mydata += skip;
1572  mydata_len -= (skip + cut);
1573  mydata_offset += skip;
1574  }
1575  }
1576  }
1577 
1578  /* run the callback */
1579  r = Callback(cb_data, mydata, mydata_len);
1580  BUG_ON(r < 0);
1581 
1582  if (return_progress) {
1583  *progress_out = (mydata_offset + mydata_len);
1584  } else {
1585  /* several blocks of data, so we need to be a bit more careful:
1586  * - if last_ack is beyond last progress, move progress forward to last_ack
1587  * - if our block matches or starts before last ack, return right edge of
1588  * our block.
1589  */
1590  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1591  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1592  uint32_t delta = stream->last_ack - stream->base_seq;
1593  /* get max absolute offset */
1594  last_ack_abs += delta;
1595  }
1596  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1597 
1598  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1599  if (mydata_offset > last_ack_abs) {
1600  /* gap between us and last ack, set progress to last ack */
1601  *progress_out = last_ack_abs;
1602  } else {
1603  *progress_out = (mydata_offset + mydata_len);
1604  }
1605  } else {
1606  *progress_out = STREAM_RAW_PROGRESS(stream);
1607  }
1608  }
1609  return r;
1610 }
1611 
1612 /** \brief access 'raw' reassembly data.
1613  *
1614  * Access data as tracked by 'raw' tracker. Data is made available to
1615  * callback that is passed to this function.
1616  *
1617  * In the case of IDS the callback may be run multiple times if data
1618  * contains gaps. It will then be run for each block of data that is
1619  * continuous.
1620  *
1621  * The callback should give on of 2 return values:
1622  * - 0 ok
1623  * - 1 done
1624  * The value 1 will break the loop if there is a block list that is
1625  * inspected.
1626  *
1627  * This function will return the 'progress' value that has been
1628  * consumed until now.
1629  *
1630  * \param ssn tcp session
1631  * \param stream tcp stream
1632  * \param Callback the function pointer to the callback function
1633  * \param cb_data callback data
1634  * \param[in] progress_in progress to work from
1635  * \param[out] progress_out absolute progress value of the data this
1636  * call handled.
1637  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1638  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1639  *
1640  * `respect_inspect_depth` is used to avoid useless inspection of too
1641  * much data.
1642  */
1643 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1644  StreamReassembleRawFunc Callback, void *cb_data,
1645  const uint64_t progress_in,
1646  uint64_t *progress_out, bool eof,
1647  bool respect_inspect_depth)
1648 {
1649  SCEnter();
1650  int r = 0;
1651 
1652  StreamingBufferBlock *iter = NULL;
1653  uint64_t progress = progress_in;
1654  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1655 
1656  /* if the app layer triggered a flush, and we're supposed to
1657  * use a minimal inspect depth, we actually take the app progress
1658  * as that is the right edge of the data. Then we take the window
1659  * of 'min_inspect_depth' before that. */
1660 
1661  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1662  respect_inspect_depth ? "true" : "false",
1663  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1664  stream->min_inspect_depth);
1665 
1666  if (respect_inspect_depth &&
1668  && stream->min_inspect_depth)
1669  {
1670  progress = STREAM_APP_PROGRESS(stream);
1671  if (stream->min_inspect_depth >= progress) {
1672  progress = 0;
1673  } else {
1674  progress -= stream->min_inspect_depth;
1675  }
1676 
1677  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1678 
1679  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1680  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1681  }
1682 
1683  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1684 
1685  /* get window of data that is acked */
1686  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1687  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1688  uint32_t delta = stream->last_ack - stream->base_seq;
1689  /* get max absolute offset */
1690  last_ack_abs += delta;
1691  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1692  }
1693 
1694  /* loop through available buffers. On no packet loss we'll have a single
1695  * iteration. On missing data we'll walk the blocks */
1696  while (1) {
1697  const uint8_t *mydata;
1698  uint32_t mydata_len;
1699  uint64_t mydata_offset = 0;
1700 
1701  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1702  if (mydata_len == 0) {
1703  SCLogDebug("no data");
1704  break;
1705  }
1706  //PrintRawDataFp(stdout, mydata, mydata_len);
1707 
1708  SCLogDebug("raw progress %"PRIu64, progress);
1709  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1710  stream, &stream->sb, mydata_len, progress);
1711 
1712  if (eof) {
1713  // inspect all remaining data, ack'd or not
1714  } else {
1715  if (last_ack_abs < progress) {
1716  SCLogDebug("nothing to do");
1717  goto end;
1718  }
1719 
1720  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1721  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1722 
1723  /* see if the buffer contains unack'd data as well */
1724  if (progress + mydata_len > last_ack_abs) {
1725  uint32_t check = mydata_len;
1726  mydata_len = last_ack_abs - progress;
1727  BUG_ON(check < mydata_len);
1728  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1729  "data is considered", mydata_len);
1730  }
1731 
1732  }
1733  if (mydata_len == 0)
1734  break;
1735 
1736  SCLogDebug("data %p len %u", mydata, mydata_len);
1737 
1738  /* we have data. */
1739  r = Callback(cb_data, mydata, mydata_len);
1740  BUG_ON(r < 0);
1741 
1742  if (mydata_offset == progress) {
1743  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1744  progress, mydata_len, progress_in + mydata_len);
1745 
1746  progress += mydata_len;
1747  SCLogDebug("raw progress now %"PRIu64, progress);
1748 
1749  /* data is beyond the progress we'd like, and before last ack. Gap. */
1750  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1751  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1752  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1753  progress = mydata_offset;
1754  SCLogDebug("raw progress now %"PRIu64, progress);
1755 
1756  } else {
1757  SCLogDebug("not increasing progress, data gap => mydata_offset "
1758  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1759  }
1760 
1761  if (iter == NULL || r == 1)
1762  break;
1763  }
1764 end:
1765  *progress_out = progress;
1766  return r;
1767 }
1768 
1770  StreamReassembleRawFunc Callback, void *cb_data,
1771  uint64_t *progress_out, bool respect_inspect_depth)
1772 {
1773  /* handle inline separately as the logic is very different */
1774  if (StreamTcpInlineMode() == TRUE) {
1775  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1776  }
1777 
1778  TcpStream *stream;
1779  if (PKT_IS_TOSERVER(p)) {
1780  stream = &ssn->client;
1781  } else {
1782  stream = &ssn->server;
1783  }
1784 
1786  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1787  {
1788  *progress_out = STREAM_RAW_PROGRESS(stream);
1789  return 0;
1790  }
1791 
1792  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1793  STREAM_RAW_PROGRESS(stream), progress_out,
1794  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1795 }
1796 
1798  StreamReassembleRawFunc Callback, void *cb_data,
1799  uint64_t progress_in,
1800  uint64_t *progress_out, bool eof)
1801 {
1802  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1803  return 0;
1804 
1805  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1806  progress_in, progress_out, eof, false);
1807 }
1808 
1809 /** \internal
1810  * \brief update app layer based on received ACK
1811  *
1812  * \retval r 0 on success, -1 on error
1813  */
1814 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1815  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1816 {
1817  SCEnter();
1818 
1819  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1820  SCReturnInt(-1);
1821 
1822  SCReturnInt(0);
1823 }
1824 
1826  TcpSession *ssn, TcpStream *stream,
1827  Packet *p, PacketQueueNoLock *pq)
1828 {
1829  SCEnter();
1830 
1831  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1832 
1833  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1834  ssn, stream, p, p->payload_len);
1835 
1836  /* default IDS: update opposing side (triggered by ACK) */
1838  /* inline and stream end and flow timeout packets trigger same dir handling */
1839  if (StreamTcpInlineMode()) {
1840  dir = UPDATE_DIR_PACKET;
1841  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1842  dir = UPDATE_DIR_PACKET;
1843  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1844  dir = UPDATE_DIR_PACKET;
1845  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1846  dir = UPDATE_DIR_PACKET;
1847  } else if (ssn->state == TCP_CLOSED) {
1848  dir = UPDATE_DIR_BOTH;
1849  }
1850 
1851  /* handle ack received */
1852  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
1853  /* we need to update the opposing stream in
1854  * StreamTcpReassembleHandleSegmentUpdateACK */
1855  TcpStream *opposing_stream = NULL;
1856  if (stream == &ssn->client) {
1857  opposing_stream = &ssn->server;
1858  } else {
1859  opposing_stream = &ssn->client;
1860  }
1861 
1862  const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
1863 
1864  if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
1865  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1866  SCReturnInt(-1);
1867  }
1868 
1869  /* StreamTcpReassembleHandleSegmentUpdateACK
1870  * may swap content of ssn->server and ssn->client structures.
1871  * We have to continue with initial content of the stream in such case */
1872  const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
1873  if (reversed_before_ack_handling != reversed_after_ack_handling) {
1874  SCLogDebug("TCP streams were swapped");
1875  stream = opposing_stream;
1876  }
1877  }
1878  /* if this segment contains data, insert it */
1879  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1880  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1881 
1882  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1883  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1884  SCReturnInt(-1);
1885  }
1886 
1887  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1888  p->flags |= PKT_STREAM_ADD;
1889  } else {
1890  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1891  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1892  ssn, stream, p->payload_len,
1893  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1894 
1895  }
1896 
1897  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1898  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1899  * was *just* set. In this case we trigger the AppLayer Truncate
1900  * logic, to inform the applayer no more data in this direction is
1901  * to be expected. */
1902  if ((stream->flags &
1905  {
1906  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1907  if (dir != UPDATE_DIR_PACKET) {
1908  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1909  "can trigger Truncate");
1910  dir = UPDATE_DIR_PACKET;
1911  }
1912  }
1913 
1914  /* in stream inline mode even if we have no data we call the reassembly
1915  * functions to handle EOF */
1916  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1917  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1918  StreamTcpInlineMode()?"true":"false",
1919  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1920  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1921  SCReturnInt(-1);
1922  }
1923  }
1924 
1925  SCReturnInt(0);
1926 }
1927 
1928 /**
1929  * \brief get a segment from the pool
1930  *
1931  * \retval seg Segment from the pool or NULL
1932  */
1934 {
1935  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1936  SCLogDebug("seg we return is %p", seg);
1937  if (seg == NULL) {
1938  /* Increment the counter to show that we are not able to serve the
1939  segment request due to memcap limit */
1941  } else {
1942  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1943  }
1944 
1945  return seg;
1946 }
1947 
1948 /**
1949  * \brief Trigger RAW stream reassembly
1950  *
1951  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1952  * reassembly from the applayer, for example upon completion of a
1953  * HTTP request.
1954  *
1955  * It sets a flag in the stream so that the next Raw call will return
1956  * the data.
1957  *
1958  * \param ssn TcpSession
1959  */
1961 {
1962 #ifdef DEBUG
1963  BUG_ON(ssn == NULL);
1964 #endif
1965 
1966  if (ssn != NULL) {
1967  if (direction == STREAM_TOSERVER) {
1969  } else {
1971  }
1972 
1973  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1974  }
1975 }
1976 
1977 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1978 {
1979 #ifdef DEBUG
1980  BUG_ON(ssn == NULL);
1981 #endif
1982 
1983  if (ssn != NULL) {
1984  if (direction == STREAM_TOSERVER) {
1985  ssn->client.min_inspect_depth = depth;
1986  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1987  } else {
1988  ssn->server.min_inspect_depth = depth;
1989  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1990  }
1991  }
1992 }
1993 
1994 #ifdef UNITTESTS
1995 /** unit tests and it's support functions below */
1996 
1997 #define SET_ISN(stream, setseq) \
1998  (stream)->isn = (setseq); \
1999  (stream)->base_seq = (setseq) + 1
2000 
2001 /** \brief The Function to create the packet with given payload, which is used
2002  * to test the reassembly of the engine.
2003  *
2004  * \param payload The variable used to store the payload contents of the
2005  * current packet.
2006  * \param value The value which current payload will have for this packet
2007  * \param payload_len The length of the filed payload for current packet.
2008  * \param len Length of the payload array
2009  */
2010 
2011 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2012  uint8_t payload_len, uint8_t len)
2013 {
2014  uint8_t i;
2015  for (i = 0; i < payload_len; i++)
2016  payload[i] = value;
2017  for (; i < len; i++)
2018  payload = NULL;
2019 }
2020 
2021 /** \brief The Function Checks the reassembled stream contents against predefined
2022  * stream contents according to OS policy used.
2023  *
2024  * \param stream_policy Predefined value of stream for different OS policies
2025  * \param stream Reassembled stream returned from the reassembly functions
2026  */
2027 
2028 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
2029 {
2030  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
2031  {
2032  //PrintRawDataFp(stdout, stream_policy, sp_size);
2033  return 0;
2034  }
2035  return 1;
2036 }
2037 
2038 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2039 {
2040  if (StreamingBufferCompareRawData(&stream->sb,
2041  data, data_len) == 0)
2042  {
2043  SCReturnInt(0);
2044  }
2045  SCLogInfo("OK");
2046  PrintRawDataFp(stdout, data, data_len);
2047  return 1;
2048 }
2049 
2050 #define MISSED_START(isn) \
2051  TcpReassemblyThreadCtx *ra_ctx = NULL; \
2052  TcpSession ssn; \
2053  ThreadVars tv; \
2054  memset(&tv, 0, sizeof(tv)); \
2055  \
2056  StreamTcpUTInit(&ra_ctx); \
2057  \
2058  StreamTcpUTSetupSession(&ssn); \
2059  StreamTcpUTSetupStream(&ssn.server, (isn)); \
2060  StreamTcpUTSetupStream(&ssn.client, (isn)); \
2061  \
2062  TcpStream *stream = &ssn.client;
2063 
2064 #define MISSED_END \
2065  StreamTcpUTClearSession(&ssn); \
2066  StreamTcpUTDeinit(ra_ctx); \
2067  PASS
2068 
2069 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2070  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2071  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2072 
2073 /**
2074  * \test Test the handling of packets missed by both IDS and the end host.
2075  * The packet is missed in the starting of the stream.
2076  *
2077  * \retval On success it returns 1 and on failure 0.
2078  */
2079 
2080 static int StreamTcpReassembleTest25 (void)
2081 {
2082  MISSED_START(6);
2083  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
2084  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
2085  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2086  MISSED_END;
2087 }
2088 
2089 /**
2090  * \test Test the handling of packets missed by both IDS and the end host.
2091  * The packet is missed in the middle of the stream.
2092  *
2093  * \retval On success it returns 1 and on failure 0.
2094  */
2095 
2096 static int StreamTcpReassembleTest26 (void)
2097 {
2098  MISSED_START(9);
2099  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2100  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2101  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2102  MISSED_END;
2103 }
2104 
2105 /**
2106  * \test Test the handling of packets missed by both IDS and the end host.
2107  * The packet is missed in the end of the stream.
2108  *
2109  * \retval On success it returns 1 and on failure 0.
2110  */
2111 
2112 static int StreamTcpReassembleTest27 (void)
2113 {
2114  MISSED_START(9);
2115  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2116  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2117  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2118  MISSED_END;
2119 }
2120 
2121 /**
2122  * \test Test the handling of packets missed by IDS, but the end host has
2123  * received it and send the acknowledgment of it. The packet is missed
2124  * in the starting of the stream.
2125  *
2126  * \retval On success it returns 1 and on failure 0.
2127  */
2128 
2129 static int StreamTcpReassembleTest28 (void)
2130 {
2131  MISSED_START(6);
2132  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2133  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2134  ssn.state = TCP_TIME_WAIT;
2135  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2136  MISSED_END;
2137 }
2138 
2139 /**
2140  * \test Test the handling of packets missed by IDS, but the end host has
2141  * received it and send the acknowledgment of it. The packet is missed
2142  * in the middle of the stream.
2143  *
2144  * \retval On success it returns 1 and on failure 0.
2145  */
2146 
2147 static int StreamTcpReassembleTest29 (void)
2148 {
2149  MISSED_START(9);
2150  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2151  ssn.state = TCP_TIME_WAIT;
2152  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2153  MISSED_END;
2154 }
2155 
2156 static int StreamTcpReassembleTest33(void)
2157 {
2158  TcpSession ssn;
2159  Packet *p = PacketGetFromAlloc();
2160  FAIL_IF(unlikely(p == NULL));
2161  Flow f;
2162  TCPHdr tcph;
2163  TcpReassemblyThreadCtx *ra_ctx = NULL;
2165  uint8_t packet[1460] = "";
2166 
2167  StreamTcpUTInit(&ra_ctx);
2169 
2170  PacketQueueNoLock pq;
2171  memset(&pq,0,sizeof(PacketQueueNoLock));
2172  memset(&f, 0, sizeof (Flow));
2173  memset(&tcph, 0, sizeof (TCPHdr));
2174  ThreadVars tv;
2175  memset(&tv, 0, sizeof (ThreadVars));
2176  FLOW_INITIALIZE(&f);
2177  f.protoctx = &ssn;
2178  f.proto = IPPROTO_TCP;
2179  p->src.family = AF_INET;
2180  p->dst.family = AF_INET;
2181  p->proto = IPPROTO_TCP;
2182  p->flow = &f;
2183  tcph.th_win = 5480;
2184  tcph.th_flags = TH_PUSH | TH_ACK;
2185  p->tcph = &tcph;
2187  p->payload = packet;
2188 
2189  p->tcph->th_seq = htonl(10);
2190  p->tcph->th_ack = htonl(31);
2191  p->payload_len = 10;
2192 
2193  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2194 
2195  p->tcph->th_seq = htonl(20);
2196  p->tcph->th_ack = htonl(31);
2197  p->payload_len = 10;
2198 
2199  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2200 
2201  p->tcph->th_seq = htonl(40);
2202  p->tcph->th_ack = htonl(31);
2203  p->payload_len = 10;
2204 
2205  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2206 
2207  p->tcph->th_seq = htonl(5);
2208  p->tcph->th_ack = htonl(31);
2209  p->payload_len = 30;
2210 
2211  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2212 
2214  StreamTcpUTDeinit(ra_ctx);
2215  SCFree(p);
2216  PASS;
2217 }
2218 
2219 static int StreamTcpReassembleTest34(void)
2220 {
2221  TcpSession ssn;
2222  Packet *p = PacketGetFromAlloc();
2223  FAIL_IF(unlikely(p == NULL));
2224  Flow f;
2225  TCPHdr tcph;
2226  TcpReassemblyThreadCtx *ra_ctx = NULL;
2228  uint8_t packet[1460] = "";
2229 
2230  StreamTcpUTInit(&ra_ctx);
2232  PacketQueueNoLock pq;
2233  memset(&pq,0,sizeof(PacketQueueNoLock));
2234  memset(&f, 0, sizeof (Flow));
2235  memset(&tcph, 0, sizeof (TCPHdr));
2236  ThreadVars tv;
2237  memset(&tv, 0, sizeof (ThreadVars));
2238  FLOW_INITIALIZE(&f);
2239  f.protoctx = &ssn;
2240  f.proto = IPPROTO_TCP;
2241  p->src.family = AF_INET;
2242  p->dst.family = AF_INET;
2243  p->proto = IPPROTO_TCP;
2244  p->flow = &f;
2245  tcph.th_win = 5480;
2246  tcph.th_flags = TH_PUSH | TH_ACK;
2247  p->tcph = &tcph;
2249  p->payload = packet;
2250  SET_ISN(&ssn.client, 857961230);
2251 
2252  p->tcph->th_seq = htonl(857961230);
2253  p->tcph->th_ack = htonl(31);
2254  p->payload_len = 304;
2255 
2256  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2257 
2258  p->tcph->th_seq = htonl(857961534);
2259  p->tcph->th_ack = htonl(31);
2260  p->payload_len = 1460;
2261 
2262  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2263 
2264  p->tcph->th_seq = htonl(857963582);
2265  p->tcph->th_ack = htonl(31);
2266  p->payload_len = 1460;
2267 
2268  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2269 
2270  p->tcph->th_seq = htonl(857960946);
2271  p->tcph->th_ack = htonl(31);
2272  p->payload_len = 1460;
2273 
2274  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2275 
2277  StreamTcpUTDeinit(ra_ctx);
2278  SCFree(p);
2279  PASS;
2280 }
2281 
2282 /** \test Test the bug 76 condition */
2283 static int StreamTcpReassembleTest37(void)
2284 {
2285  TcpSession ssn;
2286  Flow f;
2287  TCPHdr tcph;
2288  TcpReassemblyThreadCtx *ra_ctx = NULL;
2289  uint8_t packet[1460] = "";
2290  PacketQueueNoLock pq;
2291  ThreadVars tv;
2292  memset(&tv, 0, sizeof (ThreadVars));
2293 
2294  Packet *p = PacketGetFromAlloc();
2295  FAIL_IF(unlikely(p == NULL));
2296 
2297  StreamTcpUTInit(&ra_ctx);
2299  memset(&pq,0,sizeof(PacketQueueNoLock));
2300  memset(&f, 0, sizeof (Flow));
2301  memset(&tcph, 0, sizeof (TCPHdr));
2302  memset(&tv, 0, sizeof (ThreadVars));
2303 
2304  FLOW_INITIALIZE(&f);
2305  f.protoctx = &ssn;
2306  f.proto = IPPROTO_TCP;
2307  p->src.family = AF_INET;
2308  p->dst.family = AF_INET;
2309  p->proto = IPPROTO_TCP;
2310  p->flow = &f;
2311  tcph.th_win = 5480;
2312  tcph.th_flags = TH_PUSH | TH_ACK;
2313  p->tcph = &tcph;
2315  p->payload = packet;
2317 
2318  p->tcph->th_seq = htonl(3061088537UL);
2319  p->tcph->th_ack = htonl(1729548549UL);
2320  p->payload_len = 1391;
2321  ssn.client.last_ack = 3061091137UL;
2322  SET_ISN(&ssn.client, 3061091309UL);
2323 
2324  /* pre base_seq, so should be rejected */
2325  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2326 
2327  p->tcph->th_seq = htonl(3061089928UL);
2328  p->tcph->th_ack = htonl(1729548549UL);
2329  p->payload_len = 1391;
2330  ssn.client.last_ack = 3061091137UL;
2331 
2332  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2333 
2334  p->tcph->th_seq = htonl(3061091319UL);
2335  p->tcph->th_ack = htonl(1729548549UL);
2336  p->payload_len = 1391;
2337  ssn.client.last_ack = 3061091137UL;
2338 
2339  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2340 
2342  StreamTcpUTDeinit(ra_ctx);
2343  SCFree(p);
2344  PASS;
2345 }
2346 
2347 /**
2348  * \test Test to make sure that we don't return the segments until the app
2349  * layer proto has been detected and after that remove the processed
2350  * segments.
2351  *
2352  * \retval On success it returns 1 and on failure 0.
2353  */
2354 
2355 static int StreamTcpReassembleTest39 (void)
2356 {
2357  Packet *p = PacketGetFromAlloc();
2358  FAIL_IF(unlikely(p == NULL));
2359  Flow f;
2360  ThreadVars tv;
2361  StreamTcpThread stt;
2362  TCPHdr tcph;
2363  PacketQueueNoLock pq;
2364  memset(&pq,0,sizeof(PacketQueueNoLock));
2365  memset (&f, 0, sizeof(Flow));
2366  memset(&tv, 0, sizeof (ThreadVars));
2367  memset(&stt, 0, sizeof (stt));
2368  memset(&tcph, 0, sizeof (TCPHdr));
2369 
2370  FLOW_INITIALIZE(&f);
2371  f.flags = FLOW_IPV4;
2372  f.proto = IPPROTO_TCP;
2373  p->flow = &f;
2374  p->tcph = &tcph;
2375 
2376  StreamTcpUTInit(&stt.ra_ctx);
2377 
2378  /* handshake */
2379  tcph.th_win = htons(5480);
2380  tcph.th_flags = TH_SYN;
2382  p->payload_len = 0;
2383  p->payload = NULL;
2384  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2385 
2386  TcpSession *ssn = (TcpSession *)f.protoctx;
2387  FAIL_IF_NULL(ssn);
2388 
2399  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2400  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2401  FAIL_IF(ssn->data_first_seen_dir != 0);
2402 
2403  /* handshake */
2404  p->tcph->th_ack = htonl(1);
2405  p->tcph->th_flags = TH_SYN | TH_ACK;
2407  p->payload_len = 0;
2408  p->payload = NULL;
2409  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2420  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2421  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2422  FAIL_IF(ssn->data_first_seen_dir != 0);
2423 
2424  /* handshake */
2425  p->tcph->th_ack = htonl(1);
2426  p->tcph->th_seq = htonl(1);
2427  p->tcph->th_flags = TH_ACK;
2429  p->payload_len = 0;
2430  p->payload = NULL;
2431  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2442  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2443  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2444  FAIL_IF(ssn->data_first_seen_dir != 0);
2445 
2446  /* partial request */
2447  uint8_t request1[] = { 0x47, 0x45, };
2448  p->tcph->th_ack = htonl(1);
2449  p->tcph->th_seq = htonl(1);
2450  p->tcph->th_flags = TH_PUSH | TH_ACK;
2452  p->payload_len = sizeof(request1);
2453  p->payload = request1;
2454  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2465  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2466  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2467  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2469 
2470  /* response ack against partial request */
2471  p->tcph->th_ack = htonl(3);
2472  p->tcph->th_seq = htonl(1);
2473  p->tcph->th_flags = TH_ACK;
2475  p->payload_len = 0;
2476  p->payload = NULL;
2477  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2488  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2489  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2490  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2492 
2493  /* complete partial request */
2494  uint8_t request2[] = {
2495  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2496  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2497  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2498  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2499  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2500  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2501  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2502  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2503  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2504  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2505  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2506  p->tcph->th_ack = htonl(1);
2507  p->tcph->th_seq = htonl(3);
2508  p->tcph->th_flags = TH_PUSH | TH_ACK;
2510  p->payload_len = sizeof(request2);
2511  p->payload = request2;
2512  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2523  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2524  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2525  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2526  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2528 
2529  /* response - request ack */
2530  uint8_t response[] = {
2531  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2532  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2533  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2534  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2535  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2536  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2537  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2538  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2539  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2540  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2541  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2542  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2543  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2544  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2545  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2546  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2547  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2548  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2549  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2550  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2551  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2552  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2553  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2554  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2555  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2556  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2557  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2558  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2559  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2560  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2561  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2562  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2563  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2564  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2565  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2566  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2567  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2568  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2569  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2570  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2571  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2572  p->tcph->th_ack = htonl(88);
2573  p->tcph->th_seq = htonl(1);
2574  p->tcph->th_flags = TH_PUSH | TH_ACK;
2576  p->payload_len = sizeof(response);
2577  p->payload = response;
2578 
2579  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2591  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2592  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2593  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2594  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2595 
2596  /* response ack from request */
2597  p->tcph->th_ack = htonl(328);
2598  p->tcph->th_seq = htonl(88);
2599  p->tcph->th_flags = TH_ACK;
2601  p->payload_len = 0;
2602  p->payload = NULL;
2603  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2615  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2616  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2617  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2618  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2619  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2620 
2621  /* response - acking */
2622  p->tcph->th_ack = htonl(88);
2623  p->tcph->th_seq = htonl(328);
2624  p->tcph->th_flags = TH_PUSH | TH_ACK;
2626  p->payload_len = 0;
2627  p->payload = NULL;
2628  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2640  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2641  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2642  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2643  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2644  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2645 
2646  /* response ack from request */
2647  p->tcph->th_ack = htonl(328);
2648  p->tcph->th_seq = htonl(88);
2649  p->tcph->th_flags = TH_ACK;
2651  p->payload_len = 0;
2652  p->payload = NULL;
2653  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2665  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2666  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2667  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2668  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2669 
2670  /* response - acking the request again*/
2671  p->tcph->th_ack = htonl(88);
2672  p->tcph->th_seq = htonl(328);
2673  p->tcph->th_flags = TH_PUSH | TH_ACK;
2675  p->payload_len = 0;
2676  p->payload = NULL;
2677  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2689  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2690  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2691  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2692  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2693 
2694  /*** New Request ***/
2695 
2696  /* partial request */
2697  p->tcph->th_ack = htonl(328);
2698  p->tcph->th_seq = htonl(88);
2699  p->tcph->th_flags = TH_PUSH | TH_ACK;
2701  p->payload_len = sizeof(request1);
2702  p->payload = request1;
2703  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2715  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2716  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2717  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2718  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2719  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2720 
2721  /* response ack against partial request */
2722  p->tcph->th_ack = htonl(90);
2723  p->tcph->th_seq = htonl(328);
2724  p->tcph->th_flags = TH_ACK;
2726  p->payload_len = 0;
2727  p->payload = NULL;
2728  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2729  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2741  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2742  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2743  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2744  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2745  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2746 
2747  /* complete request */
2748  p->tcph->th_ack = htonl(328);
2749  p->tcph->th_seq = htonl(90);
2750  p->tcph->th_flags = TH_PUSH | TH_ACK;
2752  p->payload_len = sizeof(request2);
2753  p->payload = request2;
2754  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2766  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2767  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2768  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2769  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2770  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2771  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2772 
2773  /* response ack against second partial request */
2774  p->tcph->th_ack = htonl(175);
2775  p->tcph->th_seq = htonl(328);
2776  p->tcph->th_flags = TH_ACK;
2778  p->payload_len = 0;
2779  p->payload = NULL;
2780 
2781  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2793  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2794  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2795  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2796  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2797  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2798  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2799 
2800  /* response acking a request */
2801  p->tcph->th_ack = htonl(175);
2802  p->tcph->th_seq = htonl(328);
2803  p->tcph->th_flags = TH_ACK;
2805  p->payload_len = 0;
2806  p->payload = NULL;
2807  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2813 
2816 
2817  /* request acking a response */
2818  p->tcph->th_ack = htonl(328);
2819  p->tcph->th_seq = htonl(175);
2820  p->tcph->th_flags = TH_ACK;
2822  p->payload_len = 0;
2823  p->payload = NULL;
2824  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2825 
2826  StreamTcpSessionClear(ssn);
2828  SCFree(p);
2829  PASS;
2830 }
2831 
2832 /**
2833  * \test Test to make sure that we sent all the segments from the initial
2834  * segments to app layer until we have detected the app layer proto.
2835  *
2836  * \retval On success it returns 1 and on failure 0.
2837  */
2838 
2839 static int StreamTcpReassembleTest40 (void)
2840 {
2841  Packet *p = PacketGetFromAlloc();
2842  FAIL_IF_NULL(p);
2843  Flow *f = NULL;
2844  TCPHdr tcph;
2845  TcpSession ssn;
2846  PacketQueueNoLock pq;
2847  memset(&pq,0,sizeof(PacketQueueNoLock));
2848  memset(&tcph, 0, sizeof (TCPHdr));
2849  ThreadVars tv;
2850  memset(&tv, 0, sizeof (ThreadVars));
2851 
2852  StreamTcpInitConfig(true);
2854 
2856  FAIL_IF_NULL(ra_ctx);
2857 
2858  uint8_t httpbuf1[] = "P";
2859  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2860  uint8_t httpbuf3[] = "O";
2861  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2862  uint8_t httpbuf4[] = "S";
2863  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2864  uint8_t httpbuf5[] = "T \r\n";
2865  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2866 
2867  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2868  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2869 
2870  SET_ISN(&ssn.server, 9);
2871  ssn.server.last_ack = 10;
2872  SET_ISN(&ssn.client, 9);
2873  ssn.client.isn = 9;
2874 
2875  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2876  FAIL_IF_NULL(f);
2877  f->protoctx = &ssn;
2878  f->proto = IPPROTO_TCP;
2879  p->flow = f;
2880 
2881  tcph.th_win = htons(5480);
2882  tcph.th_seq = htonl(10);
2883  tcph.th_ack = htonl(10);
2884  tcph.th_flags = TH_ACK|TH_PUSH;
2885  p->tcph = &tcph;
2887  p->payload = httpbuf1;
2888  p->payload_len = httplen1;
2889  ssn.state = TCP_ESTABLISHED;
2890  TcpStream *s = &ssn.client;
2891  SCLogDebug("1 -- start");
2892  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2893 
2895  p->payload = httpbuf2;
2896  p->payload_len = httplen2;
2897  tcph.th_seq = htonl(10);
2898  tcph.th_ack = htonl(11);
2899  s = &ssn.server;
2900  ssn.server.last_ack = 11;
2901  SCLogDebug("2 -- start");
2902  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2903 
2905  p->payload = httpbuf3;
2906  p->payload_len = httplen3;
2907  tcph.th_seq = htonl(11);
2908  tcph.th_ack = htonl(55);
2909  s = &ssn.client;
2910  ssn.client.last_ack = 55;
2911  SCLogDebug("3 -- start");
2912  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2913 
2915  p->payload = httpbuf2;
2916  p->payload_len = httplen2;
2917  tcph.th_seq = htonl(55);
2918  tcph.th_ack = htonl(12);
2919  s = &ssn.server;
2920  ssn.server.last_ack = 12;
2921  SCLogDebug("4 -- start");
2922  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2923 
2924  /* check is have the segment in the list and flagged or not */
2925  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2926  FAIL_IF_NULL(seg);
2927  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2928 
2930  p->payload = httpbuf4;
2931  p->payload_len = httplen4;
2932  tcph.th_seq = htonl(12);
2933  tcph.th_ack = htonl(100);
2934  s = &ssn.client;
2935  ssn.client.last_ack = 100;
2936  SCLogDebug("5 -- start");
2937  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2938 
2940  p->payload = httpbuf2;
2941  p->payload_len = httplen2;
2942  tcph.th_seq = htonl(100);
2943  tcph.th_ack = htonl(13);
2944  s = &ssn.server;
2945  ssn.server.last_ack = 13;
2946  SCLogDebug("6 -- start");
2947  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2948 
2950  p->payload = httpbuf5;
2951  p->payload_len = httplen5;
2952  tcph.th_seq = htonl(13);
2953  tcph.th_ack = htonl(145);
2954  s = &ssn.client;
2955  ssn.client.last_ack = 145;
2956  SCLogDebug("7 -- start");
2957  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2958 
2960  p->payload = httpbuf2;
2961  p->payload_len = httplen2;
2962  tcph.th_seq = htonl(145);
2963  tcph.th_ack = htonl(16);
2964  s = &ssn.server;
2965  ssn.server.last_ack = 16;
2966  SCLogDebug("8 -- start");
2967  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2968  FAIL_IF(f->alproto != ALPROTO_HTTP1);
2969 
2972  StreamTcpFreeConfig(true);
2973  SCFree(p);
2974  UTHFreeFlow(f);
2975  PASS;
2976 }
2977 
2978 /** \test Test the memcap incrementing/decrementing and memcap check */
2979 static int StreamTcpReassembleTest44(void)
2980 {
2981  StreamTcpInitConfig(true);
2982  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2984  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2986  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2988  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2989  StreamTcpFreeConfig(true);
2990  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2991  PASS;
2992 }
2993 
2994 /**
2995  * \test Test to make sure that reassembly_depth is enforced.
2996  *
2997  * \retval On success it returns 1 and on failure 0.
2998  */
2999 
3000 static int StreamTcpReassembleTest45 (void)
3001 {
3002  TcpReassemblyThreadCtx *ra_ctx = NULL;
3003  TcpSession ssn;
3004  ThreadVars tv;
3005  memset(&tv, 0, sizeof(tv));
3006  uint8_t payload[100] = {0};
3007  uint16_t payload_size = 100;
3008 
3009  StreamTcpUTInit(&ra_ctx);
3011 
3013  ssn.reassembly_depth = 100;
3014  StreamTcpUTSetupStream(&ssn.server, 100);
3015  StreamTcpUTSetupStream(&ssn.client, 100);
3016 
3017  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3018  FAIL_IF(r != 0);
3020 
3021  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3022  FAIL_IF(r != 0);
3024 
3028  StreamTcpUTDeinit(ra_ctx);
3029  PASS;
3030 }
3031 
3032 /**
3033  * \test Test the unlimited config value of reassembly depth.
3034  *
3035  * \retval On success it returns 1 and on failure 0.
3036  */
3037 
3038 static int StreamTcpReassembleTest46 (void)
3039 {
3040  int result = 0;
3041  TcpReassemblyThreadCtx *ra_ctx = NULL;
3042  TcpSession ssn;
3043  ThreadVars tv;
3044  memset(&tv, 0, sizeof(tv));
3045  uint8_t payload[100] = {0};
3046  uint16_t payload_size = 100;
3047 
3048  StreamTcpUTInit(&ra_ctx);
3050 
3052  StreamTcpUTSetupStream(&ssn.server, 100);
3053  StreamTcpUTSetupStream(&ssn.client, 100);
3054 
3055  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3056  if (r != 0)
3057  goto end;
3059  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3060  goto end;
3061  }
3062 
3063  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3064  if (r != 0)
3065  goto end;
3067  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3068  goto end;
3069  }
3070 
3071  result = 1;
3072 end:
3076  StreamTcpUTDeinit(ra_ctx);
3077  return result;
3078 }
3079 
3080 /**
3081  * \test Test to make sure we detect the sequence wrap around and continue
3082  * stream reassembly properly.
3083  *
3084  * \retval On success it returns 1 and on failure 0.
3085  */
3086 
3087 static int StreamTcpReassembleTest47 (void)
3088 {
3089  Packet *p = PacketGetFromAlloc();
3090  FAIL_IF(unlikely(p == NULL));
3091  Flow *f = NULL;
3092  TCPHdr tcph;
3093  TcpSession ssn;
3094  ThreadVars tv;
3095  PacketQueueNoLock pq;
3096  memset(&pq,0,sizeof(PacketQueueNoLock));
3097  memset(&tcph, 0, sizeof (TCPHdr));
3098  memset(&tv, 0, sizeof (ThreadVars));
3099  StreamTcpInitConfig(true);
3102 
3103  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3104  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3105 
3106  SET_ISN(&ssn.server, 572799781UL);
3107  ssn.server.last_ack = 572799782UL;
3108 
3109  SET_ISN(&ssn.client, 4294967289UL);
3110  ssn.client.last_ack = 21;
3111 
3112  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3113  FAIL_IF(f == NULL);
3114  f->protoctx = &ssn;
3115  f->proto = IPPROTO_TCP;
3116  p->flow = f;
3117 
3118  tcph.th_win = htons(5480);
3119  ssn.state = TCP_ESTABLISHED;
3120  TcpStream *s = NULL;
3121  uint8_t cnt = 0;
3122 
3123  for (cnt=0; cnt < httplen1; cnt++) {
3124  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3125  tcph.th_ack = htonl(572799782UL);
3126  tcph.th_flags = TH_ACK|TH_PUSH;
3127  p->tcph = &tcph;
3129  p->payload = &httpbuf1[cnt];
3130  p->payload_len = 1;
3131  s = &ssn.client;
3132 
3133  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3134 
3136  p->payload = NULL;
3137  p->payload_len = 0;
3138  tcph.th_seq = htonl(572799782UL);
3139  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3140  tcph.th_flags = TH_ACK;
3141  p->tcph = &tcph;
3142  s = &ssn.server;
3143 
3144  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3145  }
3146 
3147  FAIL_IF(f->alproto != ALPROTO_HTTP1);
3148 
3151  StreamTcpFreeConfig(true);
3152  SCFree(p);
3153  UTHFreeFlow(f);
3154  PASS;
3155 }
3156 
3157 /** \test 3 in order segments in inline reassembly */
3158 static int StreamTcpReassembleInlineTest01(void)
3159 {
3160  int ret = 0;
3161  TcpReassemblyThreadCtx *ra_ctx = NULL;
3162  ThreadVars tv;
3163  TcpSession ssn;
3164  Flow f;
3165 
3166  memset(&tv, 0x00, sizeof(tv));
3167 
3168  StreamTcpUTInit(&ra_ctx);
3171  StreamTcpUTSetupStream(&ssn.client, 1);
3172  FLOW_INITIALIZE(&f);
3173 
3174  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3175  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3176  if (p == NULL) {
3177  printf("couldn't get a packet: ");
3178  goto end;
3179  }
3180  p->tcph->th_seq = htonl(12);
3181  p->flow = &f;
3182 
3183  FLOWLOCK_WRLOCK(&f);
3184  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3185  printf("failed to add segment 1: ");
3186  goto end;
3187  }
3188  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3189  printf("failed to add segment 2: ");
3190  goto end;
3191  }
3192  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3193  printf("failed to add segment 3: ");
3194  goto end;
3195  }
3196  ssn.client.next_seq = 17;
3197  ret = 1;
3198 end:
3199  FLOWLOCK_UNLOCK(&f);
3200  FLOW_DESTROY(&f);
3201  UTHFreePacket(p);
3203  StreamTcpUTDeinit(ra_ctx);
3204  return ret;
3205 }
3206 
3207 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3208  * test the sliding window reassembly.
3209  */
3210 static int StreamTcpReassembleInlineTest02(void)
3211 {
3212  int ret = 0;
3213  TcpReassemblyThreadCtx *ra_ctx = NULL;
3214  ThreadVars tv;
3215  TcpSession ssn;
3216  Flow f;
3217 
3218  memset(&tv, 0x00, sizeof(tv));
3219 
3220  StreamTcpUTInit(&ra_ctx);
3223  StreamTcpUTSetupStream(&ssn.client, 1);
3224  FLOW_INITIALIZE(&f);
3225 
3226  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3227  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3228  if (p == NULL) {
3229  printf("couldn't get a packet: ");
3230  goto end;
3231  }
3232  p->tcph->th_seq = htonl(12);
3233  p->flow = &f;
3234 
3235  FLOWLOCK_WRLOCK(&f);
3236  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3237  printf("failed to add segment 1: ");
3238  goto end;
3239  }
3240  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3241  printf("failed to add segment 2: ");
3242  goto end;
3243  }
3244  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3245  printf("failed to add segment 3: ");
3246  goto end;
3247  }
3248  ssn.client.next_seq = 17;
3249  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3250  printf("failed to add segment 4: ");
3251  goto end;
3252  }
3253  ssn.client.next_seq = 22;
3254  ret = 1;
3255 end:
3256  FLOWLOCK_UNLOCK(&f);
3257  FLOW_DESTROY(&f);
3258  UTHFreePacket(p);
3260  StreamTcpUTDeinit(ra_ctx);
3261  return ret;
3262 }
3263 
3264 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3265  * test the sliding window reassembly with a small window size so that we
3266  * cutting off at the start (left edge)
3267  */
3268 static int StreamTcpReassembleInlineTest03(void)
3269 {
3270  int ret = 0;
3271  TcpReassemblyThreadCtx *ra_ctx = NULL;
3272  ThreadVars tv;
3273  TcpSession ssn;
3274  Flow f;
3275 
3276  memset(&tv, 0x00, sizeof(tv));
3277 
3278  StreamTcpUTInit(&ra_ctx);
3281  StreamTcpUTSetupStream(&ssn.client, 1);
3282  FLOW_INITIALIZE(&f);
3283 
3285 
3286  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3287  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3288  if (p == NULL) {
3289  printf("couldn't get a packet: ");
3290  goto end;
3291  }
3292  p->tcph->th_seq = htonl(12);
3293  p->flow = &f;
3295 
3296  FLOWLOCK_WRLOCK(&f);
3297  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3298  printf("failed to add segment 1: ");
3299  goto end;
3300  }
3301  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3302  printf("failed to add segment 2: ");
3303  goto end;
3304  }
3305  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3306  printf("failed to add segment 3: ");
3307  goto end;
3308  }
3309  ssn.client.next_seq = 17;
3310  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3311  printf("failed to add segment 4: ");
3312  goto end;
3313  }
3314  ssn.client.next_seq = 22;
3315 
3316  p->tcph->th_seq = htonl(17);
3317  ret = 1;
3318 end:
3319  FLOWLOCK_UNLOCK(&f);
3320  FLOW_DESTROY(&f);
3321  UTHFreePacket(p);
3323  StreamTcpUTDeinit(ra_ctx);
3324  return ret;
3325 }
3326 
3327 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3328  * test the sliding window reassembly with a small window size so that we
3329  * cutting off at the start (left edge) with small packet overlap.
3330  */
3331 static int StreamTcpReassembleInlineTest04(void)
3332 {
3333  int ret = 0;
3334  TcpReassemblyThreadCtx *ra_ctx = NULL;
3335  ThreadVars tv;
3336  TcpSession ssn;
3337  Flow f;
3338 
3339  memset(&tv, 0x00, sizeof(tv));
3340 
3341  StreamTcpUTInit(&ra_ctx);
3344  StreamTcpUTSetupStream(&ssn.client, 1);
3345  FLOW_INITIALIZE(&f);
3346 
3348 
3349  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3350  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3351  if (p == NULL) {
3352  printf("couldn't get a packet: ");
3353  goto end;
3354  }
3355  p->tcph->th_seq = htonl(12);
3356  p->flow = &f;
3358 
3359  FLOWLOCK_WRLOCK(&f);
3360  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3361  printf("failed to add segment 1: ");
3362  goto end;
3363  }
3364  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3365  printf("failed to add segment 2: ");
3366  goto end;
3367  }
3368  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3369  printf("failed to add segment 3: ");
3370  goto end;
3371  }
3372  ssn.client.next_seq = 17;
3373  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3374  printf("failed to add segment 4: ");
3375  goto end;
3376  }
3377  ssn.client.next_seq = 22;
3378 
3379  p->tcph->th_seq = htonl(17);
3380  ret = 1;
3381 end:
3382  FLOWLOCK_UNLOCK(&f);
3383  FLOW_DESTROY(&f);
3384  UTHFreePacket(p);
3386  StreamTcpUTDeinit(ra_ctx);
3387  return ret;
3388 }
3389 
3390 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3391  * test the sliding window reassembly with a small window size so that we
3392  * cutting off at the start (left edge). Test if the first segment is
3393  * removed from the list.
3394  */
3395 static int StreamTcpReassembleInlineTest08(void)
3396 {
3397  TcpReassemblyThreadCtx *ra_ctx = NULL;
3398  ThreadVars tv;
3399  memset(&tv, 0x00, sizeof(tv));
3400  TcpSession ssn;
3401  Flow f;
3402  StreamTcpUTInit(&ra_ctx);
3405  StreamTcpUTSetupStream(&ssn.client, 1);
3406  FLOW_INITIALIZE(&f);
3407 
3409  f.protoctx = &ssn;
3410 
3411  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3412  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3413  FAIL_IF(p == NULL);
3414  p->tcph->th_seq = htonl(12);
3415  p->flow = &f;
3417 
3418  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3419  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3420  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3421  ssn.client.next_seq = 17;
3422  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3423  ssn.client.next_seq = 22;
3424  p->tcph->th_seq = htonl(17);
3426 
3427  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3428  FAIL_IF_NULL(seg);
3429  FAIL_IF_NOT(seg->seq == 2);
3430 
3431  FLOW_DESTROY(&f);
3432  UTHFreePacket(p);
3434  StreamTcpUTDeinit(ra_ctx);
3435  PASS;
3436 }
3437 
3438 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3439  * test the sliding window reassembly with a small window size so that we
3440  * cutting off at the start (left edge). Test if the first segment is
3441  * removed from the list.
3442  */
3443 static int StreamTcpReassembleInlineTest09(void)
3444 {
3445  int ret = 0;
3446  TcpReassemblyThreadCtx *ra_ctx = NULL;
3447  ThreadVars tv;
3448  TcpSession ssn;
3449  Flow f;
3450 
3451  memset(&tv, 0x00, sizeof(tv));
3452 
3453  StreamTcpUTInit(&ra_ctx);
3456  StreamTcpUTSetupStream(&ssn.client, 1);
3457  FLOW_INITIALIZE(&f);
3458 
3460 
3461  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3462  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3463  if (p == NULL) {
3464  printf("couldn't get a packet: ");
3465  goto end;
3466  }
3467  p->tcph->th_seq = htonl(17);
3468  p->flow = &f;
3470 
3471  FLOWLOCK_WRLOCK(&f);
3472  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3473  printf("failed to add segment 1: ");
3474  goto end;
3475  }
3476  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3477  printf("failed to add segment 2: ");
3478  goto end;
3479  }
3480  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3481  printf("failed to add segment 3: ");
3482  goto end;
3483  }
3484  ssn.client.next_seq = 12;
3485  ssn.client.last_ack = 10;
3486 
3487  /* close the GAP and see if we properly reassemble and update base_seq */
3488  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3489  printf("failed to add segment 4: ");
3490  goto end;
3491  }
3492  ssn.client.next_seq = 22;
3493 
3494  p->tcph->th_seq = htonl(12);
3495 
3496  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3497  FAIL_IF_NULL(seg);
3498  FAIL_IF_NOT(seg->seq == 2);
3499 
3500  ret = 1;
3501 end:
3502  FLOWLOCK_UNLOCK(&f);
3503  FLOW_DESTROY(&f);
3504  UTHFreePacket(p);
3506  StreamTcpUTDeinit(ra_ctx);
3507  return ret;
3508 }
3509 
3510 /** \test App Layer reassembly.
3511  */
3512 static int StreamTcpReassembleInlineTest10(void)
3513 {
3514  int ret = 0;
3515  TcpReassemblyThreadCtx *ra_ctx = NULL;
3516  ThreadVars tv;
3517  TcpSession ssn;
3518  Flow *f = NULL;
3519  Packet *p = NULL;
3520 
3521  memset(&tv, 0x00, sizeof(tv));
3522 
3523  StreamTcpUTInit(&ra_ctx);
3526  StreamTcpUTSetupStream(&ssn.server, 1);
3527  ssn.server.last_ack = 2;
3528  StreamTcpUTSetupStream(&ssn.client, 1);
3529  ssn.client.last_ack = 2;
3531 
3532  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3533  if (f == NULL)
3534  goto end;
3535  f->protoctx = &ssn;
3536  f->proto = IPPROTO_TCP;
3537 
3538  uint8_t stream_payload1[] = "GE";
3539  uint8_t stream_payload2[] = "T /";
3540  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3541 
3542  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3543  if (p == NULL) {
3544  printf("couldn't get a packet: ");
3545  goto end;
3546  }
3547  p->tcph->th_seq = htonl(7);
3548  p->flow = f;
3550 
3551  FLOWLOCK_WRLOCK(f);
3552  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3553  printf("failed to add segment 1: ");
3554  goto end;
3555  }
3556  ssn.client.next_seq = 4;
3557 
3558  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3559  if (r < 0) {
3560  printf("StreamTcpReassembleAppLayer failed: ");
3561  goto end;
3562  }
3563 
3564  /* ssn.server.ra_app_base_seq should be isn here. */
3565  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3566  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3567  goto end;
3568  }
3569 
3570  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3571  printf("failed to add segment 2: ");
3572  goto end;
3573  }
3574  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3575  printf("failed to add segment 3: ");
3576  goto end;
3577  }
3578  ssn.client.next_seq = 19;
3579 
3580  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3581  if (r < 0) {
3582  printf("StreamTcpReassembleAppLayer failed: ");
3583  goto end;
3584  }
3585 
3586  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3587 
3588  ret = 1;
3589 end:
3590  UTHFreePacket(p);
3592  StreamTcpUTDeinit(ra_ctx);
3593  FLOWLOCK_UNLOCK(f);
3594  UTHFreeFlow(f);
3595  return ret;
3596 }
3597 
3598 /** \test test insert with overlap
3599  */
3600 static int StreamTcpReassembleInsertTest01(void)
3601 {
3602  TcpReassemblyThreadCtx *ra_ctx = NULL;
3603  ThreadVars tv;
3604  TcpSession ssn;
3605  Flow f;
3606 
3607  memset(&tv, 0x00, sizeof(tv));
3608 
3609  StreamTcpUTInit(&ra_ctx);
3611  StreamTcpUTSetupStream(&ssn.client, 1);
3613  FLOW_INITIALIZE(&f);
3614 
3615  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3616  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3617  FAIL_IF(p == NULL);
3618  p->tcph->th_seq = htonl(12);
3619  p->flow = &f;
3620 
3621  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3622  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3623  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3624  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3625  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3626  ssn.client.next_seq = 21;
3627 
3628  FLOW_DESTROY(&f);
3629  UTHFreePacket(p);
3631  StreamTcpUTDeinit(ra_ctx);
3632  PASS;
3633 }
3634 
3635 /** \test test insert with overlaps
3636  */
3637 static int StreamTcpReassembleInsertTest02(void)
3638 {
3639  int ret = 0;
3640  TcpReassemblyThreadCtx *ra_ctx = NULL;
3641  ThreadVars tv;
3642  TcpSession ssn;
3643 
3644  memset(&tv, 0x00, sizeof(tv));
3645 
3646  StreamTcpUTInit(&ra_ctx);
3648  StreamTcpUTSetupStream(&ssn.client, 1);
3649 
3650  int i;
3651  for (i = 2; i < 10; i++) {
3652  int len;
3653  len = i % 2;
3654  if (len == 0)
3655  len = 1;
3656  int seq;
3657  seq = i * 10;
3658  if (seq < 2)
3659  seq = 2;
3660 
3661  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3662  printf("failed to add segment 1: ");
3663  goto end;
3664  }
3665  }
3666  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3667  printf("failed to add segment 2: ");
3668  goto end;
3669  }
3670 
3671  ret = 1;
3672 end:
3674  StreamTcpUTDeinit(ra_ctx);
3675  return ret;
3676 }
3677 
3678 /** \test test insert with overlaps
3679  */
3680 static int StreamTcpReassembleInsertTest03(void)
3681 {
3682  int ret = 0;
3683  TcpReassemblyThreadCtx *ra_ctx = NULL;
3684  ThreadVars tv;
3685  TcpSession ssn;
3686 
3687  memset(&tv, 0x00, sizeof(tv));
3688 
3689  StreamTcpUTInit(&ra_ctx);
3691  StreamTcpUTSetupStream(&ssn.client, 1);
3692 
3693  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3694  printf("failed to add segment 2: ");
3695  goto end;
3696  }
3697 
3698  int i;
3699  for (i = 2; i < 10; i++) {
3700  int len;
3701  len = i % 2;
3702  if (len == 0)
3703  len = 1;
3704  int seq;
3705  seq = i * 10;
3706  if (seq < 2)
3707  seq = 2;
3708 
3709  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3710  printf("failed to add segment 2: ");
3711  goto end;
3712  }
3713  }
3714  ret = 1;
3715 end:
3717  StreamTcpUTDeinit(ra_ctx);
3718  return ret;
3719 }
3720 
3722 #endif /* UNITTESTS */
3723 
3724 /** \brief The Function Register the Unit tests to test the reassembly engine
3725  * for various OS policies.
3726  */
3727 
3729 {
3730 #ifdef UNITTESTS
3731  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3732  StreamTcpReassembleTest25);
3733  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3734  StreamTcpReassembleTest26);
3735  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3736  StreamTcpReassembleTest27);
3737  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3738  StreamTcpReassembleTest28);
3739  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3740  StreamTcpReassembleTest29);
3741  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3742  StreamTcpReassembleTest33);
3743  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3744  StreamTcpReassembleTest34);
3745  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3746  StreamTcpReassembleTest37);
3747  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3748  StreamTcpReassembleTest39);
3749  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3750  StreamTcpReassembleTest40);
3751  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3752  StreamTcpReassembleTest44);
3753  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3754  StreamTcpReassembleTest45);
3755  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3756  StreamTcpReassembleTest46);
3757  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3758  StreamTcpReassembleTest47);
3759 
3760  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3761  StreamTcpReassembleInlineTest01);
3762  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3763  StreamTcpReassembleInlineTest02);
3764  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3765  StreamTcpReassembleInlineTest03);
3766  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3767  StreamTcpReassembleInlineTest04);
3768  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3769  StreamTcpReassembleInlineTest08);
3770  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3771  StreamTcpReassembleInlineTest09);
3772 
3773  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3774  StreamTcpReassembleInlineTest10);
3775 
3776  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3777  StreamTcpReassembleInsertTest01);
3778  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3779  StreamTcpReassembleInsertTest02);
3780  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3781  StreamTcpReassembleInsertTest03);
3782 
3786  StreamTcpReassembleRawRegisterTests();
3787 #endif /* UNITTESTS */
3788 }
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:249
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:57
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:55
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:436
TcpStream_
Definition: stream-tcp-private.h:94
StreamingBuffer_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:97
len
uint8_t len
Definition: app-layer-dnp3.h:2
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:149
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:101
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:101
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:1933
StreamingBufferConfig_::Malloc
void *(* Malloc)(size_t size)
Definition: util-streaming-buffer.h:71
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:169
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:302
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:92
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:70
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1769
STREAM_MIDSTREAM
#define STREAM_MIDSTREAM
Definition: stream.h:35
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:95
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:82
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:72
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:104
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:516
stream-tcp.h
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:493
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
Definition: stream-tcp-reassemble.c:175
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:564
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:123
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:1997
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:572
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1000
StreamTcpThread_
Definition: stream-tcp.h:70
Flow_::proto
uint8_t proto
Definition: flow.h:375
Packet_::payload
uint8_t * payload
Definition: decode.h:551
StreamTcpInlineMode
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6359
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:1977
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:61
Packet_::flags
uint32_t flags
Definition: decode.h:449
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:175
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:98
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:60
Flow_
Flow data structure.
Definition: flow.h:353
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:66
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:68
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:1960
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:144
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:38
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1370
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:867
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:225
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:143
StreamNeedsReassembly
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:803
MIN
#define MIN(x, y)
Definition: suricata-common.h:372
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:64
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:95
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
StreamingBufferConfig_::flags
uint32_t flags
Definition: util-streaming-buffer.h:68
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:150
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:884
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:445
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:119
APP_LAYER_PARSER_EOF_TS
#define APP_LAYER_PARSER_EOF_TS
Definition: app-layer-parser.h:41
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:242
Flow_::protoctx
void * protoctx
Definition: flow.h:451
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:711
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:98
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:552
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:32
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:204
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:578
util-unittest.h
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:174
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:2069
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
TcpSession_::flags
uint16_t flags
Definition: stream-tcp-private.h:269
FLOWLOCK_UNLOCK
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:270
STREAM_START
#define STREAM_START
Definition: stream.h:29
StreamTcpReassembleInit
int StreamTcpReassembleInit(bool quiet)
Definition: stream-tcp-reassemble.c:399
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:158
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:275
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:103
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:485
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:434
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
StreamTcpInitConfig
void StreamTcpInitConfig(bool)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:521
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1213
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:270
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:353
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:574
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:265
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:92
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:171
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:276
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1147
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1154
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:131
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
FLOWLOCK_WRLOCK
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:267
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:118
StreamReassembleLog
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
Definition: stream-tcp-reassemble.c:1797
util-print.h
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:50
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:859
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:4853
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1326
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:262
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:87
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:2011
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:808
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
app-layer-parser.h
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP se...
Definition: stream-tcp-reassemble.c:146
TRUE
#define TRUE
Definition: suricata-common.h:33
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueueNoLock *pq)
Definition: stream-tcp-reassemble.c:1825
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:277
TCP_GET_SEQ
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:112
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:105
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
FALSE
#define FALSE
Definition: suricata-common.h:34
SCReturn
#define SCReturn
Definition: util-debug.h:302
stream.h
TcpSegment
Definition: stream-tcp-private.h:61
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:278
Packet_
Definition: decode.h:414
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:215
STREAM_DEPTH
#define STREAM_DEPTH
Definition: stream.h:34
stream-tcp-private.h
PoolThreadReturn
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
Definition: util-pool-thread.c:187
APP_LAYER_PARSER_EOF_TC
#define APP_LAYER_PARSER_EOF_TC
Definition: app-layer-parser.h:42
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:306
conf.h
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:276
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:907
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:105
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:475
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
StreamDataAvailableForProtoDetect
uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
Definition: stream-tcp-reassemble.c:590
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
STREAM_GAP
#define STREAM_GAP
Definition: stream.h:33
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:315
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:226
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:265
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
StreamTcpReassembleFree
void StreamTcpReassembleFree(bool quiet)
Definition: stream-tcp-reassemble.c:415
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:70
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:115
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:79
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:526
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:56
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:103
StreamingBuffer_
Definition: util-streaming-buffer.h:95
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:614
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:63
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:150
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:787
StringParseUint32
int StringParseUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:313
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
STREAMTCP_FLAG_CLOSED_BY_RST
#define STREAMTCP_FLAG_CLOSED_BY_RST
Definition: stream-tcp-private.h:168
Packet_::flow
struct Flow_ * flow
Definition: decode.h:451
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:241
StreamTcpFreeConfig
void StreamTcpFreeConfig(bool quiet)
Definition: stream-tcp.c:670
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
flags
uint8_t flags
Definition: decode-gre.h:0
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:337
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:240
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:531
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:112
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:30
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:89
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:2050
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:122
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:286
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:217
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:149
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:102
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:150
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
StreamTcpReassembleIncrMemuse
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:98
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:461
ConfNode_
Definition: conf.h:32
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:485
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:881
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:238
Flow_::flags
uint32_t flags
Definition: flow.h:431
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:768
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:74
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:50
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:132
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:282
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:2064
stream-tcp-util.h
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:146
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
STREAM_EOF
#define STREAM_EOF
Definition: stream.h:30
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:60
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:57
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:168
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:188
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:200
suricata.h
Address_::family
char family
Definition: decode.h:120
Packet_::dst
Address dst
Definition: decode.h:419
StreamTcpStateAsString
const char * StreamTcpStateAsString(const enum TcpState state)
Definition: stream-tcp.c:6374
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:239
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
AppLayerParserStateSetFlag
void AppLayerParserStateSetFlag(AppLayerParserState *pstate, uint8_t flag)
Definition: app-layer-parser.c:1646
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:219
STREAMING_BUFFER_NOFLAGS
#define STREAMING_BUFFER_NOFLAGS
Definition: util-streaming-buffer.h:64
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3728
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:90
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:132
FlowChangeProto
int FlowChangeProto(Flow *f)
Check if change proto flag is set for flow.
Definition: flow.c:212
StreamTcpReassembleDecrMemuse
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:111
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
TcpSession_
Definition: stream-tcp-private.h:260
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:265
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:160
flow.h
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:462
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:460
FLOW_DIR_REVERSED
#define FLOW_DIR_REVERSED
Definition: flow.h:110
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:206
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:143
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:285
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:85
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:83
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:66
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:761
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:82
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:130
Packet_::src
Address src
Definition: decode.h:418
debug.h
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:73
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:213
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:133
app-layer.h
StreamTcpCheckStreamContents
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
Definition: stream-tcp-reassemble.c:2028