suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 
64 #include "detect-engine-state.h"
65 
66 #include "util-profiling.h"
67 #include "util-validate.h"
68 
69 #ifdef DEBUG
70 static SCMutex segment_pool_memuse_mutex;
71 static uint64_t segment_pool_memuse = 0;
72 static uint64_t segment_pool_memcnt = 0;
73 #endif
74 
75 static PoolThread *segment_thread_pool = NULL;
76 /* init only, protect initializing and growing pool */
77 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
78 
79 /* Memory use counter */
80 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
81 
82 /* prototypes */
84 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
85 
87 {
88  SC_ATOMIC_INIT(ra_memuse);
89 }
90 
91 /**
92  * \brief Function to Increment the memory usage counter for the TCP reassembly
93  * segments
94  *
95  * \param size Size of the TCP segment and its payload length memory allocated
96  */
97 void StreamTcpReassembleIncrMemuse(uint64_t size)
98 {
99  (void) SC_ATOMIC_ADD(ra_memuse, size);
100  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
101  return;
102 }
103 
104 /**
105  * \brief Function to Decrease the memory usage counter for the TCP reassembly
106  * segments
107  *
108  * \param size Size of the TCP segment and its payload length memory allocated
109  */
110 void StreamTcpReassembleDecrMemuse(uint64_t size)
111 {
112 #ifdef UNITTESTS
113  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
114  if (RunmodeIsUnittests()) {
115  BUG_ON(presize > UINT_MAX);
116  }
117 #endif
118 
119  (void) SC_ATOMIC_SUB(ra_memuse, size);
120 
121 #ifdef UNITTESTS
122  if (RunmodeIsUnittests()) {
123  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
124  BUG_ON(postsize > presize);
125  }
126 #endif
127  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
128  return;
129 }
130 
132 {
133  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
134  return smemuse;
135 }
136 
137 /**
138  * \brief Function to Check the reassembly memory usage counter against the
139  * allowed max memory usgae for TCP segments.
140  *
141  * \param size Size of the TCP segment and its payload length memory allocated
142  * \retval 1 if in bounds
143  * \retval 0 if not in bounds
144  */
146 {
147  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
148  if (memcapcopy == 0 ||
149  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
150  return 1;
151  return 0;
152 }
153 
154 /**
155  * \brief Update memcap value
156  *
157  * \param size new memcap value
158  */
160 {
161  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
162  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
163  return 1;
164  }
165 
166  return 0;
167 }
168 
169 /**
170  * \brief Return memcap value
171  *
172  * \return memcap memcap value
173  */
175 {
176  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
177  return memcapcopy;
178 }
179 
180 /* memory functions for the streaming buffer API */
181 
182 /*
183  void *(*Malloc)(size_t size);
184 */
185 static void *ReassembleMalloc(size_t size)
186 {
187  if (StreamTcpReassembleCheckMemcap(size) == 0)
188  return NULL;
189  void *ptr = SCMalloc(size);
190  if (ptr == NULL)
191  return NULL;
193  return ptr;
194 }
195 
196 /*
197  void *(*Calloc)(size_t n, size_t size);
198 */
199 static void *ReassembleCalloc(size_t n, size_t size)
200 {
201  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
202  return NULL;
203  void *ptr = SCCalloc(n, size);
204  if (ptr == NULL)
205  return NULL;
207  return ptr;
208 }
209 
210 /*
211  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
212 */
213 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
214 {
215  if (size > orig_size) {
216  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
217  return NULL;
218  }
219  void *nptr = SCRealloc(optr, size);
220  if (nptr == NULL)
221  return NULL;
222 
223  if (size > orig_size) {
224  StreamTcpReassembleIncrMemuse(size - orig_size);
225  } else {
226  StreamTcpReassembleDecrMemuse(orig_size - size);
227  }
228  return nptr;
229 }
230 
231 /*
232  void (*Free)(void *ptr, size_t size);
233 */
234 static void ReassembleFree(void *ptr, size_t size)
235 {
236  SCFree(ptr);
238 }
239 
240 /** \brief alloc a tcp segment pool entry */
241 static void *TcpSegmentPoolAlloc(void)
242 {
243  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
244  return NULL;
245  }
246 
247  TcpSegment *seg = NULL;
248 
249  seg = SCMalloc(sizeof (TcpSegment));
250  if (unlikely(seg == NULL))
251  return NULL;
252  return seg;
253 }
254 
255 static int TcpSegmentPoolInit(void *data, void *initdata)
256 {
257  TcpSegment *seg = (TcpSegment *) data;
258 
259  /* do this before the can bail, so TcpSegmentPoolCleanup
260  * won't have uninitialized memory to consider. */
261  memset(seg, 0, sizeof (TcpSegment));
262 
263  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
264  return 0;
265  }
266 
267 #ifdef DEBUG
268  SCMutexLock(&segment_pool_memuse_mutex);
269  segment_pool_memuse += sizeof(TcpSegment);
270  segment_pool_memcnt++;
271  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
272  SCMutexUnlock(&segment_pool_memuse_mutex);
273 #endif
274 
275  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
276  return 1;
277 }
278 
279 /** \brief clean up a tcp segment pool entry */
280 static void TcpSegmentPoolCleanup(void *ptr)
281 {
282  if (ptr == NULL)
283  return;
284 
285  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
286 
287 #ifdef DEBUG
288  SCMutexLock(&segment_pool_memuse_mutex);
289  segment_pool_memuse -= sizeof(TcpSegment);
290  segment_pool_memcnt--;
291  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
292  SCMutexUnlock(&segment_pool_memuse_mutex);
293 #endif
294 }
295 
296 /**
297  * \brief Function to return the segment back to the pool.
298  *
299  * \param seg Segment which will be returned back to the pool.
300  */
302 {
303  if (seg == NULL)
304  return;
305 
306  PoolThreadReturn(segment_thread_pool, seg);
307 }
308 
309 /**
310  * \brief return all segments in this stream into the pool(s)
311  *
312  * \param stream the stream to cleanup
313  */
315 {
316  TcpSegment *seg = NULL, *safe = NULL;
317  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
318  {
319  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
321  }
322 }
323 
324 #ifdef UNITTESTS
325 /** \internal
326  * \brief check if segments falls before stream 'offset' */
327 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
328 {
329  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
330  return 1;
331  return 0;
332 }
333 #endif
334 
335 /** \param f locked flow */
337 {
338  if (f->protoctx == NULL)
339  return;
340 
341  TcpSession *ssn = (TcpSession *)f->protoctx;
345 }
346 
347 /** \param f locked flow */
349 {
350  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
351  return 0;
352 
353  TcpSession *ssn = (TcpSession *)f->protoctx;
355 }
356 
357 static int StreamTcpReassemblyConfig(char quiet)
358 {
359  uint32_t segment_prealloc = 2048;
360  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
361  if (seg) {
362  uint32_t prealloc = 0;
363  if (StringParseUint32(&prealloc, 10, strlen(seg->val), seg->val) < 0)
364  {
365  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
366  "%s is invalid", seg->val);
367  return -1;
368  }
369  segment_prealloc = prealloc;
370  }
371  if (!quiet)
372  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
373  stream_config.prealloc_segments = segment_prealloc;
374 
375  int overlap_diff_data = 0;
376  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
377  if (overlap_diff_data) {
379  }
380  if (StreamTcpInlineMode() == TRUE) {
382  }
383 
386  stream_config.sbcnf.Malloc = ReassembleMalloc;
387  stream_config.sbcnf.Calloc = ReassembleCalloc;
388  stream_config.sbcnf.Realloc = ReassembleRealloc;
389  stream_config.sbcnf.Free = ReassembleFree;
390 
391  return 0;
392 }
393 
394 int StreamTcpReassembleInit(char quiet)
395 {
396  /* init the memcap/use tracker */
398 
399  if (StreamTcpReassemblyConfig(quiet) < 0)
400  return -1;
401 
402 #ifdef DEBUG
403  SCMutexInit(&segment_pool_memuse_mutex, NULL);
404 #endif
405  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
407  return 0;
408 }
409 
410 void StreamTcpReassembleFree(char quiet)
411 {
412  SCMutexLock(&segment_thread_pool_mutex);
413  if (segment_thread_pool != NULL) {
414  PoolThreadFree(segment_thread_pool);
415  segment_thread_pool = NULL;
416  }
417  SCMutexUnlock(&segment_thread_pool_mutex);
418  SCMutexDestroy(&segment_thread_pool_mutex);
419 
420 #ifdef DEBUG
421  if (segment_pool_memuse > 0)
422  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
423  if (segment_pool_memcnt > 0)
424  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
425  SCMutexDestroy(&segment_pool_memuse_mutex);
426 #endif
427 }
428 
430 {
431  SCEnter();
433  if (unlikely(ra_ctx == NULL))
434  return NULL;
435 
436  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
437 
438  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
439 
440  SCMutexLock(&segment_thread_pool_mutex);
441  if (segment_thread_pool == NULL) {
442  segment_thread_pool = PoolThreadInit(1, /* thread */
443  0, /* unlimited */
445  sizeof(TcpSegment),
446  TcpSegmentPoolAlloc,
447  TcpSegmentPoolInit, NULL,
448  TcpSegmentPoolCleanup, NULL);
449  ra_ctx->segment_thread_pool_id = 0;
450  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
451  PoolThreadSize(segment_thread_pool),
452  ra_ctx->segment_thread_pool_id);
453  } else {
454  /* grow segment_thread_pool until we have a element for our thread id */
455  ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
456  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
457  PoolThreadSize(segment_thread_pool),
458  ra_ctx->segment_thread_pool_id);
459  }
460  SCMutexUnlock(&segment_thread_pool_mutex);
461  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
462  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
464  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
465  }
466 
467  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
468 }
469 
471 {
472  SCEnter();
474  SCFree(ra_ctx);
475  SCReturn;
476 }
477 
478 /**
479  * \brief check if stream in pkt direction has depth reached
480  *
481  * \param p packet with *LOCKED* flow
482  *
483  * \retval 1 stream has depth reached
484  * \retval 0 stream does not have depth reached
485  */
487 {
488  if (p->flow != NULL && p->flow->protoctx != NULL) {
489  TcpSession *ssn = p->flow->protoctx;
490  TcpStream *stream;
491  if (p->flowflags & FLOW_PKT_TOSERVER) {
492  stream = &ssn->client;
493  } else {
494  stream = &ssn->server;
495  }
496 
497  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
498  }
499 
500  return 0;
501 }
502 
503 /**
504  * \internal
505  * \brief Function to Check the reassembly depth valuer against the
506  * allowed max depth of the stream reassembly for TCP streams.
507  *
508  * \param stream stream direction
509  * \param seq sequence number where "size" starts
510  * \param size size of the segment that is added
511  *
512  * \retval size Part of the size that fits in the depth, 0 if none
513  */
514 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
515  uint32_t seq, uint32_t size)
516 {
517  SCEnter();
518 
519  /* if the configured depth value is 0, it means there is no limit on
520  reassembly depth. Otherwise carry on my boy ;) */
521  if (ssn->reassembly_depth == 0) {
522  SCReturnUInt(size);
523  }
524 
525  /* if the final flag is set, we're not accepting anymore */
527  SCReturnUInt(0);
528  }
529 
530  uint64_t seg_depth;
531  if (SEQ_GT(stream->base_seq, seq)) {
532  if (SEQ_LEQ(seq+size, stream->base_seq)) {
533  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
534  stream->base_seq, seq, seq+size);
535  SCReturnUInt(0);
536  }
537 
538  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
539  } else {
540  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
541  }
542 
543  /* if the base_seq has moved passed the depth window we stop
544  * checking and just reject the rest of the packets including
545  * retransmissions. Saves us the hassle of dealing with sequence
546  * wraps as well */
547  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
548  stream->base_seq, seg_depth,
549  ssn->reassembly_depth);
550 
551  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
552  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
554  SCReturnUInt(0);
555  }
556  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
557  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
558 #if 0
559  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
560  (stream->base_seq_offset + stream->base_seq + size),
561  (stream->isn + ssn->reassembly_depth));
562 #endif
563  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
564  /* packet (partly?) fits the depth window */
565 
566  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
567  /* complete fit */
568  SCReturnUInt(size);
569  } else {
571  /* partial fit, return only what fits */
572  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
573  DEBUG_VALIDATE_BUG_ON(part > size);
574  if (part > size)
575  part = size;
576  SCReturnUInt(part);
577  }
578  }
579 
580  SCReturnUInt(0);
581 }
582 
583 /**
584  * \brief Insert a packets TCP data into the stream reassembly engine.
585  *
586  * \retval 0 good segment, as far as we checked.
587  * \retval -1 badness, reason to drop in inline mode
588  *
589  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
590  * or it wasn't added because of reassembly depth.
591  *
592  */
594  TcpSession *ssn, TcpStream *stream, Packet *p)
595 {
596  SCEnter();
597 
598  if (ssn->data_first_seen_dir == 0) {
599  if (PKT_IS_TOSERVER(p)) {
601  } else {
603  }
604  }
605 
606  /* If the OS policy is not set then set the OS policy for this stream */
607  if (stream->os_policy == 0) {
608  StreamTcpSetOSPolicy(stream, p);
609  }
610 
613  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
614  SCReturnInt(0);
615  }
616 
617  /* If we have reached the defined depth for either of the stream, then stop
618  reassembling the TCP session */
619  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
620  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
621 
623  /* increment stream depth counter */
625  }
626  if (size == 0) {
627  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
628  SCReturnInt(0);
629  }
630 
632  if (size > p->payload_len)
633  size = p->payload_len;
634 
635  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
636  if (seg == NULL) {
637  SCLogDebug("segment_pool is empty");
639  SCReturnInt(-1);
640  }
641 
642  TCP_SEG_LEN(seg) = size;
643  seg->seq = TCP_GET_SEQ(p);
644 
645  /* HACK: for TFO SYN packets the seq for data starts at + 1 */
646  if (TCP_HAS_TFO(p) && p->payload_len && p->tcph->th_flags == TH_SYN)
647  seg->seq += 1;
648 
649  /* proto detection skipped, but now we do get data. Set event. */
650  if (RB_EMPTY(&stream->seg_tree) &&
652 
655  }
656 
657  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
658  SCLogDebug("StreamTcpReassembleInsertSegment failed");
659  SCReturnInt(-1);
660  }
661  SCReturnInt(0);
662 }
663 
664 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
665  Packet *p)
666 {
667  uint8_t flag = 0;
668 
670  flag |= STREAM_START;
671  }
672 
673  if (ssn->state == TCP_CLOSED) {
674  flag |= STREAM_EOF;
675  }
676 
677  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
678  flag |= STREAM_MIDSTREAM;
679  }
680 
681  if (p->flags & PKT_PSEUDO_STREAM_END) {
682  flag |= STREAM_EOF;
683  }
684 
685  if (&ssn->client == stream) {
686  flag |= STREAM_TOSERVER;
687  } else {
688  flag |= STREAM_TOCLIENT;
689  }
691  flag |= STREAM_DEPTH;
692  }
693  return flag;
694 }
695 
696 /**
697  * \brief Check the minimum size limits for reassembly.
698  *
699  * \retval 0 don't reassemble yet
700  * \retval 1 do reassemble
701  */
702 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
703  const TcpStream *stream, const Packet *p)
704 {
705  SCEnter();
706 
707  /* if any of these flags is set we always inspect immediately */
708 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
709  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
710  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
711  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
712 
715  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
716  "is set, so not expecting any new data segments");
717  }
719  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
720  }
722  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
723  "so no new segments will be considered");
724  }
725  SCReturnInt(1);
726  }
727 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
728 
729  /* some states mean we reassemble no matter how much data we have */
730  if (ssn->state > TCP_TIME_WAIT)
731  SCReturnInt(1);
732 
733  if (p->flags & PKT_PSEUDO_STREAM_END)
734  SCReturnInt(1);
735 
736  /* check if we have enough data to do raw reassembly */
737  if (PKT_IS_TOSERVER(p)) {
738  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
739  uint32_t delta = stream->last_ack - stream->base_seq;
740  /* get max absolute offset */
741  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
742 
743  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
744  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
745  SCReturnInt(1);
746  } else {
747  SCLogDebug("toserver min chunk len not yet reached: "
748  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
749  "%"PRIu32"", stream->last_ack, stream->base_seq,
750  (stream->last_ack - stream->base_seq),
752  SCReturnInt(0);
753  }
754  }
755  } else {
756  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
757  uint32_t delta = stream->last_ack - stream->base_seq;
758  /* get max absolute offset */
759  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
760 
761  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
762 
763  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
764  SCReturnInt(1);
765  } else {
766  SCLogDebug("toclient min chunk len not yet reached: "
767  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
768  "%"PRIu32"", stream->last_ack, stream->base_seq,
769  (stream->last_ack - stream->base_seq),
771  SCReturnInt(0);
772  }
773  }
774  }
775 
776  SCReturnInt(0);
777 }
778 
779 /**
780  * \brief see what if any work the TCP session still needs
781  */
782 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
783 {
784  const TcpStream *stream = NULL;
785 #ifdef DEBUG
786  const char *dirstr = NULL;
787 #endif
788  if (direction == STREAM_TOSERVER) {
789  stream = &ssn->client;
790 #ifdef DEBUG
791  dirstr = "client";
792 #endif
793  } else {
794  stream = &ssn->server;
795 #ifdef DEBUG
796  dirstr = "server";
797 #endif
798  }
799 
800  int use_app = 1;
801  int use_raw = 1;
802 
804  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
805  {
806  // app is dead
807  use_app = 0;
808  }
809 
811  // raw is dead
812  use_raw = 0;
813  }
814 
815  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
816 
817  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
818  dirstr,
819  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
820  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
821  right_edge);
822  if (use_raw) {
823  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
824  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
826  }
827  }
828  if (use_app) {
829  if (right_edge > STREAM_APP_PROGRESS(stream)) {
830  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
832  }
833  }
834 
835  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
837 }
838 
839 #ifdef DEBUG
840 static uint64_t GetStreamSize(TcpStream *stream)
841 {
842  if (stream) {
843  uint64_t size = 0;
844  uint32_t cnt = 0;
845 
846  TcpSegment *seg;
847  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
848  cnt++;
849  size += (uint64_t)TCP_SEG_LEN(seg);
850  }
851 
852  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
853  return size;
854  }
855  return (uint64_t)0;
856 }
857 
858 static void GetSessionSize(TcpSession *ssn, Packet *p)
859 {
860  uint64_t size = 0;
861  if (ssn) {
862  size = GetStreamSize(&ssn->client);
863  size += GetStreamSize(&ssn->server);
864 
865  //if (size > 900000)
866  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
867  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
868  }
869 }
870 #endif
871 
872 static StreamingBufferBlock *GetBlock(StreamingBuffer *sb, const uint64_t offset)
873 {
874  StreamingBufferBlock *blk = sb->head;
875  if (blk == NULL)
876  return NULL;
877 
878  for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
879  if (blk->offset >= offset)
880  return blk;
881  else if ((blk->offset + blk->len) > offset) {
882  return blk;
883  }
884  }
885  return NULL;
886 }
887 
888 /** \internal
889  *
890  * Get buffer, or first part of the buffer if data gaps exist.
891  *
892  * \brief get stream data from offset
893  * \param offset stream offset */
894 static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset)
895 {
896  const uint8_t *mydata;
897  uint32_t mydata_len;
898 
899  if (RB_EMPTY(&stream->sb.sbb_tree)) {
900  SCLogDebug("getting one blob");
901 
902  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
903 
904  *data = mydata;
905  *data_len = mydata_len;
906  } else {
907  StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
908  if (blk == NULL) {
909  *data = NULL;
910  *data_len = 0;
911  return;
912  }
913 
914  /* block at expected offset */
915  if (blk->offset == offset) {
916 
917  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
918 
919  /* block past out offset */
920  } else if (blk->offset > offset) {
921  SCLogDebug("gap, want data at offset %"PRIu64", "
922  "got data at %"PRIu64". GAP of size %"PRIu64,
923  offset, blk->offset, blk->offset - offset);
924  *data = NULL;
925  *data_len = blk->offset - offset;
926 
927  /* block starts before offset, but ends after */
928  } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
929  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
930  offset, blk->offset, blk->len);
931  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
932  SCLogDebug("data %p, data_len %u", *data, *data_len);
933  } else {
934  *data = NULL;
935  *data_len = 0;
936  }
937  }
938 }
939 
940 /** \internal
941  * \brief check to see if we should declare a GAP
942  * Call this when the app layer didn't get data at the requested
943  * offset.
944  */
945 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
946 {
947  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
948  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
949 
950  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
951  /* get window of data that is acked */
952  const uint32_t delta = stream->last_ack - stream->base_seq;
953  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
954  /* get max absolute offset */
955  last_ack_abs += delta;
956 
957  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
958  last_ack_abs -= ackadded;
959 
960  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
961  SCLogDebug("next_seq %u", stream->next_seq);
962 
963  /* if last_ack_abs is beyond the app_progress data that we haven't seen
964  * has been ack'd. This looks like a GAP. */
965  if (last_ack_abs > app_progress) {
966  /* however, we can accept ACKs a bit too liberally. If last_ack
967  * is beyond next_seq, we only consider it a gap now if we do
968  * already have data beyond the gap. */
969  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
970  if (RB_EMPTY(&stream->sb.sbb_tree)) {
971  SCLogDebug("packet %"PRIu64": no GAP. "
972  "next_seq %u < last_ack %u, but no data in list",
973  p->pcap_cnt, stream->next_seq, stream->last_ack);
974  return false;
975  } else {
976  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
977  const StreamingBufferBlock *blk = stream->sb.head;
978  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
979  /* ack'd data after the gap */
980  SCLogDebug("packet %"PRIu64": GAP. "
981  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
982  p->pcap_cnt, stream->next_seq, stream->last_ack);
983  return true;
984  }
985  }
986  }
987 
988  SCLogDebug("packet %"PRIu64": GAP! "
989  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
990  "but we have no data.",
991  p->pcap_cnt, last_ack_abs, app_progress);
992  return true;
993  }
994  }
995  SCLogDebug("packet %"PRIu64": no GAP. "
996  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
997  p->pcap_cnt, last_ack_abs, app_progress);
998  return false;
999 }
1000 
1001 /** \internal
1002  * \brief get stream buffer and update the app-layer
1003  * \param stream pointer to pointer as app-layer can switch flow dir
1004  * \retval 0 success
1005  */
1006 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1007  TcpReassemblyThreadCtx *ra_ctx,
1008  TcpSession *ssn, TcpStream **stream,
1009  Packet *p, enum StreamUpdateDir dir)
1010 {
1011  uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1012 
1013  SCLogDebug("app progress %"PRIu64, app_progress);
1014  SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1015 
1016  const uint8_t *mydata;
1017  uint32_t mydata_len;
1018 
1019  while (1) {
1020  GetAppBuffer(*stream, &mydata, &mydata_len, app_progress);
1021  DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1022  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1023  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1024 
1025  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1026  NULL, mydata_len,
1027  StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP);
1028  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1029 
1032 
1033  /* AppLayerHandleTCPData has likely updated progress. */
1034  app_progress = STREAM_APP_PROGRESS(*stream);
1035 
1036  /* a GAP also consumes 'data required'. TODO perhaps we can use
1037  * this to skip post GAP data until the start of a next record. */
1038  if ((*stream)->data_required > mydata_len) {
1039  (*stream)->data_required -= mydata_len;
1040  } else {
1041  (*stream)->data_required = 0;
1042  }
1043 
1044  if (r < 0)
1045  return 0;
1046 
1047  continue;
1048  } else if (mydata == NULL || mydata_len == 0) {
1049  /* Possibly a gap, but no new data. */
1050  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1051  SCReturnInt(0);
1052 
1053  mydata = NULL;
1054  mydata_len = 0;
1055  }
1056  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1057  break;
1058  }
1059 
1060  //PrintRawDataFp(stdout, mydata, mydata_len);
1061 
1062  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1063  *stream, &(*stream)->sb, mydata_len, app_progress);
1064 
1065  /* get window of data that is acked */
1066  if (StreamTcpInlineMode() == FALSE) {
1067  if (p->flags & PKT_PSEUDO_STREAM_END) {
1068  // fall through, we use all available data
1069  } else {
1070  uint64_t last_ack_abs = app_progress; /* absolute right edge of ack'd data */
1071  if (STREAM_LASTACK_GT_BASESEQ(*stream)) {
1072  /* get window of data that is acked */
1073  uint32_t delta = (*stream)->last_ack - (*stream)->base_seq;
1074  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > (*stream)->window);
1075  /* get max absolute offset */
1076  last_ack_abs += delta;
1077  }
1078 
1079  /* see if the buffer contains unack'd data as well */
1080  if (app_progress + mydata_len > last_ack_abs) {
1081  uint32_t check = mydata_len;
1082  mydata_len = last_ack_abs - app_progress;
1083  BUG_ON(mydata_len > check);
1084  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1085  "data is considered", mydata_len);
1086  }
1087  }
1088  }
1089  if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1090  if (mydata_len < (*stream)->data_required) {
1091  SCLogDebug("mydata_len %u data_required %u", mydata_len, (*stream)->data_required);
1092  SCReturnInt(0);
1093  }
1094  }
1095  (*stream)->data_required = 0;
1096 
1097  /* update the app-layer */
1098  (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1099  (uint8_t *)mydata, mydata_len,
1100  StreamGetAppLayerFlags(ssn, *stream, p));
1101  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1102 
1103  SCReturnInt(0);
1104 }
1105 
1106 /**
1107  * \brief Update the stream reassembly upon receiving a packet.
1108  *
1109  * For IDS mode, the stream is in the opposite direction of the packet,
1110  * as the ACK-packet is ACK'ing the stream.
1111  *
1112  * One of the utilities call by this function AppLayerHandleTCPData(),
1113  * has a feature where it will call this very same function for the
1114  * stream opposing the stream it is called with. This shouldn't cause
1115  * any issues, since processing of each stream is independent of the
1116  * other stream.
1117  */
1119  TcpSession *ssn, TcpStream *stream,
1120  Packet *p, enum StreamUpdateDir dir)
1121 {
1122  SCEnter();
1123 
1124  /* this function can be directly called by app layer protocol
1125  * detection. */
1128  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1129  SCReturnInt(0);
1130  }
1131 
1132 #ifdef DEBUG
1133  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1134  GetSessionSize(ssn, p);
1135 #endif
1136  /* if no segments are in the list or all are already processed,
1137  * and state is beyond established, we send an empty msg */
1138  if (STREAM_HAS_SEEN_DATA(stream) && STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1139  {
1140  /* send an empty EOF msg if we have no segments but TCP state
1141  * is beyond ESTABLISHED */
1142  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1143  SCLogDebug("sending empty eof message");
1144  /* send EOF to app layer */
1145  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1146  NULL, 0,
1147  StreamGetAppLayerFlags(ssn, stream, p));
1148  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1149 
1150  SCReturnInt(0);
1151  }
1152  }
1153 
1154  /* with all that out of the way, lets update the app-layer */
1155  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1156 }
1157 
1158 /** \internal
1159  * \brief get stream data from offset
1160  * \param offset stream offset */
1161 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1162  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1163 {
1164  const uint8_t *mydata;
1165  uint32_t mydata_len;
1166  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1167  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1168 
1169  uint64_t roffset = offset;
1170  if (offset)
1171  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1172  else {
1173  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1174  }
1175 
1176  *data = mydata;
1177  *data_len = mydata_len;
1178  *data_offset = roffset;
1179  } else {
1180  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1181  *iter == NULL ? "starting" : "continuing", offset);
1182  if (*iter == NULL) {
1183  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1184  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1185  SCLogDebug("*iter %p", *iter);
1186  }
1187  if (*iter == NULL) {
1188  SCLogDebug("no data");
1189  *data = NULL;
1190  *data_len = 0;
1191  *data_offset = 0;
1192  return 0;
1193  }
1194  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1195 
1196  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1197  SCLogDebug("mydata %p", mydata);
1198 
1199  if ((*iter)->offset < offset) {
1200  uint64_t delta = offset - (*iter)->offset;
1201  if (delta < mydata_len) {
1202  *data = mydata + delta;
1203  *data_len = mydata_len - delta;
1204  *data_offset = offset;
1205  } else {
1206  SCLogDebug("no data (yet)");
1207  *data = NULL;
1208  *data_len = 0;
1209  *data_offset = 0;
1210  }
1211 
1212  } else {
1213  *data = mydata;
1214  *data_len = mydata_len;
1215  *data_offset = (*iter)->offset;
1216  }
1217 
1218  *iter = SBB_RB_NEXT(*iter);
1219  SCLogDebug("*iter %p", *iter);
1220  }
1221  return 0;
1222 }
1223 
1224 /** \brief does the stream engine have data to inspect?
1225  *
1226  * Returns true if there is data to inspect. In IDS case this is
1227  * about ACK'd data in the packet's direction.
1228  *
1229  * In the IPS case this is about the packet itself.
1230  */
1232 {
1233  TcpStream *stream;
1234  if (PKT_IS_TOSERVER(p)) {
1235  stream = &ssn->client;
1236  } else {
1237  stream = &ssn->server;
1238  }
1239 
1240  if (RB_EMPTY(&stream->seg_tree)) {
1241  return false;
1242  }
1243 
1246  return false;
1247 
1248  if (StreamTcpInlineMode() == FALSE) {
1249  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1250  return false;
1251  }
1252  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1253  return true;
1254  }
1255  } else {
1256  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1257  return true;
1258  }
1259  }
1260  return false;
1261 }
1262 
1263 /** \brief update stream engine after detection
1264  *
1265  * Tasked with progressing the 'progress' for Raw reassembly.
1266  * 2 main scenario's:
1267  * 1. progress is != 0, so we use this
1268  * 2. progress is 0, meaning the detect engine didn't touch
1269  * raw at all. In this case we need to look into progressing
1270  * raw anyway.
1271  *
1272  * Additionally, this function is tasked with disabling raw
1273  * reassembly if the app-layer requested to disable it.
1274  */
1275 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1276 {
1277  TcpStream *stream;
1278  if (PKT_IS_TOSERVER(p)) {
1279  stream = &ssn->client;
1280  } else {
1281  stream = &ssn->server;
1282  }
1283 
1284  if (progress > STREAM_RAW_PROGRESS(stream)) {
1285  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1286  stream->raw_progress_rel += slide;
1288 
1289  /* if app is active and beyond raw, sync raw to app */
1290  } else if (progress == 0 &&
1291  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1293  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1294  {
1295  /* if trigger raw is set we sync the 2 trackers */
1297  {
1298  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1299  stream->raw_progress_rel += slide;
1301 
1302  /* otherwise mix in the tcp window */
1303  } else {
1304  uint64_t tcp_window = stream->window;
1305  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1306  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1307  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1308  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1309  stream->raw_progress_rel += slide;
1310  }
1311  }
1312  }
1313  /* app is dead */
1314  } else if (progress == 0) {
1315  uint64_t tcp_window = stream->window;
1316  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1317  if (tcp_window < stream_right_edge) {
1318  uint64_t new_raw = stream_right_edge - tcp_window;
1319  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1320  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1321  stream->raw_progress_rel += slide;
1322  }
1323  }
1325 
1326  } else {
1327  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1328  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1329  STREAM_RAW_PROGRESS(stream), stream->window);
1330  }
1331 
1332  /* if we were told to accept no more raw data, we can mark raw as
1333  * disabled now. */
1336  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1337  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1338  }
1339 
1340  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1341 }
1342 
1343 /** \internal
1344  * \brief get a buffer around the current packet and run the callback on it
1345  *
1346  * The inline/IPS scanning method takes the current payload and wraps it in
1347  * data from other segments.
1348  *
1349  * How much data is inspected is controlled by the available data, chunk_size
1350  * and the payload size of the packet.
1351  *
1352  * Large packets: if payload size is close to the chunk_size, where close is
1353  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1354  * used: payload_len + 33% of the chunk_size.
1355  * If the payload size if equal to or bigger than the chunk_size, we use
1356  * payload len + 33% of the chunk size.
1357  */
1358 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1359  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1360 {
1361  SCEnter();
1362  int r = 0;
1363 
1364  TcpStream *stream;
1365  if (PKT_IS_TOSERVER(p)) {
1366  stream = &ssn->client;
1367  } else {
1368  stream = &ssn->server;
1369  }
1370 
1371  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1373  {
1374  *progress_out = STREAM_RAW_PROGRESS(stream);
1375  return 0;
1376  }
1377 
1378  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1381  if (chunk_size <= p->payload_len) {
1382  chunk_size = p->payload_len + (chunk_size / 3);
1383  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1384  p->payload_len, chunk_size);
1385  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1386  chunk_size = p->payload_len + ((chunk_size / 3));
1387  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1388  p->payload_len, chunk_size);
1389  }
1390 
1391  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1392  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1393  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1394  packet_leftedge_abs, packet_rightedge_abs);
1395 
1396  const uint8_t *mydata = NULL;
1397  uint32_t mydata_len = 0;
1398  uint64_t mydata_offset = 0;
1399  /* simply return progress from the block we inspected. */
1400  bool return_progress = false;
1401 
1402  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1403  /* continues block */
1404  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1405  return_progress = true;
1406 
1407  } else {
1408  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1409  /* find our block */
1410  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1411  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1412  if (sbb) {
1413  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1414  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1415  mydata_offset = sbb->offset;
1416  }
1417  }
1418 
1419  /* this can only happen if the segment insert of our current 'p' failed */
1420  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1421  if ((mydata == NULL || mydata_len == 0) || /* no data */
1422  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1423  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1424  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1425  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1426  {
1427  /* no data, or data is incomplete or wrong: use packet data */
1428  mydata = p->payload;
1429  mydata_len = p->payload_len;
1430  mydata_offset = packet_leftedge_abs;
1431  //mydata_rightedge_abs = packet_rightedge_abs;
1432  } else {
1433  /* adjust buffer to match chunk_size */
1434  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1435  if (mydata_len > chunk_size) {
1436  uint32_t excess = mydata_len - chunk_size;
1437  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1438 
1439  if (mydata_rightedge_abs == packet_rightedge_abs) {
1440  mydata += excess;
1441  mydata_len -= excess;
1442  mydata_offset += excess;
1443  SCLogDebug("cutting front of the buffer with %u", excess);
1444  } else if (mydata_offset == packet_leftedge_abs) {
1445  mydata_len -= excess;
1446  SCLogDebug("cutting tail of the buffer with %u", excess);
1447  } else {
1448  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1449  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1450  SCLogDebug("before %u after %u", before, after);
1451 
1452  if (after >= (chunk_size - p->payload_len) / 2) {
1453  // more trailing data than we need
1454 
1455  if (before >= (chunk_size - p->payload_len) / 2) {
1456  // also more heading data, divide evenly
1457  before = after = (chunk_size - p->payload_len) / 2;
1458  } else {
1459  // heading data is less than requested, give the
1460  // rest to the trailing data
1461  after = (chunk_size - p->payload_len) - before;
1462  }
1463  } else {
1464  // less trailing data than requested
1465 
1466  if (before >= (chunk_size - p->payload_len) / 2) {
1467  before = (chunk_size - p->payload_len) - after;
1468  } else {
1469  // both smaller than their requested size
1470  }
1471  }
1472 
1473  /* adjust the buffer */
1474  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1475  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1476  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1477  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1478  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1479  mydata += skip;
1480  mydata_len -= (skip + cut);
1481  mydata_offset += skip;
1482  }
1483  }
1484  }
1485 
1486  /* run the callback */
1487  r = Callback(cb_data, mydata, mydata_len);
1488  BUG_ON(r < 0);
1489 
1490  if (return_progress) {
1491  *progress_out = (mydata_offset + mydata_len);
1492  } else {
1493  /* several blocks of data, so we need to be a bit more careful:
1494  * - if last_ack is beyond last progress, move progress forward to last_ack
1495  * - if our block matches or starts before last ack, return right edge of
1496  * our block.
1497  */
1498  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1499  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1500  uint32_t delta = stream->last_ack - stream->base_seq;
1501  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1502  /* get max absolute offset */
1503  last_ack_abs += delta;
1504  }
1505  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1506 
1507  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1508  if (mydata_offset > last_ack_abs) {
1509  /* gap between us and last ack, set progress to last ack */
1510  *progress_out = last_ack_abs;
1511  } else {
1512  *progress_out = (mydata_offset + mydata_len);
1513  }
1514  } else {
1515  *progress_out = STREAM_RAW_PROGRESS(stream);
1516  }
1517  }
1518  return r;
1519 }
1520 
1521 /** \brief access 'raw' reassembly data.
1522  *
1523  * Access data as tracked by 'raw' tracker. Data is made available to
1524  * callback that is passed to this function.
1525  *
1526  * In the case of IDS the callback may be run multiple times if data
1527  * contains gaps. It will then be run for each block of data that is
1528  * continuous.
1529  *
1530  * The callback should give on of 2 return values:
1531  * - 0 ok
1532  * - 1 done
1533  * The value 1 will break the loop if there is a block list that is
1534  * inspected.
1535  *
1536  * This function will return the 'progress' value that has been
1537  * consumed until now.
1538  *
1539  * \param ssn tcp session
1540  * \param stream tcp stream
1541  * \param Callback the function pointer to the callback function
1542  * \param cb_data callback data
1543  * \param[in] progress_in progress to work from
1544  * \param[out] progress_out absolute progress value of the data this
1545  * call handled.
1546  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1547  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1548  *
1549  * `respect_inspect_depth` is used to avoid useless inspection of too
1550  * much data.
1551  */
1552 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1553  StreamReassembleRawFunc Callback, void *cb_data,
1554  const uint64_t progress_in,
1555  uint64_t *progress_out, bool eof,
1556  bool respect_inspect_depth)
1557 {
1558  SCEnter();
1559  int r = 0;
1560 
1561  StreamingBufferBlock *iter = NULL;
1562  uint64_t progress = progress_in;
1563  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1564 
1565  /* if the app layer triggered a flush, and we're supposed to
1566  * use a minimal inspect depth, we actually take the app progress
1567  * as that is the right edge of the data. Then we take the window
1568  * of 'min_inspect_depth' before that. */
1569 
1570  SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1571  respect_inspect_depth ? "true" : "false",
1572  (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1573  stream->min_inspect_depth);
1574 
1575  if (respect_inspect_depth &&
1577  && stream->min_inspect_depth)
1578  {
1579  progress = STREAM_APP_PROGRESS(stream);
1580  if (stream->min_inspect_depth >= progress) {
1581  progress = 0;
1582  } else {
1583  progress -= stream->min_inspect_depth;
1584  }
1585 
1586  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1587 
1588  progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1589  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1590  }
1591 
1592  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1593 
1594  /* get window of data that is acked */
1595  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1596  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1597  uint32_t delta = stream->last_ack - stream->base_seq;
1598  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1599  /* get max absolute offset */
1600  last_ack_abs += delta;
1601  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1602  }
1603 
1604  /* loop through available buffers. On no packet loss we'll have a single
1605  * iteration. On missing data we'll walk the blocks */
1606  while (1) {
1607  const uint8_t *mydata;
1608  uint32_t mydata_len;
1609  uint64_t mydata_offset = 0;
1610 
1611  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1612  if (mydata_len == 0) {
1613  SCLogDebug("no data");
1614  break;
1615  }
1616  //PrintRawDataFp(stdout, mydata, mydata_len);
1617 
1618  SCLogDebug("raw progress %"PRIu64, progress);
1619  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1620  stream, &stream->sb, mydata_len, progress);
1621 
1622  if (eof) {
1623  // inspect all remaining data, ack'd or not
1624  } else {
1625  if (last_ack_abs < progress) {
1626  SCLogDebug("nothing to do");
1627  goto end;
1628  }
1629 
1630  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1631  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1632 
1633  /* see if the buffer contains unack'd data as well */
1634  if (progress + mydata_len > last_ack_abs) {
1635  uint32_t check = mydata_len;
1636  mydata_len = last_ack_abs - progress;
1637  BUG_ON(check < mydata_len);
1638  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1639  "data is considered", mydata_len);
1640  }
1641 
1642  }
1643  if (mydata_len == 0)
1644  break;
1645 
1646  SCLogDebug("data %p len %u", mydata, mydata_len);
1647 
1648  /* we have data. */
1649  r = Callback(cb_data, mydata, mydata_len);
1650  BUG_ON(r < 0);
1651 
1652  if (mydata_offset == progress) {
1653  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1654  progress, mydata_len, progress_in + mydata_len);
1655 
1656  progress += mydata_len;
1657  SCLogDebug("raw progress now %"PRIu64, progress);
1658 
1659  /* data is beyond the progress we'd like, and before last ack. Gap. */
1660  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1661  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1662  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1663  progress = mydata_offset;
1664  SCLogDebug("raw progress now %"PRIu64, progress);
1665 
1666  } else {
1667  SCLogDebug("not increasing progress, data gap => mydata_offset "
1668  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1669  }
1670 
1671  if (iter == NULL || r == 1)
1672  break;
1673  }
1674 end:
1675  *progress_out = progress;
1676  return r;
1677 }
1678 
1680  StreamReassembleRawFunc Callback, void *cb_data,
1681  uint64_t *progress_out, bool respect_inspect_depth)
1682 {
1683  /* handle inline separately as the logic is very different */
1684  if (StreamTcpInlineMode() == TRUE) {
1685  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1686  }
1687 
1688  TcpStream *stream;
1689  if (PKT_IS_TOSERVER(p)) {
1690  stream = &ssn->client;
1691  } else {
1692  stream = &ssn->server;
1693  }
1694 
1696  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1697  {
1698  *progress_out = STREAM_RAW_PROGRESS(stream);
1699  return 0;
1700  }
1701 
1702  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1703  STREAM_RAW_PROGRESS(stream), progress_out,
1704  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1705 }
1706 
1708  StreamReassembleRawFunc Callback, void *cb_data,
1709  uint64_t progress_in,
1710  uint64_t *progress_out, bool eof)
1711 {
1712  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1713  return 0;
1714 
1715  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1716  progress_in, progress_out, eof, false);
1717 }
1718 
1719 /** \internal
1720  * \brief update app layer based on received ACK
1721  *
1722  * \retval r 0 on success, -1 on error
1723  */
1724 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1725  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1726 {
1727  SCEnter();
1728 
1729  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1730  SCReturnInt(-1);
1731 
1732  SCReturnInt(0);
1733 }
1734 
1736  TcpSession *ssn, TcpStream *stream,
1737  Packet *p, PacketQueueNoLock *pq)
1738 {
1739  SCEnter();
1740 
1741  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1742 
1743  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1744  ssn, stream, p, p->payload_len);
1745 
1746  /* we need to update the opposing stream in
1747  * StreamTcpReassembleHandleSegmentUpdateACK */
1748  TcpStream *opposing_stream = NULL;
1749  if (stream == &ssn->client) {
1750  opposing_stream = &ssn->server;
1751  } else {
1752  opposing_stream = &ssn->client;
1753  }
1754 
1755  /* default IDS: update opposing side (triggered by ACK) */
1757  /* inline and stream end and flow timeout packets trigger same dir handling */
1758  if (StreamTcpInlineMode()) {
1759  dir = UPDATE_DIR_PACKET;
1760  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1761  dir = UPDATE_DIR_PACKET;
1762  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1763  dir = UPDATE_DIR_PACKET;
1764  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1765  dir = UPDATE_DIR_PACKET;
1766  } else if (ssn->state == TCP_CLOSED) {
1767  dir = UPDATE_DIR_BOTH;
1768  }
1769 
1770  /* handle ack received */
1771  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1772  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1773  {
1774  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1775  SCReturnInt(-1);
1776  }
1777 
1778  /* if this segment contains data, insert it */
1779  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1780  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1781 
1782  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1783  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1784  SCReturnInt(-1);
1785  }
1786 
1787  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1788  p->flags |= PKT_STREAM_ADD;
1789  } else {
1790  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1791  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1792  ssn, stream, p->payload_len,
1793  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1794 
1795  }
1796 
1797  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1798  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1799  * was *just* set. In this case we trigger the AppLayer Truncate
1800  * logic, to inform the applayer no more data in this direction is
1801  * to be expected. */
1802  if ((stream->flags &
1805  {
1806  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1807  if (dir != UPDATE_DIR_PACKET) {
1808  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1809  "can trigger Truncate");
1810  dir = UPDATE_DIR_PACKET;
1811  }
1812  }
1813 
1814  /* in stream inline mode even if we have no data we call the reassembly
1815  * functions to handle EOF */
1816  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1817  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1818  StreamTcpInlineMode()?"true":"false",
1819  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1820  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1821  SCReturnInt(-1);
1822  }
1823  }
1824 
1825  SCReturnInt(0);
1826 }
1827 
1828 /**
1829  * \brief get a segment from the pool
1830  *
1831  * \retval seg Segment from the pool or NULL
1832  */
1834 {
1835  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1836  SCLogDebug("seg we return is %p", seg);
1837  if (seg == NULL) {
1838  /* Increment the counter to show that we are not able to serve the
1839  segment request due to memcap limit */
1841  } else {
1842  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1843  }
1844 
1845  return seg;
1846 }
1847 
1848 /**
1849  * \brief Trigger RAW stream reassembly
1850  *
1851  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1852  * reassembly from the applayer, for example upon completion of a
1853  * HTTP request.
1854  *
1855  * It sets a flag in the stream so that the next Raw call will return
1856  * the data.
1857  *
1858  * \param ssn TcpSession
1859  */
1861 {
1862 #ifdef DEBUG
1863  BUG_ON(ssn == NULL);
1864 #endif
1865 
1866  if (ssn != NULL) {
1867  if (direction == STREAM_TOSERVER) {
1869  } else {
1871  }
1872 
1873  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1874  }
1875 }
1876 
1877 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1878 {
1879 #ifdef DEBUG
1880  BUG_ON(ssn == NULL);
1881 #endif
1882 
1883  if (ssn != NULL) {
1884  if (direction == STREAM_TOSERVER) {
1885  ssn->client.min_inspect_depth = depth;
1886  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1887  } else {
1888  ssn->server.min_inspect_depth = depth;
1889  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1890  }
1891  }
1892 }
1893 
1894 #ifdef UNITTESTS
1895 /** unit tests and it's support functions below */
1896 
1897 #define SET_ISN(stream, setseq) \
1898  (stream)->isn = (setseq); \
1899  (stream)->base_seq = (setseq) + 1
1900 
1901 /** \brief The Function to create the packet with given payload, which is used
1902  * to test the reassembly of the engine.
1903  *
1904  * \param payload The variable used to store the payload contents of the
1905  * current packet.
1906  * \param value The value which current payload will have for this packet
1907  * \param payload_len The length of the filed payload for current packet.
1908  * \param len Length of the payload array
1909  */
1910 
1911 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1912  uint8_t payload_len, uint8_t len)
1913 {
1914  uint8_t i;
1915  for (i = 0; i < payload_len; i++)
1916  payload[i] = value;
1917  for (; i < len; i++)
1918  payload = NULL;
1919 }
1920 
1921 /** \brief The Function Checks the reassembled stream contents against predefined
1922  * stream contents according to OS policy used.
1923  *
1924  * \param stream_policy Predefined value of stream for different OS policies
1925  * \param stream Reassembled stream returned from the reassembly functions
1926  */
1927 
1928 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
1929 {
1930  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
1931  {
1932  //PrintRawDataFp(stdout, stream_policy, sp_size);
1933  return 0;
1934  }
1935  return 1;
1936 }
1937 
1938 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
1939 {
1940  if (StreamingBufferCompareRawData(&stream->sb,
1941  data, data_len) == 0)
1942  {
1943  SCReturnInt(0);
1944  }
1945  SCLogInfo("OK");
1946  PrintRawDataFp(stdout, data, data_len);
1947  return 1;
1948 }
1949 
1950 #define MISSED_START(isn) \
1951  TcpReassemblyThreadCtx *ra_ctx = NULL; \
1952  TcpSession ssn; \
1953  ThreadVars tv; \
1954  memset(&tv, 0, sizeof(tv)); \
1955  \
1956  StreamTcpUTInit(&ra_ctx); \
1957  \
1958  StreamTcpUTSetupSession(&ssn); \
1959  StreamTcpUTSetupStream(&ssn.server, (isn)); \
1960  StreamTcpUTSetupStream(&ssn.client, (isn)); \
1961  \
1962  TcpStream *stream = &ssn.client;
1963 
1964 #define MISSED_END \
1965  StreamTcpUTClearSession(&ssn); \
1966  StreamTcpUTDeinit(ra_ctx); \
1967  PASS
1968 
1969 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
1970  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
1971  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
1972 
1973 /**
1974  * \test Test the handling of packets missed by both IDS and the end host.
1975  * The packet is missed in the starting of the stream.
1976  *
1977  * \retval On success it returns 1 and on failure 0.
1978  */
1979 
1980 static int StreamTcpReassembleTest25 (void)
1981 {
1982  MISSED_START(6);
1983  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
1984  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
1985  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
1986  MISSED_END;
1987 }
1988 
1989 /**
1990  * \test Test the handling of packets missed by both IDS and the end host.
1991  * The packet is missed in the middle of the stream.
1992  *
1993  * \retval On success it returns 1 and on failure 0.
1994  */
1995 
1996 static int StreamTcpReassembleTest26 (void)
1997 {
1998  MISSED_START(9);
1999  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2000  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2001  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2002  MISSED_END;
2003 }
2004 
2005 /**
2006  * \test Test the handling of packets missed by both IDS and the end host.
2007  * The packet is missed in the end of the stream.
2008  *
2009  * \retval On success it returns 1 and on failure 0.
2010  */
2011 
2012 static int StreamTcpReassembleTest27 (void)
2013 {
2014  MISSED_START(9);
2015  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2016  MISSED_STEP(13, "BB", 2, "AAABB", 5);
2017  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2018  MISSED_END;
2019 }
2020 
2021 /**
2022  * \test Test the handling of packets missed by IDS, but the end host has
2023  * received it and send the acknowledgment of it. The packet is missed
2024  * in the starting of the stream.
2025  *
2026  * \retval On success it returns 1 and on failure 0.
2027  */
2028 
2029 static int StreamTcpReassembleTest28 (void)
2030 {
2031  MISSED_START(6);
2032  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2033  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2034  ssn.state = TCP_TIME_WAIT;
2035  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2036  MISSED_END;
2037 }
2038 
2039 /**
2040  * \test Test the handling of packets missed by IDS, but the end host has
2041  * received it and send the acknowledgment of it. The packet is missed
2042  * in the middle of the stream.
2043  *
2044  * \retval On success it returns 1 and on failure 0.
2045  */
2046 
2047 static int StreamTcpReassembleTest29 (void)
2048 {
2049  MISSED_START(9);
2050  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2051  ssn.state = TCP_TIME_WAIT;
2052  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2053  MISSED_END;
2054 }
2055 
2056 static int StreamTcpReassembleTest33(void)
2057 {
2058  TcpSession ssn;
2059  Packet *p = PacketGetFromAlloc();
2060  FAIL_IF(unlikely(p == NULL));
2061  Flow f;
2062  TCPHdr tcph;
2063  TcpReassemblyThreadCtx *ra_ctx = NULL;
2065  uint8_t packet[1460] = "";
2066 
2067  StreamTcpUTInit(&ra_ctx);
2069 
2070  PacketQueueNoLock pq;
2071  memset(&pq,0,sizeof(PacketQueueNoLock));
2072  memset(&f, 0, sizeof (Flow));
2073  memset(&tcph, 0, sizeof (TCPHdr));
2074  ThreadVars tv;
2075  memset(&tv, 0, sizeof (ThreadVars));
2076  FLOW_INITIALIZE(&f);
2077  f.protoctx = &ssn;
2078  f.proto = IPPROTO_TCP;
2079  p->src.family = AF_INET;
2080  p->dst.family = AF_INET;
2081  p->proto = IPPROTO_TCP;
2082  p->flow = &f;
2083  tcph.th_win = 5480;
2084  tcph.th_flags = TH_PUSH | TH_ACK;
2085  p->tcph = &tcph;
2087  p->payload = packet;
2088 
2089  p->tcph->th_seq = htonl(10);
2090  p->tcph->th_ack = htonl(31);
2091  p->payload_len = 10;
2092 
2093  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2094 
2095  p->tcph->th_seq = htonl(20);
2096  p->tcph->th_ack = htonl(31);
2097  p->payload_len = 10;
2098 
2099  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2100 
2101  p->tcph->th_seq = htonl(40);
2102  p->tcph->th_ack = htonl(31);
2103  p->payload_len = 10;
2104 
2105  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2106 
2107  p->tcph->th_seq = htonl(5);
2108  p->tcph->th_ack = htonl(31);
2109  p->payload_len = 30;
2110 
2111  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2112 
2114  StreamTcpUTDeinit(ra_ctx);
2115  SCFree(p);
2116  PASS;
2117 }
2118 
2119 static int StreamTcpReassembleTest34(void)
2120 {
2121  TcpSession ssn;
2122  Packet *p = PacketGetFromAlloc();
2123  FAIL_IF(unlikely(p == NULL));
2124  Flow f;
2125  TCPHdr tcph;
2126  TcpReassemblyThreadCtx *ra_ctx = NULL;
2128  uint8_t packet[1460] = "";
2129 
2130  StreamTcpUTInit(&ra_ctx);
2132  PacketQueueNoLock pq;
2133  memset(&pq,0,sizeof(PacketQueueNoLock));
2134  memset(&f, 0, sizeof (Flow));
2135  memset(&tcph, 0, sizeof (TCPHdr));
2136  ThreadVars tv;
2137  memset(&tv, 0, sizeof (ThreadVars));
2138  FLOW_INITIALIZE(&f);
2139  f.protoctx = &ssn;
2140  f.proto = IPPROTO_TCP;
2141  p->src.family = AF_INET;
2142  p->dst.family = AF_INET;
2143  p->proto = IPPROTO_TCP;
2144  p->flow = &f;
2145  tcph.th_win = 5480;
2146  tcph.th_flags = TH_PUSH | TH_ACK;
2147  p->tcph = &tcph;
2149  p->payload = packet;
2150  SET_ISN(&ssn.client, 857961230);
2151 
2152  p->tcph->th_seq = htonl(857961230);
2153  p->tcph->th_ack = htonl(31);
2154  p->payload_len = 304;
2155 
2156  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2157 
2158  p->tcph->th_seq = htonl(857961534);
2159  p->tcph->th_ack = htonl(31);
2160  p->payload_len = 1460;
2161 
2162  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2163 
2164  p->tcph->th_seq = htonl(857963582);
2165  p->tcph->th_ack = htonl(31);
2166  p->payload_len = 1460;
2167 
2168  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2169 
2170  p->tcph->th_seq = htonl(857960946);
2171  p->tcph->th_ack = htonl(31);
2172  p->payload_len = 1460;
2173 
2174  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2175 
2177  StreamTcpUTDeinit(ra_ctx);
2178  SCFree(p);
2179  PASS;
2180 }
2181 
2182 /** \test Test the bug 76 condition */
2183 static int StreamTcpReassembleTest37(void)
2184 {
2185  TcpSession ssn;
2186  Flow f;
2187  TCPHdr tcph;
2188  TcpReassemblyThreadCtx *ra_ctx = NULL;
2189  uint8_t packet[1460] = "";
2190  PacketQueueNoLock pq;
2191  ThreadVars tv;
2192  memset(&tv, 0, sizeof (ThreadVars));
2193 
2194  Packet *p = PacketGetFromAlloc();
2195  FAIL_IF(unlikely(p == NULL));
2196 
2197  StreamTcpUTInit(&ra_ctx);
2199  memset(&pq,0,sizeof(PacketQueueNoLock));
2200  memset(&f, 0, sizeof (Flow));
2201  memset(&tcph, 0, sizeof (TCPHdr));
2202  memset(&tv, 0, sizeof (ThreadVars));
2203 
2204  FLOW_INITIALIZE(&f);
2205  f.protoctx = &ssn;
2206  f.proto = IPPROTO_TCP;
2207  p->src.family = AF_INET;
2208  p->dst.family = AF_INET;
2209  p->proto = IPPROTO_TCP;
2210  p->flow = &f;
2211  tcph.th_win = 5480;
2212  tcph.th_flags = TH_PUSH | TH_ACK;
2213  p->tcph = &tcph;
2215  p->payload = packet;
2217 
2218  p->tcph->th_seq = htonl(3061088537UL);
2219  p->tcph->th_ack = htonl(1729548549UL);
2220  p->payload_len = 1391;
2221  ssn.client.last_ack = 3061091137UL;
2222  SET_ISN(&ssn.client, 3061091309UL);
2223 
2224  /* pre base_seq, so should be rejected */
2225  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2226 
2227  p->tcph->th_seq = htonl(3061089928UL);
2228  p->tcph->th_ack = htonl(1729548549UL);
2229  p->payload_len = 1391;
2230  ssn.client.last_ack = 3061091137UL;
2231 
2232  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2233 
2234  p->tcph->th_seq = htonl(3061091319UL);
2235  p->tcph->th_ack = htonl(1729548549UL);
2236  p->payload_len = 1391;
2237  ssn.client.last_ack = 3061091137UL;
2238 
2239  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2240 
2242  StreamTcpUTDeinit(ra_ctx);
2243  SCFree(p);
2244  PASS;
2245 }
2246 
2247 /**
2248  * \test Test to make sure that we don't return the segments until the app
2249  * layer proto has been detected and after that remove the processed
2250  * segments.
2251  *
2252  * \retval On success it returns 1 and on failure 0.
2253  */
2254 
2255 static int StreamTcpReassembleTest39 (void)
2256 {
2257  Packet *p = PacketGetFromAlloc();
2258  FAIL_IF(unlikely(p == NULL));
2259  Flow f;
2260  ThreadVars tv;
2261  StreamTcpThread stt;
2262  TCPHdr tcph;
2263  PacketQueueNoLock pq;
2264  memset(&pq,0,sizeof(PacketQueueNoLock));
2265  memset (&f, 0, sizeof(Flow));
2266  memset(&tv, 0, sizeof (ThreadVars));
2267  memset(&stt, 0, sizeof (stt));
2268  memset(&tcph, 0, sizeof (TCPHdr));
2269 
2270  FLOW_INITIALIZE(&f);
2271  f.flags = FLOW_IPV4;
2272  f.proto = IPPROTO_TCP;
2273  p->flow = &f;
2274  p->tcph = &tcph;
2275 
2276  StreamTcpUTInit(&stt.ra_ctx);
2277 
2278  /* handshake */
2279  tcph.th_win = htons(5480);
2280  tcph.th_flags = TH_SYN;
2282  p->payload_len = 0;
2283  p->payload = NULL;
2284  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2285 
2286  TcpSession *ssn = (TcpSession *)f.protoctx;
2287  FAIL_IF_NULL(ssn);
2288 
2299  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2300  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2301  FAIL_IF(ssn->data_first_seen_dir != 0);
2302 
2303  /* handshake */
2304  p->tcph->th_ack = htonl(1);
2305  p->tcph->th_flags = TH_SYN | TH_ACK;
2307  p->payload_len = 0;
2308  p->payload = NULL;
2309  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2320  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2321  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2322  FAIL_IF(ssn->data_first_seen_dir != 0);
2323 
2324  /* handshake */
2325  p->tcph->th_ack = htonl(1);
2326  p->tcph->th_seq = htonl(1);
2327  p->tcph->th_flags = TH_ACK;
2329  p->payload_len = 0;
2330  p->payload = NULL;
2331  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2342  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2343  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2344  FAIL_IF(ssn->data_first_seen_dir != 0);
2345 
2346  /* partial request */
2347  uint8_t request1[] = { 0x47, 0x45, };
2348  p->tcph->th_ack = htonl(1);
2349  p->tcph->th_seq = htonl(1);
2350  p->tcph->th_flags = TH_PUSH | TH_ACK;
2352  p->payload_len = sizeof(request1);
2353  p->payload = request1;
2354  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2365  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2366  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2367  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2369 
2370  /* response ack against partial request */
2371  p->tcph->th_ack = htonl(3);
2372  p->tcph->th_seq = htonl(1);
2373  p->tcph->th_flags = TH_ACK;
2375  p->payload_len = 0;
2376  p->payload = NULL;
2377  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2388  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2389  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2390  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2392 
2393  /* complete partial request */
2394  uint8_t request2[] = {
2395  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2396  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2397  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2398  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2399  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2400  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2401  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2402  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2403  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2404  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2405  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2406  p->tcph->th_ack = htonl(1);
2407  p->tcph->th_seq = htonl(3);
2408  p->tcph->th_flags = TH_PUSH | TH_ACK;
2410  p->payload_len = sizeof(request2);
2411  p->payload = request2;
2412  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2423  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2424  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2425  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2426  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2428 
2429  /* response - request ack */
2430  uint8_t response[] = {
2431  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2432  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2433  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2434  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2435  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2436  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2437  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2438  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2439  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2440  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2441  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2442  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2443  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2444  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2445  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2446  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2447  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2448  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2449  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2450  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2451  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2452  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2453  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2454  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2455  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2456  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2457  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2458  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2459  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2460  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2461  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2462  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2463  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2464  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2465  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2466  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2467  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2468  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2469  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2470  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2471  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2472  p->tcph->th_ack = htonl(88);
2473  p->tcph->th_seq = htonl(1);
2474  p->tcph->th_flags = TH_PUSH | TH_ACK;
2476  p->payload_len = sizeof(response);
2477  p->payload = response;
2478 
2479  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2491  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2492  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2493  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2494  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2495 
2496  /* response ack from request */
2497  p->tcph->th_ack = htonl(328);
2498  p->tcph->th_seq = htonl(88);
2499  p->tcph->th_flags = TH_ACK;
2501  p->payload_len = 0;
2502  p->payload = NULL;
2503  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2515  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2516  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2517  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2518  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2519  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2520 
2521  /* response - acking */
2522  p->tcph->th_ack = htonl(88);
2523  p->tcph->th_seq = htonl(328);
2524  p->tcph->th_flags = TH_PUSH | TH_ACK;
2526  p->payload_len = 0;
2527  p->payload = NULL;
2528  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2540  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2541  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2542  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2543  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2544  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2545 
2546  /* response ack from request */
2547  p->tcph->th_ack = htonl(328);
2548  p->tcph->th_seq = htonl(88);
2549  p->tcph->th_flags = TH_ACK;
2551  p->payload_len = 0;
2552  p->payload = NULL;
2553  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2565  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2566  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2567  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2568  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2569 
2570  /* response - acking the request again*/
2571  p->tcph->th_ack = htonl(88);
2572  p->tcph->th_seq = htonl(328);
2573  p->tcph->th_flags = TH_PUSH | TH_ACK;
2575  p->payload_len = 0;
2576  p->payload = NULL;
2577  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2589  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2590  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2591  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2592  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2593 
2594  /*** New Request ***/
2595 
2596  /* partial request */
2597  p->tcph->th_ack = htonl(328);
2598  p->tcph->th_seq = htonl(88);
2599  p->tcph->th_flags = TH_PUSH | TH_ACK;
2601  p->payload_len = sizeof(request1);
2602  p->payload = request1;
2603  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2615  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2616  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2617  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2618  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2619  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2620 
2621  /* response ack against partial request */
2622  p->tcph->th_ack = htonl(90);
2623  p->tcph->th_seq = htonl(328);
2624  p->tcph->th_flags = TH_ACK;
2626  p->payload_len = 0;
2627  p->payload = NULL;
2628  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2629  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2641  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2642  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2643  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2644  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2645  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2646 
2647  /* complete request */
2648  p->tcph->th_ack = htonl(328);
2649  p->tcph->th_seq = htonl(90);
2650  p->tcph->th_flags = TH_PUSH | TH_ACK;
2652  p->payload_len = sizeof(request2);
2653  p->payload = request2;
2654  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2666  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2667  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2668  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2669  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2670  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2671  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2672 
2673  /* response ack against second partial request */
2674  p->tcph->th_ack = htonl(175);
2675  p->tcph->th_seq = htonl(328);
2676  p->tcph->th_flags = TH_ACK;
2678  p->payload_len = 0;
2679  p->payload = NULL;
2680 
2681  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2693  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2694  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2695  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2696  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2697  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2698  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2699 
2700  /* response acking a request */
2701  p->tcph->th_ack = htonl(175);
2702  p->tcph->th_seq = htonl(328);
2703  p->tcph->th_flags = TH_ACK;
2705  p->payload_len = 0;
2706  p->payload = NULL;
2707  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2713 
2716 
2717  /* request acking a response */
2718  p->tcph->th_ack = htonl(328);
2719  p->tcph->th_seq = htonl(175);
2720  p->tcph->th_flags = TH_ACK;
2722  p->payload_len = 0;
2723  p->payload = NULL;
2724  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2725 
2726  StreamTcpSessionClear(ssn);
2728  SCFree(p);
2729  PASS;
2730 }
2731 
2732 /**
2733  * \test Test to make sure that we sent all the segments from the initial
2734  * segments to app layer until we have detected the app layer proto.
2735  *
2736  * \retval On success it returns 1 and on failure 0.
2737  */
2738 
2739 static int StreamTcpReassembleTest40 (void)
2740 {
2741  Packet *p = PacketGetFromAlloc();
2742  FAIL_IF_NULL(p);
2743  Flow *f = NULL;
2744  TCPHdr tcph;
2745  TcpSession ssn;
2746  PacketQueueNoLock pq;
2747  memset(&pq,0,sizeof(PacketQueueNoLock));
2748  memset(&tcph, 0, sizeof (TCPHdr));
2749  ThreadVars tv;
2750  memset(&tv, 0, sizeof (ThreadVars));
2751 
2754 
2756  FAIL_IF_NULL(ra_ctx);
2757 
2758  uint8_t httpbuf1[] = "P";
2759  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2760  uint8_t httpbuf3[] = "O";
2761  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2762  uint8_t httpbuf4[] = "S";
2763  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2764  uint8_t httpbuf5[] = "T \r\n";
2765  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2766 
2767  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2768  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2769 
2770  SET_ISN(&ssn.server, 9);
2771  ssn.server.last_ack = 10;
2772  SET_ISN(&ssn.client, 9);
2773  ssn.client.isn = 9;
2774 
2775  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2776  FAIL_IF_NULL(f);
2777  f->protoctx = &ssn;
2778  f->proto = IPPROTO_TCP;
2779  p->flow = f;
2780 
2781  tcph.th_win = htons(5480);
2782  tcph.th_seq = htonl(10);
2783  tcph.th_ack = htonl(10);
2784  tcph.th_flags = TH_ACK|TH_PUSH;
2785  p->tcph = &tcph;
2787  p->payload = httpbuf1;
2788  p->payload_len = httplen1;
2789  ssn.state = TCP_ESTABLISHED;
2790  TcpStream *s = &ssn.client;
2791  SCLogDebug("1 -- start");
2792  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2793 
2795  p->payload = httpbuf2;
2796  p->payload_len = httplen2;
2797  tcph.th_seq = htonl(10);
2798  tcph.th_ack = htonl(11);
2799  s = &ssn.server;
2800  ssn.server.last_ack = 11;
2801  SCLogDebug("2 -- start");
2802  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2803 
2805  p->payload = httpbuf3;
2806  p->payload_len = httplen3;
2807  tcph.th_seq = htonl(11);
2808  tcph.th_ack = htonl(55);
2809  s = &ssn.client;
2810  ssn.client.last_ack = 55;
2811  SCLogDebug("3 -- start");
2812  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2813 
2815  p->payload = httpbuf2;
2816  p->payload_len = httplen2;
2817  tcph.th_seq = htonl(55);
2818  tcph.th_ack = htonl(12);
2819  s = &ssn.server;
2820  ssn.server.last_ack = 12;
2821  SCLogDebug("4 -- start");
2822  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2823 
2824  /* check is have the segment in the list and flagged or not */
2825  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2826  FAIL_IF_NULL(seg);
2827  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2828 
2830  p->payload = httpbuf4;
2831  p->payload_len = httplen4;
2832  tcph.th_seq = htonl(12);
2833  tcph.th_ack = htonl(100);
2834  s = &ssn.client;
2835  ssn.client.last_ack = 100;
2836  SCLogDebug("5 -- start");
2837  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2838 
2840  p->payload = httpbuf2;
2841  p->payload_len = httplen2;
2842  tcph.th_seq = htonl(100);
2843  tcph.th_ack = htonl(13);
2844  s = &ssn.server;
2845  ssn.server.last_ack = 13;
2846  SCLogDebug("6 -- start");
2847  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2848 
2850  p->payload = httpbuf5;
2851  p->payload_len = httplen5;
2852  tcph.th_seq = htonl(13);
2853  tcph.th_ack = htonl(145);
2854  s = &ssn.client;
2855  ssn.client.last_ack = 145;
2856  SCLogDebug("7 -- start");
2857  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2858 
2860  p->payload = httpbuf2;
2861  p->payload_len = httplen2;
2862  tcph.th_seq = htonl(145);
2863  tcph.th_ack = htonl(16);
2864  s = &ssn.server;
2865  ssn.server.last_ack = 16;
2866  SCLogDebug("8 -- start");
2867  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2868  FAIL_IF(f->alproto != ALPROTO_HTTP);
2869 
2873  SCFree(p);
2874  UTHFreeFlow(f);
2875  PASS;
2876 }
2877 
2878 /** \test Test the memcap incrementing/decrementing and memcap check */
2879 static int StreamTcpReassembleTest44(void)
2880 {
2882  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2884  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2886  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2888  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2890  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2891  PASS;
2892 }
2893 
2894 /**
2895  * \test Test to make sure that reassembly_depth is enforced.
2896  *
2897  * \retval On success it returns 1 and on failure 0.
2898  */
2899 
2900 static int StreamTcpReassembleTest45 (void)
2901 {
2902  TcpReassemblyThreadCtx *ra_ctx = NULL;
2903  TcpSession ssn;
2904  ThreadVars tv;
2905  memset(&tv, 0, sizeof(tv));
2906  uint8_t payload[100] = {0};
2907  uint16_t payload_size = 100;
2908 
2909  StreamTcpUTInit(&ra_ctx);
2911 
2913  ssn.reassembly_depth = 100;
2914  StreamTcpUTSetupStream(&ssn.server, 100);
2915  StreamTcpUTSetupStream(&ssn.client, 100);
2916 
2917  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2918  FAIL_IF(r != 0);
2920 
2921  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2922  FAIL_IF(r != 0);
2924 
2928  StreamTcpUTDeinit(ra_ctx);
2929  PASS;
2930 }
2931 
2932 /**
2933  * \test Test the unlimited config value of reassembly depth.
2934  *
2935  * \retval On success it returns 1 and on failure 0.
2936  */
2937 
2938 static int StreamTcpReassembleTest46 (void)
2939 {
2940  int result = 0;
2941  TcpReassemblyThreadCtx *ra_ctx = NULL;
2942  TcpSession ssn;
2943  ThreadVars tv;
2944  memset(&tv, 0, sizeof(tv));
2945  uint8_t payload[100] = {0};
2946  uint16_t payload_size = 100;
2947 
2948  StreamTcpUTInit(&ra_ctx);
2950 
2952  StreamTcpUTSetupStream(&ssn.server, 100);
2953  StreamTcpUTSetupStream(&ssn.client, 100);
2954 
2955  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2956  if (r != 0)
2957  goto end;
2959  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2960  goto end;
2961  }
2962 
2963  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2964  if (r != 0)
2965  goto end;
2967  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2968  goto end;
2969  }
2970 
2971  result = 1;
2972 end:
2976  StreamTcpUTDeinit(ra_ctx);
2977  return result;
2978 }
2979 
2980 /**
2981  * \test Test to make sure we detect the sequence wrap around and continue
2982  * stream reassembly properly.
2983  *
2984  * \retval On success it returns 1 and on failure 0.
2985  */
2986 
2987 static int StreamTcpReassembleTest47 (void)
2988 {
2989  Packet *p = PacketGetFromAlloc();
2990  FAIL_IF(unlikely(p == NULL));
2991  Flow *f = NULL;
2992  TCPHdr tcph;
2993  TcpSession ssn;
2994  ThreadVars tv;
2995  PacketQueueNoLock pq;
2996  memset(&pq,0,sizeof(PacketQueueNoLock));
2997  memset(&tcph, 0, sizeof (TCPHdr));
2998  memset(&tv, 0, sizeof (ThreadVars));
3002 
3003  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3004  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3005 
3006  SET_ISN(&ssn.server, 572799781UL);
3007  ssn.server.last_ack = 572799782UL;
3008 
3009  SET_ISN(&ssn.client, 4294967289UL);
3010  ssn.client.last_ack = 21;
3011 
3012  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3013  FAIL_IF(f == NULL);
3014  f->protoctx = &ssn;
3015  f->proto = IPPROTO_TCP;
3016  p->flow = f;
3017 
3018  tcph.th_win = htons(5480);
3019  ssn.state = TCP_ESTABLISHED;
3020  TcpStream *s = NULL;
3021  uint8_t cnt = 0;
3022 
3023  for (cnt=0; cnt < httplen1; cnt++) {
3024  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3025  tcph.th_ack = htonl(572799782UL);
3026  tcph.th_flags = TH_ACK|TH_PUSH;
3027  p->tcph = &tcph;
3029  p->payload = &httpbuf1[cnt];
3030  p->payload_len = 1;
3031  s = &ssn.client;
3032 
3033  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3034 
3036  p->payload = NULL;
3037  p->payload_len = 0;
3038  tcph.th_seq = htonl(572799782UL);
3039  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3040  tcph.th_flags = TH_ACK;
3041  p->tcph = &tcph;
3042  s = &ssn.server;
3043 
3044  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3045  }
3046 
3047  FAIL_IF(f->alproto != ALPROTO_HTTP);
3048 
3052  SCFree(p);
3053  UTHFreeFlow(f);
3054  PASS;
3055 }
3056 
3057 /** \test 3 in order segments in inline reassembly */
3058 static int StreamTcpReassembleInlineTest01(void)
3059 {
3060  int ret = 0;
3061  TcpReassemblyThreadCtx *ra_ctx = NULL;
3062  ThreadVars tv;
3063  TcpSession ssn;
3064  Flow f;
3065 
3066  memset(&tv, 0x00, sizeof(tv));
3067 
3068  StreamTcpUTInit(&ra_ctx);
3071  StreamTcpUTSetupStream(&ssn.client, 1);
3072  FLOW_INITIALIZE(&f);
3073 
3074  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3075  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3076  if (p == NULL) {
3077  printf("couldn't get a packet: ");
3078  goto end;
3079  }
3080  p->tcph->th_seq = htonl(12);
3081  p->flow = &f;
3082 
3083  FLOWLOCK_WRLOCK(&f);
3084  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3085  printf("failed to add segment 1: ");
3086  goto end;
3087  }
3088  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3089  printf("failed to add segment 2: ");
3090  goto end;
3091  }
3092  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3093  printf("failed to add segment 3: ");
3094  goto end;
3095  }
3096  ssn.client.next_seq = 17;
3097  ret = 1;
3098 end:
3099  FLOWLOCK_UNLOCK(&f);
3100  FLOW_DESTROY(&f);
3101  UTHFreePacket(p);
3103  StreamTcpUTDeinit(ra_ctx);
3104  return ret;
3105 }
3106 
3107 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3108  * test the sliding window reassembly.
3109  */
3110 static int StreamTcpReassembleInlineTest02(void)
3111 {
3112  int ret = 0;
3113  TcpReassemblyThreadCtx *ra_ctx = NULL;
3114  ThreadVars tv;
3115  TcpSession ssn;
3116  Flow f;
3117 
3118  memset(&tv, 0x00, sizeof(tv));
3119 
3120  StreamTcpUTInit(&ra_ctx);
3123  StreamTcpUTSetupStream(&ssn.client, 1);
3124  FLOW_INITIALIZE(&f);
3125 
3126  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3127  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3128  if (p == NULL) {
3129  printf("couldn't get a packet: ");
3130  goto end;
3131  }
3132  p->tcph->th_seq = htonl(12);
3133  p->flow = &f;
3134 
3135  FLOWLOCK_WRLOCK(&f);
3136  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3137  printf("failed to add segment 1: ");
3138  goto end;
3139  }
3140  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3141  printf("failed to add segment 2: ");
3142  goto end;
3143  }
3144  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3145  printf("failed to add segment 3: ");
3146  goto end;
3147  }
3148  ssn.client.next_seq = 17;
3149  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3150  printf("failed to add segment 4: ");
3151  goto end;
3152  }
3153  ssn.client.next_seq = 22;
3154  ret = 1;
3155 end:
3156  FLOWLOCK_UNLOCK(&f);
3157  FLOW_DESTROY(&f);
3158  UTHFreePacket(p);
3160  StreamTcpUTDeinit(ra_ctx);
3161  return ret;
3162 }
3163 
3164 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3165  * test the sliding window reassembly with a small window size so that we
3166  * cutting off at the start (left edge)
3167  */
3168 static int StreamTcpReassembleInlineTest03(void)
3169 {
3170  int ret = 0;
3171  TcpReassemblyThreadCtx *ra_ctx = NULL;
3172  ThreadVars tv;
3173  TcpSession ssn;
3174  Flow f;
3175 
3176  memset(&tv, 0x00, sizeof(tv));
3177 
3178  StreamTcpUTInit(&ra_ctx);
3181  StreamTcpUTSetupStream(&ssn.client, 1);
3182  FLOW_INITIALIZE(&f);
3183 
3185 
3186  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3187  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3188  if (p == NULL) {
3189  printf("couldn't get a packet: ");
3190  goto end;
3191  }
3192  p->tcph->th_seq = htonl(12);
3193  p->flow = &f;
3195 
3196  FLOWLOCK_WRLOCK(&f);
3197  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3198  printf("failed to add segment 1: ");
3199  goto end;
3200  }
3201  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3202  printf("failed to add segment 2: ");
3203  goto end;
3204  }
3205  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3206  printf("failed to add segment 3: ");
3207  goto end;
3208  }
3209  ssn.client.next_seq = 17;
3210  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3211  printf("failed to add segment 4: ");
3212  goto end;
3213  }
3214  ssn.client.next_seq = 22;
3215 
3216  p->tcph->th_seq = htonl(17);
3217  ret = 1;
3218 end:
3219  FLOWLOCK_UNLOCK(&f);
3220  FLOW_DESTROY(&f);
3221  UTHFreePacket(p);
3223  StreamTcpUTDeinit(ra_ctx);
3224  return ret;
3225 }
3226 
3227 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3228  * test the sliding window reassembly with a small window size so that we
3229  * cutting off at the start (left edge) with small packet overlap.
3230  */
3231 static int StreamTcpReassembleInlineTest04(void)
3232 {
3233  int ret = 0;
3234  TcpReassemblyThreadCtx *ra_ctx = NULL;
3235  ThreadVars tv;
3236  TcpSession ssn;
3237  Flow f;
3238 
3239  memset(&tv, 0x00, sizeof(tv));
3240 
3241  StreamTcpUTInit(&ra_ctx);
3244  StreamTcpUTSetupStream(&ssn.client, 1);
3245  FLOW_INITIALIZE(&f);
3246 
3248 
3249  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3250  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3251  if (p == NULL) {
3252  printf("couldn't get a packet: ");
3253  goto end;
3254  }
3255  p->tcph->th_seq = htonl(12);
3256  p->flow = &f;
3258 
3259  FLOWLOCK_WRLOCK(&f);
3260  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3261  printf("failed to add segment 1: ");
3262  goto end;
3263  }
3264  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3265  printf("failed to add segment 2: ");
3266  goto end;
3267  }
3268  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3269  printf("failed to add segment 3: ");
3270  goto end;
3271  }
3272  ssn.client.next_seq = 17;
3273  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3274  printf("failed to add segment 4: ");
3275  goto end;
3276  }
3277  ssn.client.next_seq = 22;
3278 
3279  p->tcph->th_seq = htonl(17);
3280  ret = 1;
3281 end:
3282  FLOWLOCK_UNLOCK(&f);
3283  FLOW_DESTROY(&f);
3284  UTHFreePacket(p);
3286  StreamTcpUTDeinit(ra_ctx);
3287  return ret;
3288 }
3289 
3290 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3291  * test the sliding window reassembly with a small window size so that we
3292  * cutting off at the start (left edge). Test if the first segment is
3293  * removed from the list.
3294  */
3295 static int StreamTcpReassembleInlineTest08(void)
3296 {
3297  TcpReassemblyThreadCtx *ra_ctx = NULL;
3298  ThreadVars tv;
3299  memset(&tv, 0x00, sizeof(tv));
3300  TcpSession ssn;
3301  Flow f;
3302  StreamTcpUTInit(&ra_ctx);
3305  StreamTcpUTSetupStream(&ssn.client, 1);
3306  FLOW_INITIALIZE(&f);
3307 
3310  f.protoctx = &ssn;
3311 
3312  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3313  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3314  FAIL_IF(p == NULL);
3315  p->tcph->th_seq = htonl(12);
3316  p->flow = &f;
3318 
3319  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3320  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3321  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3322  ssn.client.next_seq = 17;
3323  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3324  ssn.client.next_seq = 22;
3325  p->tcph->th_seq = htonl(17);
3327 
3328  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3329  FAIL_IF_NULL(seg);
3330  FAIL_IF_NOT(seg->seq == 2);
3331 
3332  FLOW_DESTROY(&f);
3333  UTHFreePacket(p);
3335  StreamTcpUTDeinit(ra_ctx);
3336  PASS;
3337 }
3338 
3339 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3340  * test the sliding window reassembly with a small window size so that we
3341  * cutting off at the start (left edge). Test if the first segment is
3342  * removed from the list.
3343  */
3344 static int StreamTcpReassembleInlineTest09(void)
3345 {
3346  int ret = 0;
3347  TcpReassemblyThreadCtx *ra_ctx = NULL;
3348  ThreadVars tv;
3349  TcpSession ssn;
3350  Flow f;
3351 
3352  memset(&tv, 0x00, sizeof(tv));
3353 
3354  StreamTcpUTInit(&ra_ctx);
3357  StreamTcpUTSetupStream(&ssn.client, 1);
3358  FLOW_INITIALIZE(&f);
3359 
3362 
3363  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3364  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3365  if (p == NULL) {
3366  printf("couldn't get a packet: ");
3367  goto end;
3368  }
3369  p->tcph->th_seq = htonl(17);
3370  p->flow = &f;
3372 
3373  FLOWLOCK_WRLOCK(&f);
3374  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3375  printf("failed to add segment 1: ");
3376  goto end;
3377  }
3378  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3379  printf("failed to add segment 2: ");
3380  goto end;
3381  }
3382  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3383  printf("failed to add segment 3: ");
3384  goto end;
3385  }
3386  ssn.client.next_seq = 12;
3387  ssn.client.last_ack = 10;
3388 
3389  /* close the GAP and see if we properly reassemble and update base_seq */
3390  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3391  printf("failed to add segment 4: ");
3392  goto end;
3393  }
3394  ssn.client.next_seq = 22;
3395 
3396  p->tcph->th_seq = htonl(12);
3397 
3398  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3399  FAIL_IF_NULL(seg);
3400  FAIL_IF_NOT(seg->seq == 2);
3401 
3402  ret = 1;
3403 end:
3404  FLOWLOCK_UNLOCK(&f);
3405  FLOW_DESTROY(&f);
3406  UTHFreePacket(p);
3408  StreamTcpUTDeinit(ra_ctx);
3409  return ret;
3410 }
3411 
3412 /** \test App Layer reassembly.
3413  */
3414 static int StreamTcpReassembleInlineTest10(void)
3415 {
3416  int ret = 0;
3417  TcpReassemblyThreadCtx *ra_ctx = NULL;
3418  ThreadVars tv;
3419  TcpSession ssn;
3420  Flow *f = NULL;
3421  Packet *p = NULL;
3422 
3423  memset(&tv, 0x00, sizeof(tv));
3424 
3425  StreamTcpUTInit(&ra_ctx);
3428  StreamTcpUTSetupStream(&ssn.server, 1);
3429  ssn.server.last_ack = 2;
3430  StreamTcpUTSetupStream(&ssn.client, 1);
3431  ssn.client.last_ack = 2;
3433 
3434  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3435  if (f == NULL)
3436  goto end;
3437  f->protoctx = &ssn;
3438  f->proto = IPPROTO_TCP;
3439 
3440  uint8_t stream_payload1[] = "GE";
3441  uint8_t stream_payload2[] = "T /";
3442  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3443 
3444  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3445  if (p == NULL) {
3446  printf("couldn't get a packet: ");
3447  goto end;
3448  }
3449  p->tcph->th_seq = htonl(7);
3450  p->flow = f;
3452 
3453  FLOWLOCK_WRLOCK(f);
3454  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3455  printf("failed to add segment 1: ");
3456  goto end;
3457  }
3458  ssn.client.next_seq = 4;
3459 
3460  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3461  if (r < 0) {
3462  printf("StreamTcpReassembleAppLayer failed: ");
3463  goto end;
3464  }
3465 
3466  /* ssn.server.ra_app_base_seq should be isn here. */
3467  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3468  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3469  goto end;
3470  }
3471 
3472  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3473  printf("failed to add segment 2: ");
3474  goto end;
3475  }
3476  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3477  printf("failed to add segment 3: ");
3478  goto end;
3479  }
3480  ssn.client.next_seq = 19;
3481 
3482  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3483  if (r < 0) {
3484  printf("StreamTcpReassembleAppLayer failed: ");
3485  goto end;
3486  }
3487 
3488  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3489 
3490  ret = 1;
3491 end:
3492  UTHFreePacket(p);
3494  StreamTcpUTDeinit(ra_ctx);
3495  FLOWLOCK_UNLOCK(f);
3496  UTHFreeFlow(f);
3497  return ret;
3498 }
3499 
3500 /** \test test insert with overlap
3501  */
3502 static int StreamTcpReassembleInsertTest01(void)
3503 {
3504  TcpReassemblyThreadCtx *ra_ctx = NULL;
3505  ThreadVars tv;
3506  TcpSession ssn;
3507  Flow f;
3508 
3509  memset(&tv, 0x00, sizeof(tv));
3510 
3511  StreamTcpUTInit(&ra_ctx);
3513  StreamTcpUTSetupStream(&ssn.client, 1);
3515  FLOW_INITIALIZE(&f);
3516 
3517  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3518  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3519  FAIL_IF(p == NULL);
3520  p->tcph->th_seq = htonl(12);
3521  p->flow = &f;
3522 
3523  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3524  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3525  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3526  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3527  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3528  ssn.client.next_seq = 21;
3529 
3530  FLOW_DESTROY(&f);
3531  UTHFreePacket(p);
3533  StreamTcpUTDeinit(ra_ctx);
3534  PASS;
3535 }
3536 
3537 /** \test test insert with overlaps
3538  */
3539 static int StreamTcpReassembleInsertTest02(void)
3540 {
3541  int ret = 0;
3542  TcpReassemblyThreadCtx *ra_ctx = NULL;
3543  ThreadVars tv;
3544  TcpSession ssn;
3545 
3546  memset(&tv, 0x00, sizeof(tv));
3547 
3548  StreamTcpUTInit(&ra_ctx);
3550  StreamTcpUTSetupStream(&ssn.client, 1);
3551 
3552  int i;
3553  for (i = 2; i < 10; i++) {
3554  int len;
3555  len = i % 2;
3556  if (len == 0)
3557  len = 1;
3558  int seq;
3559  seq = i * 10;
3560  if (seq < 2)
3561  seq = 2;
3562 
3563  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3564  printf("failed to add segment 1: ");
3565  goto end;
3566  }
3567  }
3568  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3569  printf("failed to add segment 2: ");
3570  goto end;
3571  }
3572 
3573  ret = 1;
3574 end:
3576  StreamTcpUTDeinit(ra_ctx);
3577  return ret;
3578 }
3579 
3580 /** \test test insert with overlaps
3581  */
3582 static int StreamTcpReassembleInsertTest03(void)
3583 {
3584  int ret = 0;
3585  TcpReassemblyThreadCtx *ra_ctx = NULL;
3586  ThreadVars tv;
3587  TcpSession ssn;
3588 
3589  memset(&tv, 0x00, sizeof(tv));
3590 
3591  StreamTcpUTInit(&ra_ctx);
3593  StreamTcpUTSetupStream(&ssn.client, 1);
3594 
3595  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3596  printf("failed to add segment 2: ");
3597  goto end;
3598  }
3599 
3600  int i;
3601  for (i = 2; i < 10; i++) {
3602  int len;
3603  len = i % 2;
3604  if (len == 0)
3605  len = 1;
3606  int seq;
3607  seq = i * 10;
3608  if (seq < 2)
3609  seq = 2;
3610 
3611  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3612  printf("failed to add segment 2: ");
3613  goto end;
3614  }
3615  }
3616  ret = 1;
3617 end:
3619  StreamTcpUTDeinit(ra_ctx);
3620  return ret;
3621 }
3622 
3624 #endif /* UNITTESTS */
3625 
3626 /** \brief The Function Register the Unit tests to test the reassembly engine
3627  * for various OS policies.
3628  */
3629 
3631 {
3632 #ifdef UNITTESTS
3633  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3634  StreamTcpReassembleTest25);
3635  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3636  StreamTcpReassembleTest26);
3637  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3638  StreamTcpReassembleTest27);
3639  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3640  StreamTcpReassembleTest28);
3641  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3642  StreamTcpReassembleTest29);
3643  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3644  StreamTcpReassembleTest33);
3645  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3646  StreamTcpReassembleTest34);
3647  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3648  StreamTcpReassembleTest37);
3649  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3650  StreamTcpReassembleTest39);
3651  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3652  StreamTcpReassembleTest40);
3653  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3654  StreamTcpReassembleTest44);
3655  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3656  StreamTcpReassembleTest45);
3657  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3658  StreamTcpReassembleTest46);
3659  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3660  StreamTcpReassembleTest47);
3661 
3662  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3663  StreamTcpReassembleInlineTest01);
3664  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3665  StreamTcpReassembleInlineTest02);
3666  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3667  StreamTcpReassembleInlineTest03);
3668  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3669  StreamTcpReassembleInlineTest04);
3670  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3671  StreamTcpReassembleInlineTest08);
3672  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3673  StreamTcpReassembleInlineTest09);
3674 
3675  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3676  StreamTcpReassembleInlineTest10);
3677 
3678  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3679  StreamTcpReassembleInsertTest01);
3680  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3681  StreamTcpReassembleInsertTest02);
3682  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3683  StreamTcpReassembleInsertTest03);
3684 
3688  StreamTcpReassembleRawRegisterTests();
3689 #endif /* UNITTESTS */
3690 }
STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION
Definition: stream-tcp.h:172
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:249
APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
PoolThreadInit
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
per thread Pool, initialization function
Definition: util-pool-thread.c:43
UPDATE_DIR_BOTH
@ UPDATE_DIR_BOTH
Definition: stream-tcp-reassemble.h:57
UPDATE_DIR_PACKET
@ UPDATE_DIR_PACKET
Definition: stream-tcp-reassemble.h:55
util-byte.h
tm-threads.h
StreamTcpUTAddPayload
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
Definition: stream-tcp-util.c:97
Packet_::proto
uint8_t proto
Definition: decode.h:433
TcpStream_
Definition: stream-tcp-private.h:94
TCP_FIN_WAIT1
@ TCP_FIN_WAIT1
Definition: stream-tcp-private.h:144
len
uint8_t len
Definition: app-layer-dnp3.h:2
StreamTcpUTDeinit
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-util.c:51
StreamTcpInlineRegisterTests
void StreamTcpInlineRegisterTests(void)
Definition: stream-tcp-inline.c:149
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:101
TcpStream_::isn
uint32_t isn
Definition: stream-tcp-private.h:101
StreamTcpGetSegment
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
Definition: stream-tcp-reassemble.c:1833
StreamingBufferConfig_::Malloc
void *(* Malloc)(size_t size)
Definition: util-streaming-buffer.h:71
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:169
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:301
PoolThreadExpand
int PoolThreadExpand(PoolThread *pt)
expand pool by one for a new thread
Definition: util-pool-thread.c:92
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:70
StreamReassembleRaw
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
Definition: stream-tcp-reassemble.c:1679
STREAM_MIDSTREAM
#define STREAM_MIDSTREAM
Definition: stream.h:35
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
TCP_HAS_TFO
#define TCP_HAS_TFO(p)
Definition: decode-tcp.h:95
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:82
stream-tcp-inline.h
StreamTcpUTClearStream
void StreamTcpUTClearStream(TcpStream *s)
Definition: stream-tcp-util.c:91
flow-util.h
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:286
StreamingBufferConfig_::Calloc
void *(* Calloc)(size_t n, size_t size)
Definition: util-streaming-buffer.h:72
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:104
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
stream-tcp.h
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:355
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
StreamTcpReassembleDepthReached
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
Definition: stream-tcp-reassemble.c:486
StreamTcpReassembleGetMemcap
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
Definition: stream-tcp-reassemble.c:174
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:557
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:123
SET_ISN
#define SET_ISN(stream, setseq)
Definition: stream-tcp-reassemble.c:1897
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:564
seq
uint32_t seq
Definition: stream-tcp-private.h:2
StatsRegisterGlobalCounter
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1003
StreamTcpThread_
Definition: stream-tcp.h:70
Flow_::proto
uint8_t proto
Definition: flow.h:361
Packet_::payload
uint8_t * payload
Definition: decode.h:543
StreamTcpInlineMode
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6259
StreamTcpReassemblySetMinInspectDepth
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
Definition: stream-tcp-reassemble.c:1877
TcpReassemblyThreadCtx_::app_tctx
void * app_tctx
Definition: stream-tcp-reassemble.h:61
Packet_::flags
uint32_t flags
Definition: decode.h:446
ConfGetNode
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
threads.h
TH_RST
#define TH_RST
Definition: decode-tcp.h:36
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:98
TcpStreamCnf_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp.h:60
Flow_
Flow data structure.
Definition: flow.h:343
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:66
TcpReassemblyThreadCtx_::counter_tcp_stream_depth
uint16_t counter_tcp_stream_depth
Definition: stream-tcp-reassemble.h:68
StreamTcpReassembleTriggerRawReassembly
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
Definition: stream-tcp-reassemble.c:1860
TH_FIN
#define TH_FIN
Definition: decode-tcp.h:34
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:304
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:38
TcpStreamCnf_::sbcnf
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
TCP_CLOSING
@ TCP_CLOSING
Definition: stream-tcp-private.h:149
StreamTcpUtilRegisterTests
void StreamTcpUtilRegisterTests(void)
Definition: stream-tcp-util.c:245
StreamReassembleRawUpdateProgress
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
Definition: stream-tcp-reassemble.c:1275
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:849
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
FLOW_PKT_TOSERVER
#define FLOW_PKT_TOSERVER
Definition: flow.h:218
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
StreamNeedsReassembly
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
Definition: stream-tcp-reassemble.c:782
MIN
#define MIN(x, y)
Definition: suricata-common.h:377
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:64
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:95
SCMUTEX_INITIALIZER
#define SCMUTEX_INITIALIZER
Definition: threads-debug.h:121
StreamingBufferConfig_::flags
uint32_t flags
Definition: util-streaming-buffer.h:68
PoolThreadFree
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
Definition: util-pool-thread.c:150
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:866
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:442
UTHBuildPacketReal
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
Definition: util-unittest-helper.c:241
Flow_::protoctx
void * protoctx
Definition: flow.h:416
StreamTcpListRegisterTests
void StreamTcpListRegisterTests(void)
Definition: stream-tcp-list.c:711
StreamTcpUTInitInline
void StreamTcpUTInitInline(void)
Definition: stream-tcp-util.c:58
FLOW_IPV4
#define FLOW_IPV4
Definition: flow.h:94
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:544
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:32
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:204
Packet_::app_layer_events
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:570
util-unittest.h
MISSED_STEP
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
Definition: stream-tcp-reassemble.c:1969
util-unittest-helper.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
TcpSession_::flags
uint16_t flags
Definition: stream-tcp-private.h:269
FLOWLOCK_UNLOCK
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:260
STREAM_START
#define STREAM_START
Definition: stream.h:29
STREAMTCP_FLAG_MIDSTREAM
#define STREAMTCP_FLAG_MIDSTREAM
Definition: stream-tcp-private.h:158
FLOW_IS_PM_DONE
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:265
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:103
StreamTcpReassembleInitThreadCtx
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
Definition: stream-tcp-reassemble.c:429
StreamTcpUTInit
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
Definition: stream-tcp-util.c:44
UTHBuildFlow
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
Definition: util-unittest-helper.c:520
StreamTcpReassembleAppLayer
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
Definition: stream-tcp-reassemble.c:1118
TCP_TIME_WAIT
@ TCP_TIME_WAIT
Definition: stream-tcp-private.h:146
FLOW_INITIALIZE
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
TcpSession_::reassembly_depth
uint32_t reassembly_depth
Definition: stream-tcp-private.h:270
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
util-device.h
util-debug.h
StreamTcpAppLayerIsDisabled
int StreamTcpAppLayerIsDisabled(Flow *f)
Definition: stream-tcp-reassemble.c:348
AppLayerHandleTCPData
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:570
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:260
STREAM_HAS_SEEN_DATA
#define STREAM_HAS_SEEN_DATA(stream)
Definition: stream-tcp-private.h:92
FLOW_IS_PP_DONE
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:266
PKT_STREAM_ADD
#define PKT_STREAM_ADD
Definition: decode.h:1084
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1088
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:131
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
FLOWLOCK_WRLOCK
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:257
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:118
StreamReassembleLog
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
Definition: stream-tcp-reassemble.c:1707
util-print.h
APPLAYER_PROTO_DETECTION_SKIPPED
@ APPLAYER_PROTO_DETECTION_SKIPPED
Definition: app-layer-events.h:50
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
AppLayerGetCtxThread
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:830
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
StreamTcpPacket
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueueNoLock *pq)
Definition: stream-tcp.c:4763
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:150
StreamTcpUTSetupStream
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
Definition: stream-tcp-util.c:79
StreamReassembleRawHasDataReady
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
Definition: stream-tcp-reassemble.c:1231
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:262
TH_ACK
#define TH_ACK
Definition: decode-tcp.h:38
StreamTcpReassembleInitMemuse
void StreamTcpReassembleInitMemuse(void)
Definition: stream-tcp-reassemble.c:86
stream-tcp-list.h
StreamTcpCreateTestPacket
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
Definition: stream-tcp-reassemble.c:1911
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:801
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
SC_ERR_INVALID_ARGUMENT
@ SC_ERR_INVALID_ARGUMENT
Definition: util-error.h:43
StreamTcpReassembleCheckMemcap
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
Definition: stream-tcp-reassemble.c:145
TRUE
#define TRUE
Definition: suricata-common.h:33
StreamTcpReassembleHandleSegment
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueueNoLock *pq)
Definition: stream-tcp-reassemble.c:1735
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:119
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:282
TCP_GET_SEQ
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:112
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
stream-tcp-reassemble.c
util-profiling.h
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:313
StreamTcpSessionClear
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
FALSE
#define FALSE
Definition: suricata-common.h:34
SCReturn
#define SCReturn
Definition: util-debug.h:302
stream.h
TcpSegment
Definition: stream-tcp-private.h:61
StreamTcpReassembleFree
void StreamTcpReassembleFree(char quiet)
Definition: stream-tcp-reassemble.c:410
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:278
Packet_
Definition: decode.h:411
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
Definition: stream-tcp-private.h:215
STREAM_DEPTH
#define STREAM_DEPTH
Definition: stream.h:34
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
StreamTcpFreeConfig
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:669
stream-tcp-private.h
PoolThreadReturn
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
Definition: util-pool-thread.c:187
SCReturnUInt
#define SCReturnUInt(x)
Definition: util-debug.h:306
TCP_ESTABLISHED
@ TCP_ESTABLISHED
Definition: stream-tcp-private.h:143
conf.h
StreamTcpSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:276
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:889
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:105
StreamTcpReassembleFreeThreadCtx
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
Definition: stream-tcp-reassemble.c:470
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
STREAM_GAP
#define STREAM_GAP
Definition: stream.h:33
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:314
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:219
RunmodeIsUnittests
int RunmodeIsUnittests(void)
Definition: suricata.c:264
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
StreamTcpReassembleInit
int StreamTcpReassembleInit(char quiet)
Definition: stream-tcp-reassemble.c:394
TcpReassemblyThreadCtx_::counter_tcp_reass_gap
uint16_t counter_tcp_reass_gap
Definition: stream-tcp-reassemble.h:70
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:115
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:79
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
util-host-os-info.h
UTHFreeFlow
void UTHFreeFlow(Flow *flow)
Definition: util-unittest-helper.c:525
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
UPDATE_DIR_OPPOSING
@ UPDATE_DIR_OPPOSING
Definition: stream-tcp-reassemble.h:56
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:103
StreamingBuffer_
Definition: util-streaming-buffer.h:95
StreamTcpUTClearSession
void StreamTcpUTClearSession(TcpSession *ssn)
Definition: stream-tcp-util.c:71
StreamTcpReassembleHandleSegmentHandleData
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
Definition: stream-tcp-reassemble.c:593
TcpReassemblyThreadCtx_::segment_thread_pool_id
int segment_thread_pool_id
Definition: stream-tcp-reassemble.h:63
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:769
StringParseUint32
int StringParseUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:304
TH_PUSH
#define TH_PUSH
Definition: decode-tcp.h:37
Packet_::flow
struct Flow_ * flow
Definition: decode.h:448
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:241
TH_SYN
#define TH_SYN
Definition: decode-tcp.h:35
StreamTcpDisableAppLayer
void StreamTcpDisableAppLayer(Flow *f)
Definition: stream-tcp-reassemble.c:336
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:240
Packet_::tcph
TCPHdr * tcph
Definition: decode.h:525
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:112
AppLayerDecoderEventsSetEventRaw
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
Definition: app-layer-events.c:89
MISSED_START
#define MISSED_START(isn)
Definition: stream-tcp-reassemble.c:1950
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:122
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
Definition: stream-tcp-private.h:217
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
StreamTcpUTAddSegmentWithPayload
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
Definition: stream-tcp-util.c:113
STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
@ STREAM_HAS_UNPROCESSED_SEGMENTS_NONE
Definition: stream-tcp.h:169
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
app-layer-events.h
TcpStream_::next_seq
uint32_t next_seq
Definition: stream-tcp-private.h:102
threadvars.h
util-validate.h
PacketGetFromAlloc
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:144
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
StreamTcpUTSetupSession
void StreamTcpUTSetupSession(TcpSession *ssn)
Definition: stream-tcp-util.c:62
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
StreamTcpReassembleIncrMemuse
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:97
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:426
ConfNode_
Definition: conf.h:32
UTHFreePacket
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself.
Definition: util-unittest-helper.c:484
StreamTcpUTAddSegmentWithByte
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
Definition: stream-tcp-util.c:136
AppLayerDestroyCtxThread
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:852
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:238
Flow_::flags
uint32_t flags
Definition: flow.h:396
StreamTcpSetOSPolicy
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet.
Definition: stream-tcp.c:767
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:74
payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:1
PoolThread_
Definition: util-pool-thread.h:50
ALPROTO_HTTP
@ ALPROTO_HTTP
Definition: app-layer-protos.h:30
StreamTcpReassembleMemuseGlobalCounter
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Definition: stream-tcp-reassemble.c:131
StreamTcpDisableAppLayerReassembly
#define StreamTcpDisableAppLayerReassembly(ssn)
Definition: stream-tcp-private.h:282
MISSED_END
#define MISSED_END
Definition: stream-tcp-reassemble.c:1964
stream-tcp-util.h
StreamTcpInitConfig
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
StreamTcpThread_::ra_ctx
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
STREAM_EOF
#define STREAM_EOF
Definition: stream.h:30
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:60
TcpStreamCnf_::prealloc_segments
uint32_t prealloc_segments
Definition: stream-tcp.h:57
app-layer-protos.h
PoolThreadGetById
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
Definition: util-pool-thread.c:168
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:187
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:200
suricata.h
Address_::family
char family
Definition: decode.h:114
Packet_::dst
Address dst
Definition: decode.h:416
STREAM_REASSEMBLY_SEQ_GAP
@ STREAM_REASSEMBLY_SEQ_GAP
Definition: decode-events.h:261
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:239
StreamUpdateDir
StreamUpdateDir
Definition: stream-tcp-reassemble.h:54
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:219
STREAMING_BUFFER_NOFLAGS
#define STREAMING_BUFFER_NOFLAGS
Definition: util-streaming-buffer.h:64
STREAMTCP_STREAM_FLAG_GAP
#define STREAMTCP_STREAM_FLAG_GAP
Definition: stream-tcp-private.h:198
StreamTcpReassembleRegisterTests
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition: stream-tcp-reassemble.c:3630
STREAM_RIGHT_EDGE
#define STREAM_RIGHT_EDGE(stream)
Definition: stream-tcp-private.h:90
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:132
StreamTcpReassembleDecrMemuse
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
Definition: stream-tcp-reassemble.c:110
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:347
TcpSession_
Definition: stream-tcp-private.h:260
TcpSession_::data_first_seen_dir
int8_t data_first_seen_dir
Definition: stream-tcp-private.h:265
StreamTcpReassembleSetMemcap
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
Definition: stream-tcp-reassemble.c:159
flow.h
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:427
StreamReassembleRawFunc
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:425
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
Definition: stream-tcp-private.h:206
PoolThreadSize
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of 'threads', so array elements)
Definition: util-pool-thread.c:143
STREAM_REASSEMBLY_NO_SEGMENT
@ STREAM_REASSEMBLY_NO_SEGMENT
Definition: decode-events.h:260
util-pool.h
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:85
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:83
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:66
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:743
SCMutex
#define SCMutex
Definition: threads-debug.h:114
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:82
FLOW_DESTROY
#define FLOW_DESTROY(f)
Definition: flow-util.h:121
Packet_::src
Address src
Definition: decode.h:415
debug.h
StreamingBufferConfig_::Realloc
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
Definition: util-streaming-buffer.h:73
STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
Definition: stream-tcp-private.h:213
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:133
app-layer.h
StreamTcpCheckStreamContents
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
Definition: stream-tcp-reassemble.c:1928