suricata
stream-tcp-reassemble.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22  * \author Victor Julien <victor@inliniac.net>
23  *
24  * Reference:
25  * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26  *
27  */
28 
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36 
37 #include "flow-util.h"
38 
39 #include "threadvars.h"
40 #include "tm-threads.h"
41 
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49 
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56 
57 #include "stream.h"
58 
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 
64 #include "detect-engine-state.h"
65 
66 #include "util-profiling.h"
67 #include "util-validate.h"
68 
69 #ifdef DEBUG
70 static SCMutex segment_pool_memuse_mutex;
71 static uint64_t segment_pool_memuse = 0;
72 static uint64_t segment_pool_memcnt = 0;
73 #endif
74 
75 static PoolThread *segment_thread_pool = NULL;
76 /* init only, protect initializing and growing pool */
77 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
78 
79 /* Memory use counter */
80 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
81 
82 /* prototypes */
84 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86 
88 {
89  SC_ATOMIC_INIT(ra_memuse);
90 }
91 
92 /**
93  * \brief Function to Increment the memory usage counter for the TCP reassembly
94  * segments
95  *
96  * \param size Size of the TCP segment and its payload length memory allocated
97  */
98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100  (void) SC_ATOMIC_ADD(ra_memuse, size);
101  SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102  return;
103 }
104 
105 /**
106  * \brief Function to Decrease the memory usage counter for the TCP reassembly
107  * segments
108  *
109  * \param size Size of the TCP segment and its payload length memory allocated
110  */
111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114  uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115  if (RunmodeIsUnittests()) {
116  BUG_ON(presize > UINT_MAX);
117  }
118 #endif
119 
120  (void) SC_ATOMIC_SUB(ra_memuse, size);
121 
122 #ifdef UNITTESTS
123  if (RunmodeIsUnittests()) {
124  uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125  BUG_ON(postsize > presize);
126  }
127 #endif
128  SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129  return;
130 }
131 
133 {
134  uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135  return smemuse;
136 }
137 
138 /**
139  * \brief Function to Check the reassembly memory usage counter against the
140  * allowed max memory usgae for TCP segments.
141  *
142  * \param size Size of the TCP segment and its payload length memory allocated
143  * \retval 1 if in bounds
144  * \retval 0 if not in bounds
145  */
147 {
148  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149  if (memcapcopy == 0 ||
150  (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151  return 1;
152  return 0;
153 }
154 
155 /**
156  * \brief Update memcap value
157  *
158  * \param size new memcap value
159  */
161 {
162  if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163  SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164  return 1;
165  }
166 
167  return 0;
168 }
169 
170 /**
171  * \brief Return memcap value
172  *
173  * \return memcap memcap value
174  */
176 {
177  uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178  return memcapcopy;
179 }
180 
181 /* memory functions for the streaming buffer API */
182 
183 /*
184  void *(*Malloc)(size_t size);
185 */
186 static void *ReassembleMalloc(size_t size)
187 {
188  if (StreamTcpReassembleCheckMemcap(size) == 0)
189  return NULL;
190  void *ptr = SCMalloc(size);
191  if (ptr == NULL)
192  return NULL;
194  return ptr;
195 }
196 
197 /*
198  void *(*Calloc)(size_t n, size_t size);
199 */
200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202  if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203  return NULL;
204  void *ptr = SCCalloc(n, size);
205  if (ptr == NULL)
206  return NULL;
208  return ptr;
209 }
210 
211 /*
212  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216  if (size > orig_size) {
217  if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218  return NULL;
219  }
220  void *nptr = SCRealloc(optr, size);
221  if (nptr == NULL)
222  return NULL;
223 
224  if (size > orig_size) {
225  StreamTcpReassembleIncrMemuse(size - orig_size);
226  } else {
227  StreamTcpReassembleDecrMemuse(orig_size - size);
228  }
229  return nptr;
230 }
231 
232 /*
233  void (*Free)(void *ptr, size_t size);
234 */
235 static void ReassembleFree(void *ptr, size_t size)
236 {
237  SCFree(ptr);
239 }
240 
241 /** \brief alloc a tcp segment pool entry */
242 static void *TcpSegmentPoolAlloc(void)
243 {
244  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245  return NULL;
246  }
247 
248  TcpSegment *seg = NULL;
249 
250  seg = SCMalloc(sizeof (TcpSegment));
251  if (unlikely(seg == NULL))
252  return NULL;
253  return seg;
254 }
255 
256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258  TcpSegment *seg = (TcpSegment *) data;
259 
260  /* do this before the can bail, so TcpSegmentPoolCleanup
261  * won't have uninitialized memory to consider. */
262  memset(seg, 0, sizeof (TcpSegment));
263 
264  if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265  return 0;
266  }
267 
268 #ifdef DEBUG
269  SCMutexLock(&segment_pool_memuse_mutex);
270  segment_pool_memuse += sizeof(TcpSegment);
271  segment_pool_memcnt++;
272  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273  SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275 
276  StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277  return 1;
278 }
279 
280 /** \brief clean up a tcp segment pool entry */
281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283  if (ptr == NULL)
284  return;
285 
286  StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287 
288 #ifdef DEBUG
289  SCMutexLock(&segment_pool_memuse_mutex);
290  segment_pool_memuse -= sizeof(TcpSegment);
291  segment_pool_memcnt--;
292  SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293  SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296 
297 /**
298  * \brief Function to return the segment back to the pool.
299  *
300  * \param seg Segment which will be returned back to the pool.
301  */
303 {
304  if (seg == NULL)
305  return;
306 
307  PoolThreadReturn(segment_thread_pool, seg);
308 }
309 
310 /**
311  * \brief return all segments in this stream into the pool(s)
312  *
313  * \param stream the stream to cleanup
314  */
316 {
317  TcpSegment *seg = NULL, *safe = NULL;
318  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319  {
320  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
322  }
323 }
324 
325 #ifdef UNITTESTS
326 /** \internal
327  * \brief check if segments falls before stream 'offset' */
328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330  if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331  return 1;
332  return 0;
333 }
334 #endif
335 
336 /** \param f locked flow */
338 {
339  if (f->protoctx == NULL)
340  return;
341 
342  TcpSession *ssn = (TcpSession *)f->protoctx;
346 }
347 
348 /** \param f locked flow */
350 {
351  if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
352  return 0;
353 
354  TcpSession *ssn = (TcpSession *)f->protoctx;
356 }
357 
358 static int StreamTcpReassemblyConfig(char quiet)
359 {
360  uint32_t segment_prealloc = 2048;
361  ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
362  if (seg) {
363  uint32_t prealloc = 0;
364  if (ByteExtractStringUint32(&prealloc, 10, strlen(seg->val), seg->val) == -1)
365  {
366  SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
367  "%s is invalid", seg->val);
368  return -1;
369  }
370  segment_prealloc = prealloc;
371  }
372  if (!quiet)
373  SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
374  stream_config.prealloc_segments = segment_prealloc;
375 
376  int overlap_diff_data = 0;
377  ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
378  if (overlap_diff_data) {
380  }
381  if (StreamTcpInlineMode() == TRUE) {
383  }
384 
387  stream_config.sbcnf.Malloc = ReassembleMalloc;
388  stream_config.sbcnf.Calloc = ReassembleCalloc;
389  stream_config.sbcnf.Realloc = ReassembleRealloc;
390  stream_config.sbcnf.Free = ReassembleFree;
391 
392  return 0;
393 }
394 
395 int StreamTcpReassembleInit(char quiet)
396 {
397  /* init the memcap/use tracker */
399 
400  if (StreamTcpReassemblyConfig(quiet) < 0)
401  return -1;
402 
403 #ifdef DEBUG
404  SCMutexInit(&segment_pool_memuse_mutex, NULL);
405 #endif
406  StatsRegisterGlobalCounter("tcp.reassembly_memuse",
408  return 0;
409 }
410 
411 void StreamTcpReassembleFree(char quiet)
412 {
413  SCMutexLock(&segment_thread_pool_mutex);
414  if (segment_thread_pool != NULL) {
415  PoolThreadFree(segment_thread_pool);
416  segment_thread_pool = NULL;
417  }
418  SCMutexUnlock(&segment_thread_pool_mutex);
419  SCMutexDestroy(&segment_thread_pool_mutex);
420 
421 #ifdef DEBUG
422  if (segment_pool_memuse > 0)
423  SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
424  if (segment_pool_memcnt > 0)
425  SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
426  SCMutexDestroy(&segment_pool_memuse_mutex);
427 #endif
428 }
429 
431 {
432  SCEnter();
434  if (unlikely(ra_ctx == NULL))
435  return NULL;
436 
437  memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
438 
439  ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
440 
441  SCMutexLock(&segment_thread_pool_mutex);
442  if (segment_thread_pool == NULL) {
443  segment_thread_pool = PoolThreadInit(1, /* thread */
444  0, /* unlimited */
446  sizeof(TcpSegment),
447  TcpSegmentPoolAlloc,
448  TcpSegmentPoolInit, NULL,
449  TcpSegmentPoolCleanup, NULL);
450  ra_ctx->segment_thread_pool_id = 0;
451  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
452  PoolThreadSize(segment_thread_pool),
453  ra_ctx->segment_thread_pool_id);
454  } else {
455  /* grow segment_thread_pool until we have a element for our thread id */
456  ra_ctx->segment_thread_pool_id = PoolThreadGrow(segment_thread_pool,
457  0, /* unlimited */
459  sizeof(TcpSegment),
460  TcpSegmentPoolAlloc,
461  TcpSegmentPoolInit, NULL,
462  TcpSegmentPoolCleanup, NULL);
463  SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
464  PoolThreadSize(segment_thread_pool),
465  ra_ctx->segment_thread_pool_id);
466  }
467  SCMutexUnlock(&segment_thread_pool_mutex);
468  if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
469  SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
471  SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
472  }
473 
474  SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
475 }
476 
478 {
479  SCEnter();
481  SCFree(ra_ctx);
482  SCReturn;
483 }
484 
485 /**
486  * \brief check if stream in pkt direction has depth reached
487  *
488  * \param p packet with *LOCKED* flow
489  *
490  * \retval 1 stream has depth reached
491  * \retval 0 stream does not have depth reached
492  */
494 {
495  if (p->flow != NULL && p->flow->protoctx != NULL) {
496  TcpSession *ssn = p->flow->protoctx;
497  TcpStream *stream;
498  if (p->flowflags & FLOW_PKT_TOSERVER) {
499  stream = &ssn->client;
500  } else {
501  stream = &ssn->server;
502  }
503 
504  return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
505  }
506 
507  return 0;
508 }
509 
510 /**
511  * \internal
512  * \brief Function to Check the reassembly depth valuer against the
513  * allowed max depth of the stream reassmbly for TCP streams.
514  *
515  * \param stream stream direction
516  * \param seq sequence number where "size" starts
517  * \param size size of the segment that is added
518  *
519  * \retval size Part of the size that fits in the depth, 0 if none
520  */
521 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
522  uint32_t seq, uint32_t size)
523 {
524  SCEnter();
525 
526  /* if the configured depth value is 0, it means there is no limit on
527  reassembly depth. Otherwise carry on my boy ;) */
528  if (ssn->reassembly_depth == 0) {
529  SCReturnUInt(size);
530  }
531 
532  /* if the final flag is set, we're not accepting anymore */
534  SCReturnUInt(0);
535  }
536 
537  uint64_t seg_depth;
538  if (SEQ_GT(stream->base_seq, seq)) {
539  if (SEQ_LEQ(seq+size, stream->base_seq)) {
540  SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
541  stream->base_seq, seq, seq+size);
542  SCReturnUInt(0);
543  }
544 
545  seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
546  } else {
547  seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
548  }
549 
550  /* if the base_seq has moved passed the depth window we stop
551  * checking and just reject the rest of the packets including
552  * retransmissions. Saves us the hassle of dealing with sequence
553  * wraps as well */
554  SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
555  stream->base_seq, seg_depth,
556  ssn->reassembly_depth);
557 
558  if (seg_depth > (uint64_t)ssn->reassembly_depth) {
559  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
561  SCReturnUInt(0);
562  }
563  SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
564  SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
565 #if 0
566  SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
567  (stream->base_seq_offset + stream->base_seq + size),
568  (stream->isn + ssn->reassembly_depth));
569 #endif
570  if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
571  /* packet (partly?) fits the depth window */
572 
573  if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
574  /* complete fit */
575  SCReturnUInt(size);
576  } else {
578  /* partial fit, return only what fits */
579  uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
580  DEBUG_VALIDATE_BUG_ON(part > size);
581  if (part > size)
582  part = size;
583  SCReturnUInt(part);
584  }
585  }
586 
587  SCReturnUInt(0);
588 }
589 
590 /**
591  * \brief Insert a packets TCP data into the stream reassembly engine.
592  *
593  * \retval 0 good segment, as far as we checked.
594  * \retval -1 badness, reason to drop in inline mode
595  *
596  * If the retval is 0 the segment is inserted correctly, or overlap is handled,
597  * or it wasn't added because of reassembly depth.
598  *
599  */
601  TcpSession *ssn, TcpStream *stream, Packet *p)
602 {
603  SCEnter();
604 
605  if (ssn->data_first_seen_dir == 0) {
606  if (PKT_IS_TOSERVER(p)) {
608  } else {
610  }
611  }
612 
613  /* If the OS policy is not set then set the OS policy for this stream */
614  if (stream->os_policy == 0) {
615  StreamTcpSetOSPolicy(stream, p);
616  }
617 
620  SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
621  SCReturnInt(0);
622  }
623 
624  /* If we have reached the defined depth for either of the stream, then stop
625  reassembling the TCP session */
626  uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
627  SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
628 
630  /* increment stream depth counter */
631  StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
632  }
633  if (size == 0) {
634  SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
635  SCReturnInt(0);
636  }
637 
639  if (size > p->payload_len)
640  size = p->payload_len;
641 
642  TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
643  if (seg == NULL) {
644  SCLogDebug("segment_pool is empty");
646  SCReturnInt(-1);
647  }
648 
649  TCP_SEG_LEN(seg) = size;
650  seg->seq = TCP_GET_SEQ(p);
651 
652  /* proto detection skipped, but now we do get data. Set event. */
653  if (RB_EMPTY(&stream->seg_tree) &&
655 
658  }
659 
660  if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
661  SCLogDebug("StreamTcpReassembleInsertSegment failed");
662  SCReturnInt(-1);
663  }
664  SCReturnInt(0);
665 }
666 
667 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
668  Packet *p, enum StreamUpdateDir dir)
669 {
670  uint8_t flag = 0;
671 
673  flag |= STREAM_START;
674  }
675 
676  if (ssn->state == TCP_CLOSED) {
677  flag |= STREAM_EOF;
678  }
679 
680  if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
681  flag |= STREAM_MIDSTREAM;
682  }
683 
684  if (p->flags & PKT_PSEUDO_STREAM_END) {
685  flag |= STREAM_EOF;
686  }
687 
688  if (dir == UPDATE_DIR_OPPOSING) {
689  if (p->flowflags & FLOW_PKT_TOSERVER) {
690  flag |= STREAM_TOCLIENT;
691  } else {
692  flag |= STREAM_TOSERVER;
693  }
694  } else {
695  if (p->flowflags & FLOW_PKT_TOSERVER) {
696  flag |= STREAM_TOSERVER;
697  } else {
698  flag |= STREAM_TOCLIENT;
699  }
700  }
701 
703  flag |= STREAM_DEPTH;
704  }
705  return flag;
706 }
707 
708 /**
709  * \brief Check the minimum size limits for reassembly.
710  *
711  * \retval 0 don't reassemble yet
712  * \retval 1 do reassemble
713  */
714 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
715  const TcpStream *stream, const Packet *p)
716 {
717  SCEnter();
718 
719  /* if any of these flags is set we always inspect immediately */
720 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
721  ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
722  | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
723  | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
724 
727  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
728  "is set, so not expecting any new data segments");
729  }
731  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
732  }
734  SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
735  "so no new segments will be considered");
736  }
737  SCReturnInt(1);
738  }
739 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
740 
741  /* some states mean we reassemble no matter how much data we have */
742  if (ssn->state > TCP_TIME_WAIT)
743  SCReturnInt(1);
744 
745  if (p->flags & PKT_PSEUDO_STREAM_END)
746  SCReturnInt(1);
747 
748  /* check if we have enough data to do raw reassembly */
749  if (PKT_IS_TOSERVER(p)) {
750  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
751  uint32_t delta = stream->last_ack - stream->base_seq;
752  /* get max absolute offset */
753  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
754 
755  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
756  if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
757  SCReturnInt(1);
758  } else {
759  SCLogDebug("toserver min chunk len not yet reached: "
760  "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
761  "%"PRIu32"", stream->last_ack, stream->base_seq,
762  (stream->last_ack - stream->base_seq),
764  SCReturnInt(0);
765  }
766  }
767  } else {
768  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
769  uint32_t delta = stream->last_ack - stream->base_seq;
770  /* get max absolute offset */
771  uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
772 
773  int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
774 
775  if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
776  SCReturnInt(1);
777  } else {
778  SCLogDebug("toclient min chunk len not yet reached: "
779  "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
780  "%"PRIu32"", stream->last_ack, stream->base_seq,
781  (stream->last_ack - stream->base_seq),
783  SCReturnInt(0);
784  }
785  }
786  }
787 
788  SCReturnInt(0);
789 }
790 
791 /**
792  * \brief see what if any work the TCP session still needs
793  */
794 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
795 {
796  const TcpStream *stream = NULL;
797 #ifdef DEBUG
798  const char *dirstr = NULL;
799 #endif
800  if (direction == STREAM_TOSERVER) {
801  stream = &ssn->client;
802 #ifdef DEBUG
803  dirstr = "client";
804 #endif
805  } else {
806  stream = &ssn->server;
807 #ifdef DEBUG
808  dirstr = "server";
809 #endif
810  }
811 
812  int use_app = 1;
813  int use_raw = 1;
814 
816  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
817  {
818  // app is dead
819  use_app = 0;
820  }
821 
823  // raw is dead
824  use_raw = 0;
825  }
826 
827  uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
828 
829  SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
830  dirstr,
831  STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
832  STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
833  right_edge);
834  if (use_raw) {
835  if (right_edge > STREAM_RAW_PROGRESS(stream)) {
836  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
838  }
839  }
840  if (use_app) {
841  if (right_edge > STREAM_APP_PROGRESS(stream)) {
842  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
844  }
845  }
846 
847  SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
849 }
850 
851 #ifdef DEBUG
852 static uint64_t GetStreamSize(TcpStream *stream)
853 {
854  if (stream) {
855  uint64_t size = 0;
856  uint32_t cnt = 0;
857 
858  TcpSegment *seg;
859  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
860  cnt++;
861  size += (uint64_t)TCP_SEG_LEN(seg);
862  }
863 
864  SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
865  return size;
866  }
867  return (uint64_t)0;
868 }
869 
870 static void GetSessionSize(TcpSession *ssn, Packet *p)
871 {
872  uint64_t size = 0;
873  if (ssn) {
874  size = GetStreamSize(&ssn->client);
875  size += GetStreamSize(&ssn->server);
876 
877  //if (size > 900000)
878  // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
879  SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
880  }
881 }
882 #endif
883 
884 /** \internal
885  *
886  * Get buffer, or first part of the buffer if data gaps exist.
887  *
888  * \brief get stream data from offset
889  * \param offset stream offset */
890 static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len, uint64_t offset)
891 {
892  const uint8_t *mydata;
893  uint32_t mydata_len;
894 
895  if (RB_EMPTY(&stream->sb.sbb_tree)) {
896  SCLogDebug("getting one blob");
897 
898  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
899 
900  *data = mydata;
901  *data_len = mydata_len;
902  } else {
903  StreamingBufferBlock *blk = stream->sb.head;
904 
905  if (blk->offset > offset) {
906  SCLogDebug("gap, want data at offset %"PRIu64", "
907  "got data at %"PRIu64". GAP of size %"PRIu64,
908  offset, blk->offset, blk->offset - offset);
909  *data = NULL;
910  *data_len = blk->offset - offset;
911 
912  } else if (offset >= (blk->offset + blk->len)) {
913 
914  *data = NULL;
915  StreamingBufferBlock *nblk = SBB_RB_NEXT(blk);
916  *data_len = nblk ? nblk->offset - offset : 0;
917  if (nblk) {
918  SCLogDebug("gap, want data at offset %"PRIu64", "
919  "got data at %"PRIu64". GAP of size %"PRIu64,
920  offset, nblk->offset, nblk->offset - offset);
921  }
922 
923  } else if (offset > blk->offset && offset < (blk->offset + blk->len)) {
924  SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
925  offset, blk->offset, blk->len);
926  StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
927  SCLogDebug("data %p, data_len %u", *data, *data_len);
928  } else {
929  StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
930  }
931  }
932 }
933 
934 /** \internal
935  * \brief check to see if we should declare a GAP
936  * Call this when the app layer didn't get data at the requested
937  * offset.
938  */
939 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
940 {
941  const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
942  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
943 
944  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
945  /* get window of data that is acked */
946  const uint32_t delta = stream->last_ack - stream->base_seq;
947  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
948  /* get max absolute offset */
949  last_ack_abs += delta;
950 
951  const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
952  last_ack_abs -= ackadded;
953 
954  SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
955  SCLogDebug("next_seq %u", stream->next_seq);
956 
957  /* if last_ack_abs is beyond the app_progress data that we haven't seen
958  * has been ack'd. This looks like a GAP. */
959  if (last_ack_abs > app_progress) {
960  /* however, we can accept ACKs a bit too liberally. If last_ack
961  * is beyond next_seq, we only consider it a gap now if we do
962  * already have data beyond the gap. */
963  if (SEQ_GT(stream->last_ack, stream->next_seq)) {
964  if (RB_EMPTY(&stream->sb.sbb_tree)) {
965  SCLogDebug("packet %"PRIu64": no GAP. "
966  "next_seq %u < last_ack %u, but no data in list",
967  p->pcap_cnt, stream->next_seq, stream->last_ack);
968  return false;
969  } else {
970  const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
971  const StreamingBufferBlock *blk = stream->sb.head;
972  if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
973  /* ack'd data after the gap */
974  SCLogDebug("packet %"PRIu64": GAP. "
975  "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
976  p->pcap_cnt, stream->next_seq, stream->last_ack);
977  return true;
978  }
979  }
980  }
981 
982  SCLogDebug("packet %"PRIu64": GAP! "
983  "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
984  "but we have no data.",
985  p->pcap_cnt, last_ack_abs, app_progress);
986  return true;
987  }
988  }
989  SCLogDebug("packet %"PRIu64": no GAP. "
990  "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
991  p->pcap_cnt, last_ack_abs, app_progress);
992  return false;
993 }
994 
995 /** \internal
996  * \brief get stream buffer and update the app-layer
997  * \retval 0 success
998  */
999 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1000  TcpReassemblyThreadCtx *ra_ctx,
1001  TcpSession *ssn, TcpStream *stream,
1002  Packet *p, enum StreamUpdateDir dir)
1003 {
1004  uint64_t app_progress = STREAM_APP_PROGRESS(stream);
1005 
1006  SCLogDebug("app progress %"PRIu64, app_progress);
1007  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1008 
1009  const uint8_t *mydata;
1010  uint32_t mydata_len;
1011 
1012  while (1) {
1013  GetAppBuffer(stream, &mydata, &mydata_len, app_progress);
1014  if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, stream, p)) {
1015  SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1016 
1017  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1018  NULL, mydata_len,
1019  StreamGetAppLayerFlags(ssn, stream, p, dir)|STREAM_GAP);
1020  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1021 
1023  StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
1024 
1025  stream->app_progress_rel += mydata_len;
1026  app_progress += mydata_len;
1027  if (r < 0)
1028  break;
1029 
1030  continue;
1031  } else if (mydata == NULL || mydata_len == 0) {
1032  /* Possibly a gap, but no new data. */
1033  return 0;
1034  }
1035  SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1036  break;
1037  }
1038 
1039  //PrintRawDataFp(stdout, mydata, mydata_len);
1040 
1041  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1042  stream, &stream->sb, mydata_len, app_progress);
1043 
1044  /* get window of data that is acked */
1045  if (StreamTcpInlineMode() == FALSE) {
1046  if (p->flags & PKT_PSEUDO_STREAM_END) {
1047  // fall through, we use all available data
1048  } else {
1049  uint64_t last_ack_abs = app_progress; /* absolute right edge of ack'd data */
1050  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1051  /* get window of data that is acked */
1052  uint32_t delta = stream->last_ack - stream->base_seq;
1053  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1054  /* get max absolute offset */
1055  last_ack_abs += delta;
1056  }
1057 
1058  /* see if the buffer contains unack'd data as well */
1059  if (app_progress + mydata_len > last_ack_abs) {
1060  uint32_t check = mydata_len;
1061  mydata_len = last_ack_abs - app_progress;
1062  BUG_ON(mydata_len > check);
1063  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1064  "data is considered", mydata_len);
1065  }
1066  }
1067  }
1068 
1069  /* update the app-layer */
1070  int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1071  (uint8_t *)mydata, mydata_len,
1072  StreamGetAppLayerFlags(ssn, stream, p, dir));
1073  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1074 
1075  /* see if we can update the progress */
1076  if (r == 0 && mydata_len > 0 &&
1078  {
1079  SCLogDebug("app progress %"PRIu64" increasing with data len %u to %"PRIu64,
1080  app_progress, mydata_len, app_progress + mydata_len);
1081 
1082  stream->app_progress_rel += mydata_len;
1083  SCLogDebug("app progress now %"PRIu64, STREAM_APP_PROGRESS(stream));
1084  } else {
1085  SCLogDebug("NOT UPDATED app progress still %"PRIu64, app_progress);
1086  }
1087 
1088  SCReturnInt(0);
1089 }
1090 
1091 /**
1092  * \brief Update the stream reassembly upon receiving a packet.
1093  *
1094  * For IDS mode, the stream is in the opposite direction of the packet,
1095  * as the ACK-packet is ACK'ing the stream.
1096  *
1097  * One of the utilities call by this function AppLayerHandleTCPData(),
1098  * has a feature where it will call this very same function for the
1099  * stream opposing the stream it is called with. This shouldn't cause
1100  * any issues, since processing of each stream is independent of the
1101  * other stream.
1102  */
1104  TcpSession *ssn, TcpStream *stream,
1105  Packet *p, enum StreamUpdateDir dir)
1106 {
1107  SCEnter();
1108 
1109  /* this function can be directly called by app layer protocol
1110  * detection. */
1113  SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1114  SCReturnInt(0);
1115  }
1116 
1117 #ifdef DEBUG
1118  SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1119  GetSessionSize(ssn, p);
1120 #endif
1121  /* if no segments are in the list or all are already processed,
1122  * and state is beyond established, we send an empty msg */
1123  if (STREAM_HAS_SEEN_DATA(stream) && STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1124  {
1125  /* send an empty EOF msg if we have no segments but TCP state
1126  * is beyond ESTABLISHED */
1127  if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1128  SCLogDebug("sending empty eof message");
1129  /* send EOF to app layer */
1130  AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1131  NULL, 0,
1132  StreamGetAppLayerFlags(ssn, stream, p, dir));
1133  AppLayerProfilingStore(ra_ctx->app_tctx, p);
1134 
1135  SCReturnInt(0);
1136  }
1137  }
1138 
1139  /* with all that out of the way, lets update the app-layer */
1140  return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, stream, p, dir);
1141 }
1142 
1143 /** \internal
1144  * \brief get stream data from offset
1145  * \param offset stream offset */
1146 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1147  StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1148 {
1149  const uint8_t *mydata;
1150  uint32_t mydata_len;
1151  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1152  SCLogDebug("getting one blob for offset %"PRIu64, offset);
1153 
1154  uint64_t roffset = offset;
1155  if (offset)
1156  StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1157  else {
1158  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1159  }
1160 
1161  *data = mydata;
1162  *data_len = mydata_len;
1163  *data_offset = roffset;
1164  } else {
1165  SCLogDebug("multiblob %s. Want offset %"PRIu64,
1166  *iter == NULL ? "starting" : "continuing", offset);
1167  if (*iter == NULL) {
1168  StreamingBufferBlock key = { .offset = offset, .len = 0 };
1169  *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1170  SCLogDebug("*iter %p", *iter);
1171  }
1172  if (*iter == NULL) {
1173  SCLogDebug("no data");
1174  *data = NULL;
1175  *data_len = 0;
1176  *data_offset = 0;
1177  return 0;
1178  }
1179  SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1180 
1181  StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1182  SCLogDebug("mydata %p", mydata);
1183 
1184  if ((*iter)->offset < offset) {
1185  uint64_t delta = offset - (*iter)->offset;
1186  if (delta < mydata_len) {
1187  *data = mydata + delta;
1188  *data_len = mydata_len - delta;
1189  *data_offset = offset;
1190  } else {
1191  SCLogDebug("no data (yet)");
1192  *data = NULL;
1193  *data_len = 0;
1194  *data_offset = 0;
1195  }
1196 
1197  } else {
1198  *data = mydata;
1199  *data_len = mydata_len;
1200  *data_offset = (*iter)->offset;
1201  }
1202 
1203  *iter = SBB_RB_NEXT(*iter);
1204  SCLogDebug("*iter %p", *iter);
1205  }
1206  return 0;
1207 }
1208 
1209 /** \brief does the stream engine have data to inspect?
1210  *
1211  * Returns true if there is data to inspect. In IDS case this is
1212  * about ACK'd data in the packet's direction.
1213  *
1214  * In the IPS case this is about the packet itself.
1215  */
1217 {
1218  TcpStream *stream;
1219  if (PKT_IS_TOSERVER(p)) {
1220  stream = &ssn->client;
1221  } else {
1222  stream = &ssn->server;
1223  }
1224 
1225  if (RB_EMPTY(&stream->seg_tree)) {
1226  return false;
1227  }
1228 
1231  return false;
1232 
1233  if (StreamTcpInlineMode() == FALSE) {
1234  if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1235  return false;
1236  }
1237  if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1238  return true;
1239  }
1240  } else {
1241  if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1242  return true;
1243  }
1244  }
1245  return false;
1246 }
1247 
1248 /** \brief update stream engine after detection
1249  *
1250  * Tasked with progressing the 'progress' for Raw reassembly.
1251  * 2 main scenario's:
1252  * 1. progress is != 0, so we use this
1253  * 2. progress is 0, meaning the detect engine didn't touch
1254  * raw at all. In this case we need to look into progressing
1255  * raw anyway.
1256  *
1257  * Additionally, this function is tasked with disabling raw
1258  * reassembly if the app-layer requested to disable it.
1259  */
1260 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1261 {
1262  TcpStream *stream;
1263  if (PKT_IS_TOSERVER(p)) {
1264  stream = &ssn->client;
1265  } else {
1266  stream = &ssn->server;
1267  }
1268 
1269  if (progress > STREAM_RAW_PROGRESS(stream)) {
1270  uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1271  stream->raw_progress_rel += slide;
1273 
1274  /* if app is active and beyond raw, sync raw to app */
1275  } else if (progress == 0 &&
1276  STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1278  !(stream->flags & STREAMTCP_STREAM_FLAG_GAP))
1279  {
1280  /* if trigger raw is set we sync the 2 trackers */
1282  {
1283  uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1284  stream->raw_progress_rel += slide;
1286 
1287  /* otherwise mix in the tcp window */
1288  } else {
1289  uint64_t tcp_window = stream->window;
1290  if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1291  uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1292  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1293  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1294  stream->raw_progress_rel += slide;
1295  }
1296  }
1297  }
1298  /* app is dead */
1299  } else if (progress == 0) {
1300  uint64_t tcp_window = stream->window;
1301  uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1302  if (tcp_window < stream_right_edge) {
1303  uint64_t new_raw = stream_right_edge - tcp_window;
1304  if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1305  uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1306  stream->raw_progress_rel += slide;
1307  }
1308  }
1310 
1311  } else {
1312  SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1313  p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1314  STREAM_RAW_PROGRESS(stream), stream->window);
1315  }
1316 
1317  /* if we were told to accept no more raw data, we can mark raw as
1318  * disabled now. */
1321  SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1322  "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1323  }
1324 
1325  SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1326 }
1327 
1328 /** \internal
1329  * \brief get a buffer around the current packet and run the callback on it
1330  *
1331  * The inline/IPS scanning method takes the current payload and wraps it in
1332  * data from other segments.
1333  *
1334  * How much data is inspected is controlled by the available data, chunk_size
1335  * and the payload size of the packet.
1336  *
1337  * Large packets: if payload size is close to the chunk_size, where close is
1338  * defined as more than 67% of the chunk_size, a larger chunk_size will be
1339  * used: payload_len + 33% of the chunk_size.
1340  * If the payload size if equal to or bigger than the chunk_size, we use
1341  * payload len + 33% of the chunk size.
1342  */
1343 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1344  StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1345 {
1346  SCEnter();
1347  int r = 0;
1348 
1349  TcpStream *stream;
1350  if (PKT_IS_TOSERVER(p)) {
1351  stream = &ssn->client;
1352  } else {
1353  stream = &ssn->server;
1354  }
1355 
1356  if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1358  {
1359  *progress_out = STREAM_RAW_PROGRESS(stream);
1360  return 0;
1361  }
1362 
1363  uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1366  if (chunk_size <= p->payload_len) {
1367  chunk_size = p->payload_len + (chunk_size / 3);
1368  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1369  p->payload_len, chunk_size);
1370  } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1371  chunk_size = p->payload_len + ((chunk_size / 3));
1372  SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1373  p->payload_len, chunk_size);
1374  }
1375 
1376  uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1377  uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1378  SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1379  packet_leftedge_abs, packet_rightedge_abs);
1380 
1381  const uint8_t *mydata = NULL;
1382  uint32_t mydata_len = 0;
1383  uint64_t mydata_offset = 0;
1384  /* simply return progress from the block we inspected. */
1385  bool return_progress = false;
1386 
1387  if (RB_EMPTY(&stream->sb.sbb_tree)) {
1388  /* continues block */
1389  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1390  return_progress = true;
1391 
1392  } else {
1393  SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1394  /* find our block */
1395  StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1396  StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1397  if (sbb) {
1398  SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1399  StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1400  mydata_offset = sbb->offset;
1401  }
1402  }
1403 
1404  /* this can only happen if the segment insert of our current 'p' failed */
1405  uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1406  if ((mydata == NULL || mydata_len == 0) || /* no data */
1407  (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1408  packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1409  (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1410  packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1411  {
1412  /* no data, or data is incomplete or wrong: use packet data */
1413  mydata = p->payload;
1414  mydata_len = p->payload_len;
1415  mydata_offset = packet_leftedge_abs;
1416  //mydata_rightedge_abs = packet_rightedge_abs;
1417  } else {
1418  /* adjust buffer to match chunk_size */
1419  SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1420  if (mydata_len > chunk_size) {
1421  uint32_t excess = mydata_len - chunk_size;
1422  SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1423 
1424  if (mydata_rightedge_abs == packet_rightedge_abs) {
1425  mydata += excess;
1426  mydata_len -= excess;
1427  mydata_offset += excess;
1428  SCLogDebug("cutting front of the buffer with %u", excess);
1429  } else if (mydata_offset == packet_leftedge_abs) {
1430  mydata_len -= excess;
1431  SCLogDebug("cutting tail of the buffer with %u", excess);
1432  } else {
1433  uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1434  uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1435  SCLogDebug("before %u after %u", before, after);
1436 
1437  if (after >= (chunk_size - p->payload_len) / 2) {
1438  // more trailing data than we need
1439 
1440  if (before >= (chunk_size - p->payload_len) / 2) {
1441  // also more heading data, devide evenly
1442  before = after = (chunk_size - p->payload_len) / 2;
1443  } else {
1444  // heading data is less than requested, give the
1445  // rest to the trailing data
1446  after = (chunk_size - p->payload_len) - before;
1447  }
1448  } else {
1449  // less trailing data than requested
1450 
1451  if (before >= (chunk_size - p->payload_len) / 2) {
1452  before = (chunk_size - p->payload_len) - after;
1453  } else {
1454  // both smaller than their requested size
1455  }
1456  }
1457 
1458  /* adjust the buffer */
1459  uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1460  uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1461  DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1462  DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1463  DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1464  mydata += skip;
1465  mydata_len -= (skip + cut);
1466  mydata_offset += skip;
1467  }
1468  }
1469  }
1470 
1471  /* run the callback */
1472  r = Callback(cb_data, mydata, mydata_len);
1473  BUG_ON(r < 0);
1474 
1475  if (return_progress) {
1476  *progress_out = (mydata_offset + mydata_len);
1477  } else {
1478  /* several blocks of data, so we need to be a bit more careful:
1479  * - if last_ack is beyond last progress, move progress forward to last_ack
1480  * - if our block matches or starts before last ack, return right edge of
1481  * our block.
1482  */
1483  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1484  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1485  uint32_t delta = stream->last_ack - stream->base_seq;
1486  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1487  /* get max absolute offset */
1488  last_ack_abs += delta;
1489  }
1490  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1491 
1492  if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1493  if (mydata_offset > last_ack_abs) {
1494  /* gap between us and last ack, set progress to last ack */
1495  *progress_out = last_ack_abs;
1496  } else {
1497  *progress_out = (mydata_offset + mydata_len);
1498  }
1499  } else {
1500  *progress_out = STREAM_RAW_PROGRESS(stream);
1501  }
1502  }
1503  return r;
1504 }
1505 
1506 /** \brief access 'raw' reassembly data.
1507  *
1508  * Access data as tracked by 'raw' tracker. Data is made available to
1509  * callback that is passed to this function.
1510  *
1511  * In the case of IDS the callback may be run multiple times if data
1512  * contains gaps. It will then be run for each block of data that is
1513  * continuous.
1514  *
1515  * The callback should give on of 2 return values:
1516  * - 0 ok
1517  * - 1 done
1518  * The value 1 will break the loop if there is a block list that is
1519  * inspected.
1520  *
1521  * This function will return the 'progress' value that has been
1522  * consumed until now.
1523  *
1524  * \param ssn tcp session
1525  * \param stream tcp stream
1526  * \param Callback the function pointer to the callback function
1527  * \param cb_data callback data
1528  * \param[in] progress_in progress to work from
1529  * \param[out] progress_out absolute progress value of the data this
1530  * call handled.
1531  * \param eof we're wrapping up so inspect all data we have, incl unACKd
1532  * \param respect_inspect_depth use Stream::min_inspect_depth if set
1533  *
1534  * `respect_inspect_depth` is used to avoid useless inspection of too
1535  * much data.
1536  */
1537 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1538  StreamReassembleRawFunc Callback, void *cb_data,
1539  const uint64_t progress_in,
1540  uint64_t *progress_out, bool eof,
1541  bool respect_inspect_depth)
1542 {
1543  SCEnter();
1544  int r = 0;
1545 
1546  StreamingBufferBlock *iter = NULL;
1547  uint64_t progress = progress_in;
1548  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1549 
1550  /* if the app layer triggered a flush, and we're supposed to
1551  * use a minimal inspect depth, we actually take the app progress
1552  * as that is the right edge of the data. Then we take the window
1553  * of 'min_inspect_depth' before that. */
1554  if (respect_inspect_depth &&
1556  && stream->min_inspect_depth)
1557  {
1558  progress = STREAM_APP_PROGRESS(stream);
1559  if (stream->min_inspect_depth >= progress) {
1560  progress = 0;
1561  } else {
1562  progress -= stream->min_inspect_depth;
1563  }
1564  SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1565 
1566  SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1567  }
1568 
1569  SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1570 
1571  /* get window of data that is acked */
1572  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1573  SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1574  uint32_t delta = stream->last_ack - stream->base_seq;
1575  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
1576  /* get max absolute offset */
1577  last_ack_abs += delta;
1578  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1579  }
1580 
1581  /* loop through available buffers. On no packet loss we'll have a single
1582  * iteration. On missing data we'll walk the blocks */
1583  while (1) {
1584  const uint8_t *mydata;
1585  uint32_t mydata_len;
1586  uint64_t mydata_offset = 0;
1587 
1588  GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1589  if (mydata_len == 0) {
1590  SCLogDebug("no data");
1591  break;
1592  }
1593  //PrintRawDataFp(stdout, mydata, mydata_len);
1594 
1595  SCLogDebug("raw progress %"PRIu64, progress);
1596  SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1597  stream, &stream->sb, mydata_len, progress);
1598 
1599  if (eof) {
1600  // inspect all remaining data, ack'd or not
1601  } else {
1602  if (last_ack_abs < progress) {
1603  SCLogDebug("nothing to do");
1604  goto end;
1605  }
1606 
1607  SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1608  SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1609 
1610  /* see if the buffer contains unack'd data as well */
1611  if (progress + mydata_len > last_ack_abs) {
1612  uint32_t check = mydata_len;
1613  mydata_len = last_ack_abs - progress;
1614  BUG_ON(check < mydata_len);
1615  SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1616  "data is considered", mydata_len);
1617  }
1618 
1619  }
1620  if (mydata_len == 0)
1621  break;
1622 
1623  SCLogDebug("data %p len %u", mydata, mydata_len);
1624 
1625  /* we have data. */
1626  r = Callback(cb_data, mydata, mydata_len);
1627  BUG_ON(r < 0);
1628 
1629  if (mydata_offset == progress) {
1630  SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1631  progress, mydata_len, progress_in + mydata_len);
1632 
1633  progress += mydata_len;
1634  SCLogDebug("raw progress now %"PRIu64, progress);
1635 
1636  /* data is beyond the progress we'd like, and before last ack. Gap. */
1637  } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1638  SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1639  SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1640  progress = mydata_offset;
1641  SCLogDebug("raw progress now %"PRIu64, progress);
1642 
1643  } else {
1644  SCLogDebug("not increasing progress, data gap => mydata_offset "
1645  "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1646  }
1647 
1648  if (iter == NULL || r == 1)
1649  break;
1650  }
1651 end:
1652  *progress_out = progress;
1653  return r;
1654 }
1655 
1657  StreamReassembleRawFunc Callback, void *cb_data,
1658  uint64_t *progress_out, bool respect_inspect_depth)
1659 {
1660  /* handle inline seperately as the logic is very different */
1661  if (StreamTcpInlineMode() == TRUE) {
1662  return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1663  }
1664 
1665  TcpStream *stream;
1666  if (PKT_IS_TOSERVER(p)) {
1667  stream = &ssn->client;
1668  } else {
1669  stream = &ssn->server;
1670  }
1671 
1673  StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1674  {
1675  *progress_out = STREAM_RAW_PROGRESS(stream);
1676  return 0;
1677  }
1678 
1679  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1680  STREAM_RAW_PROGRESS(stream), progress_out,
1681  (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1682 }
1683 
1685  StreamReassembleRawFunc Callback, void *cb_data,
1686  uint64_t progress_in,
1687  uint64_t *progress_out, bool eof)
1688 {
1689  if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1690  return 0;
1691 
1692  return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1693  progress_in, progress_out, eof, false);
1694 }
1695 
1696 /** \internal
1697  * \brief update app layer based on received ACK
1698  *
1699  * \retval r 0 on success, -1 on error
1700  */
1701 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1702  TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1703 {
1704  SCEnter();
1705 
1706  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1707  SCReturnInt(-1);
1708 
1709  SCReturnInt(0);
1710 }
1711 
1713  TcpSession *ssn, TcpStream *stream,
1714  Packet *p, PacketQueue *pq)
1715 {
1716  SCEnter();
1717 
1718  DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1719 
1720  SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1721  ssn, stream, p, p->payload_len);
1722 
1723  /* we need to update the opposing stream in
1724  * StreamTcpReassembleHandleSegmentUpdateACK */
1725  TcpStream *opposing_stream = NULL;
1726  if (stream == &ssn->client) {
1727  opposing_stream = &ssn->server;
1728  } else {
1729  opposing_stream = &ssn->client;
1730  }
1731 
1732  /* default IDS: update opposing side (triggered by ACK) */
1734  /* inline and stream end and flow timeout packets trigger same dir handling */
1735  if (StreamTcpInlineMode()) {
1736  dir = UPDATE_DIR_PACKET;
1737  } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1738  dir = UPDATE_DIR_PACKET;
1739  } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1740  dir = UPDATE_DIR_PACKET;
1741  } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1742  dir = UPDATE_DIR_PACKET;
1743  } else if (ssn->state == TCP_CLOSED) {
1744  dir = UPDATE_DIR_BOTH;
1745  }
1746 
1747  /* handle ack received */
1748  if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH) &&
1749  StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0)
1750  {
1751  SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1752  SCReturnInt(-1);
1753  }
1754 
1755  /* if this segment contains data, insert it */
1756  if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1757  SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1758 
1759  if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1760  SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1761  SCReturnInt(-1);
1762  }
1763 
1764  SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1765  p->flags |= PKT_STREAM_ADD;
1766  } else {
1767  SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1768  " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1769  ssn, stream, p->payload_len,
1770  (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1771 
1772  }
1773 
1774  /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1775  * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1776  * was *just* set. In this case we trigger the AppLayer Truncate
1777  * logic, to inform the applayer no more data in this direction is
1778  * to be expected. */
1779  if ((stream->flags &
1782  {
1783  SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1784  if (dir != UPDATE_DIR_PACKET) {
1785  SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1786  "can trigger Truncate");
1787  dir = UPDATE_DIR_PACKET;
1788  }
1789  }
1790 
1791  /* in stream inline mode even if we have no data we call the reassembly
1792  * functions to handle EOF */
1793  if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1794  SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1795  StreamTcpInlineMode()?"true":"false",
1796  (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1797  if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1798  SCReturnInt(-1);
1799  }
1800  }
1801 
1802  SCReturnInt(0);
1803 }
1804 
1805 /**
1806  * \brief get a segment from the pool
1807  *
1808  * \retval seg Segment from the pool or NULL
1809  */
1811 {
1812  TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1813  SCLogDebug("seg we return is %p", seg);
1814  if (seg == NULL) {
1815  /* Increment the counter to show that we are not able to serve the
1816  segment request due to memcap limit */
1818  } else {
1819  memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1820  }
1821 
1822  return seg;
1823 }
1824 
1825 /**
1826  * \brief Trigger RAW stream reassembly
1827  *
1828  * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1829  * reassembly from the applayer, for example upon completion of a
1830  * HTTP request.
1831  *
1832  * It sets a flag in the stream so that the next Raw call will return
1833  * the data.
1834  *
1835  * \param ssn TcpSession
1836  */
1838 {
1839 #ifdef DEBUG
1840  BUG_ON(ssn == NULL);
1841 #endif
1842 
1843  if (ssn != NULL) {
1844  if (direction == STREAM_TOSERVER) {
1846  } else {
1848  }
1849 
1850  SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1851  }
1852 }
1853 
1854 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1855 {
1856 #ifdef DEBUG
1857  BUG_ON(ssn == NULL);
1858 #endif
1859 
1860  if (ssn != NULL) {
1861  if (direction == STREAM_TOSERVER) {
1862  ssn->client.min_inspect_depth = depth;
1863  SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1864  } else {
1865  ssn->server.min_inspect_depth = depth;
1866  SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1867  }
1868  }
1869 }
1870 
1871 #ifdef UNITTESTS
1872 /** unit tests and it's support functions below */
1873 
1874 #define SET_ISN(stream, setseq) \
1875  (stream)->isn = (setseq); \
1876  (stream)->base_seq = (setseq) + 1
1877 
1878 /** \brief The Function to create the packet with given payload, which is used
1879  * to test the reassembly of the engine.
1880  *
1881  * \param payload The variable used to store the payload contents of the
1882  * current packet.
1883  * \param value The value which current payload will have for this packet
1884  * \param payload_len The length of the filed payload for current packet.
1885  * \param len Length of the payload array
1886  */
1887 
1888 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
1889  uint8_t payload_len, uint8_t len)
1890 {
1891  uint8_t i;
1892  for (i = 0; i < payload_len; i++)
1893  payload[i] = value;
1894  for (; i < len; i++)
1895  payload = NULL;
1896 }
1897 
1898 /** \brief The Function Checks the reassembled stream contents against predefined
1899  * stream contents according to OS policy used.
1900  *
1901  * \param stream_policy Predefined value of stream for different OS policies
1902  * \param stream Reassembled stream returned from the reassembly functions
1903  */
1904 
1905 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
1906 {
1907  if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
1908  {
1909  //PrintRawDataFp(stdout, stream_policy, sp_size);
1910  return 0;
1911  }
1912  return 1;
1913 }
1914 
1915 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
1916 {
1917  if (StreamingBufferCompareRawData(&stream->sb,
1918  data, data_len) == 0)
1919  {
1920  SCReturnInt(0);
1921  }
1922  SCLogInfo("OK");
1923  PrintRawDataFp(stdout, data, data_len);
1924  return 1;
1925 }
1926 
1927 #define MISSED_START(isn) \
1928  TcpReassemblyThreadCtx *ra_ctx = NULL; \
1929  TcpSession ssn; \
1930  ThreadVars tv; \
1931  memset(&tv, 0, sizeof(tv)); \
1932  \
1933  StreamTcpUTInit(&ra_ctx); \
1934  \
1935  StreamTcpUTSetupSession(&ssn); \
1936  StreamTcpUTSetupStream(&ssn.server, (isn)); \
1937  StreamTcpUTSetupStream(&ssn.client, (isn)); \
1938  \
1939  TcpStream *stream = &ssn.client;
1940 
1941 #define MISSED_END \
1942  StreamTcpUTClearSession(&ssn); \
1943  StreamTcpUTDeinit(ra_ctx); \
1944  PASS
1945 
1946 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
1947  StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
1948  FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
1949 
1950 /**
1951  * \test Test the handling of packets missed by both IDS and the end host.
1952  * The packet is missed in the starting of the stream.
1953  *
1954  * \retval On success it returns 1 and on failure 0.
1955  */
1956 
1957 static int StreamTcpReassembleTest25 (void)
1958 {
1959  MISSED_START(6);
1960  MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
1961  MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
1962  MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
1963  MISSED_END;
1964 }
1965 
1966 /**
1967  * \test Test the handling of packets missed by both IDS and the end host.
1968  * The packet is missed in the middle of the stream.
1969  *
1970  * \retval On success it returns 1 and on failure 0.
1971  */
1972 
1973 static int StreamTcpReassembleTest26 (void)
1974 {
1975  MISSED_START(9);
1976  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1977  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
1978  MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
1979  MISSED_END;
1980 }
1981 
1982 /**
1983  * \test Test the handling of packets missed by both IDS and the end host.
1984  * The packet is missed in the end of the stream.
1985  *
1986  * \retval On success it returns 1 and on failure 0.
1987  */
1988 
1989 static int StreamTcpReassembleTest27 (void)
1990 {
1991  MISSED_START(9);
1992  MISSED_STEP(10, "AAA", 3, "AAA", 3);
1993  MISSED_STEP(13, "BB", 2, "AAABB", 5);
1994  MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
1995  MISSED_END;
1996 }
1997 
1998 /**
1999  * \test Test the handling of packets missed by IDS, but the end host has
2000  * received it and send the acknowledgment of it. The packet is missed
2001  * in the starting of the stream.
2002  *
2003  * \retval On success it returns 1 and on failure 0.
2004  */
2005 
2006 static int StreamTcpReassembleTest28 (void)
2007 {
2008  MISSED_START(6);
2009  MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2010  MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2011  ssn.state = TCP_TIME_WAIT;
2012  MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2013  MISSED_END;
2014 }
2015 
2016 /**
2017  * \test Test the handling of packets missed by IDS, but the end host has
2018  * received it and send the acknowledgment of it. The packet is missed
2019  * in the middle of the stream.
2020  *
2021  * \retval On success it returns 1 and on failure 0.
2022  */
2023 
2024 static int StreamTcpReassembleTest29 (void)
2025 {
2026  MISSED_START(9);
2027  MISSED_STEP(10, "AAA", 3, "AAA", 3);
2028  ssn.state = TCP_TIME_WAIT;
2029  MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2030  MISSED_END;
2031 }
2032 
2033 static int StreamTcpReassembleTest33(void)
2034 {
2035  TcpSession ssn;
2036  Packet *p = PacketGetFromAlloc();
2037  FAIL_IF(unlikely(p == NULL));
2038  Flow f;
2039  TCPHdr tcph;
2040  TcpReassemblyThreadCtx *ra_ctx = NULL;
2042  uint8_t packet[1460] = "";
2043 
2044  StreamTcpUTInit(&ra_ctx);
2046 
2047  PacketQueue pq;
2048  memset(&pq,0,sizeof(PacketQueue));
2049  memset(&f, 0, sizeof (Flow));
2050  memset(&tcph, 0, sizeof (TCPHdr));
2051  ThreadVars tv;
2052  memset(&tv, 0, sizeof (ThreadVars));
2053  FLOW_INITIALIZE(&f);
2054  f.protoctx = &ssn;
2055  f.proto = IPPROTO_TCP;
2056  p->src.family = AF_INET;
2057  p->dst.family = AF_INET;
2058  p->proto = IPPROTO_TCP;
2059  p->flow = &f;
2060  tcph.th_win = 5480;
2061  tcph.th_flags = TH_PUSH | TH_ACK;
2062  p->tcph = &tcph;
2064  p->payload = packet;
2065 
2066  p->tcph->th_seq = htonl(10);
2067  p->tcph->th_ack = htonl(31);
2068  p->payload_len = 10;
2069 
2070  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2071 
2072  p->tcph->th_seq = htonl(20);
2073  p->tcph->th_ack = htonl(31);
2074  p->payload_len = 10;
2075 
2076  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2077 
2078  p->tcph->th_seq = htonl(40);
2079  p->tcph->th_ack = htonl(31);
2080  p->payload_len = 10;
2081 
2082  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2083 
2084  p->tcph->th_seq = htonl(5);
2085  p->tcph->th_ack = htonl(31);
2086  p->payload_len = 30;
2087 
2088  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2089 
2091  StreamTcpUTDeinit(ra_ctx);
2092  SCFree(p);
2093  PASS;
2094 }
2095 
2096 static int StreamTcpReassembleTest34(void)
2097 {
2098  TcpSession ssn;
2099  Packet *p = PacketGetFromAlloc();
2100  FAIL_IF(unlikely(p == NULL));
2101  Flow f;
2102  TCPHdr tcph;
2103  TcpReassemblyThreadCtx *ra_ctx = NULL;
2105  uint8_t packet[1460] = "";
2106 
2107  StreamTcpUTInit(&ra_ctx);
2109  PacketQueue pq;
2110  memset(&pq,0,sizeof(PacketQueue));
2111  memset(&f, 0, sizeof (Flow));
2112  memset(&tcph, 0, sizeof (TCPHdr));
2113  ThreadVars tv;
2114  memset(&tv, 0, sizeof (ThreadVars));
2115  FLOW_INITIALIZE(&f);
2116  f.protoctx = &ssn;
2117  f.proto = IPPROTO_TCP;
2118  p->src.family = AF_INET;
2119  p->dst.family = AF_INET;
2120  p->proto = IPPROTO_TCP;
2121  p->flow = &f;
2122  tcph.th_win = 5480;
2123  tcph.th_flags = TH_PUSH | TH_ACK;
2124  p->tcph = &tcph;
2126  p->payload = packet;
2127  SET_ISN(&ssn.client, 857961230);
2128 
2129  p->tcph->th_seq = htonl(857961230);
2130  p->tcph->th_ack = htonl(31);
2131  p->payload_len = 304;
2132 
2133  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2134 
2135  p->tcph->th_seq = htonl(857961534);
2136  p->tcph->th_ack = htonl(31);
2137  p->payload_len = 1460;
2138 
2139  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2140 
2141  p->tcph->th_seq = htonl(857963582);
2142  p->tcph->th_ack = htonl(31);
2143  p->payload_len = 1460;
2144 
2145  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2146 
2147  p->tcph->th_seq = htonl(857960946);
2148  p->tcph->th_ack = htonl(31);
2149  p->payload_len = 1460;
2150 
2151  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2152 
2154  StreamTcpUTDeinit(ra_ctx);
2155  SCFree(p);
2156  PASS;
2157 }
2158 
2159 /** \test Test the bug 76 condition */
2160 static int StreamTcpReassembleTest37(void)
2161 {
2162  TcpSession ssn;
2163  Flow f;
2164  TCPHdr tcph;
2165  TcpReassemblyThreadCtx *ra_ctx = NULL;
2166  uint8_t packet[1460] = "";
2167  PacketQueue pq;
2168  ThreadVars tv;
2169  memset(&tv, 0, sizeof (ThreadVars));
2170 
2171  Packet *p = PacketGetFromAlloc();
2172  FAIL_IF(unlikely(p == NULL));
2173 
2174  StreamTcpUTInit(&ra_ctx);
2176  memset(&pq,0,sizeof(PacketQueue));
2177  memset(&f, 0, sizeof (Flow));
2178  memset(&tcph, 0, sizeof (TCPHdr));
2179  memset(&tv, 0, sizeof (ThreadVars));
2180 
2181  FLOW_INITIALIZE(&f);
2182  f.protoctx = &ssn;
2183  f.proto = IPPROTO_TCP;
2184  p->src.family = AF_INET;
2185  p->dst.family = AF_INET;
2186  p->proto = IPPROTO_TCP;
2187  p->flow = &f;
2188  tcph.th_win = 5480;
2189  tcph.th_flags = TH_PUSH | TH_ACK;
2190  p->tcph = &tcph;
2192  p->payload = packet;
2194 
2195  p->tcph->th_seq = htonl(3061088537UL);
2196  p->tcph->th_ack = htonl(1729548549UL);
2197  p->payload_len = 1391;
2198  ssn.client.last_ack = 3061091137UL;
2199  SET_ISN(&ssn.client, 3061091309UL);
2200 
2201  /* pre base_seq, so should be rejected */
2202  FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2203 
2204  p->tcph->th_seq = htonl(3061089928UL);
2205  p->tcph->th_ack = htonl(1729548549UL);
2206  p->payload_len = 1391;
2207  ssn.client.last_ack = 3061091137UL;
2208 
2209  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2210 
2211  p->tcph->th_seq = htonl(3061091319UL);
2212  p->tcph->th_ack = htonl(1729548549UL);
2213  p->payload_len = 1391;
2214  ssn.client.last_ack = 3061091137UL;
2215 
2216  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2217 
2219  StreamTcpUTDeinit(ra_ctx);
2220  SCFree(p);
2221  PASS;
2222 }
2223 
2224 /**
2225  * \test Test to make sure that we don't return the segments until the app
2226  * layer proto has been detected and after that remove the processed
2227  * segments.
2228  *
2229  * \retval On success it returns 1 and on failure 0.
2230  */
2231 
2232 static int StreamTcpReassembleTest39 (void)
2233 {
2234  Packet *p = PacketGetFromAlloc();
2235  FAIL_IF(unlikely(p == NULL));
2236  Flow f;
2237  ThreadVars tv;
2238  StreamTcpThread stt;
2239  TCPHdr tcph;
2240  PacketQueue pq;
2241  memset(&pq,0,sizeof(PacketQueue));
2242  memset (&f, 0, sizeof(Flow));
2243  memset(&tv, 0, sizeof (ThreadVars));
2244  memset(&stt, 0, sizeof (stt));
2245  memset(&tcph, 0, sizeof (TCPHdr));
2246 
2247  FLOW_INITIALIZE(&f);
2248  f.flags = FLOW_IPV4;
2249  f.proto = IPPROTO_TCP;
2250  p->flow = &f;
2251  p->tcph = &tcph;
2252 
2253  StreamTcpUTInit(&stt.ra_ctx);
2254 
2255  /* handshake */
2256  tcph.th_win = htons(5480);
2257  tcph.th_flags = TH_SYN;
2259  p->payload_len = 0;
2260  p->payload = NULL;
2261  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2262 
2263  TcpSession *ssn = (TcpSession *)f.protoctx;
2264  FAIL_IF_NULL(ssn);
2265 
2276  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2277  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2278  FAIL_IF(ssn->data_first_seen_dir != 0);
2279 
2280  /* handshake */
2281  p->tcph->th_ack = htonl(1);
2282  p->tcph->th_flags = TH_SYN | TH_ACK;
2284  p->payload_len = 0;
2285  p->payload = NULL;
2286  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2297  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2298  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2299  FAIL_IF(ssn->data_first_seen_dir != 0);
2300 
2301  /* handshake */
2302  p->tcph->th_ack = htonl(1);
2303  p->tcph->th_seq = htonl(1);
2304  p->tcph->th_flags = TH_ACK;
2306  p->payload_len = 0;
2307  p->payload = NULL;
2308  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2319  FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2320  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2321  FAIL_IF(ssn->data_first_seen_dir != 0);
2322 
2323  /* partial request */
2324  uint8_t request1[] = { 0x47, 0x45, };
2325  p->tcph->th_ack = htonl(1);
2326  p->tcph->th_seq = htonl(1);
2327  p->tcph->th_flags = TH_PUSH | TH_ACK;
2329  p->payload_len = sizeof(request1);
2330  p->payload = request1;
2331  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2342  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2343  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2344  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2346 
2347  /* response ack against partial request */
2348  p->tcph->th_ack = htonl(3);
2349  p->tcph->th_seq = htonl(1);
2350  p->tcph->th_flags = TH_ACK;
2352  p->payload_len = 0;
2353  p->payload = NULL;
2354  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2365  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2366  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2367  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2369 
2370  /* complete partial request */
2371  uint8_t request2[] = {
2372  0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2373  0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2374  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2375  0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2376  0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2377  0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2378  0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2379  0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2380  0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2381  0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2382  0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2383  p->tcph->th_ack = htonl(1);
2384  p->tcph->th_seq = htonl(3);
2385  p->tcph->th_flags = TH_PUSH | TH_ACK;
2387  p->payload_len = sizeof(request2);
2388  p->payload = request2;
2389  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2400  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2401  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2402  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2403  FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2405 
2406  /* response - request ack */
2407  uint8_t response[] = {
2408  0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2409  0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2410  0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2411  0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2412  0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2413  0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2414  0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2415  0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2416  0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2417  0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2418  0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2419  0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2420  0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2421  0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2422  0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2423  0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2424  0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2425  0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2426  0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2427  0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2428  0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2429  0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2430  0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2431  0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2432  0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2433  0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2434  0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2435  0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2436  0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2437  0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2438  0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2439  0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2440  0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2441  0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2442  0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2443  0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2444  0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2445  0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2446  0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2447  0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2448  0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2449  p->tcph->th_ack = htonl(88);
2450  p->tcph->th_seq = htonl(1);
2451  p->tcph->th_flags = TH_PUSH | TH_ACK;
2453  p->payload_len = sizeof(response);
2454  p->payload = response;
2455 
2456  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2468  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2469  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2470  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2471  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2472 
2473  /* response ack from request */
2474  p->tcph->th_ack = htonl(328);
2475  p->tcph->th_seq = htonl(88);
2476  p->tcph->th_flags = TH_ACK;
2478  p->payload_len = 0;
2479  p->payload = NULL;
2480  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2492  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2493  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2494  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2495  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2496  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2497 
2498  /* response - acking */
2499  p->tcph->th_ack = htonl(88);
2500  p->tcph->th_seq = htonl(328);
2501  p->tcph->th_flags = TH_PUSH | TH_ACK;
2503  p->payload_len = 0;
2504  p->payload = NULL;
2505  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2517  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2518  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2519  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2520  FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2521  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2522 
2523  /* response ack from request */
2524  p->tcph->th_ack = htonl(328);
2525  p->tcph->th_seq = htonl(88);
2526  p->tcph->th_flags = TH_ACK;
2528  p->payload_len = 0;
2529  p->payload = NULL;
2530  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2542  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2543  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2544  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2545  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2546 
2547  /* response - acking the request again*/
2548  p->tcph->th_ack = htonl(88);
2549  p->tcph->th_seq = htonl(328);
2550  p->tcph->th_flags = TH_PUSH | TH_ACK;
2552  p->payload_len = 0;
2553  p->payload = NULL;
2554  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2566  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2567  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2568  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2569  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2570 
2571  /*** New Request ***/
2572 
2573  /* partial request */
2574  p->tcph->th_ack = htonl(328);
2575  p->tcph->th_seq = htonl(88);
2576  p->tcph->th_flags = TH_PUSH | TH_ACK;
2578  p->payload_len = sizeof(request1);
2579  p->payload = request1;
2580  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2592  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2593  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2594  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2595  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2596  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2597 
2598  /* response ack against partial request */
2599  p->tcph->th_ack = htonl(90);
2600  p->tcph->th_seq = htonl(328);
2601  p->tcph->th_flags = TH_ACK;
2603  p->payload_len = 0;
2604  p->payload = NULL;
2605  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2606  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2618  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2619  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2620  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2621  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2622  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2623 
2624  /* complete request */
2625  p->tcph->th_ack = htonl(328);
2626  p->tcph->th_seq = htonl(90);
2627  p->tcph->th_flags = TH_PUSH | TH_ACK;
2629  p->payload_len = sizeof(request2);
2630  p->payload = request2;
2631  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2643  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2644  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2645  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2646  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2647  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2648  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2649 
2650  /* response ack against second partial request */
2651  p->tcph->th_ack = htonl(175);
2652  p->tcph->th_seq = htonl(328);
2653  p->tcph->th_flags = TH_ACK;
2655  p->payload_len = 0;
2656  p->payload = NULL;
2657 
2658  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2670  FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2671  FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2672  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2673  FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2674  FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2675  FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2676 
2677  /* response acking a request */
2678  p->tcph->th_ack = htonl(175);
2679  p->tcph->th_seq = htonl(328);
2680  p->tcph->th_flags = TH_ACK;
2682  p->payload_len = 0;
2683  p->payload = NULL;
2684  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2690 
2693 
2694  /* request acking a response */
2695  p->tcph->th_ack = htonl(328);
2696  p->tcph->th_seq = htonl(175);
2697  p->tcph->th_flags = TH_ACK;
2699  p->payload_len = 0;
2700  p->payload = NULL;
2701  FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2702 
2703  StreamTcpSessionClear(ssn);
2705  SCFree(p);
2706  PASS;
2707 }
2708 
2709 /**
2710  * \test Test to make sure that we sent all the segments from the initial
2711  * segments to app layer until we have detected the app layer proto.
2712  *
2713  * \retval On success it returns 1 and on failure 0.
2714  */
2715 
2716 static int StreamTcpReassembleTest40 (void)
2717 {
2718  Packet *p = PacketGetFromAlloc();
2719  FAIL_IF_NULL(p);
2720  Flow *f = NULL;
2721  TCPHdr tcph;
2722  TcpSession ssn;
2723  PacketQueue pq;
2724  memset(&pq,0,sizeof(PacketQueue));
2725  memset(&tcph, 0, sizeof (TCPHdr));
2726  ThreadVars tv;
2727  memset(&tv, 0, sizeof (ThreadVars));
2728 
2731 
2733  FAIL_IF_NULL(ra_ctx);
2734 
2735  uint8_t httpbuf1[] = "P";
2736  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2737  uint8_t httpbuf3[] = "O";
2738  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2739  uint8_t httpbuf4[] = "S";
2740  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2741  uint8_t httpbuf5[] = "T \r\n";
2742  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2743 
2744  uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2745  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2746 
2747  SET_ISN(&ssn.server, 9);
2748  ssn.server.last_ack = 10;
2749  SET_ISN(&ssn.client, 9);
2750  ssn.client.isn = 9;
2751 
2752  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2753  FAIL_IF_NULL(f);
2754  f->protoctx = &ssn;
2755  f->proto = IPPROTO_TCP;
2756  p->flow = f;
2757 
2758  tcph.th_win = htons(5480);
2759  tcph.th_seq = htonl(10);
2760  tcph.th_ack = htonl(10);
2761  tcph.th_flags = TH_ACK|TH_PUSH;
2762  p->tcph = &tcph;
2764  p->payload = httpbuf1;
2765  p->payload_len = httplen1;
2766  ssn.state = TCP_ESTABLISHED;
2767  TcpStream *s = &ssn.client;
2768  SCLogDebug("1 -- start");
2769  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2770 
2772  p->payload = httpbuf2;
2773  p->payload_len = httplen2;
2774  tcph.th_seq = htonl(10);
2775  tcph.th_ack = htonl(11);
2776  s = &ssn.server;
2777  ssn.server.last_ack = 11;
2778  SCLogDebug("2 -- start");
2779  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2780 
2782  p->payload = httpbuf3;
2783  p->payload_len = httplen3;
2784  tcph.th_seq = htonl(11);
2785  tcph.th_ack = htonl(55);
2786  s = &ssn.client;
2787  ssn.client.last_ack = 55;
2788  SCLogDebug("3 -- start");
2789  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2790 
2792  p->payload = httpbuf2;
2793  p->payload_len = httplen2;
2794  tcph.th_seq = htonl(55);
2795  tcph.th_ack = htonl(12);
2796  s = &ssn.server;
2797  ssn.server.last_ack = 12;
2798  SCLogDebug("4 -- start");
2799  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2800 
2801  /* check is have the segment in the list and flagged or not */
2802  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2803  FAIL_IF_NULL(seg);
2804  FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2805 
2807  p->payload = httpbuf4;
2808  p->payload_len = httplen4;
2809  tcph.th_seq = htonl(12);
2810  tcph.th_ack = htonl(100);
2811  s = &ssn.client;
2812  ssn.client.last_ack = 100;
2813  SCLogDebug("5 -- start");
2814  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2815 
2817  p->payload = httpbuf2;
2818  p->payload_len = httplen2;
2819  tcph.th_seq = htonl(100);
2820  tcph.th_ack = htonl(13);
2821  s = &ssn.server;
2822  ssn.server.last_ack = 13;
2823  SCLogDebug("6 -- start");
2824  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2825 
2827  p->payload = httpbuf5;
2828  p->payload_len = httplen5;
2829  tcph.th_seq = htonl(13);
2830  tcph.th_ack = htonl(145);
2831  s = &ssn.client;
2832  ssn.client.last_ack = 145;
2833  SCLogDebug("7 -- start");
2834  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2835 
2837  p->payload = httpbuf2;
2838  p->payload_len = httplen2;
2839  tcph.th_seq = htonl(145);
2840  tcph.th_ack = htonl(16);
2841  s = &ssn.server;
2842  ssn.server.last_ack = 16;
2843  SCLogDebug("8 -- start");
2844  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2845  FAIL_IF(f->alproto != ALPROTO_HTTP);
2846 
2850  SCFree(p);
2851  UTHFreeFlow(f);
2852  PASS;
2853 }
2854 
2855 /** \test Test the memcap incrementing/decrementing and memcap check */
2856 static int StreamTcpReassembleTest44(void)
2857 {
2859  uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2861  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2863  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2865  FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2867  FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2868  PASS;
2869 }
2870 
2871 /**
2872  * \test Test to make sure that reassembly_depth is enforced.
2873  *
2874  * \retval On success it returns 1 and on failure 0.
2875  */
2876 
2877 static int StreamTcpReassembleTest45 (void)
2878 {
2879  TcpReassemblyThreadCtx *ra_ctx = NULL;
2880  TcpSession ssn;
2881  ThreadVars tv;
2882  memset(&tv, 0, sizeof(tv));
2883  uint8_t payload[100] = {0};
2884  uint16_t payload_size = 100;
2885 
2886  StreamTcpUTInit(&ra_ctx);
2888 
2890  ssn.reassembly_depth = 100;
2891  StreamTcpUTSetupStream(&ssn.server, 100);
2892  StreamTcpUTSetupStream(&ssn.client, 100);
2893 
2894  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2895  FAIL_IF(r != 0);
2897 
2898  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2899  FAIL_IF(r != 0);
2901 
2905  StreamTcpUTDeinit(ra_ctx);
2906  PASS;
2907 }
2908 
2909 /**
2910  * \test Test the unlimited config value of reassembly depth.
2911  *
2912  * \retval On success it returns 1 and on failure 0.
2913  */
2914 
2915 static int StreamTcpReassembleTest46 (void)
2916 {
2917  int result = 0;
2918  TcpReassemblyThreadCtx *ra_ctx = NULL;
2919  TcpSession ssn;
2920  ThreadVars tv;
2921  memset(&tv, 0, sizeof(tv));
2922  uint8_t payload[100] = {0};
2923  uint16_t payload_size = 100;
2924 
2925  StreamTcpUTInit(&ra_ctx);
2927 
2929  StreamTcpUTSetupStream(&ssn.server, 100);
2930  StreamTcpUTSetupStream(&ssn.client, 100);
2931 
2932  int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
2933  if (r != 0)
2934  goto end;
2936  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2937  goto end;
2938  }
2939 
2940  r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
2941  if (r != 0)
2942  goto end;
2944  printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
2945  goto end;
2946  }
2947 
2948  result = 1;
2949 end:
2953  StreamTcpUTDeinit(ra_ctx);
2954  return result;
2955 }
2956 
2957 /**
2958  * \test Test to make sure we detect the sequence wrap around and continue
2959  * stream reassembly properly.
2960  *
2961  * \retval On success it returns 1 and on failure 0.
2962  */
2963 
2964 static int StreamTcpReassembleTest47 (void)
2965 {
2966  Packet *p = PacketGetFromAlloc();
2967  FAIL_IF(unlikely(p == NULL));
2968  Flow *f = NULL;
2969  TCPHdr tcph;
2970  TcpSession ssn;
2971  ThreadVars tv;
2972  PacketQueue pq;
2973  memset(&pq,0,sizeof(PacketQueue));
2974  memset(&tcph, 0, sizeof (TCPHdr));
2975  memset(&tv, 0, sizeof (ThreadVars));
2979 
2980  uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
2981  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2982 
2983  SET_ISN(&ssn.server, 572799781UL);
2984  ssn.server.last_ack = 572799782UL;
2985 
2986  SET_ISN(&ssn.client, 4294967289UL);
2987  ssn.client.last_ack = 21;
2988 
2989  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2990  FAIL_IF(f == NULL);
2991  f->protoctx = &ssn;
2992  f->proto = IPPROTO_TCP;
2993  p->flow = f;
2994 
2995  tcph.th_win = htons(5480);
2996  ssn.state = TCP_ESTABLISHED;
2997  TcpStream *s = NULL;
2998  uint8_t cnt = 0;
2999 
3000  for (cnt=0; cnt < httplen1; cnt++) {
3001  tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3002  tcph.th_ack = htonl(572799782UL);
3003  tcph.th_flags = TH_ACK|TH_PUSH;
3004  p->tcph = &tcph;
3006  p->payload = &httpbuf1[cnt];
3007  p->payload_len = 1;
3008  s = &ssn.client;
3009 
3010  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3011 
3013  p->payload = NULL;
3014  p->payload_len = 0;
3015  tcph.th_seq = htonl(572799782UL);
3016  tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3017  tcph.th_flags = TH_ACK;
3018  p->tcph = &tcph;
3019  s = &ssn.server;
3020 
3021  FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3022  }
3023 
3024  FAIL_IF(f->alproto != ALPROTO_HTTP);
3025 
3029  SCFree(p);
3030  UTHFreeFlow(f);
3031  PASS;
3032 }
3033 
3034 /** \test 3 in order segments in inline reassembly */
3035 static int StreamTcpReassembleInlineTest01(void)
3036 {
3037  int ret = 0;
3038  TcpReassemblyThreadCtx *ra_ctx = NULL;
3039  ThreadVars tv;
3040  TcpSession ssn;
3041  Flow f;
3042 
3043  memset(&tv, 0x00, sizeof(tv));
3044 
3045  StreamTcpUTInit(&ra_ctx);
3048  StreamTcpUTSetupStream(&ssn.client, 1);
3049  FLOW_INITIALIZE(&f);
3050 
3051  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3052  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3053  if (p == NULL) {
3054  printf("couldn't get a packet: ");
3055  goto end;
3056  }
3057  p->tcph->th_seq = htonl(12);
3058  p->flow = &f;
3059 
3060  FLOWLOCK_WRLOCK(&f);
3061  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3062  printf("failed to add segment 1: ");
3063  goto end;
3064  }
3065  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3066  printf("failed to add segment 2: ");
3067  goto end;
3068  }
3069  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3070  printf("failed to add segment 3: ");
3071  goto end;
3072  }
3073  ssn.client.next_seq = 17;
3074  ret = 1;
3075 end:
3076  FLOWLOCK_UNLOCK(&f);
3077  FLOW_DESTROY(&f);
3078  UTHFreePacket(p);
3080  StreamTcpUTDeinit(ra_ctx);
3081  return ret;
3082 }
3083 
3084 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3085  * test the sliding window reassembly.
3086  */
3087 static int StreamTcpReassembleInlineTest02(void)
3088 {
3089  int ret = 0;
3090  TcpReassemblyThreadCtx *ra_ctx = NULL;
3091  ThreadVars tv;
3092  TcpSession ssn;
3093  Flow f;
3094 
3095  memset(&tv, 0x00, sizeof(tv));
3096 
3097  StreamTcpUTInit(&ra_ctx);
3100  StreamTcpUTSetupStream(&ssn.client, 1);
3101  FLOW_INITIALIZE(&f);
3102 
3103  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3104  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3105  if (p == NULL) {
3106  printf("couldn't get a packet: ");
3107  goto end;
3108  }
3109  p->tcph->th_seq = htonl(12);
3110  p->flow = &f;
3111 
3112  FLOWLOCK_WRLOCK(&f);
3113  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3114  printf("failed to add segment 1: ");
3115  goto end;
3116  }
3117  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3118  printf("failed to add segment 2: ");
3119  goto end;
3120  }
3121  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3122  printf("failed to add segment 3: ");
3123  goto end;
3124  }
3125  ssn.client.next_seq = 17;
3126  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3127  printf("failed to add segment 4: ");
3128  goto end;
3129  }
3130  ssn.client.next_seq = 22;
3131  ret = 1;
3132 end:
3133  FLOWLOCK_UNLOCK(&f);
3134  FLOW_DESTROY(&f);
3135  UTHFreePacket(p);
3137  StreamTcpUTDeinit(ra_ctx);
3138  return ret;
3139 }
3140 
3141 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3142  * test the sliding window reassembly with a small window size so that we
3143  * cutting off at the start (left edge)
3144  */
3145 static int StreamTcpReassembleInlineTest03(void)
3146 {
3147  int ret = 0;
3148  TcpReassemblyThreadCtx *ra_ctx = NULL;
3149  ThreadVars tv;
3150  TcpSession ssn;
3151  Flow f;
3152 
3153  memset(&tv, 0x00, sizeof(tv));
3154 
3155  StreamTcpUTInit(&ra_ctx);
3158  StreamTcpUTSetupStream(&ssn.client, 1);
3159  FLOW_INITIALIZE(&f);
3160 
3162 
3163  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3164  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3165  if (p == NULL) {
3166  printf("couldn't get a packet: ");
3167  goto end;
3168  }
3169  p->tcph->th_seq = htonl(12);
3170  p->flow = &f;
3172 
3173  FLOWLOCK_WRLOCK(&f);
3174  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3175  printf("failed to add segment 1: ");
3176  goto end;
3177  }
3178  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3179  printf("failed to add segment 2: ");
3180  goto end;
3181  }
3182  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3183  printf("failed to add segment 3: ");
3184  goto end;
3185  }
3186  ssn.client.next_seq = 17;
3187  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3188  printf("failed to add segment 4: ");
3189  goto end;
3190  }
3191  ssn.client.next_seq = 22;
3192 
3193  p->tcph->th_seq = htonl(17);
3194  ret = 1;
3195 end:
3196  FLOWLOCK_UNLOCK(&f);
3197  FLOW_DESTROY(&f);
3198  UTHFreePacket(p);
3200  StreamTcpUTDeinit(ra_ctx);
3201  return ret;
3202 }
3203 
3204 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3205  * test the sliding window reassembly with a small window size so that we
3206  * cutting off at the start (left edge) with small packet overlap.
3207  */
3208 static int StreamTcpReassembleInlineTest04(void)
3209 {
3210  int ret = 0;
3211  TcpReassemblyThreadCtx *ra_ctx = NULL;
3212  ThreadVars tv;
3213  TcpSession ssn;
3214  Flow f;
3215 
3216  memset(&tv, 0x00, sizeof(tv));
3217 
3218  StreamTcpUTInit(&ra_ctx);
3221  StreamTcpUTSetupStream(&ssn.client, 1);
3222  FLOW_INITIALIZE(&f);
3223 
3225 
3226  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3227  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3228  if (p == NULL) {
3229  printf("couldn't get a packet: ");
3230  goto end;
3231  }
3232  p->tcph->th_seq = htonl(12);
3233  p->flow = &f;
3235 
3236  FLOWLOCK_WRLOCK(&f);
3237  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3238  printf("failed to add segment 1: ");
3239  goto end;
3240  }
3241  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3242  printf("failed to add segment 2: ");
3243  goto end;
3244  }
3245  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3246  printf("failed to add segment 3: ");
3247  goto end;
3248  }
3249  ssn.client.next_seq = 17;
3250  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3251  printf("failed to add segment 4: ");
3252  goto end;
3253  }
3254  ssn.client.next_seq = 22;
3255 
3256  p->tcph->th_seq = htonl(17);
3257  ret = 1;
3258 end:
3259  FLOWLOCK_UNLOCK(&f);
3260  FLOW_DESTROY(&f);
3261  UTHFreePacket(p);
3263  StreamTcpUTDeinit(ra_ctx);
3264  return ret;
3265 }
3266 
3267 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3268  * test the sliding window reassembly with a small window size so that we
3269  * cutting off at the start (left edge). Test if the first segment is
3270  * removed from the list.
3271  */
3272 static int StreamTcpReassembleInlineTest08(void)
3273 {
3274  TcpReassemblyThreadCtx *ra_ctx = NULL;
3275  ThreadVars tv;
3276  memset(&tv, 0x00, sizeof(tv));
3277  TcpSession ssn;
3278  Flow f;
3279  StreamTcpUTInit(&ra_ctx);
3282  StreamTcpUTSetupStream(&ssn.client, 1);
3283  FLOW_INITIALIZE(&f);
3284 
3287  f.protoctx = &ssn;
3288 
3289  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3290  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3291  FAIL_IF(p == NULL);
3292  p->tcph->th_seq = htonl(12);
3293  p->flow = &f;
3295 
3296  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3297  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3298  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3299  ssn.client.next_seq = 17;
3300  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3301  ssn.client.next_seq = 22;
3302  p->tcph->th_seq = htonl(17);
3304 
3305  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3306  FAIL_IF_NULL(seg);
3307  FAIL_IF_NOT(seg->seq == 2);
3308 
3309  FLOW_DESTROY(&f);
3310  UTHFreePacket(p);
3312  StreamTcpUTDeinit(ra_ctx);
3313  PASS;
3314 }
3315 
3316 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3317  * test the sliding window reassembly with a small window size so that we
3318  * cutting off at the start (left edge). Test if the first segment is
3319  * removed from the list.
3320  */
3321 static int StreamTcpReassembleInlineTest09(void)
3322 {
3323  int ret = 0;
3324  TcpReassemblyThreadCtx *ra_ctx = NULL;
3325  ThreadVars tv;
3326  TcpSession ssn;
3327  Flow f;
3328 
3329  memset(&tv, 0x00, sizeof(tv));
3330 
3331  StreamTcpUTInit(&ra_ctx);
3334  StreamTcpUTSetupStream(&ssn.client, 1);
3335  FLOW_INITIALIZE(&f);
3336 
3339 
3340  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3341  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3342  if (p == NULL) {
3343  printf("couldn't get a packet: ");
3344  goto end;
3345  }
3346  p->tcph->th_seq = htonl(17);
3347  p->flow = &f;
3349 
3350  FLOWLOCK_WRLOCK(&f);
3351  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3352  printf("failed to add segment 1: ");
3353  goto end;
3354  }
3355  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3356  printf("failed to add segment 2: ");
3357  goto end;
3358  }
3359  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3360  printf("failed to add segment 3: ");
3361  goto end;
3362  }
3363  ssn.client.next_seq = 12;
3364  ssn.client.last_ack = 10;
3365 
3366  /* close the GAP and see if we properly reassemble and update base_seq */
3367  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3368  printf("failed to add segment 4: ");
3369  goto end;
3370  }
3371  ssn.client.next_seq = 22;
3372 
3373  p->tcph->th_seq = htonl(12);
3374 
3375  TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3376  FAIL_IF_NULL(seg);
3377  FAIL_IF_NOT(seg->seq == 2);
3378 
3379  ret = 1;
3380 end:
3381  FLOWLOCK_UNLOCK(&f);
3382  FLOW_DESTROY(&f);
3383  UTHFreePacket(p);
3385  StreamTcpUTDeinit(ra_ctx);
3386  return ret;
3387 }
3388 
3389 /** \test App Layer reassembly.
3390  */
3391 static int StreamTcpReassembleInlineTest10(void)
3392 {
3393  int ret = 0;
3394  TcpReassemblyThreadCtx *ra_ctx = NULL;
3395  ThreadVars tv;
3396  TcpSession ssn;
3397  Flow *f = NULL;
3398  Packet *p = NULL;
3399 
3400  memset(&tv, 0x00, sizeof(tv));
3401 
3402  StreamTcpUTInit(&ra_ctx);
3405  StreamTcpUTSetupStream(&ssn.server, 1);
3406  ssn.server.last_ack = 2;
3407  StreamTcpUTSetupStream(&ssn.client, 1);
3408  ssn.client.last_ack = 2;
3410 
3411  f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3412  if (f == NULL)
3413  goto end;
3414  f->protoctx = &ssn;
3415  f->proto = IPPROTO_TCP;
3416 
3417  uint8_t stream_payload1[] = "GE";
3418  uint8_t stream_payload2[] = "T /";
3419  uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3420 
3421  p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3422  if (p == NULL) {
3423  printf("couldn't get a packet: ");
3424  goto end;
3425  }
3426  p->tcph->th_seq = htonl(7);
3427  p->flow = f;
3429 
3430  FLOWLOCK_WRLOCK(f);
3431  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3432  printf("failed to add segment 1: ");
3433  goto end;
3434  }
3435  ssn.client.next_seq = 4;
3436 
3437  int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3438  if (r < 0) {
3439  printf("StreamTcpReassembleAppLayer failed: ");
3440  goto end;
3441  }
3442 
3443  /* ssn.server.ra_app_base_seq should be isn here. */
3444  if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3445  printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3446  goto end;
3447  }
3448 
3449  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3450  printf("failed to add segment 2: ");
3451  goto end;
3452  }
3453  if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3454  printf("failed to add segment 3: ");
3455  goto end;
3456  }
3457  ssn.client.next_seq = 19;
3458 
3459  r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3460  if (r < 0) {
3461  printf("StreamTcpReassembleAppLayer failed: ");
3462  goto end;
3463  }
3464 
3465  FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3466 
3467  ret = 1;
3468 end:
3469  UTHFreePacket(p);
3471  StreamTcpUTDeinit(ra_ctx);
3472  FLOWLOCK_UNLOCK(f);
3473  UTHFreeFlow(f);
3474  return ret;
3475 }
3476 
3477 /** \test test insert with overlap
3478  */
3479 static int StreamTcpReassembleInsertTest01(void)
3480 {
3481  TcpReassemblyThreadCtx *ra_ctx = NULL;
3482  ThreadVars tv;
3483  TcpSession ssn;
3484  Flow f;
3485 
3486  memset(&tv, 0x00, sizeof(tv));
3487 
3488  StreamTcpUTInit(&ra_ctx);
3490  StreamTcpUTSetupStream(&ssn.client, 1);
3492  FLOW_INITIALIZE(&f);
3493 
3494  uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3495  Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3496  FAIL_IF(p == NULL);
3497  p->tcph->th_seq = htonl(12);
3498  p->flow = &f;
3499 
3500  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3501  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3502  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3503  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3504  FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3505  ssn.client.next_seq = 21;
3506 
3507  FLOW_DESTROY(&f);
3508  UTHFreePacket(p);
3510  StreamTcpUTDeinit(ra_ctx);
3511  PASS;
3512 }
3513 
3514 /** \test test insert with overlaps
3515  */
3516 static int StreamTcpReassembleInsertTest02(void)
3517 {
3518  int ret = 0;
3519  TcpReassemblyThreadCtx *ra_ctx = NULL;
3520  ThreadVars tv;
3521  TcpSession ssn;
3522 
3523  memset(&tv, 0x00, sizeof(tv));
3524 
3525  StreamTcpUTInit(&ra_ctx);
3527  StreamTcpUTSetupStream(&ssn.client, 1);
3528 
3529  int i;
3530  for (i = 2; i < 10; i++) {
3531  int len;
3532  len = i % 2;
3533  if (len == 0)
3534  len = 1;
3535  int seq;
3536  seq = i * 10;
3537  if (seq < 2)
3538  seq = 2;
3539 
3540  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3541  printf("failed to add segment 1: ");
3542  goto end;
3543  }
3544  }
3545  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3546  printf("failed to add segment 2: ");
3547  goto end;
3548  }
3549 
3550  ret = 1;
3551 end:
3553  StreamTcpUTDeinit(ra_ctx);
3554  return ret;
3555 }
3556 
3557 /** \test test insert with overlaps
3558  */
3559 static int StreamTcpReassembleInsertTest03(void)
3560 {
3561  int ret = 0;
3562  TcpReassemblyThreadCtx *ra_ctx = NULL;
3563  ThreadVars tv;
3564  TcpSession ssn;
3565 
3566  memset(&tv, 0x00, sizeof(tv));
3567 
3568  StreamTcpUTInit(&ra_ctx);
3570  StreamTcpUTSetupStream(&ssn.client, 1);
3571 
3572  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3573  printf("failed to add segment 2: ");
3574  goto end;
3575  }
3576 
3577  int i;
3578  for (i = 2; i < 10; i++) {
3579  int len;
3580  len = i % 2;
3581  if (len == 0)
3582  len = 1;
3583  int seq;
3584  seq = i * 10;
3585  if (seq < 2)
3586  seq = 2;
3587 
3588  if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3589  printf("failed to add segment 2: ");
3590  goto end;
3591  }
3592  }
3593  ret = 1;
3594 end:
3596  StreamTcpUTDeinit(ra_ctx);
3597  return ret;
3598 }
3599 
3601 #endif /* UNITTESTS */
3602 
3603 /** \brief The Function Register the Unit tests to test the reassembly engine
3604  * for various OS policies.
3605  */
3606 
3608 {
3609 #ifdef UNITTESTS
3610  UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3611  StreamTcpReassembleTest25);
3612  UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3613  StreamTcpReassembleTest26);
3614  UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3615  StreamTcpReassembleTest27);
3616  UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3617  StreamTcpReassembleTest28);
3618  UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3619  StreamTcpReassembleTest29);
3620  UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3621  StreamTcpReassembleTest33);
3622  UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3623  StreamTcpReassembleTest34);
3624  UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3625  StreamTcpReassembleTest37);
3626  UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3627  StreamTcpReassembleTest39);
3628  UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3629  StreamTcpReassembleTest40);
3630  UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3631  StreamTcpReassembleTest44);
3632  UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3633  StreamTcpReassembleTest45);
3634  UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3635  StreamTcpReassembleTest46);
3636  UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3637  StreamTcpReassembleTest47);
3638 
3639  UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3640  StreamTcpReassembleInlineTest01);
3641  UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3642  StreamTcpReassembleInlineTest02);
3643  UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3644  StreamTcpReassembleInlineTest03);
3645  UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3646  StreamTcpReassembleInlineTest04);
3647  UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3648  StreamTcpReassembleInlineTest08);
3649  UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3650  StreamTcpReassembleInlineTest09);
3651 
3652  UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3653  StreamTcpReassembleInlineTest10);
3654 
3655  UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3656  StreamTcpReassembleInsertTest01);
3657  UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3658  StreamTcpReassembleInsertTest02);
3659  UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3660  StreamTcpReassembleInsertTest03);
3661 
3665  StreamTcpReassembleRawRegisterTests();
3666 #endif /* UNITTESTS */
3667 }
TcpSegment * StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *)
get a segment from the pool
#define SCMutex
StreamingBufferSegment sbseg
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
#define SCLogDebug(...)
Definition: util-debug.h:335
void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx)
void *(* Malloc)(size_t size)
int StreamTcpAppLayerIsDisabled(Flow *f)
struct Flow_ * flow
Definition: decode.h:443
AppProto alproto_tc
Definition: flow.h:406
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
uint64_t StreamTcpReassembleGetMemcap()
Return memcap value.
#define MISSED_STEP(seq, seg, seglen, buf, buflen)
AppLayerDecoderEvents * app_layer_events
Definition: decode.h:567
#define STREAM_BASE_OFFSET(stream)
int StreamTcpReassembleInit(char quiet)
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:343
#define FALSE
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:242
StreamingBuffer sb
#define PASS
Pass the test.
#define unlikely(expr)
Definition: util-optimize.h:35
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
#define STREAMTCP_STREAM_FLAG_GAP
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
update stream engine after detection
void StreamTcpListRegisterTests(void)
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define TH_RST
Definition: decode-tcp.h:37
void StreamTcpSetOSPolicy(TcpStream *, Packet *)
Function to set the OS policy for the given stream based on the destination of the received packet...
Definition: stream-tcp.c:770
int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream *stream, uint8_t *data, uint32_t data_len, uint8_t flags)
handle TCP data for the app-layer.
Definition: app-layer.c:551
void StreamTcpDisableAppLayer(Flow *f)
#define TH_FIN
Definition: decode-tcp.h:35
#define STREAMING_BUFFER_NOFLAGS
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:107
uint64_t offset
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Address dst
Definition: decode.h:411
int(* StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len)
Definition: stream-tcp.h:130
void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
Trigger RAW stream reassembly.
#define SCReturnUInt(x)
Definition: util-debug.h:343
void AppLayerDecoderEventsSetEventRaw(AppLayerDecoderEvents **sevents, uint8_t event)
Set an app layer decoder event.
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED
int ByteExtractStringUint32(uint32_t *res, int base, uint16_t len, const char *str)
Definition: util-byte.c:244
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
does the stream engine have data to inspect?
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t)
The Function to create the packet with given payload, which is used to test the reassembly of the eng...
void StreamTcpFreeConfig(char quiet)
Definition: stream-tcp.c:669
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:239
uint64_t pcap_cnt
Definition: decode.h:561
#define STREAM_RAW_PROGRESS(stream)
void StreamTcpReassembleInitMemuse(void)
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:124
char * val
Definition: conf.h:34
TCPHdr * tcph
Definition: decode.h:520
#define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED
#define TRUE
#define SCMutexLock(mut)
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
StreamingBufferBlock * head
#define STREAMTCP_FLAG_MIDSTREAM
void * protoctx
Definition: flow.h:395
void *(* Calloc)(size_t n, size_t size)
void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
void PoolThreadFree(PoolThread *pt)
destroy the thread pool
int StreamReassembleLog(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, uint64_t progress_in, uint64_t *progress_out, bool eof)
#define STREAM_DEPTH
Definition: stream.h:34
TcpReassemblyThreadCtx * ra_ctx
Definition: stream-tcp.h:103
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
uint32_t min_inspect_depth
void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx)
int StreamReassembleRaw(TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth)
int StreamTcpPacket(ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueue *pq)
Definition: stream-tcp.c:4674
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it&#39;s lock.
Definition: util-atomic.h:81
#define StreamTcpSetEvent(p, e)
AppProto alproto_ts
Definition: flow.h:405
Flow * UTHBuildFlow(int family, const char *src, const char *dst, Port sp, Port dp)
#define SCCalloc(nm, a)
Definition: util-mem.h:197
#define SEQ_GEQ(a, b)
#define TCP_GET_SEQ(p)
Definition: decode-tcp.h:106
#define STREAM_GAP
Definition: stream.h:33
#define SCMutexUnlock(mut)
AppLayerThreadCtx * AppLayerGetCtxThread(ThreadVars *tv)
Creates a new app layer thread context.
Definition: app-layer.c:803
uint32_t seq
uint32_t raw_progress_rel
#define FLOW_IS_PM_DONE(f, dir)
Definition: flow.h:247
char family
Definition: decode.h:109
uint8_t proto
Definition: decode.h:428
void StreamTcpReassembleDecrMemuse(uint64_t size)
Function to Decrease the memory usage counter for the TCP reassembly segments.
#define SCMUTEX_INITIALIZER
#define TCP_SEG_LEN(seg)
void StreamTcpReassembleIncrMemuse(uint64_t size)
Function to Increment the memory usage counter for the TCP reassembly segments.
#define STREAM_HAS_SEEN_DATA(stream)
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
uint32_t app_progress_rel
Data structures and function prototypes for keeping state for the detection engine.
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define FLOW_DESTROY(f)
Definition: flow-util.h:115
void StreamTcpReassembleRegisterTests(void)
The Function Register the Unit tests to test the reassembly engine for various OS policies...
void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
#define STREAM_EOF
Definition: stream.h:30
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define StreamTcpDisableAppLayerReassembly(ssn)
void StreamTcpSessionClear(void *ssnptr)
Function to return the stream back to the pool. It returns the segments in the stream to the segment ...
Definition: stream-tcp.c:249
void PoolThreadReturn(PoolThread *pt, void *data)
return data to thread pool
TcpReassemblyThreadCtx * StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
void *(* Realloc)(void *ptr, size_t orig_size, size_t size)
#define SCEnter(...)
Definition: util-debug.h:337
void StreamTcpInitConfig(char)
To initialize the stream global configuration data.
Definition: stream-tcp.c:365
uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
Packet * UTHBuildPacketReal(uint8_t *payload, uint16_t payload_len, uint8_t ipproto, const char *src, const char *dst, uint16_t sport, uint16_t dport)
UTHBuildPacketReal is a function that create tcp/udp packets for unittests specifying ip and port sou...
void StreamTcpReassembleFree(char quiet)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:437
#define STREAM_TOCLIENT
Definition: stream.h:32
#define PKT_IS_TOSERVER(p)
Definition: decode.h:255
#define FLOW_PKT_TOSERVER
Definition: flow.h:200
uint32_t reassembly_depth
#define TH_PUSH
Definition: decode-tcp.h:38
#define SEQ_GT(a, b)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:163
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1093
int RunmodeIsUnittests(void)
Definition: suricata.c:259
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
Definition: counters.c:1000
#define SCMutexInit(mut, mutattrs)
int8_t data_first_seen_dir
#define StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream)
#define STREAMTCP_STREAM_FLAG_TRIGGER_RAW
#define SCReturnInt(x)
Definition: util-debug.h:341
int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, PacketQueue *pq)
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
void StreamTcpUTInitInline(void)
#define SEQ_LEQ(a, b)
#define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED
uint32_t reassembly_depth
Definition: stream-tcp.h:60
#define SCRealloc(x, a)
Definition: util-mem.h:182
int StreamTcpReassembleSetMemcap(uint64_t size)
Update memcap value.
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define RB_MIN(name, x)
Definition: tree.h:780
Definition: conf.h:32
int PoolThreadGrow(PoolThread *pt, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
grow a thread pool by one
#define SCMalloc(a)
Definition: util-mem.h:166
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:207
struct TCPSEG seg_tree
uint16_t payload_len
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define TH_SYN
Definition: decode-tcp.h:36
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
block of continues data
#define SCFree(a)
Definition: util-mem.h:228
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
#define FLOW_IS_PP_DONE(f, dir)
Definition: flow.h:248
#define RB_REMOVE(name, x, y)
Definition: tree.h:775
void StreamTcpUtilRegisterTests(void)
void AppLayerDestroyCtxThread(AppLayerThreadCtx *app_tctx)
Destroys the context created by AppLayeGetCtxThread().
Definition: app-layer.c:825
int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
Insert a packets TCP data into the stream reassembly engine.
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
#define STREAM_START
Definition: stream.h:29
uint32_t prealloc_segments
Definition: stream-tcp.h:57
#define FLOW_INITIALIZE(f)
Definition: flow-util.h:39
#define PKT_STREAM_ADD
Definition: decode.h:1089
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn)
void UTHFreePacket(Packet *p)
UTHFreePacket: function to release the allocated data from UTHBuildPacket and the packet itself...
void StreamTcpInlineRegisterTests(void)
SC_ATOMIC_DECLARE(uint64_t, ra_memuse)
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:192
void StreamTcpReassemblePseudoPacketCreate(TcpStream *, Packet *, PacketQueue *)
void * PoolThreadGetById(PoolThread *pt, uint16_t id)
get data from thread pool by thread id
ConfNode * ConfGetNode(const char *name)
Get a ConfNode by name.
Definition: conf.c:176
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig sbcnf
Definition: stream-tcp.h:67
int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
The Function Checks the reassembled stream contents against predefined stream contents according to O...
int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir)
Update the stream reassembly upon receiving a packet.
int StreamTcpUTAddSegmentWithPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
#define SCReturn
Definition: util-debug.h:339
void StreamTcpUTSetupSession(TcpSession *ssn)
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define MISSED_END
int StreamTcpReassembleDepthReached(Packet *p)
check if stream in pkt direction has depth reached
#define FLOW_PKT_TOCLIENT
Definition: flow.h:201
AppProto alproto
application level protocol
Definition: flow.h:404
int PoolThreadSize(PoolThread *pt)
get size of PoolThread (number of &#39;threads&#39;, so array elements)
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
uint32_t flags
Definition: decode.h:441
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
void StreamTcpReassembleConfigEnableOverlapCheck(void)
uint16_t payload_len
Definition: decode.h:541
#define STREAM_APP_PROGRESS(stream)
#define SEQ_LT(a, b)
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6282
int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len)
#define STREAM_RIGHT_EDGE(stream)
int StreamTcpReassembleCheckMemcap(uint64_t size)
Function to Check the reassembly memory usage counter against the allowed max memory usgae for TCP se...
void StreamTcpUTClearSession(TcpSession *ssn)
#define MISSED_START(isn)
Flow data structure.
Definition: flow.h:324
#define FLOW_IPV4
Definition: flow.h:93
uint8_t * payload
Definition: decode.h:540
PoolThread * PoolThreadInit(int threads, uint32_t size, uint32_t prealloc_size, uint32_t elt_size, void *(*Alloc)(void), int(*Init)(void *, void *), void *InitData, void(*Cleanup)(void *), void(*Free)(void *))
initialize a thread pool
uint32_t flags
Definition: flow.h:374
#define APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER
Definition: app-layer.h:36
int StreamTcpUTAddPayload(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, uint32_t seq, uint8_t *payload, uint16_t len)
wrapper for StreamTcpReassembleHandleSegmentHandleData
void StreamTcpUTClearStream(TcpStream *s)
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
void(* Free)(void *ptr, size_t size)
Packet * PacketGetFromAlloc(void)
Get a malloced packet.
Definition: decode.c:140
Address src
Definition: decode.h:410
void UTHFreeFlow(Flow *flow)
#define STREAM_MIDSTREAM
Definition: stream.h:35
#define DEBUG_VALIDATE_BUG_ON(exp)
#define SET_ISN(stream, setseq)
#define SCMutexDestroy
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.