suricata
stream-tcp-list.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /** \file
19  *
20  * Segment list functions for insertions, overlap handling, removal and
21  * more.
22  */
23 
24 #include "suricata-common.h"
25 #include "stream-tcp-private.h"
26 #include "stream-tcp.h"
27 #include "stream-tcp-reassemble.h"
28 #include "stream-tcp-inline.h"
29 #include "stream-tcp-list.h"
30 #include "util-streaming-buffer.h"
31 #include "util-print.h"
32 #include "util-validate.h"
33 
34 static void StreamTcpRemoveSegmentFromStream(TcpStream *stream, TcpSegment *seg);
35 
36 static int check_overlap_different_data = 0;
37 
39 {
40  check_overlap_different_data = 1;
41 }
42 
43 /*
44  * Inserts and overlap handling
45  */
46 
48 
49 int TcpSegmentCompare(struct TcpSegment *a, struct TcpSegment *b)
50 {
51  if (SEQ_GT(a->seq, b->seq))
52  return 1;
53  else if (SEQ_LT(a->seq, b->seq))
54  return -1;
55  else {
56  if (a->payload_len == b->payload_len)
57  return 0;
58  else if (a->payload_len > b->payload_len)
59  return 1;
60  else
61  return -1;
62  }
63 }
64 
65 /** \internal
66  * \brief insert segment data into the streaming buffer
67  * \param seg segment to store stream offset in
68  * \param data segment data after overlap handling (if any)
69  * \param data_len data length
70  *
71  * \return 0 on success
72  * \return -1 on memory allocation error
73  * \return negative value on other errors
74  */
75 static inline int InsertSegmentDataCustom(TcpStream *stream, TcpSegment *seg, uint8_t *data, uint16_t data_len)
76 {
77  uint64_t stream_offset;
78  uint16_t data_offset;
79 
80  if (likely(SEQ_GEQ(seg->seq, stream->base_seq))) {
81  stream_offset = STREAM_BASE_OFFSET(stream) + (seg->seq - stream->base_seq);
82  data_offset = 0;
83  } else {
84  /* segment is partly before base_seq */
85  data_offset = stream->base_seq - seg->seq;
87  }
88 
89  SCLogDebug("stream %p buffer %p, stream_offset %"PRIu64", "
90  "data_offset %"PRIu16", SEQ %u BASE %u, data_len %u",
91  stream, &stream->sb, stream_offset,
92  data_offset, seg->seq, stream->base_seq, data_len);
93  BUG_ON(data_offset > data_len);
94  if (data_len == data_offset) {
95  SCReturnInt(0);
96  }
97 
98  int ret = StreamingBufferInsertAt(
99  &stream->sb, &seg->sbseg, data + data_offset, data_len - data_offset, stream_offset);
100  if (ret != 0) {
101  SCReturnInt(ret);
102  }
103 #ifdef DEBUG
104  {
105  const uint8_t *mydata;
106  uint32_t mydata_len;
107  uint64_t mydata_offset;
108  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
109 
110  SCLogDebug("stream %p seg %p data in buffer %p of len %u and offset %"PRIu64,
111  stream, seg, &stream->sb, mydata_len, mydata_offset);
112  //PrintRawDataFp(stdout, mydata, mydata_len);
113  }
114 #endif
115  SCReturnInt(0);
116 }
117 
118 /** \internal
119  * \brief check if this segments overlaps with an in-tree seg.
120  * \retval true
121  * \retval false
122  */
123 static inline bool CheckOverlap(struct TCPSEG *tree, TcpSegment *seg)
124 {
125  const uint32_t re = SEG_SEQ_RIGHT_EDGE(seg);
126  SCLogDebug("start. SEQ %u payload_len %u. Right edge: %u. Seg %p",
127  seg->seq, seg->payload_len, re, seg);
128 
129  /* check forward */
130  TcpSegment *next = TCPSEG_RB_NEXT(seg);
131  if (next) {
132  // next has same seq, so data must overlap
133  if (SEQ_EQ(next->seq, seg->seq))
134  return true;
135  // our right edge is beyond next seq, overlap
136  if (SEQ_GT(re, next->seq))
137  return true;
138  }
139  /* check backwards */
140  TcpSegment *prev = TCPSEG_RB_PREV(seg);
141  if (prev) {
142  // prev has same seq, so data must overlap
143  if (SEQ_EQ(prev->seq, seg->seq))
144  return true;
145  // prev's right edge is beyond our seq, overlap
146  const uint32_t prev_re = SEG_SEQ_RIGHT_EDGE(prev);
147  if (SEQ_GT(prev_re, seg->seq))
148  return true;
149  }
150 
151  SCLogDebug("no overlap");
152  return false;
153 }
154 
155 /** \internal
156  * \brief insert the segment into the proper place in the tree
157  * don't worry about the data or overlaps
158  *
159  * \retval 2 not inserted, data overlap
160  * \retval 1 inserted with overlap detected
161  * \retval 0 inserted, no overlap
162  * \retval -1 error
163  */
164 static int DoInsertSegment (TcpStream *stream, TcpSegment *seg, TcpSegment **dup_seg, Packet *p)
165 {
166  /* before our base_seq we don't insert it in our list */
167  if (SEQ_LEQ(SEG_SEQ_RIGHT_EDGE(seg), stream->base_seq))
168  {
169  SCLogDebug("not inserting: SEQ+payload %"PRIu32", last_ack %"PRIu32", "
170  "base_seq %"PRIu32, (seg->seq + TCP_SEG_LEN(seg)),
171  stream->last_ack, stream->base_seq);
173  return -1;
174  }
175 
176  /* fast track */
177  if (RB_EMPTY(&stream->seg_tree)) {
178  SCLogDebug("empty tree, inserting seg %p seq %" PRIu32 ", "
179  "len %" PRIu32 "", seg, seg->seq, TCP_SEG_LEN(seg));
180  TCPSEG_RB_INSERT(&stream->seg_tree, seg);
181  stream->segs_right_edge = SEG_SEQ_RIGHT_EDGE(seg);
182  return 0;
183  }
184 
185  /* insert and then check if there was any overlap with other segments */
186  TcpSegment *res = TCPSEG_RB_INSERT(&stream->seg_tree, seg);
187  if (res) {
188  SCLogDebug("seg has a duplicate in the tree seq %u/%u",
189  res->seq, res->payload_len);
190  /* exact duplicate SEQ + payload_len */
191  *dup_seg = res;
192  return 2; // duplicate has overlap by definition.
193  } else {
194  if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), stream->segs_right_edge))
195  stream->segs_right_edge = SEG_SEQ_RIGHT_EDGE(seg);
196 
197  /* insert succeeded, now check if we overlap with someone */
198  if (CheckOverlap(&stream->seg_tree, seg) == true) {
199  SCLogDebug("seg %u has overlap in the tree", seg->seq);
200  return 1;
201  }
202  }
203  SCLogDebug("seg %u: no overlap", seg->seq);
204  return 0;
205 }
206 
207 /** \internal
208  * \brief handle overlap per list segment
209  *
210  * For a list segment handle the overlap according to the policy.
211  *
212  * The 'buf' parameter points to the memory that will be inserted into
213  * the stream after the overlap checks are complete. As it will
214  * unconditionally overwrite whats in the stream now, the overlap
215  * policies are applied to this buffer. It starts with the 'new' data,
216  * so when the policy states 'old' data has to be used, 'buf' is
217  * updated to contain the 'old' data here.
218  *
219  * \param buf stack allocated buffer sized p->payload_len that will be
220  * inserted into the stream buffer
221  *
222  * \retval 1 if data was different
223  * \retval 0 data was the same or we didn't check for differences
224  */
225 static int DoHandleDataOverlap(TcpStream *stream, const TcpSegment *list,
226  const TcpSegment *seg, uint8_t *buf, Packet *p)
227 {
228  SCLogDebug("handle overlap for segment %p seq %u len %u re %u, "
229  "list segment %p seq %u len %u re %u", seg, seg->seq,
231  list, list->seq, TCP_SEG_LEN(list), SEG_SEQ_RIGHT_EDGE(list));
232 
233  int data_is_different = 0;
234  int use_new_data = 0;
235 
236  if (StreamTcpInlineMode()) {
237  SCLogDebug("inline mode");
238  if (StreamTcpInlineSegmentCompare(stream, p, list) != 0) {
239  SCLogDebug("already accepted data not the same as packet data, rewrite packet");
240  StreamTcpInlineSegmentReplacePacket(stream, p, list);
241  data_is_different = 1;
242 
243  /* in inline mode we check for different data unconditionally,
244  * but setting events still depends on config */
245  if (check_overlap_different_data) {
247  }
248  }
249 
250  /* IDS mode */
251  } else {
252  if (check_overlap_different_data) {
253  if (StreamTcpInlineSegmentCompare(stream, p, list) != 0) {
254  SCLogDebug("data is different from what is in the list");
255  data_is_different = 1;
256  }
257  } else {
258  /* if we're not checking, assume it's different */
259  data_is_different = 1;
260  }
261 
262  /* apply overlap policies */
263 
264  if (stream->os_policy == OS_POLICY_LAST) {
265  /* buf will start with LAST data (from the segment),
266  * so if policy is LAST we're now done here. */
267  return (check_overlap_different_data && data_is_different);
268  }
269 
270  /* start at the same seq */
271  if (SEQ_EQ(seg->seq, list->seq)) {
272  SCLogDebug("seg starts at list segment");
273 
274  if (SEQ_LT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
275  SCLogDebug("seg ends before list end, end overlapped by list");
276  } else {
277  if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
278  SCLogDebug("seg ends beyond list end, list overlapped and more");
279  switch (stream->os_policy) {
280  case OS_POLICY_LINUX:
281  if (data_is_different) {
282  use_new_data = 1;
283  }
284  break;
285  }
286  } else {
287  SCLogDebug("full overlap");
288  }
289 
290  switch (stream->os_policy) {
291  case OS_POLICY_OLD_LINUX:
292  case OS_POLICY_SOLARIS:
293  case OS_POLICY_HPUX11:
294  if (data_is_different) {
295  use_new_data = 1;
296  }
297  break;
298  }
299  }
300 
301  /* new seg starts before list segment */
302  } else if (SEQ_LT(seg->seq, list->seq)) {
303  SCLogDebug("seg starts before list segment");
304 
305  if (SEQ_LT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
306  SCLogDebug("seg ends before list end, end overlapped by list");
307  } else {
308  if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
309  SCLogDebug("seg starts before and fully overlaps list and beyond");
310  } else {
311  SCLogDebug("seg starts before and fully overlaps list");
312  }
313 
314  switch (stream->os_policy) {
315  case OS_POLICY_SOLARIS:
316  case OS_POLICY_HPUX11:
317  if (data_is_different) {
318  use_new_data = 1;
319  }
320  break;
321  }
322  }
323 
324  switch (stream->os_policy) {
325  case OS_POLICY_BSD:
326  case OS_POLICY_HPUX10:
327  case OS_POLICY_IRIX:
328  case OS_POLICY_WINDOWS:
330  case OS_POLICY_OLD_LINUX:
331  case OS_POLICY_LINUX:
332  case OS_POLICY_MACOS:
333  if (data_is_different) {
334  use_new_data = 1;
335  }
336  break;
337  }
338 
339  /* new seg starts after list segment */
340  } else { //if (SEQ_GT(seg->seq, list->seq)) {
341  SCLogDebug("seg starts after list segment");
342 
343  if (SEQ_EQ(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
344  SCLogDebug("seg after and is fully overlapped by list");
345  } else if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
346  SCLogDebug("seg starts after list and ends after list");
347 
348  switch (stream->os_policy) {
349  case OS_POLICY_SOLARIS:
350  case OS_POLICY_HPUX11:
351  if (data_is_different) {
352  use_new_data = 1;
353  }
354  break;
355  }
356  } else {
357  SCLogDebug("seg starts after list and ends before list end");
358 
359  }
360  }
361  }
362 
363  SCLogDebug("data_is_different %s, use_new_data %s",
364  data_is_different ? "yes" : "no",
365  use_new_data ? "yes" : "no");
366 
367  /* if the data is different and we don't want to use the new (seg)
368  * data, we have to update buf with the list data */
369  if (data_is_different && !use_new_data) {
370  /* we need to copy list into seg */
371  uint16_t list_offset = 0;
372  uint16_t seg_offset = 0;
373  uint32_t list_len;
374  uint16_t seg_len = p->payload_len;
375  uint32_t list_seq = list->seq;
376 
377  const uint8_t *list_data;
378  StreamingBufferSegmentGetData(&stream->sb, &list->sbseg, &list_data, &list_len);
379  if (list_data == NULL || list_len == 0)
380  return 0;
381  BUG_ON(list_len > USHRT_MAX);
382 
383  /* if list seg is partially before base_seq, list_len (from stream) and
384  * TCP_SEG_LEN(list) will not be the same */
385  if (SEQ_GEQ(list->seq, stream->base_seq)) {
386  ;
387  } else {
388  list_seq = stream->base_seq;
389  list_len = SEG_SEQ_RIGHT_EDGE(list) - stream->base_seq;
390  }
391 
392  if (SEQ_LT(seg->seq, list_seq)) {
393  seg_offset = list_seq - seg->seq;
394  seg_len -= seg_offset;
395  } else if (SEQ_GT(seg->seq, list_seq)) {
396  list_offset = seg->seq - list_seq;
397  list_len -= list_offset;
398  }
399 
400  if (SEQ_LT(seg->seq + seg_offset + seg_len, list_seq + list_offset + list_len)) {
401  list_len -= (list_seq + list_offset + list_len) - (seg->seq + seg_offset + seg_len);
402  }
403  SCLogDebug("here goes nothing: list %u %u, seg %u %u", list_offset, list_len, seg_offset, seg_len);
404 
405  //PrintRawDataFp(stdout, list_data + list_offset, list_len);
406  //PrintRawDataFp(stdout, buf + seg_offset, seg_len);
407 
408  memcpy(buf + seg_offset, list_data + list_offset, list_len);
409  //PrintRawDataFp(stdout, buf, p->payload_len);
410  }
411  return (check_overlap_different_data && data_is_different);
412 }
413 
414 /** \internal
415  * \brief walk segment tree backwards to see if there are overlaps
416  *
417  * Walk back from the current segment which is already in the tree.
418  * We walk until we can't possibly overlap anymore.
419  */
420 static int DoHandleDataCheckBackwards(TcpStream *stream,
421  TcpSegment *seg, uint8_t *buf, Packet *p)
422 {
423  int retval = 0;
424 
425  SCLogDebug("check tree backwards: insert data for segment %p seq %u len %u re %u",
426  seg, seg->seq, TCP_SEG_LEN(seg), SEG_SEQ_RIGHT_EDGE(seg));
427 
428  /* check backwards */
429  TcpSegment *tree_seg = NULL, *s = seg;
430  RB_FOREACH_REVERSE_FROM(tree_seg, TCPSEG, s) {
431  if (tree_seg == seg)
432  continue;
433 
434  int overlap = 0;
435  if (SEQ_LEQ(SEG_SEQ_RIGHT_EDGE(tree_seg), stream->base_seq)) {
436  // segment entirely before base_seq
437  ;
438  } else if (SEQ_LEQ(tree_seg->seq + tree_seg->payload_len, seg->seq)) {
439  SCLogDebug("list segment too far to the left, no more overlap will be found");
440  break;
441  } else if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(tree_seg), seg->seq)) {
442  overlap = 1;
443  }
444 
445  SCLogDebug("(back) tree seg %u len %u re %u overlap? %s",
446  tree_seg->seq, TCP_SEG_LEN(tree_seg),
447  SEG_SEQ_RIGHT_EDGE(tree_seg), overlap ? "yes" : "no");
448 
449  if (overlap) {
450  retval |= DoHandleDataOverlap(stream, tree_seg, seg, buf, p);
451  }
452  }
453  return retval;
454 }
455 
456 /** \internal
457  * \brief walk segment tree in forward direction to see if there are overlaps
458  *
459  * Walk forward from the current segment which is already in the tree.
460  * We walk until the next segs start with a SEQ beyond our right edge.
461  */
462 static int DoHandleDataCheckForward(TcpStream *stream,
463  TcpSegment *seg, uint8_t *buf, Packet *p)
464 {
465  int retval = 0;
466 
467  uint32_t seg_re = SEG_SEQ_RIGHT_EDGE(seg);
468 
469  SCLogDebug("check list forward: insert data for segment %p seq %u len %u re %u",
470  seg, seg->seq, TCP_SEG_LEN(seg), seg_re);
471 
472  TcpSegment *tree_seg = NULL, *s = seg;
473  RB_FOREACH_FROM(tree_seg, TCPSEG, s) {
474  if (tree_seg == seg)
475  continue;
476 
477  int overlap = 0;
478  if (SEQ_GT(seg_re, tree_seg->seq))
479  overlap = 1;
480  else if (SEQ_LEQ(seg_re, tree_seg->seq)) {
481  SCLogDebug("tree segment %u too far ahead, "
482  "no more overlaps can happen", tree_seg->seq);
483  break;
484  }
485 
486  SCLogDebug("(fwd) in-tree seg %u len %u re %u overlap? %s",
487  tree_seg->seq, TCP_SEG_LEN(tree_seg),
488  SEG_SEQ_RIGHT_EDGE(tree_seg), overlap ? "yes" : "no");
489 
490  if (overlap) {
491  retval |= DoHandleDataOverlap(stream, tree_seg, seg, buf, p);
492  }
493  }
494  return retval;
495 }
496 
497 /**
498  * \param dup_seg in-tree duplicate of `seg`
499  */
500 static int DoHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
501  TcpStream *stream, TcpSegment *seg, TcpSegment *tree_seg, Packet *p)
502 {
503  int result = 0;
504  TcpSegment *handle = seg;
505 
506  SCLogDebug("insert data for segment %p seq %u len %u re %u",
507  seg, seg->seq, TCP_SEG_LEN(seg), SEG_SEQ_RIGHT_EDGE(seg));
508 
509  /* create temporary buffer to contain the data we will insert. Overlap
510  * handling may update it. By using this we don't have to track whether
511  * parts of the data are already inserted or not. */
512  uint8_t buf[p->payload_len];
513  memcpy(buf, p->payload, p->payload_len);
514 
515  /* if tree_seg is set, we have an exact duplicate that we need to check */
516  if (tree_seg) {
517  DoHandleDataOverlap(stream, tree_seg, seg, buf, p);
518  handle = tree_seg;
519  }
520 
521  const bool is_head = !(TCPSEG_RB_PREV(handle));
522  const bool is_tail = !(TCPSEG_RB_NEXT(handle));
523 
524  /* new list head */
525  if (is_head && !is_tail) {
526  result = DoHandleDataCheckForward(stream, handle, buf, p);
527 
528  /* new list tail */
529  } else if (!is_head && is_tail) {
530  result = DoHandleDataCheckBackwards(stream, handle, buf, p);
531 
532  /* middle of the list */
533  } else if (!is_head && !is_tail) {
534  result = DoHandleDataCheckBackwards(stream, handle, buf, p);
535  result |= DoHandleDataCheckForward(stream, handle, buf, p);
536  }
537 
538  /* we had an overlap with different data */
539  if (result) {
542  }
543 
544  /* insert the temp buffer now that we've (possibly) updated
545  * it to account for the overlap policies */
546  int res = InsertSegmentDataCustom(stream, handle, buf, p->payload_len);
547  if (res < 0) {
548  if (res == -1) {
550  }
551  return -1;
552  }
553 
554  return 0;
555 }
556 
557 /**
558  * \retval -1 segment not inserted
559  *
560  * \param seg segment, this function takes total ownership
561  *
562  * In case of error, this function returns the segment to the pool
563  */
565  TcpStream *stream, TcpSegment *seg, Packet *p,
566  uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
567 {
568  SCEnter();
569 
570  TcpSegment *dup_seg = NULL;
571 
572  /* insert segment into list. Note: doesn't handle the data */
573  int r = DoInsertSegment (stream, seg, &dup_seg, p);
574  SCLogDebug("DoInsertSegment returned %d", r);
575  if (r < 0) {
578  SCReturnInt(-1);
579  }
580 
581  if (likely(r == 0)) {
582  /* no overlap, straight data insert */
583  int res = InsertSegmentDataCustom(stream, seg, pkt_data, pkt_datalen);
584  if (res < 0) {
585  if (res == -1) {
587  }
589  StreamTcpRemoveSegmentFromStream(stream, seg);
591  SCReturnInt(-1);
592  }
593 
594  } else if (r == 1 || r == 2) {
595  SCLogDebug("overlap (%s%s)", r == 1 ? "normal" : "", r == 2 ? "duplicate" : "");
596 
597  if (r == 2) {
598  SCLogDebug("dup_seg %p", dup_seg);
599  }
600 
601  /* XXX should we exclude 'retransmissions' here? */
603 
604  /* now let's consider the data in the overlap case */
605  int res = DoHandleData(tv, ra_ctx, stream, seg, dup_seg, p);
606  if (res < 0) {
608 
609  if (r == 1) // r == 2 mean seg wasn't added to stream
610  StreamTcpRemoveSegmentFromStream(stream, seg);
611 
613  SCReturnInt(-1);
614  }
615  if (r == 2) {
616  SCLogDebug("duplicate segment %u/%u, discard it",
617  seg->seq, seg->payload_len);
618 
620 #ifdef DEBUG
621  if (SCLogDebugEnabled()) {
622  TcpSegment *s = NULL, *safe = NULL;
623  RB_FOREACH_SAFE(s, TCPSEG, &stream->seg_tree, safe)
624  {
625  SCLogDebug("tree: seg %p, SEQ %"PRIu32", LEN %"PRIu16", SUM %"PRIu32"%s%s%s",
626  s, s->seq, TCP_SEG_LEN(s),
627  (uint32_t)(s->seq + TCP_SEG_LEN(s)),
628  s->seq == seg->seq ? " DUPLICATE" : "",
629  TCPSEG_RB_PREV(s) == NULL ? " HEAD" : "",
630  TCPSEG_RB_NEXT(s) == NULL ? " TAIL" : "");
631  }
632  }
633 #endif
634  }
635  }
636 
637  SCReturnInt(0);
638 }
639 
640 
641 /*
642  * Pruning & removal
643  */
644 
645 
646 static inline bool SegmentInUse(const TcpStream *stream, const TcpSegment *seg)
647 {
648  /* if proto detect isn't done, we're not returning */
649  if (!(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
651  SCReturnInt(true);
652  }
653  }
654 
655  SCReturnInt(false);
656 }
657 
658 
659 /** \internal
660  * \brief check if we can remove a segment from our segment list
661  *
662  * \retval true
663  * \retval false
664  */
665 static inline bool StreamTcpReturnSegmentCheck(const TcpStream *stream, const TcpSegment *seg)
666 {
667  if (SegmentInUse(stream, seg)) {
668  SCReturnInt(false);
669  }
670 
671  if (!(StreamingBufferSegmentIsBeforeWindow(&stream->sb, &seg->sbseg))) {
672  SCReturnInt(false);
673  }
674 
675  SCReturnInt(true);
676 }
677 
678 static inline uint64_t GetLeftEdge(TcpSession *ssn, TcpStream *stream)
679 {
680  bool use_app = true;
681  bool use_raw = true;
682  bool use_log = true;
683 
684  uint64_t left_edge = 0;
686  use_app = false; // app is dead
687  }
688 
690  use_raw = false; // raw is dead
691  }
693  use_log = false;
694  }
695 
696  if (use_raw) {
697  uint64_t raw_progress = STREAM_RAW_PROGRESS(stream);
698 
699  if (StreamTcpInlineMode() == TRUE) {
700  uint32_t chunk_size = (stream == &ssn->client) ?
703  if (raw_progress < (uint64_t)chunk_size) {
704  raw_progress = 0;
705  } else {
706  raw_progress -= (uint64_t)chunk_size;
707  }
708  }
709 
710  /* apply min inspect depth: if it is set we need to keep data
711  * before the raw progress. */
712  if (use_app && stream->min_inspect_depth) {
713  if (raw_progress < stream->min_inspect_depth)
714  raw_progress = 0;
715  else
716  raw_progress -= stream->min_inspect_depth;
717 
718  SCLogDebug("stream->min_inspect_depth %u, raw_progress %"PRIu64,
719  stream->min_inspect_depth, raw_progress);
720  }
721 
722  if (use_app) {
723  left_edge = MIN(STREAM_APP_PROGRESS(stream), raw_progress);
724  SCLogDebug("left_edge %"PRIu64", using both app:%"PRIu64", raw:%"PRIu64,
725  left_edge, STREAM_APP_PROGRESS(stream), raw_progress);
726  } else {
727  left_edge = raw_progress;
728  SCLogDebug("left_edge %"PRIu64", using only raw:%"PRIu64,
729  left_edge, raw_progress);
730  }
731  } else if (use_app) {
732  left_edge = STREAM_APP_PROGRESS(stream);
733  SCLogDebug("left_edge %"PRIu64", using only app:%"PRIu64,
734  left_edge, STREAM_APP_PROGRESS(stream));
735  } else {
736  left_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
737  SCLogDebug("no app & raw: left_edge %"PRIu64" (full stream)", left_edge);
738  }
739 
740  if (use_log) {
741  if (use_app || use_raw) {
742  left_edge = MIN(left_edge, STREAM_LOG_PROGRESS(stream));
743  } else {
744  left_edge = STREAM_LOG_PROGRESS(stream);
745  }
746  }
747 
748  /* in inline mode keep at least unack'd segments so we can check for overlaps */
749  if (StreamTcpInlineMode() == TRUE) {
750  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
751  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
752  /* get window of data that is acked */
753  const uint32_t delta = stream->last_ack - stream->base_seq;
754  /* get max absolute offset */
755  last_ack_abs += delta;
756  }
757  left_edge = MIN(left_edge, last_ack_abs);
758 
759  /* if we're told to look for overlaps with different data we should
760  * consider data that is ack'd as well. Injected packets may have
761  * been ack'd or injected packet may be too late. */
762  } else if (check_overlap_different_data) {
763  const uint32_t window = stream->window ? stream->window : 4096;
764  if (window < left_edge)
765  left_edge -= window;
766  else
767  left_edge = 0;
768 
769  SCLogDebug("stream:%p left_edge %"PRIu64, stream, left_edge);
770  }
771 
772  if (left_edge > 0) {
773  /* we know left edge based on the progress values now,
774  * lets adjust it to make sure in-use segments still have
775  * data */
776  TcpSegment *seg = NULL;
777  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
778  if (TCP_SEG_OFFSET(seg) > left_edge) {
779  SCLogDebug("seg beyond left_edge, we're done");
780  break;
781  }
782 
783  if (SegmentInUse(stream, seg)) {
784  left_edge = TCP_SEG_OFFSET(seg);
785  SCLogDebug("in-use seg before left_edge, adjust to %"PRIu64" and bail", left_edge);
786  break;
787  }
788  }
789  }
790 
791  return left_edge;
792 }
793 
794 static void StreamTcpRemoveSegmentFromStream(TcpStream *stream, TcpSegment *seg)
795 {
796  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
797 }
798 
799 /** \brief Remove idle TcpSegments from TcpSession
800  *
801  * Checks app progress and raw progress and progresses them
802  * if needed, slides the streaming buffer, then gets rid of
803  * excess segments.
804  *
805  * \param f flow
806  * \param flags direction flags
807  */
809 {
810  SCEnter();
811 
812  if (f == NULL || f->protoctx == NULL) {
813  SCReturn;
814  }
815 
816  TcpSession *ssn = f->protoctx;
817  TcpStream *stream = NULL;
818 
819  if (flags & STREAM_TOSERVER) {
820  stream = &ssn->client;
821  } else if (flags & STREAM_TOCLIENT) {
822  stream = &ssn->server;
823  } else {
824  SCReturn;
825  }
826 
828  return;
829  }
830 
833  SCLogDebug("ssn %p / stream %p: reassembly depth reached, "
834  "STREAMTCP_STREAM_FLAG_NOREASSEMBLY set", ssn, stream);
836  StreamingBufferClear(&stream->sb);
837  return;
838 
839  } else if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) &&
841  SCLogDebug("ssn %p / stream %p: both app and raw are done, "
842  "STREAMTCP_STREAM_FLAG_NOREASSEMBLY set", ssn, stream);
845  StreamingBufferClear(&stream->sb);
846  return;
847  }
848 
849  const uint64_t left_edge = GetLeftEdge(ssn, stream);
850  if (left_edge && left_edge > STREAM_BASE_OFFSET(stream)) {
851  uint32_t slide = left_edge - STREAM_BASE_OFFSET(stream);
852  SCLogDebug("buffer sliding %u to offset %"PRIu64, slide, left_edge);
853  StreamingBufferSlideToOffset(&stream->sb, left_edge);
854  stream->base_seq += slide;
855 
856  if (slide <= stream->app_progress_rel) {
857  stream->app_progress_rel -= slide;
858  } else {
859  stream->app_progress_rel = 0;
860  }
861  if (slide <= stream->raw_progress_rel) {
862  stream->raw_progress_rel -= slide;
863  } else {
864  stream->raw_progress_rel = 0;
865  }
866  if (slide <= stream->log_progress_rel) {
867  stream->log_progress_rel -= slide;
868  } else {
869  stream->log_progress_rel = 0;
870  }
871 
872  SCLogDebug("stream base_seq %u at stream offset %"PRIu64,
873  stream->base_seq, STREAM_BASE_OFFSET(stream));
874  }
875 
876  /* loop through the segments and remove all not in use */
877  TcpSegment *seg = NULL, *safe = NULL;
878  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
879  {
880  SCLogDebug("seg %p, SEQ %"PRIu32", LEN %"PRIu16", SUM %"PRIu32,
881  seg, seg->seq, TCP_SEG_LEN(seg),
882  (uint32_t)(seg->seq + TCP_SEG_LEN(seg)));
883 
884  if (StreamTcpReturnSegmentCheck(stream, seg) == 0) {
885  SCLogDebug("not removing segment");
886  break;
887  }
888 
889  StreamTcpRemoveSegmentFromStream(stream, seg);
891  SCLogDebug("removed segment");
892  continue;
893  }
894 
895  SCReturn;
896 }
897 
898 
899 /*
900  * unittests
901  */
902 
903 #ifdef UNITTESTS
904 #include "tests/stream-tcp-list.c"
905 #endif
StreamTcpSetEvent
#define StreamTcpSetEvent(p, e)
Definition: stream-tcp-private.h:249
TcpStream_
Definition: stream-tcp-private.h:94
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:101
RB_GENERATE
RB_GENERATE(TCPSEG, TcpSegment, rb, TcpSegmentCompare)
StatsIncr
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:169
StreamTcpSegmentReturntoPool
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
Definition: stream-tcp-reassemble.c:302
TCP_SEG_LEN
#define TCP_SEG_LEN(seg)
Definition: stream-tcp-private.h:82
stream-tcp-inline.h
stream-tcp.h
OS_POLICY_WINDOWS2K3
@ OS_POLICY_WINDOWS2K3
Definition: stream-tcp-reassemble.h:49
RB_REMOVE
#define RB_REMOVE(name, x, y)
Definition: tree.h:773
StreamTcpReassembleInsertSegment
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
Definition: stream-tcp-list.c:564
TcpStream_::seg_tree
struct TCPSEG seg_tree
Definition: stream-tcp-private.h:123
TcpStream_::log_progress_rel
uint32_t log_progress_rel
Definition: stream-tcp-private.h:116
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
Packet_::payload
uint8_t * payload
Definition: decode.h:551
StreamTcpInlineMode
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6381
TcpStream_::os_policy
uint8_t os_policy
Definition: stream-tcp-private.h:98
Flow_
Flow data structure.
Definition: flow.h:353
TcpSegment::sbseg
StreamingBufferSegment sbseg
Definition: stream-tcp-private.h:66
OS_POLICY_LAST
@ OS_POLICY_LAST
Definition: stream-tcp-reassemble.h:51
StreamTcpReassembleConfigEnableOverlapCheck
void StreamTcpReassembleConfigEnableOverlapCheck(void)
Definition: stream-tcp-list.c:38
STREAM_REASSEMBLY_SEGMENT_BEFORE_BASE_SEQ
@ STREAM_REASSEMBLY_SEGMENT_BEFORE_BASE_SEQ
Definition: decode-events.h:285
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:867
StreamTcpInlineSegmentReplacePacket
void StreamTcpInlineSegmentReplacePacket(const TcpStream *stream, Packet *p, const TcpSegment *seg)
Replace (part of) the payload portion of a packet by the data in a TCP segment.
Definition: stream-tcp-inline.c:112
TcpReassemblyThreadCtx_::counter_tcp_reass_overlap
uint16_t counter_tcp_reass_overlap
Definition: stream-tcp-reassemble.h:73
MIN
#define MIN(x, y)
Definition: suricata-common.h:372
stream-tcp-reassemble.h
TcpSegment::seq
uint32_t seq
Definition: stream-tcp-private.h:64
TcpStream_::flags
uint16_t flags
Definition: stream-tcp-private.h:95
TcpReassemblyThreadCtx_::counter_tcp_reass_overlap_diff_data
uint16_t counter_tcp_reass_overlap_diff_data
Definition: stream-tcp-reassemble.h:75
TcpStreamCnf_::streaming_log_api
bool streaming_log_api
Definition: stream-tcp.h:65
stream_config
TcpStreamCnf stream_config
Definition: stream-tcp.c:119
Flow_::protoctx
void * protoctx
Definition: flow.h:451
STREAM_REASSEMBLY_OVERLAP_DIFFERENT_DATA
@ STREAM_REASSEMBLY_OVERLAP_DIFFERENT_DATA
Definition: decode-events.h:288
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
Packet_::payload_len
uint16_t payload_len
Definition: decode.h:552
STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
Definition: stream-tcp-private.h:204
TcpSession_::flags
uint16_t flags
Definition: stream-tcp-private.h:269
STREAM_LOG_PROGRESS
#define STREAM_LOG_PROGRESS(stream)
Definition: stream-tcp-private.h:134
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb)
Definition: util-streaming-buffer.c:132
TCP_SEG_OFFSET
#define TCP_SEG_OFFSET(seg)
Definition: stream-tcp-private.h:83
TcpReassemblyThreadCtx_::counter_tcp_reass_data_normal_fail
uint16_t counter_tcp_reass_data_normal_fail
Definition: stream-tcp-reassemble.h:77
TcpStream_::last_ack
uint32_t last_ack
Definition: stream-tcp-private.h:103
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:749
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:131
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
res
PoolThreadReserved res
Definition: stream-tcp-private.h:0
TcpStream_::min_inspect_depth
uint32_t min_inspect_depth
Definition: stream-tcp-private.h:118
TcpSegmentCompare
int TcpSegmentCompare(struct TcpSegment *a, struct TcpSegment *b)
compare function for the Segment tree
Definition: stream-tcp-list.c:49
util-print.h
OS_POLICY_IRIX
@ OS_POLICY_IRIX
Definition: stream-tcp-reassemble.h:45
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
OS_POLICY_LINUX
@ OS_POLICY_LINUX
Definition: stream-tcp-reassemble.h:40
TcpStream_::segs_right_edge
uint32_t segs_right_edge
Definition: stream-tcp-private.h:124
stream-tcp-list.h
StreamTcpPruneSession
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.
Definition: stream-tcp-list.c:808
TRUE
#define TRUE
Definition: suricata-common.h:33
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
SCReturn
#define SCReturn
Definition: util-debug.h:302
TcpSegment
Definition: stream-tcp-private.h:61
StreamTcpIsSetStreamFlagAppProtoDetectionCompleted
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
Definition: stream-tcp-private.h:278
Packet_
Definition: decode.h:414
stream-tcp-private.h
TcpStream_::window
uint32_t window
Definition: stream-tcp-private.h:105
SEG_SEQ_RIGHT_EDGE
#define SEG_SEQ_RIGHT_EDGE(seg)
Definition: stream-tcp-private.h:85
StreamTcpReturnStreamSegments
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
Definition: stream-tcp-reassemble.c:315
TcpSegment::payload_len
uint16_t payload_len
Definition: stream-tcp-private.h:63
TcpReassemblyThreadCtx_::counter_tcp_reass_list_fail
uint16_t counter_tcp_reass_list_fail
Definition: stream-tcp-reassemble.h:79
OS_POLICY_SOLARIS
@ OS_POLICY_SOLARIS
Definition: stream-tcp-reassemble.h:42
OS_POLICY_BSD
@ OS_POLICY_BSD
Definition: stream-tcp-reassemble.h:37
TcpStream_::raw_progress_rel
uint32_t raw_progress_rel
Definition: stream-tcp-private.h:115
stream-tcp-list.c
SEQ_GEQ
#define SEQ_GEQ(a, b)
Definition: stream-tcp-private.h:241
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
SEQ_GT
#define SEQ_GT(a, b)
Definition: stream-tcp-private.h:240
util-streaming-buffer.h
TcpStream_::base_seq
uint32_t base_seq
Definition: stream-tcp-private.h:112
TcpStream_::sb
StreamingBuffer sb
Definition: stream-tcp-private.h:122
STREAM_TOCLIENT
#define STREAM_TOCLIENT
Definition: stream.h:32
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:272
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
OS_POLICY_HPUX11
@ OS_POLICY_HPUX11
Definition: stream-tcp-reassemble.h:44
TcpStreamCnf_::reassembly_toclient_chunk_size
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
util-validate.h
StreamTcpInlineSegmentCompare
int StreamTcpInlineSegmentCompare(const TcpStream *stream, const Packet *p, const TcpSegment *seg)
Compare the shared data portion of two segments.
Definition: stream-tcp-inline.c:47
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:271
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:521
OS_POLICY_MACOS
@ OS_POLICY_MACOS
Definition: stream-tcp-reassemble.h:46
SEQ_LT
#define SEQ_LT(a, b)
Definition: stream-tcp-private.h:238
TcpStream_::app_progress_rel
uint32_t app_progress_rel
Definition: stream-tcp-private.h:114
TcpReassemblyThreadCtx_::counter_tcp_reass_data_overlap_fail
uint16_t counter_tcp_reass_data_overlap_fail
Definition: stream-tcp-reassemble.h:78
OS_POLICY_WINDOWS
@ OS_POLICY_WINDOWS
Definition: stream-tcp-reassemble.h:47
SEQ_EQ
#define SEQ_EQ(a, b)
Definition: stream-tcp-private.h:237
TcpReassemblyThreadCtx_
Definition: stream-tcp-reassemble.h:60
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:679
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:188
STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
Definition: stream-tcp-private.h:200
SEQ_LEQ
#define SEQ_LEQ(a, b)
Definition: stream-tcp-private.h:239
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
STREAMTCP_STREAM_FLAG_DISABLE_RAW
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
Definition: stream-tcp-private.h:219
likely
#define likely(expr)
Definition: util-optimize.h:32
OS_POLICY_OLD_LINUX
@ OS_POLICY_OLD_LINUX
Definition: stream-tcp-reassemble.h:39
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:132
TcpSession_
Definition: stream-tcp-private.h:260
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:820
OS_POLICY_HPUX10
@ OS_POLICY_HPUX10
Definition: stream-tcp-reassemble.h:43
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
TcpReassemblyThreadCtx_::counter_tcp_segment_memcap
uint16_t counter_tcp_segment_memcap
Definition: stream-tcp-reassemble.h:66
SCLogDebugEnabled
int SCLogDebugEnabled(void)
Returns whether debug messages are enabled to be logged or not.
Definition: util-debug.c:656
TcpStreamCnf_::reassembly_toserver_chunk_size
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
STREAM_RAW_PROGRESS
#define STREAM_RAW_PROGRESS(stream)
Definition: stream-tcp-private.h:133