suricata
stream-tcp-list.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /** \file
19  *
20  * Segment list functions for insertions, overlap handling, removal and
21  * more.
22  */
23 
24 #include "suricata-common.h"
25 #include "stream-tcp-private.h"
26 #include "stream-tcp.h"
27 #include "stream-tcp-reassemble.h"
28 #include "stream-tcp-inline.h"
29 #include "stream-tcp-list.h"
30 #include "util-streaming-buffer.h"
31 #include "util-print.h"
32 #include "util-validate.h"
33 
34 static void StreamTcpRemoveSegmentFromStream(TcpStream *stream, TcpSegment *seg);
35 
36 static int check_overlap_different_data = 0;
37 
39 {
40  check_overlap_different_data = 1;
41 }
42 
43 /*
44  * Inserts and overlap handling
45  */
46 
48 
49 int TcpSegmentCompare(struct TcpSegment *a, struct TcpSegment *b)
50 {
51  if (SEQ_GT(a->seq, b->seq))
52  return 1;
53  else if (SEQ_LT(a->seq, b->seq))
54  return -1;
55  else {
56  if (a->payload_len == b->payload_len)
57  return 0;
58  else if (a->payload_len > b->payload_len)
59  return 1;
60  else
61  return -1;
62  }
63 }
64 
65 /** \internal
66  * \brief insert segment data into the streaming buffer
67  * \param seg segment to store stream offset in
68  * \param data segment data after overlap handling (if any)
69  * \param data_len data length
70  */
71 static inline int InsertSegmentDataCustom(TcpStream *stream, TcpSegment *seg, uint8_t *data, uint16_t data_len)
72 {
73  uint64_t stream_offset;
74  uint16_t data_offset;
75 
76  if (likely(SEQ_GEQ(seg->seq, stream->base_seq))) {
77  stream_offset = STREAM_BASE_OFFSET(stream) + (seg->seq - stream->base_seq);
78  data_offset = 0;
79  } else {
80  /* segment is partly before base_seq */
81  data_offset = stream->base_seq - seg->seq;
82  stream_offset = STREAM_BASE_OFFSET(stream);
83  }
84 
85  SCLogDebug("stream %p buffer %p, stream_offset %"PRIu64", "
86  "data_offset %"PRIu16", SEQ %u BASE %u, data_len %u",
87  stream, &stream->sb, stream_offset,
88  data_offset, seg->seq, stream->base_seq, data_len);
89  BUG_ON(data_offset > data_len);
90  if (data_len == data_offset) {
91  SCReturnInt(0);
92  }
93 
94  if (StreamingBufferInsertAt(&stream->sb, &seg->sbseg,
95  data + data_offset,
96  data_len - data_offset,
97  stream_offset) != 0) {
98  SCReturnInt(-1);
99  }
100 #ifdef DEBUG
101  {
102  const uint8_t *mydata;
103  uint32_t mydata_len;
104  uint64_t mydata_offset;
105  StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
106 
107  SCLogDebug("stream %p seg %p data in buffer %p of len %u and offset %"PRIu64,
108  stream, seg, &stream->sb, mydata_len, mydata_offset);
109  //PrintRawDataFp(stdout, mydata, mydata_len);
110  }
111 #endif
112  SCReturnInt(0);
113 }
114 
115 /** \internal
116  * \brief check if this segments overlaps with an in-tree seg.
117  * \retval true
118  * \retval false
119  */
120 static inline bool CheckOverlap(struct TCPSEG *tree, TcpSegment *seg)
121 {
122  const uint32_t re = SEG_SEQ_RIGHT_EDGE(seg);
123  SCLogDebug("start. SEQ %u payload_len %u. Right edge: %u. Seg %p",
124  seg->seq, seg->payload_len, re, seg);
125 
126  /* check forward */
127  TcpSegment *next = TCPSEG_RB_NEXT(seg);
128  if (next) {
129  // next has same seq, so data must overlap
130  if (SEQ_EQ(next->seq, seg->seq))
131  return true;
132  // our right edge is beyond next seq, overlap
133  if (SEQ_GT(re, next->seq))
134  return true;
135  }
136  /* check backwards */
137  TcpSegment *prev = TCPSEG_RB_PREV(seg);
138  if (prev) {
139  // prev has same seq, so data must overlap
140  if (SEQ_EQ(prev->seq, seg->seq))
141  return true;
142  // prev's right edge is beyond our seq, overlap
143  const uint32_t prev_re = SEG_SEQ_RIGHT_EDGE(prev);
144  if (SEQ_GT(prev_re, prev->seq))
145  return true;
146  }
147 
148  SCLogDebug("no overlap");
149  return false;
150 }
151 
152 /** \internal
153  * \brief insert the segment into the proper place in the tree
154  * don't worry about the data or overlaps
155  *
156  * \retval 2 not inserted, data overlap
157  * \retval 1 inserted with overlap detected
158  * \retval 0 inserted, no overlap
159  * \retval -1 error
160  */
161 static int DoInsertSegment (TcpStream *stream, TcpSegment *seg, TcpSegment **dup_seg, Packet *p)
162 {
163  /* before our base_seq we don't insert it in our list */
164  if (SEQ_LEQ(SEG_SEQ_RIGHT_EDGE(seg), stream->base_seq))
165  {
166  SCLogDebug("not inserting: SEQ+payload %"PRIu32", last_ack %"PRIu32", "
167  "base_seq %"PRIu32, (seg->seq + TCP_SEG_LEN(seg)),
168  stream->last_ack, stream->base_seq);
170  return -1;
171  }
172 
173  /* fast track */
174  if (RB_EMPTY(&stream->seg_tree)) {
175  SCLogDebug("empty tree, inserting seg %p seq %" PRIu32 ", "
176  "len %" PRIu32 "", seg, seg->seq, TCP_SEG_LEN(seg));
177  TCPSEG_RB_INSERT(&stream->seg_tree, seg);
178  stream->segs_right_edge = SEG_SEQ_RIGHT_EDGE(seg);
179  return 0;
180  }
181 
182  /* insert and then check if there was any overlap with other segments */
183  TcpSegment *res = TCPSEG_RB_INSERT(&stream->seg_tree, seg);
184  if (res) {
185  SCLogDebug("seg has a duplicate in the tree seq %u/%u",
186  res->seq, res->payload_len);
187  /* exact duplicate SEQ + payload_len */
188  *dup_seg = res;
189  return 2; // duplicate has overlap by definition.
190  } else {
191  if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), stream->segs_right_edge))
192  stream->segs_right_edge = SEG_SEQ_RIGHT_EDGE(seg);
193 
194  /* insert succeeded, now check if we overlap with someone */
195  if (CheckOverlap(&stream->seg_tree, seg) == true) {
196  SCLogDebug("seg %u has overlap in the tree", seg->seq);
197  return 1;
198  }
199  }
200  SCLogDebug("seg %u: no overlap", seg->seq);
201  return 0;
202 }
203 
204 /** \internal
205  * \brief handle overlap per list segment
206  *
207  * For a list segment handle the overlap according to the policy.
208  *
209  * The 'buf' parameter points to the memory that will be inserted into
210  * the stream after the overlap checks are complete. As it will
211  * unconditionally overwrite whats in the stream now, the overlap
212  * policies are applied to this buffer. It starts with the 'new' data,
213  * so when the policy states 'old' data has to be used, 'buf' is
214  * updated to contain the 'old' data here.
215  *
216  * \param buf stack allocated buffer sized p->payload_len that will be
217  * inserted into the stream buffer
218  *
219  * \retval 1 if data was different
220  * \retval 0 data was the same or we didn't check for differences
221  */
222 static int DoHandleDataOverlap(TcpStream *stream, const TcpSegment *list,
223  const TcpSegment *seg, uint8_t *buf, Packet *p)
224 {
225  SCLogDebug("handle overlap for segment %p seq %u len %u re %u, "
226  "list segment %p seq %u len %u re %u", seg, seg->seq,
228  list, list->seq, TCP_SEG_LEN(list), SEG_SEQ_RIGHT_EDGE(list));
229 
230  int data_is_different = 0;
231  int use_new_data = 0;
232 
233  if (StreamTcpInlineMode()) {
234  SCLogDebug("inline mode");
235  if (StreamTcpInlineSegmentCompare(stream, p, list) != 0) {
236  SCLogDebug("already accepted data not the same as packet data, rewrite packet");
237  StreamTcpInlineSegmentReplacePacket(stream, p, list);
238  data_is_different = 1;
239 
240  /* in inline mode we check for different data unconditionally,
241  * but setting events still depends on config */
242  if (check_overlap_different_data) {
244  }
245  }
246 
247  /* IDS mode */
248  } else {
249  if (check_overlap_different_data) {
250  if (StreamTcpInlineSegmentCompare(stream, p, list) != 0) {
251  SCLogDebug("data is different from what is in the list");
252  data_is_different = 1;
253  }
254  } else {
255  /* if we're not checking, assume it's different */
256  data_is_different = 1;
257  }
258 
259  /* apply overlap policies */
260 
261  if (stream->os_policy == OS_POLICY_LAST) {
262  /* buf will start with LAST data (from the segment),
263  * so if policy is LAST we're now done here. */
264  return (check_overlap_different_data && data_is_different);
265  }
266 
267  /* start at the same seq */
268  if (SEQ_EQ(seg->seq, list->seq)) {
269  SCLogDebug("seg starts at list segment");
270 
271  if (SEQ_LT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
272  SCLogDebug("seg ends before list end, end overlapped by list");
273  } else {
274  if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
275  SCLogDebug("seg ends beyond list end, list overlapped and more");
276  switch (stream->os_policy) {
277  case OS_POLICY_LINUX:
278  if (data_is_different) {
279  use_new_data = 1;
280  }
281  break;
282  }
283  } else {
284  SCLogDebug("full overlap");
285  }
286 
287  switch (stream->os_policy) {
288  case OS_POLICY_OLD_LINUX:
289  case OS_POLICY_SOLARIS:
290  case OS_POLICY_HPUX11:
291  if (data_is_different) {
292  use_new_data = 1;
293  }
294  break;
295  }
296  }
297 
298  /* new seg starts before list segment */
299  } else if (SEQ_LT(seg->seq, list->seq)) {
300  SCLogDebug("seg starts before list segment");
301 
302  if (SEQ_LT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
303  SCLogDebug("seg ends before list end, end overlapped by list");
304  } else {
305  if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
306  SCLogDebug("seg starts before and fully overlaps list and beyond");
307  } else {
308  SCLogDebug("seg starts before and fully overlaps list");
309  }
310 
311  switch (stream->os_policy) {
312  case OS_POLICY_SOLARIS:
313  case OS_POLICY_HPUX11:
314  if (data_is_different) {
315  use_new_data = 1;
316  }
317  break;
318  }
319  }
320 
321  switch (stream->os_policy) {
322  case OS_POLICY_BSD:
323  case OS_POLICY_HPUX10:
324  case OS_POLICY_IRIX:
325  case OS_POLICY_WINDOWS:
327  case OS_POLICY_OLD_LINUX:
328  case OS_POLICY_LINUX:
329  case OS_POLICY_MACOS:
330  if (data_is_different) {
331  use_new_data = 1;
332  }
333  break;
334  }
335 
336  /* new seg starts after list segment */
337  } else { //if (SEQ_GT(seg->seq, list->seq)) {
338  SCLogDebug("seg starts after list segment");
339 
340  if (SEQ_EQ(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
341  SCLogDebug("seg after and is fully overlapped by list");
342  } else if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(seg), SEG_SEQ_RIGHT_EDGE(list))) {
343  SCLogDebug("seg starts after list and ends after list");
344 
345  switch (stream->os_policy) {
346  case OS_POLICY_SOLARIS:
347  case OS_POLICY_HPUX11:
348  if (data_is_different) {
349  use_new_data = 1;
350  }
351  break;
352  }
353  } else {
354  SCLogDebug("seg starts after list and ends before list end");
355 
356  }
357  }
358  }
359 
360  SCLogDebug("data_is_different %s, use_new_data %s",
361  data_is_different ? "yes" : "no",
362  use_new_data ? "yes" : "no");
363 
364  /* if the data is different and we don't want to use the new (seg)
365  * data, we have to update buf with the list data */
366  if (data_is_different && !use_new_data) {
367  /* we need to copy list into seg */
368  uint16_t list_offset = 0;
369  uint16_t seg_offset = 0;
370  uint32_t list_len;
371  uint16_t seg_len = p->payload_len;
372  uint32_t list_seq = list->seq;
373 
374  const uint8_t *list_data;
375  StreamingBufferSegmentGetData(&stream->sb, &list->sbseg, &list_data, &list_len);
376  if (list_data == NULL || list_len == 0)
377  return 0;
378  BUG_ON(list_len > USHRT_MAX);
379 
380  /* if list seg is partially before base_seq, list_len (from stream) and
381  * TCP_SEG_LEN(list) will not be the same */
382  if (SEQ_GEQ(list->seq, stream->base_seq)) {
383  ;
384  } else {
385  list_seq = stream->base_seq;
386  list_len = SEG_SEQ_RIGHT_EDGE(list) - stream->base_seq;
387  }
388 
389  if (SEQ_LT(seg->seq, list_seq)) {
390  seg_offset = list_seq - seg->seq;
391  seg_len -= seg_offset;
392  } else if (SEQ_GT(seg->seq, list_seq)) {
393  list_offset = seg->seq - list_seq;
394  list_len -= list_offset;
395  }
396 
397  if (SEQ_LT(seg->seq + seg_offset + seg_len, list_seq + list_offset + list_len)) {
398  list_len -= (list_seq + list_offset + list_len) - (seg->seq + seg_offset + seg_len);
399  }
400  SCLogDebug("here goes nothing: list %u %u, seg %u %u", list_offset, list_len, seg_offset, seg_len);
401 
402  //PrintRawDataFp(stdout, list_data + list_offset, list_len);
403  //PrintRawDataFp(stdout, buf + seg_offset, seg_len);
404 
405  memcpy(buf + seg_offset, list_data + list_offset, list_len);
406  //PrintRawDataFp(stdout, buf, p->payload_len);
407  }
408  return (check_overlap_different_data && data_is_different);
409 }
410 
411 /** \internal
412  * \brief walk segment tree backwards to see if there are overlaps
413  *
414  * Walk back from the current segment which is already in the tree.
415  * We walk until we can't possibly overlap anymore.
416  */
417 static int DoHandleDataCheckBackwards(TcpStream *stream,
418  TcpSegment *seg, uint8_t *buf, Packet *p)
419 {
420  int retval = 0;
421 
422  SCLogDebug("check tree backwards: insert data for segment %p seq %u len %u re %u",
423  seg, seg->seq, TCP_SEG_LEN(seg), SEG_SEQ_RIGHT_EDGE(seg));
424 
425  /* check backwards */
426  TcpSegment *tree_seg = NULL, *s = seg;
427  RB_FOREACH_REVERSE_FROM(tree_seg, TCPSEG, s) {
428  if (tree_seg == seg)
429  continue;
430 
431  int overlap = 0;
432  if (SEQ_LEQ(SEG_SEQ_RIGHT_EDGE(tree_seg), stream->base_seq)) {
433  // segment entirely before base_seq
434  ;
435  } else if (SEQ_LEQ(tree_seg->seq + tree_seg->payload_len, seg->seq)) {
436  SCLogDebug("list segment too far to the left, no more overlap will be found");
437  break;
438  } else if (SEQ_GT(SEG_SEQ_RIGHT_EDGE(tree_seg), seg->seq)) {
439  overlap = 1;
440  }
441 
442  SCLogDebug("(back) tree seg %u len %u re %u overlap? %s",
443  tree_seg->seq, TCP_SEG_LEN(tree_seg),
444  SEG_SEQ_RIGHT_EDGE(tree_seg), overlap ? "yes" : "no");
445 
446  if (overlap) {
447  retval |= DoHandleDataOverlap(stream, tree_seg, seg, buf, p);
448  }
449  }
450  return retval;
451 }
452 
453 /** \internal
454  * \brief walk segment tree in forward direction to see if there are overlaps
455  *
456  * Walk forward from the current segment which is already in the tree.
457  * We walk until the next segs start with a SEQ beyond our right edge.
458  */
459 static int DoHandleDataCheckForward(TcpStream *stream,
460  TcpSegment *seg, uint8_t *buf, Packet *p)
461 {
462  int retval = 0;
463 
464  uint32_t seg_re = SEG_SEQ_RIGHT_EDGE(seg);
465 
466  SCLogDebug("check list forward: insert data for segment %p seq %u len %u re %u",
467  seg, seg->seq, TCP_SEG_LEN(seg), seg_re);
468 
469  TcpSegment *tree_seg = NULL, *s = seg;
470  RB_FOREACH_FROM(tree_seg, TCPSEG, s) {
471  if (tree_seg == seg)
472  continue;
473 
474  int overlap = 0;
475  if (SEQ_GT(seg_re, tree_seg->seq))
476  overlap = 1;
477  else if (SEQ_LEQ(seg_re, tree_seg->seq)) {
478  SCLogDebug("tree segment %u too far ahead, "
479  "no more overlaps can happen", tree_seg->seq);
480  break;
481  }
482 
483  SCLogDebug("(fwd) in-tree seg %u len %u re %u overlap? %s",
484  tree_seg->seq, TCP_SEG_LEN(tree_seg),
485  SEG_SEQ_RIGHT_EDGE(tree_seg), overlap ? "yes" : "no");
486 
487  if (overlap) {
488  retval |= DoHandleDataOverlap(stream, tree_seg, seg, buf, p);
489  }
490  }
491  return retval;
492 }
493 
494 /**
495  * \param dup_seg in-tree duplicate of `seg`
496  */
497 static int DoHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
498  TcpStream *stream, TcpSegment *seg, TcpSegment *tree_seg, Packet *p)
499 {
500  int result = 0;
501  TcpSegment *handle = seg;
502 
503  SCLogDebug("insert data for segment %p seq %u len %u re %u",
504  seg, seg->seq, TCP_SEG_LEN(seg), SEG_SEQ_RIGHT_EDGE(seg));
505 
506  /* create temporary buffer to contain the data we will insert. Overlap
507  * handling may update it. By using this we don't have to track whether
508  * parts of the data are already inserted or not. */
509  uint8_t buf[p->payload_len];
510  memcpy(buf, p->payload, p->payload_len);
511 
512  /* if tree_seg is set, we have an exact duplicate that we need to check */
513  if (tree_seg) {
514  DoHandleDataOverlap(stream, tree_seg, seg, buf, p);
515  handle = tree_seg;
516  }
517 
518  const bool is_head = !(TCPSEG_RB_PREV(handle));
519  const bool is_tail = !(TCPSEG_RB_NEXT(handle));
520 
521  /* new list head */
522  if (is_head && !is_tail) {
523  result = DoHandleDataCheckForward(stream, handle, buf, p);
524 
525  /* new list tail */
526  } else if (!is_head && is_tail) {
527  result = DoHandleDataCheckBackwards(stream, handle, buf, p);
528 
529  /* middle of the list */
530  } else if (!is_head && !is_tail) {
531  result = DoHandleDataCheckBackwards(stream, handle, buf, p);
532  result |= DoHandleDataCheckForward(stream, handle, buf, p);
533  }
534 
535  /* we had an overlap with different data */
536  if (result) {
539  }
540 
541  /* insert the temp buffer now that we've (possibly) updated
542  * it to account for the overlap policies */
543  if (InsertSegmentDataCustom(stream, handle, buf, p->payload_len) < 0) {
544  return -1;
545  }
546 
547  return 0;
548 }
549 
550 /**
551  * \retval -1 segment not inserted
552  *
553  * \param seg segment, this function takes total ownership
554  *
555  * In case of error, this function returns the segment to the pool
556  */
558  TcpStream *stream, TcpSegment *seg, Packet *p,
559  uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
560 {
561  SCEnter();
562 
563  TcpSegment *dup_seg = NULL;
564 
565  /* insert segment into list. Note: doesn't handle the data */
566  int r = DoInsertSegment (stream, seg, &dup_seg, p);
567  SCLogDebug("DoInsertSegment returned %d", r);
568  if (r < 0) {
571  SCReturnInt(-1);
572  }
573 
574  if (likely(r == 0)) {
575  /* no overlap, straight data insert */
576  int res = InsertSegmentDataCustom(stream, seg, pkt_data, pkt_datalen);
577  if (res < 0) {
579  StreamTcpRemoveSegmentFromStream(stream, seg);
581  SCReturnInt(-1);
582  }
583 
584  } else if (r == 1 || r == 2) {
585  SCLogDebug("overlap (%s%s)", r == 1 ? "normal" : "", r == 2 ? "duplicate" : "");
586 
587  if (r == 2) {
588  SCLogDebug("dup_seg %p", dup_seg);
589  }
590 
591  /* XXX should we exclude 'retransmissions' here? */
593 
594  /* now let's consider the data in the overlap case */
595  int res = DoHandleData(tv, ra_ctx, stream, seg, dup_seg, p);
596  if (res < 0) {
598 
599  if (r == 1) // r == 2 mean seg wasn't added to stream
600  StreamTcpRemoveSegmentFromStream(stream, seg);
601 
603  SCReturnInt(-1);
604  }
605  if (r == 2) {
606  SCLogDebug("duplicate segment %u/%u, discard it",
607  seg->seq, seg->payload_len);
608 
610 #ifdef DEBUG
611  if (SCLogDebugEnabled()) {
612  TcpSegment *s = NULL, *safe = NULL;
613  RB_FOREACH_SAFE(s, TCPSEG, &stream->seg_tree, safe)
614  {
615  SCLogDebug("tree: seg %p, SEQ %"PRIu32", LEN %"PRIu16", SUM %"PRIu32"%s%s%s",
616  s, s->seq, TCP_SEG_LEN(s),
617  (uint32_t)(s->seq + TCP_SEG_LEN(s)),
618  s->seq == seg->seq ? " DUPLICATE" : "",
619  TCPSEG_RB_PREV(s) == NULL ? " HEAD" : "",
620  TCPSEG_RB_NEXT(s) == NULL ? " TAIL" : "");
621  }
622  }
623 #endif
624  }
625  }
626 
627  SCReturnInt(0);
628 }
629 
630 
631 /*
632  * Pruning & removal
633  */
634 
635 
636 static inline bool SegmentInUse(const TcpStream *stream, const TcpSegment *seg)
637 {
638  /* if proto detect isn't done, we're not returning */
641  SCReturnInt(true);
642  }
643  }
644 
645  SCReturnInt(false);
646 }
647 
648 
649 /** \internal
650  * \brief check if we can remove a segment from our segment list
651  *
652  * \retval true
653  * \retval false
654  */
655 static inline bool StreamTcpReturnSegmentCheck(const TcpStream *stream, const TcpSegment *seg)
656 {
657  if (SegmentInUse(stream, seg)) {
658  SCReturnInt(false);
659  }
660 
661  if (!(StreamingBufferSegmentIsBeforeWindow(&stream->sb, &seg->sbseg))) {
662  SCReturnInt(false);
663  }
664 
665  SCReturnInt(true);
666 }
667 
668 static inline uint64_t GetLeftEdge(TcpSession *ssn, TcpStream *stream)
669 {
670  bool use_app = true;
671  bool use_raw = true;
672  bool use_log = true;
673 
674  uint64_t left_edge = 0;
676  (stream->flags & STREAMTCP_STREAM_FLAG_GAP))
677  {
678  use_app = false; // app is dead
679  }
680 
682  use_raw = false; // raw is dead
683  }
685  use_log = false;
686  }
687 
688  if (use_raw) {
689  uint64_t raw_progress = STREAM_RAW_PROGRESS(stream);
690 
691  if (StreamTcpInlineMode() == TRUE) {
692  uint32_t chunk_size = (stream == &ssn->client) ?
695  if (raw_progress < (uint64_t)chunk_size) {
696  raw_progress = 0;
697  } else {
698  raw_progress -= (uint64_t)chunk_size;
699  }
700  }
701 
702  /* apply min inspect depth: if it is set we need to keep data
703  * before the raw progress. */
704  if (use_app && stream->min_inspect_depth) {
705  if (raw_progress < stream->min_inspect_depth)
706  raw_progress = 0;
707  else
708  raw_progress -= stream->min_inspect_depth;
709 
710  SCLogDebug("stream->min_inspect_depth %u, raw_progress %"PRIu64,
711  stream->min_inspect_depth, raw_progress);
712  }
713 
714  if (use_app) {
715  left_edge = MIN(STREAM_APP_PROGRESS(stream), raw_progress);
716  SCLogDebug("left_edge %"PRIu64", using both app:%"PRIu64", raw:%"PRIu64,
717  left_edge, STREAM_APP_PROGRESS(stream), raw_progress);
718  } else {
719  left_edge = raw_progress;
720  SCLogDebug("left_edge %"PRIu64", using only raw:%"PRIu64,
721  left_edge, raw_progress);
722  }
723  } else if (use_app) {
724  left_edge = STREAM_APP_PROGRESS(stream);
725  SCLogDebug("left_edge %"PRIu64", using only app:%"PRIu64,
726  left_edge, STREAM_APP_PROGRESS(stream));
727  } else {
728  left_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
729  SCLogDebug("no app & raw: left_edge %"PRIu64" (full stream)", left_edge);
730  }
731 
732  if (use_log) {
733  if (use_app || use_raw) {
734  left_edge = MIN(left_edge, STREAM_LOG_PROGRESS(stream));
735  } else {
736  left_edge = STREAM_LOG_PROGRESS(stream);
737  }
738  }
739 
740  /* in inline mode keep at least unack'd segments so we can check for overlaps */
741  if (StreamTcpInlineMode() == TRUE) {
742  uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
743  if (STREAM_LASTACK_GT_BASESEQ(stream)) {
744  /* get window of data that is acked */
745  const uint32_t delta = stream->last_ack - stream->base_seq;
746  DEBUG_VALIDATE_BUG_ON(delta > 10000000ULL && delta > stream->window);
747  /* get max absolute offset */
748  last_ack_abs += delta;
749  }
750  left_edge = MIN(left_edge, last_ack_abs);
751 
752  /* if we're told to look for overlaps with different data we should
753  * consider data that is ack'd as well. Injected packets may have
754  * been ack'd or injected packet may be too late. */
755  } else if (check_overlap_different_data) {
756  const uint32_t window = stream->window ? stream->window : 4096;
757  if (window < left_edge)
758  left_edge -= window;
759  else
760  left_edge = 0;
761 
762  SCLogDebug("stream:%p left_edge %"PRIu64, stream, left_edge);
763  }
764 
765  if (left_edge > 0) {
766  /* we know left edge based on the progress values now,
767  * lets adjust it to make sure in-use segments still have
768  * data */
769  TcpSegment *seg = NULL;
770  RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
771  if (TCP_SEG_OFFSET(seg) > left_edge) {
772  SCLogDebug("seg beyond left_edge, we're done");
773  break;
774  }
775 
776  if (SegmentInUse(stream, seg)) {
777  left_edge = TCP_SEG_OFFSET(seg);
778  SCLogDebug("in-use seg before left_edge, adjust to %"PRIu64" and bail", left_edge);
779  break;
780  }
781  }
782  }
783 
784  return left_edge;
785 }
786 
787 static void StreamTcpRemoveSegmentFromStream(TcpStream *stream, TcpSegment *seg)
788 {
789  RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
790 }
791 
792 /** \brief Remove idle TcpSegments from TcpSession
793  *
794  * Checks app progress and raw progress and progresses them
795  * if needed, slides the streaming buffer, then gets rid of
796  * excess segments.
797  *
798  * \param f flow
799  * \param flags direction flags
800  */
802 {
803  SCEnter();
804 
805  if (f == NULL || f->protoctx == NULL) {
806  SCReturn;
807  }
808 
809  TcpSession *ssn = f->protoctx;
810  TcpStream *stream = NULL;
811 
812  if (flags & STREAM_TOSERVER) {
813  stream = &ssn->client;
814  } else if (flags & STREAM_TOCLIENT) {
815  stream = &ssn->server;
816  } else {
817  SCReturn;
818  }
819 
821  return;
822  }
823 
826  SCLogDebug("ssn %p / stream %p: reassembly depth reached, "
827  "STREAMTCP_STREAM_FLAG_NOREASSEMBLY set", ssn, stream);
829  StreamingBufferClear(&stream->sb);
830  return;
831 
832  } else if (((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) ||
833  (stream->flags & STREAMTCP_STREAM_FLAG_GAP)) &&
835  {
836  SCLogDebug("ssn %p / stream %p: both app and raw are done, "
837  "STREAMTCP_STREAM_FLAG_NOREASSEMBLY set", ssn, stream);
840  StreamingBufferClear(&stream->sb);
841  return;
842  }
843 
844  const uint64_t left_edge = GetLeftEdge(ssn, stream);
845  if (left_edge && left_edge > STREAM_BASE_OFFSET(stream)) {
846  uint32_t slide = left_edge - STREAM_BASE_OFFSET(stream);
847  SCLogDebug("buffer sliding %u to offset %"PRIu64, slide, left_edge);
848  StreamingBufferSlideToOffset(&stream->sb, left_edge);
849  stream->base_seq += slide;
850 
851  if (slide <= stream->app_progress_rel) {
852  stream->app_progress_rel -= slide;
853  } else {
854  stream->app_progress_rel = 0;
855  }
856  if (slide <= stream->raw_progress_rel) {
857  stream->raw_progress_rel -= slide;
858  } else {
859  stream->raw_progress_rel = 0;
860  }
861  if (slide <= stream->log_progress_rel) {
862  stream->log_progress_rel -= slide;
863  } else {
864  stream->log_progress_rel = 0;
865  }
866 
867  SCLogDebug("stream base_seq %u at stream offset %"PRIu64,
868  stream->base_seq, STREAM_BASE_OFFSET(stream));
869  }
870 
871  /* loop through the segments and remove all not in use */
872  TcpSegment *seg = NULL, *safe = NULL;
873  RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
874  {
875  SCLogDebug("seg %p, SEQ %"PRIu32", LEN %"PRIu16", SUM %"PRIu32,
876  seg, seg->seq, TCP_SEG_LEN(seg),
877  (uint32_t)(seg->seq + TCP_SEG_LEN(seg)));
878 
879  if (StreamTcpReturnSegmentCheck(stream, seg) == 0) {
880  SCLogDebug("not removing segment");
881  break;
882  }
883 
884  StreamTcpRemoveSegmentFromStream(stream, seg);
886  SCLogDebug("removed segment");
887  continue;
888  }
889 
890  SCReturn;
891 }
892 
893 
894 /*
895  * unittests
896  */
897 
898 #ifdef UNITTESTS
899 #include "tests/stream-tcp-list.c"
900 #endif
StreamingBufferSegment sbseg
uint16_t flags
#define SCLogDebug(...)
Definition: util-debug.h:335
struct HtpBodyChunk_ * next
#define STREAM_BASE_OFFSET(stream)
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
StreamingBuffer sb
#define STREAMTCP_STREAM_FLAG_GAP
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:788
#define STREAMTCP_STREAM_FLAG_DEPTH_REACHED
#define MIN(x, y)
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
void StreamingBufferClear(StreamingBuffer *sb)
#define STREAM_RAW_PROGRESS(stream)
#define TRUE
void * protoctx
Definition: flow.h:398
int SCLogDebugEnabled(void)
Returns whether debug messages are enabled to be logged or not.
Definition: util-debug.c:631
int TcpSegmentCompare(struct TcpSegment *a, struct TcpSegment *b)
compare function for the Segment tree
uint16_t reassembly_toclient_chunk_size
Definition: stream-tcp.h:63
uint32_t min_inspect_depth
RB_GENERATE(TCPSEG, TcpSegment, rb, TcpSegmentCompare)
#define StreamTcpSetEvent(p, e)
#define SEQ_GEQ(a, b)
uint32_t raw_progress_rel
#define TCP_SEG_LEN(seg)
int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p, uint32_t pkt_seq, uint8_t *pkt_data, uint16_t pkt_datalen)
uint32_t app_progress_rel
uint64_t stream_offset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, uint64_t offset)
slide to absolute offset
uint16_t payload_len
int StreamTcpInlineSegmentCompare(const TcpStream *stream, const Packet *p, const TcpSegment *seg)
Compare the shared data portion of two segments.
#define SCEnter(...)
Definition: util-debug.h:337
#define STREAM_TOCLIENT
Definition: stream.h:32
#define SEQ_GT(a, b)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
Definition: counters.c:163
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)
#define SCReturnInt(x)
Definition: util-debug.h:341
uint16_t reassembly_toserver_chunk_size
Definition: stream-tcp.h:62
#define SEQ_LEQ(a, b)
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:803
#define TCP_SEG_OFFSET(seg)
#define STREAMTCP_STREAM_FLAG_DISABLE_RAW
struct TCPSEG seg_tree
uint32_t log_progress_rel
PoolThreadReserved res
void StreamTcpInlineSegmentReplacePacket(const TcpStream *stream, Packet *p, const TcpSegment *seg)
Replace (part of) the payload portion of a packet by the data in a TCP segment.
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
bool streaming_log_api
Definition: stream-tcp.h:65
int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
#define RB_REMOVE(name, x, y)
Definition: tree.h:775
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
uint32_t segs_right_edge
void StreamTcpSegmentReturntoPool(TcpSegment *seg)
Function to return the segment back to the pool.
#define STREAM_TOSERVER
Definition: stream.h:31
#define STREAMTCP_STREAM_FLAG_NOREASSEMBLY
#define SEQ_EQ(a, b)
#define SCReturn
Definition: util-debug.h:339
Per thread variable structure.
Definition: threadvars.h:57
void StreamTcpReturnStreamSegments(TcpStream *stream)
return all segments in this stream into the pool(s)
void StreamTcpReassembleConfigEnableOverlapCheck(void)
uint16_t payload_len
Definition: decode.h:546
#define likely(expr)
Definition: util-optimize.h:32
#define STREAM_APP_PROGRESS(stream)
#define SEQ_LT(a, b)
int StreamTcpInlineMode(void)
See if stream engine is operating in inline mode.
Definition: stream-tcp.c:6283
#define SEG_SEQ_RIGHT_EDGE(seg)
Flow data structure.
Definition: flow.h:327
uint8_t * payload
Definition: decode.h:545
#define STREAM_LOG_PROGRESS(stream)
#define DEBUG_VALIDATE_BUG_ON(exp)
void StreamTcpPruneSession(Flow *f, uint8_t flags)
Remove idle TcpSegments from TcpSession.