suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-unittest.h"
21 #include "util-print.h"
22 #include "util-validate.h"
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  *
29  * \brief Streaming Buffer API
30  */
31 
32 /* memory handling wrappers. If config doesn't define it's own set of
33  * functions, use the defaults */
34 #define CALLOC(cfg, n, s) \
35  (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : SCCalloc((n), (s))
36 #define REALLOC(cfg, ptr, orig_s, s) \
37  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : SCRealloc((ptr), (s))
38 #define FREE(cfg, ptr, s) \
39  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
40 
41 static void SBBFree(StreamingBuffer *sb);
42 
44 
46 {
47  SCLogDebug("a %"PRIu64" len %u, b %"PRIu64" len %u",
48  a->offset, a->len, b->offset, b->len);
49 
50  if (a->offset > b->offset)
51  SCReturnInt(1);
52  else if (a->offset < b->offset)
53  SCReturnInt(-1);
54  else {
55  if (a->len == 0 || b->len == 0 || a->len == b->len)
56  SCReturnInt(0);
57  else if (a->len > b->len)
58  SCReturnInt(1);
59  else
60  SCReturnInt(-1);
61  }
62 }
63 
64 /* inclusive compare function that also considers the right edge,
65  * not just the offset. */
66 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
67  const uint64_t lre = lookup->offset + lookup->len;
68  const uint64_t tre = intree->offset + intree->len;
69  if (lre <= intree->offset) // entirely before
70  return -1;
71  else if (lookup->offset < tre && lre <= tre) // (some) overlap
72  return 0;
73  else
74  return 1; // entirely after
75 }
76 
78 {
79  SCLogDebug("looking up %"PRIu64, elm->offset);
80 
81  struct StreamingBufferBlock *tmp = RB_ROOT(head);
82  struct StreamingBufferBlock *res = NULL;
83  while (tmp) {
84  SCLogDebug("compare with %"PRIu64"/%u", tmp->offset, tmp->len);
85  const int comp = InclusiveCompare(elm, tmp);
86  SCLogDebug("compare result: %d", comp);
87  if (comp < 0) {
88  res = tmp;
89  tmp = RB_LEFT(tmp, rb);
90  } else if (comp > 0) {
91  tmp = RB_RIGHT(tmp, rb);
92  } else {
93  return tmp;
94  }
95  }
96  return res;
97 }
98 
99 
100 static inline int InitBuffer(StreamingBuffer *sb)
101 {
102  sb->buf = CALLOC(sb->cfg, 1, sb->cfg->buf_size);
103  if (sb->buf == NULL) {
104  return -1;
105  }
106  sb->buf_size = sb->cfg->buf_size;
107  return 0;
108 }
109 
111 {
112  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
113  if (sb != NULL) {
114  sb->buf_size = cfg->buf_size;
115  sb->cfg = cfg;
116 
117  if (cfg->buf_size > 0) {
118  if (InitBuffer(sb) == 0) {
119  return sb;
120  }
121  FREE(cfg, sb, sizeof(StreamingBuffer));
122  /* implied buf_size == 0 */
123  } else {
124  return sb;
125  }
126  }
127  return NULL;
128 }
129 
131 {
132  if (sb != NULL) {
133  SCLogDebug("sb->buf_size %u max %u", sb->buf_size, sb->buf_size_max);
134 
135  SBBFree(sb);
136  if (sb->buf != NULL) {
137  FREE(sb->cfg, sb->buf, sb->buf_size);
138  sb->buf = NULL;
139  }
140  }
141 }
142 
144 {
145  if (sb != NULL) {
147  FREE(sb->cfg, sb, sizeof(StreamingBuffer));
148  }
149 }
150 
151 #ifdef DEBUG
152 static void SBBPrintList(StreamingBuffer *sb)
153 {
154  StreamingBufferBlock *sbb = NULL;
155  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
156  SCLogDebug("sbb: offset %"PRIu64", len %u", sbb->offset, sbb->len);
157  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
158  if (next) {
159  if ((sbb->offset + sbb->len) != next->offset) {
160  SCLogDebug("gap: offset %"PRIu64", len %"PRIu64, (sbb->offset + sbb->len),
161  next->offset - (sbb->offset + sbb->len));
162  }
163  }
164  }
165 }
166 #endif
167 
168 /* setup with gap between 2 blocks
169  *
170  * [block][gap][block]
171  **/
172 static void SBBInit(StreamingBuffer *sb,
173  uint32_t rel_offset, uint32_t data_len)
174 {
176  DEBUG_VALIDATE_BUG_ON(sb->buf_offset > sb->stream_offset + rel_offset);
177 
178  /* need to set up 2: existing data block and new data block */
179  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
180  if (sbb == NULL) {
181  return;
182  }
183  sbb->offset = sb->stream_offset;
184  sbb->len = sb->buf_offset;
185 
186  StreamingBufferBlock *sbb2 = CALLOC(sb->cfg, 1, sizeof(*sbb2));
187  if (sbb2 == NULL) {
188  FREE(sb->cfg, sbb, sizeof(*sbb));
189  return;
190  }
191  sbb2->offset = sb->stream_offset + rel_offset;
192  sbb2->len = data_len;
193 
194  sb->head = sbb;
195  sb->sbb_size = sbb->len + sbb2->len;
196  SBB_RB_INSERT(&sb->sbb_tree, sbb);
197  SBB_RB_INSERT(&sb->sbb_tree, sbb2);
198 
199  SCLogDebug("sbb1 %"PRIu64", len %u, sbb2 %"PRIu64", len %u",
200  sbb->offset, sbb->len, sbb2->offset, sbb2->len);
201 #ifdef DEBUG
202  SBBPrintList(sb);
203 #endif
204  BUG_ON(sbb2->offset < sbb->len);
205 }
206 
207 /* setup with leading gap
208  *
209  * [gap][block]
210  **/
211 static void SBBInitLeadingGap(StreamingBuffer *sb,
212  uint64_t offset, uint32_t data_len)
213 {
215 
216  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
217  if (sbb == NULL)
218  return;
219  sbb->offset = offset;
220  sbb->len = data_len;
221 
222  sb->head = sbb;
223  sb->sbb_size = sbb->len;
224  SBB_RB_INSERT(&sb->sbb_tree, sbb);
225 
226  SCLogDebug("sbb %"PRIu64", len %u",
227  sbb->offset, sbb->len);
228 #ifdef DEBUG
229  SBBPrintList(sb);
230 #endif
231 }
232 
233 static inline void ConsolidateFwd(StreamingBuffer *sb,
234  struct SBB *tree, StreamingBufferBlock *sa)
235 {
236  uint64_t sa_re = sa->offset + sa->len;
237  StreamingBufferBlock *tr, *s = sa;
238  RB_FOREACH_FROM(tr, SBB, s) {
239  if (sa == tr)
240  continue;
241 
242  const uint64_t tr_re = tr->offset + tr->len;
243  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u re %"PRIu64,
244  tr, tr->offset, tr->len, tr_re);
245 
246  if (sa_re < tr->offset)
247  break; // entirely before
248 
249  /*
250  sa: [ ]
251  tr: [ ]
252  sa: [ ]
253  tr: [ ]
254  sa: [ ]
255  tr: [ ]
256  */
257  if (sa->offset >= tr->offset && sa_re <= tr_re) {
258  sb->sbb_size -= sa->len;
259  sa->len = tr->len;
260  sa->offset = tr->offset;
261  sa_re = sa->offset + sa->len;
262  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED2", tr, tr->offset, tr->len);
263  SBB_RB_REMOVE(tree, tr);
264  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
265  /*
266  sa: [ ]
267  tr: [ ]
268  sa: [ ]
269  tr: [ ]
270  sa: [ ]
271  tr: [ ]
272  */
273  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
274  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED", tr, tr->offset, tr->len);
275  SBB_RB_REMOVE(tree, tr);
276  sb->sbb_size -= tr->len;
277  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
278  /*
279  sa: [ ]
280  tr: [ ]
281  sa: [ ]
282  tr: [ ]
283  */
284  } else if (sa->offset < tr->offset && // starts before
285  sa_re >= tr->offset && sa_re < tr_re) // ends inside
286  {
287  // merge. sb->sbb_size includes both so we need to adjust that too.
288  uint32_t combined_len = sa->len + tr->len;
289  sa->len = tr_re - sa->offset;
290  sa_re = sa->offset + sa->len;
291  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED MERGED", tr, tr->offset, tr->len);
292  SBB_RB_REMOVE(tree, tr);
293  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
294  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
295  }
296  }
297 }
298 
299 static inline void ConsolidateBackward(StreamingBuffer *sb,
300  struct SBB *tree, StreamingBufferBlock *sa)
301 {
302  uint64_t sa_re = sa->offset + sa->len;
303  StreamingBufferBlock *tr, *s = sa;
304  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
305  if (sa == tr)
306  continue;
307  const uint64_t tr_re = tr->offset + tr->len;
308  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u", tr, tr->offset, tr->len);
309 
310  if (sa->offset > tr_re)
311  break; // entirely after
312 
313  if (sa->offset >= tr->offset && sa_re <= tr_re) {
314  sb->sbb_size -= sa->len; // sa entirely eclipsed so remove double accounting
315  sa->len = tr->len;
316  sa->offset = tr->offset;
317  sa_re = sa->offset + sa->len;
318  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED2", tr, tr->offset, tr->len);
319  if (sb->head == tr)
320  sb->head = sa;
321  SBB_RB_REMOVE(tree, tr);
322  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
323  /*
324  sa: [ ]
325  tr: [ ]
326  sa: [ ]
327  tr: [ ]
328  sa: [ ]
329  tr: [ ]
330  */
331  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
332  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED", tr, tr->offset, tr->len);
333  if (sb->head == tr)
334  sb->head = sa;
335  SBB_RB_REMOVE(tree, tr);
336  sb->sbb_size -= tr->len; // tr entirely eclipsed so remove double accounting
337  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
338  /*
339  sa: [ ]
340  tr: [ ]
341  sa: [ ]
342  tr: [ ]
343  */
344  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
345  // merge. sb->sbb_size includes both so we need to adjust that too.
346  uint32_t combined_len = sa->len + tr->len;
347  sa->len = sa_re - tr->offset;
348  sa->offset = tr->offset;
349  sa_re = sa->offset + sa->len;
350  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED MERGED", tr, tr->offset, tr->len);
351  if (sb->head == tr)
352  sb->head = sa;
353  SBB_RB_REMOVE(tree, tr);
354  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
355  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
356  }
357  }
358 }
359 
360 static int Insert(StreamingBuffer *sb, struct SBB *tree,
361  uint32_t rel_offset, uint32_t len)
362 {
363  SCLogDebug("* inserting: %u/%u\n", rel_offset, len);
364 
365  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
366  if (sbb == NULL)
367  return -1;
368  sbb->offset = sb->stream_offset + rel_offset;
369  sbb->len = len;
370  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
371  if (res) {
372  // exact overlap
373  SCLogDebug("* insert failed: exact match in tree with %p %"PRIu64"/%u", res, res->offset, res->len);
374  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
375  return 0;
376  }
377  sb->sbb_size += len; // may adjust based on consolidation below
378  if (SBB_RB_PREV(sbb) == NULL) {
379  sb->head = sbb;
380  } else {
381  ConsolidateBackward(sb, tree, sbb);
382  }
383  ConsolidateFwd(sb, tree, sbb);
384 #ifdef DEBUG
385  SBBPrintList(sb);
386 #endif
387  return 0;
388 }
389 
390 static void SBBUpdate(StreamingBuffer *sb,
391  uint32_t rel_offset, uint32_t data_len)
392 {
393  Insert(sb, &sb->sbb_tree, rel_offset, data_len);
394 }
395 
396 static void SBBFree(StreamingBuffer *sb)
397 {
398  StreamingBufferBlock *sbb = NULL, *safe = NULL;
399  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
400  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
401  sb->sbb_size -= sbb->len;
402  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
403  }
404  sb->head = NULL;
405 }
406 
407 static void SBBPrune(StreamingBuffer *sb)
408 {
409  SCLogDebug("pruning %p to %"PRIu64, sb, sb->stream_offset);
410  StreamingBufferBlock *sbb = NULL, *safe = NULL;
411  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
412  /* completely beyond window, we're done */
413  if (sbb->offset >= sb->stream_offset) {
414  sb->head = sbb;
415  break;
416  }
417 
418  /* partly before, partly beyond. Adjust */
419  if (sbb->offset < sb->stream_offset &&
420  sbb->offset + sbb->len > sb->stream_offset) {
421  uint32_t shrink_by = sb->stream_offset - sbb->offset;
422  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
423  if (sbb->len >= shrink_by) {
424  sbb->len -= shrink_by;
425  sbb->offset += shrink_by;
426  sb->sbb_size -= shrink_by;
428  }
429  sb->head = sbb;
430  break;
431  }
432 
433  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
434  /* either we set it again for the next sbb, or there isn't any */
435  sb->head = NULL;
436  sb->sbb_size -= sbb->len;
437  SCLogDebug("sb %p removed %p %"PRIu64", %u", sb, sbb, sbb->offset, sbb->len);
438  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
439  }
440 }
441 
442 static int WARN_UNUSED
443 GrowToSize(StreamingBuffer *sb, uint32_t size)
444 {
445  /* try to grow in multiples of sb->cfg->buf_size */
446  uint32_t x = sb->cfg->buf_size ? size % sb->cfg->buf_size : 0;
447  uint32_t base = size - x;
448  uint32_t grow = base + sb->cfg->buf_size;
449 
450  void *ptr = REALLOC(sb->cfg, sb->buf, sb->buf_size, grow);
451  if (ptr == NULL)
452  return -1;
453 
454  /* for safe printing and general caution, lets memset the
455  * new data to 0 */
456  size_t diff = grow - sb->buf_size;
457  void *new_mem = ((char *)ptr) + sb->buf_size;
458  memset(new_mem, 0, diff);
459 
460  sb->buf = ptr;
461  sb->buf_size = grow;
462  SCLogDebug("grown buffer to %u", grow);
463 #ifdef DEBUG
464  if (sb->buf_size > sb->buf_size_max) {
465  sb->buf_size_max = sb->buf_size;
466  }
467 #endif
468  return 0;
469 }
470 
471 /** \internal
472  * \brief try to double the buffer size
473  * \retval 0 ok
474  * \retval -1 failed, buffer unchanged
475  */
476 static int WARN_UNUSED Grow(StreamingBuffer *sb)
477 {
478  uint32_t grow = sb->buf_size * 2;
479  void *ptr = REALLOC(sb->cfg, sb->buf, sb->buf_size, grow);
480  if (ptr == NULL)
481  return -1;
482 
483  /* for safe printing and general caution, lets memset the
484  * new data to 0 */
485  size_t diff = grow - sb->buf_size;
486  void *new_mem = ((char *)ptr) + sb->buf_size;
487  memset(new_mem, 0, diff);
488 
489  sb->buf = ptr;
490  sb->buf_size = grow;
491  SCLogDebug("grown buffer to %u", grow);
492 #ifdef DEBUG
493  if (sb->buf_size > sb->buf_size_max) {
494  sb->buf_size_max = sb->buf_size;
495  }
496 #endif
497  return 0;
498 }
499 
500 /**
501  * \brief slide to absolute offset
502  * \todo if sliding beyond window, we could perhaps reset?
503  */
505 {
506  if (offset > sb->stream_offset &&
507  offset <= sb->stream_offset + sb->buf_offset)
508  {
509  uint32_t slide = offset - sb->stream_offset;
510  uint32_t size = sb->buf_offset - slide;
511  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide, size);
512  memmove(sb->buf, sb->buf+slide, size);
513  sb->stream_offset += slide;
514  sb->buf_offset = size;
515  SBBPrune(sb);
516  }
517 }
518 
519 void StreamingBufferSlide(StreamingBuffer *sb, uint32_t slide)
520 {
521  uint32_t size = sb->buf_offset - slide;
522  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide, size);
523  memmove(sb->buf, sb->buf+slide, size);
524  sb->stream_offset += slide;
525  sb->buf_offset = size;
526  SBBPrune(sb);
527 }
528 
529 #define DATA_FITS(sb, len) \
530  ((sb)->buf_offset + (len) <= (sb)->buf_size)
531 
532 StreamingBufferSegment *StreamingBufferAppendRaw(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
533 {
534  if (sb->buf == NULL) {
535  if (InitBuffer(sb) == -1)
536  return NULL;
537  }
538 
539  if (!DATA_FITS(sb, data_len)) {
540  if (sb->buf_size == 0) {
541  if (GrowToSize(sb, data_len) != 0)
542  return NULL;
543  } else {
544  while (!DATA_FITS(sb, data_len)) {
545  if (Grow(sb) != 0) {
546  return NULL;
547  }
548  }
549  }
550  }
551  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
552 
553  StreamingBufferSegment *seg = CALLOC(sb->cfg, 1, sizeof(StreamingBufferSegment));
554  if (seg != NULL) {
555  memcpy(sb->buf + sb->buf_offset, data, data_len);
556  seg->stream_offset = sb->stream_offset + sb->buf_offset;
557  seg->segment_len = data_len;
558  uint32_t rel_offset = sb->buf_offset;
559  sb->buf_offset += data_len;
560 
561  if (!RB_EMPTY(&sb->sbb_tree)) {
562  SBBUpdate(sb, rel_offset, data_len);
563  }
564  return seg;
565  }
566  return NULL;
567 }
568 
569 int StreamingBufferAppend(StreamingBuffer *sb, StreamingBufferSegment *seg,
570  const uint8_t *data, uint32_t data_len)
571 {
572  BUG_ON(seg == NULL);
573 
574  if (sb->buf == NULL) {
575  if (InitBuffer(sb) == -1)
576  return -1;
577  }
578 
579  if (!DATA_FITS(sb, data_len)) {
580  if (sb->buf_size == 0) {
581  if (GrowToSize(sb, data_len) != 0)
582  return -1;
583  } else {
584  while (!DATA_FITS(sb, data_len)) {
585  if (Grow(sb) != 0) {
586  return -1;
587  }
588  }
589  }
590  }
591  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
592 
593  memcpy(sb->buf + sb->buf_offset, data, data_len);
594  seg->stream_offset = sb->stream_offset + sb->buf_offset;
595  seg->segment_len = data_len;
596  uint32_t rel_offset = sb->buf_offset;
597  sb->buf_offset += data_len;
598 
599  if (!RB_EMPTY(&sb->sbb_tree)) {
600  SBBUpdate(sb, rel_offset, data_len);
601  }
602  return 0;
603 }
604 
605 /**
606  * \brief add data w/o tracking a segment
607  */
609  const uint8_t *data, uint32_t data_len)
610 {
611  if (sb->buf == NULL) {
612  if (InitBuffer(sb) == -1)
613  return -1;
614  }
615 
616  if (!DATA_FITS(sb, data_len)) {
617  if (sb->buf_size == 0) {
618  if (GrowToSize(sb, data_len) != 0)
619  return -1;
620  } else {
621  while (!DATA_FITS(sb, data_len)) {
622  if (Grow(sb) != 0) {
623  return -1;
624  }
625  }
626  }
627  }
628  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
629 
630  memcpy(sb->buf + sb->buf_offset, data, data_len);
631  uint32_t rel_offset = sb->buf_offset;
632  sb->buf_offset += data_len;
633 
634  if (!RB_EMPTY(&sb->sbb_tree)) {
635  SBBUpdate(sb, rel_offset, data_len);
636  }
637  return 0;
638 }
639 
640 #define DATA_FITS_AT_OFFSET(sb, len, offset) \
641  ((offset) + (len) <= (sb)->buf_size)
642 
643 /**
644  * \param offset offset relative to StreamingBuffer::stream_offset
645  *
646  * \return 0 in case of success
647  * \return -1 on memory allocation errors
648  * \return negative value on other errors
649  */
650 int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg,
651  const uint8_t *data, uint32_t data_len,
652  uint64_t offset)
653 {
654  BUG_ON(seg == NULL);
655  DEBUG_VALIDATE_BUG_ON(offset < sb->stream_offset);
656  if (offset < sb->stream_offset)
657  return -2;
658 
659  if (sb->buf == NULL) {
660  if (InitBuffer(sb) == -1)
661  return -1;
662  }
663 
664  uint32_t rel_offset = offset - sb->stream_offset;
665  if (!DATA_FITS_AT_OFFSET(sb, data_len, rel_offset)) {
666  if (GrowToSize(sb, (rel_offset + data_len)) != 0)
667  return -1;
668  }
669  DEBUG_VALIDATE_BUG_ON(!DATA_FITS_AT_OFFSET(sb, data_len, rel_offset));
670 
671  memcpy(sb->buf + rel_offset, data, data_len);
672  seg->stream_offset = offset;
673  seg->segment_len = data_len;
674 
675  SCLogDebug("rel_offset %u sb->stream_offset %"PRIu64", buf_offset %u",
676  rel_offset, sb->stream_offset, sb->buf_offset);
677 
678  if (RB_EMPTY(&sb->sbb_tree)) {
679  SCLogDebug("empty sbb list");
680 
681  if (sb->stream_offset == offset) {
682  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
683  /* empty list, data is exactly what is expected (append),
684  * so do nothing */
685  } else if ((rel_offset + data_len) <= sb->buf_offset) {
686  SCLogDebug("empty sbb list: block is within existing region");
687  } else {
688  if (sb->buf_offset && rel_offset == sb->buf_offset) {
689  // nothing to do
690  } else if (rel_offset < sb->buf_offset) {
691  // nothing to do
692  } else if (sb->buf_offset) {
693  /* existing data, but there is a gap between us */
694  SBBInit(sb, rel_offset, data_len);
695  } else {
696  /* gap before data in empty list */
697  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
698  SBBInitLeadingGap(sb, offset, data_len);
699  }
700  }
701  } else {
702  /* already have blocks, so append new block based on new data */
703  SBBUpdate(sb, rel_offset, data_len);
704  }
705 
706  if (rel_offset + data_len > sb->buf_offset)
707  sb->buf_offset = rel_offset + data_len;
708 
709  return 0;
710 }
711 
713  const StreamingBufferSegment *seg)
714 {
715  if (seg->stream_offset < sb->stream_offset) {
716  if (seg->stream_offset + seg->segment_len <= sb->stream_offset) {
717  return 1;
718  }
719  }
720  return 0;
721 }
722 
723 /** \brief get the data for one SBB */
725  const StreamingBufferBlock *sbb,
726  const uint8_t **data, uint32_t *data_len)
727 {
728  if (sbb->offset >= sb->stream_offset) {
729  uint64_t offset = sbb->offset - sb->stream_offset;
730  *data = sb->buf + offset;
731  if (offset + sbb->len > sb->buf_offset)
732  *data_len = sb->buf_offset - offset;
733  else
734  *data_len = sbb->len;
735  return;
736  } else {
737  uint64_t offset = sb->stream_offset - sbb->offset;
738  if (offset < sbb->len) {
739  *data = sb->buf;
740  *data_len = sbb->len - offset;
741  return;
742  }
743  }
744  *data = NULL;
745  *data_len = 0;
746  return;
747 }
748 
749 /** \brief get the data for one SBB */
751  const StreamingBufferBlock *sbb,
752  const uint8_t **data, uint32_t *data_len,
753  uint64_t offset)
754 {
755  if (offset >= sbb->offset && offset < (sbb->offset + sbb->len)) {
756  uint32_t sbblen = sbb->len - (offset - sbb->offset);
757 
758  if (offset >= sb->stream_offset) {
759  uint64_t data_offset = offset - sb->stream_offset;
760  *data = sb->buf + data_offset;
761  if (data_offset + sbblen > sb->buf_size)
762  *data_len = sb->buf_size - data_offset;
763  else
764  *data_len = sbblen;
765  BUG_ON(*data_len > sbblen);
766  return;
767  } else {
768  uint64_t data_offset = sb->stream_offset - sbb->offset;
769  if (data_offset < sbblen) {
770  *data = sb->buf;
771  *data_len = sbblen - data_offset;
772  BUG_ON(*data_len > sbblen);
773  return;
774  }
775  }
776  }
777 
778  *data = NULL;
779  *data_len = 0;
780  return;
781 }
782 
784  const StreamingBufferSegment *seg,
785  const uint8_t **data, uint32_t *data_len)
786 {
787  if (likely(sb->buf)) {
788  if (seg->stream_offset >= sb->stream_offset) {
789  uint64_t offset = seg->stream_offset - sb->stream_offset;
790  *data = sb->buf + offset;
791  if (offset + seg->segment_len > sb->buf_size)
792  *data_len = sb->buf_size - offset;
793  else
794  *data_len = seg->segment_len;
795  return;
796  } else {
797  uint64_t offset = sb->stream_offset - seg->stream_offset;
798  if (offset < seg->segment_len) {
799  *data = sb->buf;
800  *data_len = seg->segment_len - offset;
801  return;
802  }
803  }
804  }
805  *data = NULL;
806  *data_len = 0;
807  return;
808 }
809 
810 /**
811  * \retval 1 data is the same
812  * \retval 0 data is different
813  */
815  const StreamingBufferSegment *seg,
816  const uint8_t *rawdata, uint32_t rawdata_len)
817 {
818  const uint8_t *segdata = NULL;
819  uint32_t segdata_len = 0;
820  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
821  if (segdata && segdata_len &&
822  segdata_len == rawdata_len &&
823  memcmp(segdata, rawdata, segdata_len) == 0)
824  {
825  return 1;
826  }
827  return 0;
828 }
829 
831  const uint8_t **data, uint32_t *data_len,
832  uint64_t *stream_offset)
833 {
834  if (sb != NULL && sb->buf != NULL) {
835  *data = sb->buf;
836  *data_len = sb->buf_offset;
838  return 1;
839  } else {
840  *data = NULL;
841  *data_len = 0;
842  *stream_offset = 0;
843  return 0;
844  }
845 }
846 
848  const uint8_t **data, uint32_t *data_len,
849  uint64_t offset)
850 {
851  if (sb != NULL && sb->buf != NULL &&
852  offset >= sb->stream_offset &&
853  offset < (sb->stream_offset + sb->buf_offset))
854  {
855  uint32_t skip = offset - sb->stream_offset;
856  *data = sb->buf + skip;
857  *data_len = sb->buf_offset - skip;
858  return 1;
859  } else {
860  *data = NULL;
861  *data_len = 0;
862  return 0;
863  }
864 }
865 
866 /**
867  * \retval 1 data is the same
868  * \retval 0 data is different
869  */
871  const uint8_t *rawdata, uint32_t rawdata_len)
872 {
873  const uint8_t *sbdata = NULL;
874  uint32_t sbdata_len = 0;
875  uint64_t offset = 0;
876  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
877  if (offset == 0 &&
878  sbdata && sbdata_len &&
879  sbdata_len == rawdata_len &&
880  memcmp(sbdata, rawdata, sbdata_len) == 0)
881  {
882  return 1;
883  }
884  SCLogDebug("sbdata_len %u, offset %"PRIu64, sbdata_len, offset);
885  printf("got:\n");
886  PrintRawDataFp(stdout, sbdata,sbdata_len);
887  printf("wanted:\n");
888  PrintRawDataFp(stdout, rawdata,rawdata_len);
889  return 0;
890 }
891 
892 #ifdef UNITTESTS
893 static void Dump(StreamingBuffer *sb)
894 {
895  PrintRawDataFp(stdout, sb->buf, sb->buf_offset);
896 }
897 
898 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
899 {
900  const uint8_t *data = NULL;
901  uint32_t data_len = 0;
902  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
903  if (data && data_len) {
904  PrintRawDataFp(stdout, data, data_len);
905  }
906 }
907 
908 static int StreamingBufferTest02(void)
909 {
910  StreamingBufferConfig cfg = { 8, 24, NULL, NULL, NULL };
912  FAIL_IF(sb == NULL);
913 
914  StreamingBufferSegment seg1;
915  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
916  StreamingBufferSegment seg2;
917  FAIL_IF(StreamingBufferAppend(sb, &seg2, (const uint8_t *)"01234567", 8) != 0);
918  FAIL_IF(sb->stream_offset != 0);
919  FAIL_IF(sb->buf_offset != 16);
920  FAIL_IF(seg1.stream_offset != 0);
921  FAIL_IF(seg2.stream_offset != 8);
924  Dump(sb);
925  DumpSegment(sb, &seg1);
926  DumpSegment(sb, &seg2);
927  FAIL_IF_NOT_NULL(sb->head);
928  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
929 
930  StreamingBufferSlide(sb, 6);
931  FAIL_IF_NOT_NULL(sb->head);
932  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
933 
934  StreamingBufferSegment seg3;
935  FAIL_IF(StreamingBufferAppend(sb, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
936  FAIL_IF(sb->stream_offset != 6);
937  FAIL_IF(sb->buf_offset != 16);
938  FAIL_IF(seg3.stream_offset != 16);
942  Dump(sb);
943  DumpSegment(sb, &seg1);
944  DumpSegment(sb, &seg2);
945  DumpSegment(sb, &seg3);
946  FAIL_IF_NOT_NULL(sb->head);
947  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
948 
949  StreamingBufferSlide(sb, 6);
953  Dump(sb);
954  DumpSegment(sb, &seg1);
955  DumpSegment(sb, &seg2);
956  DumpSegment(sb, &seg3);
957  FAIL_IF_NOT_NULL(sb->head);
958  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
959 
961  PASS;
962 }
963 
964 static int StreamingBufferTest03(void)
965 {
966  StreamingBufferConfig cfg = { 8, 24, NULL, NULL, NULL };
968  FAIL_IF(sb == NULL);
969 
970  StreamingBufferSegment seg1;
971  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
972  StreamingBufferSegment seg2;
973  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
974  FAIL_IF(sb->stream_offset != 0);
975  FAIL_IF(sb->buf_offset != 22);
976  FAIL_IF(seg1.stream_offset != 0);
977  FAIL_IF(seg2.stream_offset != 14);
980  Dump(sb);
981  DumpSegment(sb, &seg1);
982  DumpSegment(sb, &seg2);
983  FAIL_IF_NULL(sb->head);
984  FAIL_IF_NOT(sb->sbb_size == 16);
985  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
986 
987  StreamingBufferSegment seg3;
988  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
989  FAIL_IF(sb->stream_offset != 0);
990  FAIL_IF(sb->buf_offset != 22);
991  FAIL_IF(seg3.stream_offset != 8);
995  Dump(sb);
996  DumpSegment(sb, &seg1);
997  DumpSegment(sb, &seg2);
998  DumpSegment(sb, &seg3);
999  FAIL_IF_NULL(sb->head);
1000  FAIL_IF_NOT(sb->sbb_size == 22);
1001  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1002 
1003  StreamingBufferSlide(sb, 10);
1007  Dump(sb);
1008  DumpSegment(sb, &seg1);
1009  DumpSegment(sb, &seg2);
1010  DumpSegment(sb, &seg3);
1011  FAIL_IF_NULL(sb->head);
1012  FAIL_IF_NOT(sb->sbb_size == 12);
1013  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1014 
1015  StreamingBufferFree(sb);
1016  PASS;
1017 }
1018 
1019 static int StreamingBufferTest04(void)
1020 {
1021  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL };
1023  FAIL_IF(sb == NULL);
1024 
1025  StreamingBufferSegment seg1;
1026  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1027  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
1028  StreamingBufferSegment seg2;
1029  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1030  FAIL_IF(sb->stream_offset != 0);
1031  FAIL_IF(sb->buf_offset != 22);
1032  FAIL_IF(seg1.stream_offset != 0);
1033  FAIL_IF(seg2.stream_offset != 14);
1036  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
1037  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1038  FAIL_IF(sbb1 != sb->head);
1039  FAIL_IF_NULL(sbb1);
1040  FAIL_IF(sbb1->offset != 0);
1041  FAIL_IF(sbb1->len != 8);
1042  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
1043  FAIL_IF_NULL(sbb2);
1044  FAIL_IF(sbb2 == sb->head);
1045  FAIL_IF(sbb2->offset != 14);
1046  FAIL_IF(sbb2->len != 8);
1047  Dump(sb);
1048  DumpSegment(sb, &seg1);
1049  DumpSegment(sb, &seg2);
1050  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1051 
1052  StreamingBufferSegment seg3;
1053  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1054  FAIL_IF(sb->stream_offset != 0);
1055  FAIL_IF(sb->buf_offset != 22);
1056  FAIL_IF(seg3.stream_offset != 8);
1060  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1061  FAIL_IF_NULL(sbb1);
1062  FAIL_IF(sbb1 != sb->head);
1063  FAIL_IF(sbb1->offset != 0);
1064  FAIL_IF(sbb1->len != 22);
1065  FAIL_IF(SBB_RB_NEXT(sbb1));
1066  Dump(sb);
1067  DumpSegment(sb, &seg1);
1068  DumpSegment(sb, &seg2);
1069  DumpSegment(sb, &seg3);
1070  FAIL_IF_NULL(sb->head);
1071  FAIL_IF_NOT(sb->sbb_size == 22);
1072  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1073 
1074  /* far ahead of curve: */
1075  StreamingBufferSegment seg4;
1076  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
1077  FAIL_IF(sb->stream_offset != 0);
1078  FAIL_IF(sb->buf_offset != 127);
1079  FAIL_IF(sb->buf_size != 128);
1080  FAIL_IF(seg4.stream_offset != 124);
1085  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1086  FAIL_IF_NULL(sbb1);
1087  FAIL_IF(sbb1 != sb->head);
1088  FAIL_IF(sbb1->offset != 0);
1089  FAIL_IF(sbb1->len != 22);
1090  FAIL_IF(!SBB_RB_NEXT(sbb1));
1091  Dump(sb);
1092  DumpSegment(sb, &seg1);
1093  DumpSegment(sb, &seg2);
1094  DumpSegment(sb, &seg3);
1095  DumpSegment(sb, &seg4);
1096  FAIL_IF_NULL(sb->head);
1097  FAIL_IF_NOT(sb->sbb_size == 25);
1098  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1099 
1100  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
1101  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
1102  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
1103  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
1104 
1105  StreamingBufferFree(sb);
1106  PASS;
1107 }
1108 
1109 /** \test lots of gaps in block list */
1110 static int StreamingBufferTest06(void)
1111 {
1112  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL };
1114  FAIL_IF(sb == NULL);
1115 
1116  StreamingBufferSegment seg1;
1117  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"A", 1) != 0);
1118  StreamingBufferSegment seg2;
1119  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"C", 1, 2) != 0);
1120  Dump(sb);
1121  FAIL_IF_NULL(sb->head);
1122  FAIL_IF_NOT(sb->sbb_size == 2);
1123  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1124 
1125  StreamingBufferSegment seg3;
1126  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1127  Dump(sb);
1128  FAIL_IF_NULL(sb->head);
1129  FAIL_IF_NOT(sb->sbb_size == 3);
1130  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1131 
1132  StreamingBufferSegment seg4;
1133  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1134  Dump(sb);
1135  FAIL_IF_NULL(sb->head);
1136  FAIL_IF_NOT(sb->sbb_size == 4);
1137  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1138 
1139  StreamingBufferSegment seg5;
1140  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1141  Dump(sb);
1142  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1143  FAIL_IF_NULL(sbb1);
1144  FAIL_IF(sbb1->offset != 0);
1145  FAIL_IF(sbb1->len != 10);
1146  FAIL_IF(SBB_RB_NEXT(sbb1));
1147  FAIL_IF_NULL(sb->head);
1148  FAIL_IF_NOT(sb->sbb_size == 10);
1149  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1150 
1151  StreamingBufferSegment seg6;
1152  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1153  Dump(sb);
1154  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1155  FAIL_IF_NULL(sbb1);
1156  FAIL_IF(sbb1->offset != 0);
1157  FAIL_IF(sbb1->len != 10);
1158  FAIL_IF(SBB_RB_NEXT(sbb1));
1159  FAIL_IF_NULL(sb->head);
1160  FAIL_IF_NOT(sb->sbb_size == 10);
1161  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1162 
1163  StreamingBufferFree(sb);
1164  PASS;
1165 }
1166 
1167 /** \test lots of gaps in block list */
1168 static int StreamingBufferTest07(void)
1169 {
1170  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL };
1172  FAIL_IF(sb == NULL);
1173 
1174  StreamingBufferSegment seg1;
1175  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1176  StreamingBufferSegment seg2;
1177  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1178  Dump(sb);
1179  FAIL_IF_NULL(sb->head);
1180  FAIL_IF_NOT(sb->sbb_size == 2);
1181  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1182 
1183  StreamingBufferSegment seg3;
1184  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1185  Dump(sb);
1186  FAIL_IF_NULL(sb->head);
1187  FAIL_IF_NOT(sb->sbb_size == 3);
1188  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1189 
1190  StreamingBufferSegment seg4;
1191  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1192  Dump(sb);
1193  FAIL_IF_NULL(sb->head);
1194  FAIL_IF_NOT(sb->sbb_size == 4);
1195  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1196 
1197  StreamingBufferSegment seg5;
1198  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1199  Dump(sb);
1200  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1201  FAIL_IF_NULL(sbb1);
1202  FAIL_IF(sbb1->offset != 0);
1203  FAIL_IF(sbb1->len != 10);
1204  FAIL_IF(SBB_RB_NEXT(sbb1));
1205  FAIL_IF_NULL(sb->head);
1206  FAIL_IF_NOT(sb->sbb_size == 10);
1207  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1208 
1209  StreamingBufferSegment seg6;
1210  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1211  Dump(sb);
1212  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1213  FAIL_IF_NULL(sbb1);
1214  FAIL_IF(sbb1->offset != 0);
1215  FAIL_IF(sbb1->len != 10);
1216  FAIL_IF(SBB_RB_NEXT(sbb1));
1217  FAIL_IF_NULL(sb->head);
1218  FAIL_IF_NOT(sb->sbb_size == 10);
1219  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1220 
1221  StreamingBufferFree(sb);
1222  PASS;
1223 }
1224 
1225 /** \test lots of gaps in block list */
1226 static int StreamingBufferTest08(void)
1227 {
1228  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL };
1230  FAIL_IF(sb == NULL);
1231 
1232  StreamingBufferSegment seg1;
1233  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1234  StreamingBufferSegment seg2;
1235  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1236  Dump(sb);
1237  FAIL_IF_NULL(sb->head);
1238  FAIL_IF_NOT(sb->sbb_size == 2);
1239  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1240 
1241  StreamingBufferSegment seg3;
1242  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1243  Dump(sb);
1244  FAIL_IF_NULL(sb->head);
1245  FAIL_IF_NOT(sb->sbb_size == 3);
1246  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1247 
1248  StreamingBufferSegment seg4;
1249  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1250  Dump(sb);
1251  FAIL_IF_NULL(sb->head);
1252  FAIL_IF_NOT(sb->sbb_size == 4);
1253  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1254 
1255  StreamingBufferSegment seg5;
1256  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1257  Dump(sb);
1258  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1259  FAIL_IF_NULL(sbb1);
1260  FAIL_IF(sbb1->offset != 0);
1261  FAIL_IF(sbb1->len != 10);
1262  FAIL_IF(SBB_RB_NEXT(sbb1));
1263  FAIL_IF_NULL(sb->head);
1264  FAIL_IF_NOT(sb->sbb_size == 10);
1265  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1266 
1267  StreamingBufferSegment seg6;
1268  FAIL_IF(StreamingBufferAppend(sb, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
1269  Dump(sb);
1270  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1271  FAIL_IF_NULL(sbb1);
1272  FAIL_IF(sbb1->offset != 0);
1273  FAIL_IF(sbb1->len != 20);
1274  FAIL_IF(SBB_RB_NEXT(sbb1));
1275  FAIL_IF_NULL(sb->head);
1276  FAIL_IF_NOT(sb->sbb_size == 20);
1277  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1278 
1279  StreamingBufferFree(sb);
1280  PASS;
1281 }
1282 
1283 /** \test lots of gaps in block list */
1284 static int StreamingBufferTest09(void)
1285 {
1286  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL };
1288  FAIL_IF(sb == NULL);
1289 
1290  StreamingBufferSegment seg1;
1291  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1292  StreamingBufferSegment seg2;
1293  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1294  Dump(sb);
1295  FAIL_IF_NULL(sb->head);
1296  FAIL_IF_NOT(sb->sbb_size == 2);
1297  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1298 
1299  StreamingBufferSegment seg3;
1300  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"H", 1, 7) != 0);
1301  Dump(sb);
1302  FAIL_IF_NULL(sb->head);
1303  FAIL_IF_NOT(sb->sbb_size == 3);
1304  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1305 
1306  StreamingBufferSegment seg4;
1307  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"F", 1, 5) != 0);
1308  Dump(sb);
1309  FAIL_IF_NULL(sb->head);
1310  FAIL_IF_NOT(sb->sbb_size == 4);
1311  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1312 
1313  StreamingBufferSegment seg5;
1314  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1315  Dump(sb);
1316  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1317  FAIL_IF_NULL(sbb1);
1318  FAIL_IF(sbb1->offset != 0);
1319  FAIL_IF(sbb1->len != 10);
1320  FAIL_IF(SBB_RB_NEXT(sbb1));
1321  FAIL_IF_NULL(sb->head);
1322  FAIL_IF_NOT(sb->sbb_size == 10);
1323  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1324 
1325  StreamingBufferSegment seg6;
1326  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1327  Dump(sb);
1328  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1329  FAIL_IF_NULL(sbb1);
1330  FAIL_IF(sbb1->offset != 0);
1331  FAIL_IF(sbb1->len != 10);
1332  FAIL_IF(SBB_RB_NEXT(sbb1));
1333  FAIL_IF_NULL(sb->head);
1334  FAIL_IF_NOT(sb->sbb_size == 10);
1335  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1336 
1337  StreamingBufferFree(sb);
1338  PASS;
1339 }
1340 
1341 /** \test lots of gaps in block list */
1342 static int StreamingBufferTest10(void)
1343 {
1344  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL };
1346  FAIL_IF(sb == NULL);
1347 
1348  StreamingBufferSegment seg1;
1349  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"A", 1, 0) != 0);
1350  Dump(sb);
1351  StreamingBufferSegment seg2;
1352  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1353  Dump(sb);
1354  StreamingBufferSegment seg3;
1355  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"H", 1, 7) != 0);
1356  Dump(sb);
1357  FAIL_IF_NULL(sb->head);
1358  FAIL_IF_NOT(sb->sbb_size == 3);
1359 
1360  StreamingBufferSegment seg4;
1361  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"B", 1, 1) != 0);
1362  Dump(sb);
1363  StreamingBufferSegment seg5;
1364  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"C", 1, 2) != 0);
1365  Dump(sb);
1366  StreamingBufferSegment seg6;
1367  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"G", 1, 6) != 0);
1368  Dump(sb);
1369  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1370  FAIL_IF_NULL(sb->head);
1371  FAIL_IF_NOT(sb->sbb_size == 6);
1372 
1373  StreamingBufferSegment seg7;
1374  FAIL_IF(StreamingBufferInsertAt(sb, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1375  Dump(sb);
1376  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1377  FAIL_IF_NULL(sbb1);
1378  FAIL_IF(sbb1->offset != 0);
1379  FAIL_IF(sbb1->len != 10);
1380  FAIL_IF(SBB_RB_NEXT(sbb1));
1381  FAIL_IF_NULL(sb->head);
1382  FAIL_IF_NOT(sb->sbb_size == 10);
1383 
1384  StreamingBufferSegment seg8;
1385  FAIL_IF(StreamingBufferInsertAt(sb, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1386  Dump(sb);
1387  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1388  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1389  FAIL_IF_NULL(sbb1);
1390  FAIL_IF(sbb1->offset != 0);
1391  FAIL_IF(sbb1->len != 10);
1392  FAIL_IF(SBB_RB_NEXT(sbb1));
1393  FAIL_IF_NULL(sb->head);
1394  FAIL_IF_NOT(sb->sbb_size == 10);
1395 
1396  StreamingBufferFree(sb);
1397  PASS;
1398 }
1399 
1400 #endif
1401 
1403 {
1404 #ifdef UNITTESTS
1405  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
1406  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
1407  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
1408  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
1409  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
1410  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
1411  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
1412  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
1413 #endif
1414 }
FREE
#define FREE(cfg, ptr, s)
Definition: util-streaming-buffer.c:38
StreamingBuffer_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:95
len
uint8_t len
Definition: app-layer-dnp3.h:2
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:99
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb)
Definition: util-streaming-buffer.c:143
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:102
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
Definition: util-streaming-buffer.c:608
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
StreamingBuffer_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:97
RB_LEFT
#define RB_LEFT(elm, field)
Definition: tree.h:322
StreamingBuffer_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:98
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:830
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:0
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:847
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:814
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
util-unittest.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb)
Definition: util-streaming-buffer.c:130
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
FAIL_IF_NOT_NULL
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
RB_ROOT
#define RB_ROOT(head)
Definition: tree.h:326
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:110
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:712
REALLOC
#define REALLOC(cfg, ptr, orig_s, s)
Definition: util-streaming-buffer.c:36
res
PoolThreadReserved res
Definition: stream-tcp-private.h:0
CALLOC
#define CALLOC(cfg, n, s)
Definition: util-streaming-buffer.c:34
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:143
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:103
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:870
RB_GENERATE
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:77
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:101
StreamingBuffer_
Definition: util-streaming-buffer.h:93
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:750
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
suricata-common.h
util-streaming-buffer.h
StreamingBuffer_::cfg
const StreamingBufferConfig * cfg
Definition: util-streaming-buffer.h:94
util-validate.h
StreamingBufferConfig_
Definition: util-streaming-buffer.h:64
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:504
RB_RIGHT
#define RB_RIGHT(elm, field)
Definition: tree.h:323
head
Flow * head
Definition: flow-hash.h:1
StreamingBufferAppendRaw
StreamingBufferSegment * StreamingBufferAppendRaw(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:532
StreamingBufferSlide
void StreamingBufferSlide(StreamingBuffer *sb, uint32_t slide)
Definition: util-streaming-buffer.c:519
DATA_FITS_AT_OFFSET
#define DATA_FITS_AT_OFFSET(sb, len, offset)
Definition: util-streaming-buffer.c:640
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:650
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:45
likely
#define likely(expr)
Definition: util-optimize.h:32
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:569
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:783
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:302
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:83
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:81
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:724
DATA_FITS
#define DATA_FITS(sb, len)
Definition: util-streaming-buffer.c:529
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:80
WARN_UNUSED
#define WARN_UNUSED
Definition: suricata-common.h:384
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:1402