suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-unittest.h"
21 #include "util-print.h"
22 #include "util-validate.h"
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  *
29  * \brief Streaming Buffer API
30  */
31 
32 /* memory handling wrappers. If config doesn't define it's own set of
33  * functions, use the defaults */
34 #define MALLOC(cfg, s) \
35  (cfg)->Malloc ? (cfg)->Malloc((s)) : SCMalloc((s))
36 #define CALLOC(cfg, n, s) \
37  (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : SCCalloc((n), (s))
38 #define REALLOC(cfg, ptr, orig_s, s) \
39  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : SCRealloc((ptr), (s))
40 #define FREE(cfg, ptr, s) \
41  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
42 
43 static void SBBFree(StreamingBuffer *sb);
44 
46 
48 {
49  SCLogDebug("a %"PRIu64" len %u, b %"PRIu64" len %u",
50  a->offset, a->len, b->offset, b->len);
51 
52  if (a->offset > b->offset)
53  SCReturnInt(1);
54  else if (a->offset < b->offset)
55  SCReturnInt(-1);
56  else {
57  if (a->len == 0 || b->len == 0 || a->len == b->len)
58  SCReturnInt(0);
59  else if (a->len > b->len)
60  SCReturnInt(1);
61  else
62  SCReturnInt(-1);
63  }
64 }
65 
66 /* inclusive compare function that also considers the right edge,
67  * not just the offset. */
68 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
69  const uint64_t lre = lookup->offset + lookup->len;
70  const uint64_t tre = intree->offset + intree->len;
71  if (lre <= intree->offset) // entirely before
72  return -1;
73  else if (lre >= intree->offset && lre <= tre) // (some) overlap
74  return 0;
75  else
76  return 1; // entirely after
77 }
78 
80 {
81  SCLogDebug("looking up %"PRIu64, elm->offset);
82 
83  struct StreamingBufferBlock *tmp = RB_ROOT(head);
84  struct StreamingBufferBlock *res = NULL;
85  while (tmp) {
86  SCLogDebug("compare with %"PRIu64"/%u", tmp->offset, tmp->len);
87  const int comp = InclusiveCompare(elm, tmp);
88  SCLogDebug("compare result: %d", comp);
89  if (comp < 0) {
90  res = tmp;
91  tmp = RB_LEFT(tmp, rb);
92  } else if (comp > 0) {
93  tmp = RB_RIGHT(tmp, rb);
94  } else {
95  return tmp;
96  }
97  }
98  return res;
99 }
100 
101 
102 static inline int InitBuffer(StreamingBuffer *sb)
103 {
104  sb->buf = CALLOC(sb->cfg, 1, sb->cfg->buf_size);
105  if (sb->buf == NULL) {
106  return -1;
107  }
108  sb->buf_size = sb->cfg->buf_size;
109  return 0;
110 }
111 
113 {
114  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
115  if (sb != NULL) {
116  sb->buf_size = cfg->buf_size;
117  sb->cfg = cfg;
118 
119  if (cfg->buf_size > 0) {
120  if (InitBuffer(sb) == 0) {
121  return sb;
122  }
123  FREE(cfg, sb, sizeof(StreamingBuffer));
124  /* implied buf_size == 0 */
125  } else {
126  return sb;
127  }
128  }
129  return NULL;
130 }
131 
133 {
134  if (sb != NULL) {
135  SCLogDebug("sb->buf_size %u max %u", sb->buf_size, sb->buf_size_max);
136 
137  SBBFree(sb);
138  if (sb->buf != NULL) {
139  FREE(sb->cfg, sb->buf, sb->buf_size);
140  sb->buf = NULL;
141  }
142  }
143 }
144 
146 {
147  if (sb != NULL) {
149  FREE(sb->cfg, sb, sizeof(StreamingBuffer));
150  }
151 }
152 
153 #ifdef DEBUG
154 static void SBBPrintList(StreamingBuffer *sb)
155 {
156  StreamingBufferBlock *sbb = NULL;
157  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
158  SCLogDebug("sbb: offset %"PRIu64", len %u", sbb->offset, sbb->len);
159  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
160  if (next) {
161  if ((sbb->offset + sbb->len) != next->offset) {
162  SCLogDebug("gap: offset %"PRIu64", len %"PRIu64, (sbb->offset + sbb->len),
163  next->offset - (sbb->offset + sbb->len));
164  }
165  }
166  }
167 }
168 #endif
169 
170 /* setup with gap between 2 blocks
171  *
172  * [block][gap][block]
173  **/
174 static void SBBInit(StreamingBuffer *sb,
175  uint32_t rel_offset, uint32_t data_len)
176 {
178  DEBUG_VALIDATE_BUG_ON(sb->buf_offset > sb->stream_offset + rel_offset);
179 
180  /* need to set up 2: existing data block and new data block */
181  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
182  if (sbb == NULL) {
183  return;
184  }
185  sbb->offset = sb->stream_offset;
186  sbb->len = sb->buf_offset;
187 
188  StreamingBufferBlock *sbb2 = CALLOC(sb->cfg, 1, sizeof(*sbb2));
189  if (sbb2 == NULL) {
190  FREE(sb->cfg, sbb, sizeof(*sbb));
191  return;
192  }
193  sbb2->offset = sb->stream_offset + rel_offset;
194  sbb2->len = data_len;
195 
196  sb->head = sbb;
197  SBB_RB_INSERT(&sb->sbb_tree, sbb);
198  SBB_RB_INSERT(&sb->sbb_tree, sbb2);
199 
200  SCLogDebug("sbb1 %"PRIu64", len %u, sbb2 %"PRIu64", len %u",
201  sbb->offset, sbb->len, sbb2->offset, sbb2->len);
202 #ifdef DEBUG
203  SBBPrintList(sb);
204 #endif
205  BUG_ON(sbb2->offset < sbb->len);
206 }
207 
208 /* setup with leading gap
209  *
210  * [gap][block]
211  **/
212 static void SBBInitLeadingGap(StreamingBuffer *sb,
213  uint64_t offset, uint32_t data_len)
214 {
216 
217  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
218  if (sbb == NULL)
219  return;
220  sbb->offset = offset;
221  sbb->len = data_len;
222 
223  sb->head = sbb;
224  SBB_RB_INSERT(&sb->sbb_tree, sbb);
225 
226  SCLogDebug("sbb %"PRIu64", len %u",
227  sbb->offset, sbb->len);
228 #ifdef DEBUG
229  SBBPrintList(sb);
230 #endif
231 }
232 
233 static inline void ConsolidateFwd(StreamingBuffer *sb,
234  struct SBB *tree, StreamingBufferBlock *sa)
235 {
236  uint64_t sa_re = sa->offset + sa->len;
237  StreamingBufferBlock *tr, *s = sa;
238  RB_FOREACH_FROM(tr, SBB, s) {
239  if (sa == tr)
240  continue;
241 
242  const uint64_t tr_re = tr->offset + tr->len;
243  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u re %"PRIu64,
244  tr, tr->offset, tr->len, tr_re);
245 
246  if (sa_re < tr->offset)
247  break; // entirely before
248 
249  /*
250  sa: [ ]
251  tr: [ ]
252  sa: [ ]
253  tr: [ ]
254  sa: [ ]
255  tr: [ ]
256  */
257  if (sa->offset >= tr->offset && sa_re <= tr_re) {
258  sa->len = tr->len;
259  sa->offset = tr->offset;
260  sa_re = sa->offset + sa->len;
261  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED2", tr, tr->offset, tr->len);
262  SBB_RB_REMOVE(tree, tr);
263  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
264  /*
265  sa: [ ]
266  tr: [ ]
267  sa: [ ]
268  tr: [ ]
269  sa: [ ]
270  tr: [ ]
271  */
272  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
273  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED", tr, tr->offset, tr->len);
274  SBB_RB_REMOVE(tree, tr);
275  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
276  /*
277  sa: [ ]
278  tr: [ ]
279  sa: [ ]
280  tr: [ ]
281  */
282  } else if (sa->offset < tr->offset && // starts before
283  sa_re >= tr->offset && sa_re < tr_re) // ends inside
284  {
285  // merge
286  sa->len = tr_re - sa->offset;
287  sa_re = sa->offset + sa->len;
288  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED MERGED", tr, tr->offset, tr->len);
289  SBB_RB_REMOVE(tree, tr);
290  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
291  }
292  }
293 }
294 
295 static inline void ConsolidateBackward(StreamingBuffer *sb,
296  struct SBB *tree, StreamingBufferBlock *sa)
297 {
298  uint64_t sa_re = sa->offset + sa->len;
299  StreamingBufferBlock *tr, *s = sa;
300  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
301  if (sa == tr)
302  continue;
303  const uint64_t tr_re = tr->offset + tr->len;
304  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u", tr, tr->offset, tr->len);
305 
306  if (sa->offset > tr_re)
307  break; // entirely after
308 
309  if (sa->offset >= tr->offset && sa_re <= tr_re) {
310  sa->len = tr->len;
311  sa->offset = tr->offset;
312  sa_re = sa->offset + sa->len;
313  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED2", tr, tr->offset, tr->len);
314  if (sb->head == tr)
315  sb->head = sa;
316  SBB_RB_REMOVE(tree, tr);
317  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
318  /*
319  sa: [ ]
320  tr: [ ]
321  sa: [ ]
322  tr: [ ]
323  sa: [ ]
324  tr: [ ]
325  */
326  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
327  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED", tr, tr->offset, tr->len);
328  if (sb->head == tr)
329  sb->head = sa;
330  SBB_RB_REMOVE(tree, tr);
331  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
332  /*
333  sa: [ ]
334  tr: [ ]
335  sa: [ ]
336  tr: [ ]
337  */
338  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
339  // merge
340  sa->len = sa_re - tr->offset;
341  sa->offset = tr->offset;
342  sa_re = sa->offset + sa->len;
343  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED MERGED", tr, tr->offset, tr->len);
344  if (sb->head == tr)
345  sb->head = sa;
346  SBB_RB_REMOVE(tree, tr);
347  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
348  }
349  }
350 }
351 
352 static int Insert(StreamingBuffer *sb, struct SBB *tree,
353  uint32_t rel_offset, uint32_t len)
354 {
355  SCLogDebug("* inserting: %u/%u\n", rel_offset, len);
356 
357  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
358  if (sbb == NULL)
359  return -1;
360  sbb->offset = sb->stream_offset + rel_offset;
361  sbb->len = len;
362  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
363  if (res) {
364  // exact overlap
365  SCLogDebug("* insert failed: exact match in tree with %p %"PRIu64"/%u", res, res->offset, res->len);
366  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
367  return 0;
368  }
369  if (SBB_RB_PREV(sbb) == NULL) {
370  sb->head = sbb;
371  } else {
372  ConsolidateBackward(sb, tree, sbb);
373  }
374  ConsolidateFwd(sb, tree, sbb);
375 #ifdef DEBUG
376  SBBPrintList(sb);
377 #endif
378  return 0;
379 }
380 
381 static void SBBUpdate(StreamingBuffer *sb,
382  uint32_t rel_offset, uint32_t data_len)
383 {
384  Insert(sb, &sb->sbb_tree, rel_offset, data_len);
385 }
386 
387 static void SBBFree(StreamingBuffer *sb)
388 {
389  StreamingBufferBlock *sbb = NULL, *safe = NULL;
390  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
391  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
392  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
393  }
394  sb->head = NULL;
395 }
396 
397 static void SBBPrune(StreamingBuffer *sb)
398 {
399  SCLogDebug("pruning %p to %"PRIu64, sb, sb->stream_offset);
400  StreamingBufferBlock *sbb = NULL, *safe = NULL;
401  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
402  /* completely beyond window, we're done */
403  if (sbb->offset > sb->stream_offset) {
404  sb->head = sbb;
405  break;
406  }
407 
408  /* partly before, partly beyond. Adjust */
409  if (sbb->offset < sb->stream_offset &&
410  sbb->offset + sbb->len > sb->stream_offset) {
411  uint32_t shrink_by = sb->stream_offset - sbb->offset;
412  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
413  if (sbb->len >= shrink_by) {
414  sbb->len -= shrink_by;
415  sbb->offset += shrink_by;
417  }
418  sb->head = sbb;
419  break;
420  }
421 
422  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
423  /* either we set it again for the next sbb, or there isn't any */
424  sb->head = NULL;
425  SCLogDebug("sb %p removed %p %"PRIu64", %u", sb, sbb, sbb->offset, sbb->len);
426  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
427  }
428 }
429 
430 /**
431  * \internal
432  * \brief move buffer forward by 'slide'
433  */
434 static void AutoSlide(StreamingBuffer *sb)
435 {
436  uint32_t size = sb->cfg->buf_slide;
437  uint32_t slide = sb->buf_offset - size;
438  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide, size);
439  memmove(sb->buf, sb->buf+slide, size);
440  sb->stream_offset += slide;
441  sb->buf_offset = size;
442  SBBPrune(sb);
443 }
444 
445 static int __attribute__((warn_unused_result))
446 GrowToSize(StreamingBuffer *sb, uint32_t size)
447 {
448  /* try to grow in multiples of sb->cfg->buf_size */
449  uint32_t x = sb->cfg->buf_size ? size % sb->cfg->buf_size : 0;
450  uint32_t base = size - x;
451  uint32_t grow = base + sb->cfg->buf_size;
452 
453  void *ptr = REALLOC(sb->cfg, sb->buf, sb->buf_size, grow);
454  if (ptr == NULL)
455  return -1;
456 
457  /* for safe printing and general caution, lets memset the
458  * new data to 0 */
459  size_t diff = grow - sb->buf_size;
460  void *new_mem = ((char *)ptr) + sb->buf_size;
461  memset(new_mem, 0, diff);
462 
463  sb->buf = ptr;
464  sb->buf_size = grow;
465  SCLogDebug("grown buffer to %u", grow);
466 #ifdef DEBUG
467  if (sb->buf_size > sb->buf_size_max) {
468  sb->buf_size_max = sb->buf_size;
469  }
470 #endif
471  return 0;
472 }
473 
474 /** \internal
475  * \brief try to double the buffer size
476  * \retval 0 ok
477  * \retval -1 failed, buffer unchanged
478  */
479 static int __attribute__((warn_unused_result)) Grow(StreamingBuffer *sb)
480 {
481  uint32_t grow = sb->buf_size * 2;
482  void *ptr = REALLOC(sb->cfg, sb->buf, sb->buf_size, grow);
483  if (ptr == NULL)
484  return -1;
485 
486  /* for safe printing and general caution, lets memset the
487  * new data to 0 */
488  size_t diff = grow - sb->buf_size;
489  void *new_mem = ((char *)ptr) + sb->buf_size;
490  memset(new_mem, 0, diff);
491 
492  sb->buf = ptr;
493  sb->buf_size = grow;
494  SCLogDebug("grown buffer to %u", grow);
495 #ifdef DEBUG
496  if (sb->buf_size > sb->buf_size_max) {
497  sb->buf_size_max = sb->buf_size;
498  }
499 #endif
500  return 0;
501 }
502 
503 /**
504  * \brief slide to absolute offset
505  * \todo if sliding beyond window, we could perhaps reset?
506  */
508 {
509  if (offset > sb->stream_offset &&
510  offset <= sb->stream_offset + sb->buf_offset)
511  {
512  uint32_t slide = offset - sb->stream_offset;
513  uint32_t size = sb->buf_offset - slide;
514  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide, size);
515  memmove(sb->buf, sb->buf+slide, size);
516  sb->stream_offset += slide;
517  sb->buf_offset = size;
518  SBBPrune(sb);
519  }
520 }
521 
522 void StreamingBufferSlide(StreamingBuffer *sb, uint32_t slide)
523 {
524  uint32_t size = sb->buf_offset - slide;
525  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide, size);
526  memmove(sb->buf, sb->buf+slide, size);
527  sb->stream_offset += slide;
528  sb->buf_offset = size;
529  SBBPrune(sb);
530 }
531 
532 #define DATA_FITS(sb, len) \
533  ((sb)->buf_offset + (len) <= (sb)->buf_size)
534 
535 StreamingBufferSegment *StreamingBufferAppendRaw(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
536 {
537  if (sb->buf == NULL) {
538  if (InitBuffer(sb) == -1)
539  return NULL;
540  }
541 
542  if (!DATA_FITS(sb, data_len)) {
544  AutoSlide(sb);
545  if (sb->buf_size == 0) {
546  if (GrowToSize(sb, data_len) != 0)
547  return NULL;
548  } else {
549  while (!DATA_FITS(sb, data_len)) {
550  if (Grow(sb) != 0) {
551  return NULL;
552  }
553  }
554  }
555  }
556  if (!DATA_FITS(sb, data_len)) {
557  return NULL;
558  }
559 
560  StreamingBufferSegment *seg = CALLOC(sb->cfg, 1, sizeof(StreamingBufferSegment));
561  if (seg != NULL) {
562  memcpy(sb->buf + sb->buf_offset, data, data_len);
563  seg->stream_offset = sb->stream_offset + sb->buf_offset;
564  seg->segment_len = data_len;
565  uint32_t rel_offset = sb->buf_offset;
566  sb->buf_offset += data_len;
567 
568  if (!RB_EMPTY(&sb->sbb_tree)) {
569  SBBUpdate(sb, rel_offset, data_len);
570  }
571  return seg;
572  }
573  return NULL;
574 }
575 
576 int StreamingBufferAppend(StreamingBuffer *sb, StreamingBufferSegment *seg,
577  const uint8_t *data, uint32_t data_len)
578 {
579  BUG_ON(seg == NULL);
580 
581  if (sb->buf == NULL) {
582  if (InitBuffer(sb) == -1)
583  return -1;
584  }
585 
586  if (!DATA_FITS(sb, data_len)) {
588  AutoSlide(sb);
589  if (sb->buf_size == 0) {
590  if (GrowToSize(sb, data_len) != 0)
591  return -1;
592  } else {
593  while (!DATA_FITS(sb, data_len)) {
594  if (Grow(sb) != 0) {
595  return -1;
596  }
597  }
598  }
599  }
600  if (!DATA_FITS(sb, data_len)) {
601  return -1;
602  }
603 
604  memcpy(sb->buf + sb->buf_offset, data, data_len);
605  seg->stream_offset = sb->stream_offset + sb->buf_offset;
606  seg->segment_len = data_len;
607  uint32_t rel_offset = sb->buf_offset;
608  sb->buf_offset += data_len;
609 
610  if (!RB_EMPTY(&sb->sbb_tree)) {
611  SBBUpdate(sb, rel_offset, data_len);
612  }
613  return 0;
614 }
615 
616 /**
617  * \brief add data w/o tracking a segment
618  */
620  const uint8_t *data, uint32_t data_len)
621 {
622  if (sb->buf == NULL) {
623  if (InitBuffer(sb) == -1)
624  return -1;
625  }
626 
627  if (!DATA_FITS(sb, data_len)) {
629  AutoSlide(sb);
630  if (sb->buf_size == 0) {
631  if (GrowToSize(sb, data_len) != 0)
632  return -1;
633  } else {
634  while (!DATA_FITS(sb, data_len)) {
635  if (Grow(sb) != 0) {
636  return -1;
637  }
638  }
639  }
640  }
641  if (!DATA_FITS(sb, data_len)) {
642  return -1;
643  }
644 
645  memcpy(sb->buf + sb->buf_offset, data, data_len);
646  uint32_t rel_offset = sb->buf_offset;
647  sb->buf_offset += data_len;
648 
649  if (!RB_EMPTY(&sb->sbb_tree)) {
650  SBBUpdate(sb, rel_offset, data_len);
651  }
652  return 0;
653 }
654 
655 #define DATA_FITS_AT_OFFSET(sb, len, offset) \
656  ((offset) + (len) <= (sb)->buf_size)
657 
658 /**
659  * \param offset offset relative to StreamingBuffer::stream_offset
660  */
661 int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg,
662  const uint8_t *data, uint32_t data_len,
663  uint64_t offset)
664 {
665  BUG_ON(seg == NULL);
666 
667  if (offset < sb->stream_offset)
668  return -1;
669 
670  if (sb->buf == NULL) {
671  if (InitBuffer(sb) == -1)
672  return -1;
673  }
674 
675  uint32_t rel_offset = offset - sb->stream_offset;
676  if (!DATA_FITS_AT_OFFSET(sb, data_len, rel_offset)) {
677  if (sb->cfg->flags & STREAMING_BUFFER_AUTOSLIDE) {
678  AutoSlide(sb);
679  rel_offset = offset - sb->stream_offset;
680  }
681  if (!DATA_FITS_AT_OFFSET(sb, data_len, rel_offset)) {
682  if (GrowToSize(sb, (rel_offset + data_len)) != 0)
683  return -1;
684  }
685  }
686  if (!DATA_FITS_AT_OFFSET(sb, data_len, rel_offset)) {
687  return -1;
688  }
689 
690  memcpy(sb->buf + rel_offset, data, data_len);
691  seg->stream_offset = offset;
692  seg->segment_len = data_len;
693 
694  SCLogDebug("rel_offset %u sb->stream_offset %"PRIu64", buf_offset %u",
695  rel_offset, sb->stream_offset, sb->buf_offset);
696 
697  if (RB_EMPTY(&sb->sbb_tree)) {
698  SCLogDebug("empty sbb list");
699 
700  if (sb->stream_offset == offset) {
701  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
702  /* empty list, data is exactly what is expected (append),
703  * so do nothing */
704  } else if ((rel_offset + data_len) <= sb->buf_offset) {
705  SCLogDebug("empty sbb list: block is within existing region");
706  } else {
707  if (sb->buf_offset && rel_offset == sb->buf_offset) {
708  // nothing to do
709  } else if (rel_offset < sb->buf_offset) {
710  // nothing to do
711  } else if (sb->buf_offset) {
712  /* existing data, but there is a gap between us */
713  SBBInit(sb, rel_offset, data_len);
714  } else {
715  /* gap before data in empty list */
716  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
717  SBBInitLeadingGap(sb, offset, data_len);
718  }
719  }
720  } else {
721  /* already have blocks, so append new block based on new data */
722  SBBUpdate(sb, rel_offset, data_len);
723  }
724 
725  if (rel_offset + data_len > sb->buf_offset)
726  sb->buf_offset = rel_offset + data_len;
727 
728  return 0;
729 }
730 
732  const StreamingBufferSegment *seg)
733 {
734  if (seg->stream_offset < sb->stream_offset) {
735  if (seg->stream_offset + seg->segment_len <= sb->stream_offset) {
736  return 1;
737  }
738  }
739  return 0;
740 }
741 
742 /** \brief get the data for one SBB */
744  const StreamingBufferBlock *sbb,
745  const uint8_t **data, uint32_t *data_len)
746 {
747  if (sbb->offset >= sb->stream_offset) {
748  uint64_t offset = sbb->offset - sb->stream_offset;
749  *data = sb->buf + offset;
750  if (offset + sbb->len > sb->buf_offset)
751  *data_len = sb->buf_offset - offset;
752  else
753  *data_len = sbb->len;
754  return;
755  } else {
756  uint64_t offset = sb->stream_offset - sbb->offset;
757  if (offset < sbb->len) {
758  *data = sb->buf;
759  *data_len = sbb->len - offset;
760  return;
761  }
762  }
763  *data = NULL;
764  *data_len = 0;
765  return;
766 }
767 
768 /** \brief get the data for one SBB */
770  const StreamingBufferBlock *sbb,
771  const uint8_t **data, uint32_t *data_len,
772  uint64_t offset)
773 {
774  if (offset >= sbb->offset && offset < (sbb->offset + sbb->len)) {
775  uint32_t sbblen = sbb->len - (offset - sbb->offset);
776 
777  if (offset >= sb->stream_offset) {
778  uint64_t data_offset = offset - sb->stream_offset;
779  *data = sb->buf + data_offset;
780  if (data_offset + sbblen > sb->buf_size)
781  *data_len = sb->buf_size - data_offset;
782  else
783  *data_len = sbblen;
784  BUG_ON(*data_len > sbblen);
785  return;
786  } else {
787  uint64_t data_offset = sb->stream_offset - sbb->offset;
788  if (data_offset < sbblen) {
789  *data = sb->buf;
790  *data_len = sbblen - data_offset;
791  BUG_ON(*data_len > sbblen);
792  return;
793  }
794  }
795  }
796 
797  *data = NULL;
798  *data_len = 0;
799  return;
800 }
801 
803  const StreamingBufferSegment *seg,
804  const uint8_t **data, uint32_t *data_len)
805 {
806  if (likely(sb->buf)) {
807  if (seg->stream_offset >= sb->stream_offset) {
808  uint64_t offset = seg->stream_offset - sb->stream_offset;
809  *data = sb->buf + offset;
810  if (offset + seg->segment_len > sb->buf_size)
811  *data_len = sb->buf_size - offset;
812  else
813  *data_len = seg->segment_len;
814  return;
815  } else {
816  uint64_t offset = sb->stream_offset - seg->stream_offset;
817  if (offset < seg->segment_len) {
818  *data = sb->buf;
819  *data_len = seg->segment_len - offset;
820  return;
821  }
822  }
823  }
824  *data = NULL;
825  *data_len = 0;
826  return;
827 }
828 
829 /**
830  * \retval 1 data is the same
831  * \retval 0 data is different
832  */
834  const StreamingBufferSegment *seg,
835  const uint8_t *rawdata, uint32_t rawdata_len)
836 {
837  const uint8_t *segdata = NULL;
838  uint32_t segdata_len = 0;
839  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
840  if (segdata && segdata_len &&
841  segdata_len == rawdata_len &&
842  memcmp(segdata, rawdata, segdata_len) == 0)
843  {
844  return 1;
845  }
846  return 0;
847 }
848 
850  const uint8_t **data, uint32_t *data_len,
851  uint64_t *stream_offset)
852 {
853  if (sb != NULL && sb->buf != NULL) {
854  *data = sb->buf;
855  *data_len = sb->buf_offset;
856  *stream_offset = sb->stream_offset;
857  return 1;
858  } else {
859  *data = NULL;
860  *data_len = 0;
861  *stream_offset = 0;
862  return 0;
863  }
864 }
865 
867  const uint8_t **data, uint32_t *data_len,
868  uint64_t offset)
869 {
870  if (sb != NULL && sb->buf != NULL &&
871  offset >= sb->stream_offset &&
872  offset < (sb->stream_offset + sb->buf_offset))
873  {
874  uint32_t skip = offset - sb->stream_offset;
875  *data = sb->buf + skip;
876  *data_len = sb->buf_offset - skip;
877  return 1;
878  } else {
879  *data = NULL;
880  *data_len = 0;
881  return 0;
882  }
883 }
884 
885 /**
886  * \retval 1 data is the same
887  * \retval 0 data is different
888  */
890  const uint8_t *rawdata, uint32_t rawdata_len)
891 {
892  const uint8_t *sbdata = NULL;
893  uint32_t sbdata_len = 0;
894  uint64_t offset = 0;
895  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
896  if (offset == 0 &&
897  sbdata && sbdata_len &&
898  sbdata_len == rawdata_len &&
899  memcmp(sbdata, rawdata, sbdata_len) == 0)
900  {
901  return 1;
902  }
903  SCLogDebug("sbdata_len %u, offset %"PRIu64, sbdata_len, offset);
904  printf("got:\n");
905  PrintRawDataFp(stdout, sbdata,sbdata_len);
906  printf("wanted:\n");
907  PrintRawDataFp(stdout, rawdata,rawdata_len);
908  return 0;
909 }
910 
911 #ifdef UNITTESTS
912 static void Dump(StreamingBuffer *sb)
913 {
914  PrintRawDataFp(stdout, sb->buf, sb->buf_offset);
915 }
916 
917 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
918 {
919  const uint8_t *data = NULL;
920  uint32_t data_len = 0;
921  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
922  if (data && data_len) {
923  PrintRawDataFp(stdout, data, data_len);
924  }
925 }
926 
927 static int StreamingBufferTest01(void)
928 {
929  StreamingBufferConfig cfg = { STREAMING_BUFFER_AUTOSLIDE, 8, 16, NULL, NULL, NULL, NULL };
931  FAIL_IF(sb == NULL);
932 
933  StreamingBufferSegment *seg1 = StreamingBufferAppendRaw(sb, (const uint8_t *)"ABCDEFGH", 8);
934  StreamingBufferSegment *seg2 = StreamingBufferAppendRaw(sb, (const uint8_t *)"01234567", 8);
935  FAIL_IF(sb->stream_offset != 0);
936  FAIL_IF(sb->buf_offset != 16);
937  FAIL_IF(seg1->stream_offset != 0);
938  FAIL_IF(seg2->stream_offset != 8);
941  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,seg1,(const uint8_t *)"ABCDEFGH", 8));
942  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,seg2,(const uint8_t *)"01234567", 8));
943  Dump(sb);
944  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
945 
946  StreamingBufferSegment *seg3 = StreamingBufferAppendRaw(sb, (const uint8_t *)"QWERTY", 6);
947  FAIL_IF(sb->stream_offset != 8);
948  FAIL_IF(sb->buf_offset != 14);
949  FAIL_IF(seg3->stream_offset != 16);
953  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,seg3,(const uint8_t *)"QWERTY", 6));
954  Dump(sb);
955  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
956 
957  StreamingBufferSegment *seg4 = StreamingBufferAppendRaw(sb, (const uint8_t *)"KLM", 3);
958  FAIL_IF(sb->stream_offset != 14);
959  FAIL_IF(sb->buf_offset != 11);
960  FAIL_IF(seg4->stream_offset != 22);
965  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,seg4,(const uint8_t *)"KLM", 3));
966  Dump(sb);
967  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
968 
969  StreamingBufferSegment *seg5 = StreamingBufferAppendRaw(sb, (const uint8_t *)"!@#$%^&*()_+<>?/,.;:'[]{}-=", 27);
970  FAIL_IF(sb->stream_offset != 17);
971  FAIL_IF(sb->buf_offset != 35);
972  FAIL_IF(seg5->stream_offset != 25);
978  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,seg5,(const uint8_t *)"!@#$%^&*()_+<>?/,.;:'[]{}-=", 27));
979  Dump(sb);
980  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
981 
982  StreamingBufferSegment *seg6 = StreamingBufferAppendRaw(sb, (const uint8_t *)"UVWXYZ", 6);
983  FAIL_IF(sb->stream_offset != 17);
984  FAIL_IF(sb->buf_offset != 41);
985  FAIL_IF(seg6->stream_offset != 52);
992  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,seg6,(const uint8_t *)"UVWXYZ", 6));
993  Dump(sb);
994  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
995 
996  SCFree(seg1);
997  SCFree(seg2);
998  SCFree(seg3);
999  SCFree(seg4);
1000  SCFree(seg5);
1001  SCFree(seg6);
1002  StreamingBufferFree(sb);
1003  PASS;
1004 }
1005 
1006 static int StreamingBufferTest02(void)
1007 {
1008  StreamingBufferConfig cfg = { 0, 8, 24, NULL, NULL, NULL, NULL };
1010  FAIL_IF(sb == NULL);
1011 
1012  StreamingBufferSegment seg1;
1013  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1014  StreamingBufferSegment seg2;
1015  FAIL_IF(StreamingBufferAppend(sb, &seg2, (const uint8_t *)"01234567", 8) != 0);
1016  FAIL_IF(sb->stream_offset != 0);
1017  FAIL_IF(sb->buf_offset != 16);
1018  FAIL_IF(seg1.stream_offset != 0);
1019  FAIL_IF(seg2.stream_offset != 8);
1022  Dump(sb);
1023  DumpSegment(sb, &seg1);
1024  DumpSegment(sb, &seg2);
1025  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1026 
1027  StreamingBufferSlide(sb, 6);
1028  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1029 
1030  StreamingBufferSegment seg3;
1031  FAIL_IF(StreamingBufferAppend(sb, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
1032  FAIL_IF(sb->stream_offset != 6);
1033  FAIL_IF(sb->buf_offset != 16);
1034  FAIL_IF(seg3.stream_offset != 16);
1038  Dump(sb);
1039  DumpSegment(sb, &seg1);
1040  DumpSegment(sb, &seg2);
1041  DumpSegment(sb, &seg3);
1042  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1043 
1044  StreamingBufferSlide(sb, 6);
1048  Dump(sb);
1049  DumpSegment(sb, &seg1);
1050  DumpSegment(sb, &seg2);
1051  DumpSegment(sb, &seg3);
1052  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1053 
1054  StreamingBufferFree(sb);
1055  PASS;
1056 }
1057 
1058 static int StreamingBufferTest03(void)
1059 {
1060  StreamingBufferConfig cfg = { 0, 8, 24, NULL, NULL, NULL, NULL };
1062  FAIL_IF(sb == NULL);
1063 
1064  StreamingBufferSegment seg1;
1065  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1066  StreamingBufferSegment seg2;
1067  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1068  FAIL_IF(sb->stream_offset != 0);
1069  FAIL_IF(sb->buf_offset != 22);
1070  FAIL_IF(seg1.stream_offset != 0);
1071  FAIL_IF(seg2.stream_offset != 14);
1074  Dump(sb);
1075  DumpSegment(sb, &seg1);
1076  DumpSegment(sb, &seg2);
1077  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1078 
1079  StreamingBufferSegment seg3;
1080  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1081  FAIL_IF(sb->stream_offset != 0);
1082  FAIL_IF(sb->buf_offset != 22);
1083  FAIL_IF(seg3.stream_offset != 8);
1087  Dump(sb);
1088  DumpSegment(sb, &seg1);
1089  DumpSegment(sb, &seg2);
1090  DumpSegment(sb, &seg3);
1091  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1092 
1093  StreamingBufferSlide(sb, 10);
1097  Dump(sb);
1098  DumpSegment(sb, &seg1);
1099  DumpSegment(sb, &seg2);
1100  DumpSegment(sb, &seg3);
1101  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1102 
1103  StreamingBufferFree(sb);
1104  PASS;
1105 }
1106 
1107 static int StreamingBufferTest04(void)
1108 {
1109  StreamingBufferConfig cfg = { 0, 8, 16, NULL, NULL, NULL, NULL };
1111  FAIL_IF(sb == NULL);
1112 
1113  StreamingBufferSegment seg1;
1114  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1115  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
1116  StreamingBufferSegment seg2;
1117  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1118  FAIL_IF(sb->stream_offset != 0);
1119  FAIL_IF(sb->buf_offset != 22);
1120  FAIL_IF(seg1.stream_offset != 0);
1121  FAIL_IF(seg2.stream_offset != 14);
1124  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
1125  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1126  FAIL_IF(sbb1 != sb->head);
1127  FAIL_IF_NULL(sbb1);
1128  FAIL_IF(sbb1->offset != 0);
1129  FAIL_IF(sbb1->len != 8);
1130  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
1131  FAIL_IF_NULL(sbb2);
1132  FAIL_IF(sbb2 == sb->head);
1133  FAIL_IF(sbb2->offset != 14);
1134  FAIL_IF(sbb2->len != 8);
1135  Dump(sb);
1136  DumpSegment(sb, &seg1);
1137  DumpSegment(sb, &seg2);
1138  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1139 
1140  StreamingBufferSegment seg3;
1141  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1142  FAIL_IF(sb->stream_offset != 0);
1143  FAIL_IF(sb->buf_offset != 22);
1144  FAIL_IF(seg3.stream_offset != 8);
1148  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1149  FAIL_IF_NULL(sbb1);
1150  FAIL_IF(sbb1 != sb->head);
1151  FAIL_IF(sbb1->offset != 0);
1152  FAIL_IF(sbb1->len != 22);
1153  FAIL_IF(SBB_RB_NEXT(sbb1));
1154  Dump(sb);
1155  DumpSegment(sb, &seg1);
1156  DumpSegment(sb, &seg2);
1157  DumpSegment(sb, &seg3);
1158  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1159 
1160  /* far ahead of curve: */
1161  StreamingBufferSegment seg4;
1162  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
1163  FAIL_IF(sb->stream_offset != 0);
1164  FAIL_IF(sb->buf_offset != 127);
1165  FAIL_IF(sb->buf_size != 128);
1166  FAIL_IF(seg4.stream_offset != 124);
1171  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1172  FAIL_IF_NULL(sbb1);
1173  FAIL_IF(sbb1 != sb->head);
1174  FAIL_IF(sbb1->offset != 0);
1175  FAIL_IF(sbb1->len != 22);
1176  FAIL_IF(!SBB_RB_NEXT(sbb1));
1177  Dump(sb);
1178  DumpSegment(sb, &seg1);
1179  DumpSegment(sb, &seg2);
1180  DumpSegment(sb, &seg3);
1181  DumpSegment(sb, &seg4);
1182  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1183 
1184  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
1185  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
1186  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
1187  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
1188 
1189  StreamingBufferFree(sb);
1190  PASS;
1191 }
1192 
1193 static int StreamingBufferTest05(void)
1194 {
1195  StreamingBufferConfig cfg = { STREAMING_BUFFER_AUTOSLIDE, 8, 32, NULL, NULL, NULL, NULL };
1197 
1198  StreamingBufferSegment *seg1 = StreamingBufferAppendRaw(&sb, (const uint8_t *)"AAAAAAAA", 8);
1199  StreamingBufferSegment *seg2 = StreamingBufferAppendRaw(&sb, (const uint8_t *)"BBBBBBBB", 8);
1200  StreamingBufferSegment *seg3 = StreamingBufferAppendRaw(&sb, (const uint8_t *)"CCCCCCCC", 8);
1201  StreamingBufferSegment *seg4 = StreamingBufferAppendRaw(&sb, (const uint8_t *)"DDDDDDDD", 8);
1202  FAIL_IF(sb.stream_offset != 0);
1203  FAIL_IF(sb.buf_offset != 32);
1204  FAIL_IF(seg1->stream_offset != 0);
1205  FAIL_IF(seg2->stream_offset != 8);
1206  FAIL_IF(seg3->stream_offset != 16);
1207  FAIL_IF(seg4->stream_offset != 24);
1212  FAIL_IF(!StreamingBufferSegmentCompareRawData(&sb,seg1,(const uint8_t *)"AAAAAAAA", 8));
1213  FAIL_IF(!StreamingBufferSegmentCompareRawData(&sb,seg2,(const uint8_t *)"BBBBBBBB", 8));
1214  FAIL_IF(!StreamingBufferSegmentCompareRawData(&sb,seg3,(const uint8_t *)"CCCCCCCC", 8));
1215  FAIL_IF(!StreamingBufferSegmentCompareRawData(&sb,seg4,(const uint8_t *)"DDDDDDDD", 8));
1216  Dump(&sb);
1217  FAIL_IF_NOT(sb.head == RB_MIN(SBB, &sb.sbb_tree));
1218  StreamingBufferSegment *seg5 = StreamingBufferAppendRaw(&sb, (const uint8_t *)"EEEEEEEE", 8);
1219  FAIL_IF(!StreamingBufferSegmentCompareRawData(&sb,seg5,(const uint8_t *)"EEEEEEEE", 8));
1220  Dump(&sb);
1221  FAIL_IF_NOT(sb.head == RB_MIN(SBB, &sb.sbb_tree));
1222 
1223  SCFree(seg1);
1224  SCFree(seg2);
1225  SCFree(seg3);
1226  SCFree(seg4);
1227  SCFree(seg5);
1228  StreamingBufferClear(&sb);
1229  PASS;
1230 }
1231 
1232 /** \test lots of gaps in block list */
1233 static int StreamingBufferTest06(void)
1234 {
1235  StreamingBufferConfig cfg = { 0, 8, 16, NULL, NULL, NULL, NULL };
1237  FAIL_IF(sb == NULL);
1238 
1239  StreamingBufferSegment seg1;
1240  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"A", 1) != 0);
1241  StreamingBufferSegment seg2;
1242  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"C", 1, 2) != 0);
1243  Dump(sb);
1244  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1245 
1246  StreamingBufferSegment seg3;
1247  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1248  Dump(sb);
1249  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1250 
1251  StreamingBufferSegment seg4;
1252  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1253  Dump(sb);
1254  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1255 
1256  StreamingBufferSegment seg5;
1257  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1258  Dump(sb);
1259  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1260  FAIL_IF_NULL(sbb1);
1261  FAIL_IF(sbb1->offset != 0);
1262  FAIL_IF(sbb1->len != 10);
1263  FAIL_IF(SBB_RB_NEXT(sbb1));
1264  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1265 
1266  StreamingBufferSegment seg6;
1267  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1268  Dump(sb);
1269  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1270  FAIL_IF_NULL(sbb1);
1271  FAIL_IF(sbb1->offset != 0);
1272  FAIL_IF(sbb1->len != 10);
1273  FAIL_IF(SBB_RB_NEXT(sbb1));
1274  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1275 
1276  StreamingBufferFree(sb);
1277  PASS;
1278 }
1279 
1280 /** \test lots of gaps in block list */
1281 static int StreamingBufferTest07(void)
1282 {
1283  StreamingBufferConfig cfg = { 0, 8, 16, NULL, NULL, NULL, NULL };
1285  FAIL_IF(sb == NULL);
1286 
1287  StreamingBufferSegment seg1;
1288  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1289  StreamingBufferSegment seg2;
1290  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1291  Dump(sb);
1292  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1293 
1294  StreamingBufferSegment seg3;
1295  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1296  Dump(sb);
1297  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1298 
1299  StreamingBufferSegment seg4;
1300  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1301  Dump(sb);
1302  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1303 
1304  StreamingBufferSegment seg5;
1305  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1306  Dump(sb);
1307  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1308  FAIL_IF_NULL(sbb1);
1309  FAIL_IF(sbb1->offset != 0);
1310  FAIL_IF(sbb1->len != 10);
1311  FAIL_IF(SBB_RB_NEXT(sbb1));
1312  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1313 
1314  StreamingBufferSegment seg6;
1315  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1316  Dump(sb);
1317  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1318  FAIL_IF_NULL(sbb1);
1319  FAIL_IF(sbb1->offset != 0);
1320  FAIL_IF(sbb1->len != 10);
1321  FAIL_IF(SBB_RB_NEXT(sbb1));
1322  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1323 
1324  StreamingBufferFree(sb);
1325  PASS;
1326 }
1327 
1328 /** \test lots of gaps in block list */
1329 static int StreamingBufferTest08(void)
1330 {
1331  StreamingBufferConfig cfg = { 0, 8, 16, NULL, NULL, NULL, NULL };
1333  FAIL_IF(sb == NULL);
1334 
1335  StreamingBufferSegment seg1;
1336  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1337  StreamingBufferSegment seg2;
1338  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1339  Dump(sb);
1340  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1341 
1342  StreamingBufferSegment seg3;
1343  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1344  Dump(sb);
1345  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1346 
1347  StreamingBufferSegment seg4;
1348  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1349  Dump(sb);
1350  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1351 
1352  StreamingBufferSegment seg5;
1353  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1354  Dump(sb);
1355  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1356  FAIL_IF_NULL(sbb1);
1357  FAIL_IF(sbb1->offset != 0);
1358  FAIL_IF(sbb1->len != 10);
1359  FAIL_IF(SBB_RB_NEXT(sbb1));
1360  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1361 
1362  StreamingBufferSegment seg6;
1363  FAIL_IF(StreamingBufferAppend(sb, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
1364  Dump(sb);
1365  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1366  FAIL_IF_NULL(sbb1);
1367  FAIL_IF(sbb1->offset != 0);
1368  FAIL_IF(sbb1->len != 20);
1369  FAIL_IF(SBB_RB_NEXT(sbb1));
1370  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1371 
1372  StreamingBufferFree(sb);
1373  PASS;
1374 }
1375 
1376 /** \test lots of gaps in block list */
1377 static int StreamingBufferTest09(void)
1378 {
1379  StreamingBufferConfig cfg = { 0, 8, 16, NULL, NULL, NULL, NULL };
1381  FAIL_IF(sb == NULL);
1382 
1383  StreamingBufferSegment seg1;
1384  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1385  StreamingBufferSegment seg2;
1386  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1387  Dump(sb);
1388  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1389 
1390  StreamingBufferSegment seg3;
1391  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"H", 1, 7) != 0);
1392  Dump(sb);
1393  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1394 
1395  StreamingBufferSegment seg4;
1396  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"F", 1, 5) != 0);
1397  Dump(sb);
1398  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1399 
1400  StreamingBufferSegment seg5;
1401  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1402  Dump(sb);
1403  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1404  FAIL_IF_NULL(sbb1);
1405  FAIL_IF(sbb1->offset != 0);
1406  FAIL_IF(sbb1->len != 10);
1407  FAIL_IF(SBB_RB_NEXT(sbb1));
1408  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1409 
1410  StreamingBufferSegment seg6;
1411  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1412  Dump(sb);
1413  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1414  FAIL_IF_NULL(sbb1);
1415  FAIL_IF(sbb1->offset != 0);
1416  FAIL_IF(sbb1->len != 10);
1417  FAIL_IF(SBB_RB_NEXT(sbb1));
1418  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1419 
1420  StreamingBufferFree(sb);
1421  PASS;
1422 }
1423 
1424 /** \test lots of gaps in block list */
1425 static int StreamingBufferTest10(void)
1426 {
1427  StreamingBufferConfig cfg = { 0, 8, 16, NULL, NULL, NULL, NULL };
1429  FAIL_IF(sb == NULL);
1430 
1431  StreamingBufferSegment seg1;
1432  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"A", 1, 0) != 0);
1433  Dump(sb);
1434  StreamingBufferSegment seg2;
1435  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1436  Dump(sb);
1437  StreamingBufferSegment seg3;
1438  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"H", 1, 7) != 0);
1439  Dump(sb);
1440 
1441  StreamingBufferSegment seg4;
1442  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"B", 1, 1) != 0);
1443  Dump(sb);
1444  StreamingBufferSegment seg5;
1445  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"C", 1, 2) != 0);
1446  Dump(sb);
1447  StreamingBufferSegment seg6;
1448  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"G", 1, 6) != 0);
1449  Dump(sb);
1450  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1451 
1452  StreamingBufferSegment seg7;
1453  FAIL_IF(StreamingBufferInsertAt(sb, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1454  Dump(sb);
1455  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1456  FAIL_IF_NULL(sbb1);
1457  FAIL_IF(sbb1->offset != 0);
1458  FAIL_IF(sbb1->len != 10);
1459  FAIL_IF(SBB_RB_NEXT(sbb1));
1460 
1461  StreamingBufferSegment seg8;
1462  FAIL_IF(StreamingBufferInsertAt(sb, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1463  Dump(sb);
1464  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1465  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1466  FAIL_IF_NULL(sbb1);
1467  FAIL_IF(sbb1->offset != 0);
1468  FAIL_IF(sbb1->len != 10);
1469  FAIL_IF(SBB_RB_NEXT(sbb1));
1470 
1471  StreamingBufferFree(sb);
1472  PASS;
1473 }
1474 
1475 #endif
1476 
1478 {
1479 #ifdef UNITTESTS
1480  UtRegisterTest("StreamingBufferTest01", StreamingBufferTest01);
1481  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
1482  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
1483  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
1484  UtRegisterTest("StreamingBufferTest05", StreamingBufferTest05);
1485  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
1486  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
1487  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
1488  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
1489  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
1490 #endif
1491 }
#define RB_ROOT(head)
Definition: tree.h:328
#define STREAMING_BUFFER_INITIALIZER(cfg)
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
#define SCLogDebug(...)
Definition: util-debug.h:335
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
struct HtpBodyChunk_ * next
#define STREAMING_BUFFER_AUTOSLIDE
#define BUG_ON(x)
#define REALLOC(cfg, ptr, orig_s, s)
#define RB_FOREACH(x, name, head)
Definition: tree.h:783
#define PASS
Pass the test.
#define FREE(cfg, ptr, s)
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:788
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
uint64_t offset
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
void StreamingBufferClear(StreamingBuffer *sb)
#define DATA_FITS(sb, len)
Flow * head
Definition: flow-hash.h:102
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
void StreamingBufferRegisterTests(void)
StreamingBufferBlock * head
enum @32 __attribute__
DNP3 application header.
#define CALLOC(cfg, n, s)
#define RB_LEFT(elm, field)
Definition: tree.h:324
void StreamingBufferFree(StreamingBuffer *sb)
int StreamingBufferAppend(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
uint64_t stream_offset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, uint64_t offset)
slide to absolute offset
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define RB_RIGHT(elm, field)
Definition: tree.h:325
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
#define RB_EMPTY(head)
Definition: tree.h:329
#define SCReturnInt(x)
Definition: util-debug.h:341
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:803
#define RB_MIN(name, x)
Definition: tree.h:780
const StreamingBufferConfig * cfg
StreamingBufferSegment * StreamingBufferAppendRaw(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
void StreamingBufferSlide(StreamingBuffer *sb, uint32_t slide)
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
block of continues data
#define SCFree(a)
Definition: util-mem.h:322
PoolThreadReserved res
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:793
int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
uint8_t len
#define likely(expr)
Definition: util-optimize.h:32
uint32_t segment_len
#define DATA_FITS_AT_OFFSET(sb, len, offset)
#define FAIL_IF_NOT(expr)
Fail a test if expression to true.
Definition: util-unittest.h:82
#define DEBUG_VALIDATE_BUG_ON(exp)