suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-unittest.h"
21 #include "util-print.h"
22 #include "util-validate.h"
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  *
29  * \brief Streaming Buffer API
30  */
31 
32 /* memory handling wrappers. If config doesn't define it's own set of
33  * functions, use the defaults */
34 #define MALLOC(cfg, s) \
35  (cfg)->Malloc ? (cfg)->Malloc((s)) : SCMalloc((s))
36 #define CALLOC(cfg, n, s) \
37  (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : SCCalloc((n), (s))
38 #define REALLOC(cfg, ptr, orig_s, s) \
39  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : SCRealloc((ptr), (s))
40 #define FREE(cfg, ptr, s) \
41  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
42 
43 static void SBBFree(StreamingBuffer *sb);
44 
46 
48 {
49  SCLogDebug("a %"PRIu64" len %u, b %"PRIu64" len %u",
50  a->offset, a->len, b->offset, b->len);
51 
52  if (a->offset > b->offset)
53  SCReturnInt(1);
54  else if (a->offset < b->offset)
55  SCReturnInt(-1);
56  else {
57  if (a->len == 0 || b->len == 0 || a->len == b->len)
58  SCReturnInt(0);
59  else if (a->len > b->len)
60  SCReturnInt(1);
61  else
62  SCReturnInt(-1);
63  }
64 }
65 
66 /* inclusive compare function that also considers the right edge,
67  * not just the offset. */
68 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
69  const uint64_t lre = lookup->offset + lookup->len;
70  const uint64_t tre = intree->offset + intree->len;
71  if (lre <= intree->offset) // entirely before
72  return -1;
73  else if (lookup->offset < tre && lre <= tre) // (some) overlap
74  return 0;
75  else
76  return 1; // entirely after
77 }
78 
80 {
81  SCLogDebug("looking up %"PRIu64, elm->offset);
82 
83  struct StreamingBufferBlock *tmp = RB_ROOT(head);
84  struct StreamingBufferBlock *res = NULL;
85  while (tmp) {
86  SCLogDebug("compare with %"PRIu64"/%u", tmp->offset, tmp->len);
87  const int comp = InclusiveCompare(elm, tmp);
88  SCLogDebug("compare result: %d", comp);
89  if (comp < 0) {
90  res = tmp;
91  tmp = RB_LEFT(tmp, rb);
92  } else if (comp > 0) {
93  tmp = RB_RIGHT(tmp, rb);
94  } else {
95  return tmp;
96  }
97  }
98  return res;
99 }
100 
101 
102 static inline int InitBuffer(StreamingBuffer *sb)
103 {
104  sb->buf = CALLOC(sb->cfg, 1, sb->cfg->buf_size);
105  if (sb->buf == NULL) {
106  return -1;
107  }
108  sb->buf_size = sb->cfg->buf_size;
109  return 0;
110 }
111 
113 {
114  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
115  if (sb != NULL) {
116  sb->buf_size = cfg->buf_size;
117  sb->cfg = cfg;
118 
119  if (cfg->buf_size > 0) {
120  if (InitBuffer(sb) == 0) {
121  return sb;
122  }
123  FREE(cfg, sb, sizeof(StreamingBuffer));
124  /* implied buf_size == 0 */
125  } else {
126  return sb;
127  }
128  }
129  return NULL;
130 }
131 
133 {
134  if (sb != NULL) {
135  SCLogDebug("sb->buf_size %u max %u", sb->buf_size, sb->buf_size_max);
136 
137  SBBFree(sb);
138  if (sb->buf != NULL) {
139  FREE(sb->cfg, sb->buf, sb->buf_size);
140  sb->buf = NULL;
141  }
142  }
143 }
144 
146 {
147  if (sb != NULL) {
149  FREE(sb->cfg, sb, sizeof(StreamingBuffer));
150  }
151 }
152 
153 #ifdef DEBUG
154 static void SBBPrintList(StreamingBuffer *sb)
155 {
156  StreamingBufferBlock *sbb = NULL;
157  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
158  SCLogDebug("sbb: offset %"PRIu64", len %u", sbb->offset, sbb->len);
159  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
160  if (next) {
161  if ((sbb->offset + sbb->len) != next->offset) {
162  SCLogDebug("gap: offset %"PRIu64", len %"PRIu64, (sbb->offset + sbb->len),
163  next->offset - (sbb->offset + sbb->len));
164  }
165  }
166  }
167 }
168 #endif
169 
170 /* setup with gap between 2 blocks
171  *
172  * [block][gap][block]
173  **/
174 static void SBBInit(StreamingBuffer *sb,
175  uint32_t rel_offset, uint32_t data_len)
176 {
178  DEBUG_VALIDATE_BUG_ON(sb->buf_offset > sb->stream_offset + rel_offset);
179 
180  /* need to set up 2: existing data block and new data block */
181  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
182  if (sbb == NULL) {
183  return;
184  }
185  sbb->offset = sb->stream_offset;
186  sbb->len = sb->buf_offset;
187 
188  StreamingBufferBlock *sbb2 = CALLOC(sb->cfg, 1, sizeof(*sbb2));
189  if (sbb2 == NULL) {
190  FREE(sb->cfg, sbb, sizeof(*sbb));
191  return;
192  }
193  sbb2->offset = sb->stream_offset + rel_offset;
194  sbb2->len = data_len;
195 
196  sb->head = sbb;
197  sb->sbb_size = sbb->len + sbb2->len;
198  SBB_RB_INSERT(&sb->sbb_tree, sbb);
199  SBB_RB_INSERT(&sb->sbb_tree, sbb2);
200 
201  SCLogDebug("sbb1 %"PRIu64", len %u, sbb2 %"PRIu64", len %u",
202  sbb->offset, sbb->len, sbb2->offset, sbb2->len);
203 #ifdef DEBUG
204  SBBPrintList(sb);
205 #endif
206  BUG_ON(sbb2->offset < sbb->len);
207 }
208 
209 /* setup with leading gap
210  *
211  * [gap][block]
212  **/
213 static void SBBInitLeadingGap(StreamingBuffer *sb,
214  uint64_t offset, uint32_t data_len)
215 {
217 
218  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
219  if (sbb == NULL)
220  return;
221  sbb->offset = offset;
222  sbb->len = data_len;
223 
224  sb->head = sbb;
225  sb->sbb_size = sbb->len;
226  SBB_RB_INSERT(&sb->sbb_tree, sbb);
227 
228  SCLogDebug("sbb %"PRIu64", len %u",
229  sbb->offset, sbb->len);
230 #ifdef DEBUG
231  SBBPrintList(sb);
232 #endif
233 }
234 
235 static inline void ConsolidateFwd(StreamingBuffer *sb,
236  struct SBB *tree, StreamingBufferBlock *sa)
237 {
238  uint64_t sa_re = sa->offset + sa->len;
239  StreamingBufferBlock *tr, *s = sa;
240  RB_FOREACH_FROM(tr, SBB, s) {
241  if (sa == tr)
242  continue;
243 
244  const uint64_t tr_re = tr->offset + tr->len;
245  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u re %"PRIu64,
246  tr, tr->offset, tr->len, tr_re);
247 
248  if (sa_re < tr->offset)
249  break; // entirely before
250 
251  /*
252  sa: [ ]
253  tr: [ ]
254  sa: [ ]
255  tr: [ ]
256  sa: [ ]
257  tr: [ ]
258  */
259  if (sa->offset >= tr->offset && sa_re <= tr_re) {
260  sb->sbb_size -= sa->len;
261  sa->len = tr->len;
262  sa->offset = tr->offset;
263  sa_re = sa->offset + sa->len;
264  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED2", tr, tr->offset, tr->len);
265  SBB_RB_REMOVE(tree, tr);
266  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
267  /*
268  sa: [ ]
269  tr: [ ]
270  sa: [ ]
271  tr: [ ]
272  sa: [ ]
273  tr: [ ]
274  */
275  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
276  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED", tr, tr->offset, tr->len);
277  SBB_RB_REMOVE(tree, tr);
278  sb->sbb_size -= tr->len;
279  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
280  /*
281  sa: [ ]
282  tr: [ ]
283  sa: [ ]
284  tr: [ ]
285  */
286  } else if (sa->offset < tr->offset && // starts before
287  sa_re >= tr->offset && sa_re < tr_re) // ends inside
288  {
289  // merge. sb->sbb_size includes both so we need to adjust that too.
290  uint32_t combined_len = sa->len + tr->len;
291  sa->len = tr_re - sa->offset;
292  sa_re = sa->offset + sa->len;
293  SCLogDebug("-> (fwd) tr %p %"PRIu64"/%u REMOVED MERGED", tr, tr->offset, tr->len);
294  SBB_RB_REMOVE(tree, tr);
295  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
296  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
297  }
298  }
299 }
300 
301 static inline void ConsolidateBackward(StreamingBuffer *sb,
302  struct SBB *tree, StreamingBufferBlock *sa)
303 {
304  uint64_t sa_re = sa->offset + sa->len;
305  StreamingBufferBlock *tr, *s = sa;
306  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
307  if (sa == tr)
308  continue;
309  const uint64_t tr_re = tr->offset + tr->len;
310  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u", tr, tr->offset, tr->len);
311 
312  if (sa->offset > tr_re)
313  break; // entirely after
314 
315  if (sa->offset >= tr->offset && sa_re <= tr_re) {
316  sb->sbb_size -= sa->len; // sa entirely eclipsed so remove double accounting
317  sa->len = tr->len;
318  sa->offset = tr->offset;
319  sa_re = sa->offset + sa->len;
320  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED2", tr, tr->offset, tr->len);
321  if (sb->head == tr)
322  sb->head = sa;
323  SBB_RB_REMOVE(tree, tr);
324  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
325  /*
326  sa: [ ]
327  tr: [ ]
328  sa: [ ]
329  tr: [ ]
330  sa: [ ]
331  tr: [ ]
332  */
333  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
334  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED ECLIPSED", tr, tr->offset, tr->len);
335  if (sb->head == tr)
336  sb->head = sa;
337  SBB_RB_REMOVE(tree, tr);
338  sb->sbb_size -= tr->len; // tr entirely eclipsed so remove double accounting
339  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
340  /*
341  sa: [ ]
342  tr: [ ]
343  sa: [ ]
344  tr: [ ]
345  */
346  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
347  // merge. sb->sbb_size includes both so we need to adjust that too.
348  uint32_t combined_len = sa->len + tr->len;
349  sa->len = sa_re - tr->offset;
350  sa->offset = tr->offset;
351  sa_re = sa->offset + sa->len;
352  SCLogDebug("-> (bwd) tr %p %"PRIu64"/%u REMOVED MERGED", tr, tr->offset, tr->len);
353  if (sb->head == tr)
354  sb->head = sa;
355  SBB_RB_REMOVE(tree, tr);
356  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
357  FREE(sb->cfg, tr, sizeof(StreamingBufferBlock));
358  }
359  }
360 }
361 
362 static int Insert(StreamingBuffer *sb, struct SBB *tree,
363  uint32_t rel_offset, uint32_t len)
364 {
365  SCLogDebug("* inserting: %u/%u\n", rel_offset, len);
366 
367  StreamingBufferBlock *sbb = CALLOC(sb->cfg, 1, sizeof(*sbb));
368  if (sbb == NULL)
369  return -1;
370  sbb->offset = sb->stream_offset + rel_offset;
371  sbb->len = len;
372  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
373  if (res) {
374  // exact overlap
375  SCLogDebug("* insert failed: exact match in tree with %p %"PRIu64"/%u", res, res->offset, res->len);
376  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
377  return 0;
378  }
379  sb->sbb_size += len; // may adjust based on consolidation below
380  if (SBB_RB_PREV(sbb) == NULL) {
381  sb->head = sbb;
382  } else {
383  ConsolidateBackward(sb, tree, sbb);
384  }
385  ConsolidateFwd(sb, tree, sbb);
386 #ifdef DEBUG
387  SBBPrintList(sb);
388 #endif
389  return 0;
390 }
391 
392 static void SBBUpdate(StreamingBuffer *sb,
393  uint32_t rel_offset, uint32_t data_len)
394 {
395  Insert(sb, &sb->sbb_tree, rel_offset, data_len);
396 }
397 
398 static void SBBFree(StreamingBuffer *sb)
399 {
400  StreamingBufferBlock *sbb = NULL, *safe = NULL;
401  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
402  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
403  sb->sbb_size -= sbb->len;
404  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
405  }
406  sb->head = NULL;
407 }
408 
409 static void SBBPrune(StreamingBuffer *sb)
410 {
411  SCLogDebug("pruning %p to %"PRIu64, sb, sb->stream_offset);
412  StreamingBufferBlock *sbb = NULL, *safe = NULL;
413  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
414  /* completely beyond window, we're done */
415  if (sbb->offset >= sb->stream_offset) {
416  sb->head = sbb;
417  break;
418  }
419 
420  /* partly before, partly beyond. Adjust */
421  if (sbb->offset < sb->stream_offset &&
422  sbb->offset + sbb->len > sb->stream_offset) {
423  uint32_t shrink_by = sb->stream_offset - sbb->offset;
424  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
425  if (sbb->len >= shrink_by) {
426  sbb->len -= shrink_by;
427  sbb->offset += shrink_by;
428  sb->sbb_size -= shrink_by;
430  }
431  sb->head = sbb;
432  break;
433  }
434 
435  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
436  /* either we set it again for the next sbb, or there isn't any */
437  sb->head = NULL;
438  sb->sbb_size -= sbb->len;
439  SCLogDebug("sb %p removed %p %"PRIu64", %u", sb, sbb, sbb->offset, sbb->len);
440  FREE(sb->cfg, sbb, sizeof(StreamingBufferBlock));
441  }
442 }
443 
444 static int WARN_UNUSED
445 GrowToSize(StreamingBuffer *sb, uint32_t size)
446 {
447  /* try to grow in multiples of sb->cfg->buf_size */
448  uint32_t x = sb->cfg->buf_size ? size % sb->cfg->buf_size : 0;
449  uint32_t base = size - x;
450  uint32_t grow = base + sb->cfg->buf_size;
451 
452  void *ptr = REALLOC(sb->cfg, sb->buf, sb->buf_size, grow);
453  if (ptr == NULL)
454  return -1;
455 
456  /* for safe printing and general caution, lets memset the
457  * new data to 0 */
458  size_t diff = grow - sb->buf_size;
459  void *new_mem = ((char *)ptr) + sb->buf_size;
460  memset(new_mem, 0, diff);
461 
462  sb->buf = ptr;
463  sb->buf_size = grow;
464  SCLogDebug("grown buffer to %u", grow);
465 #ifdef DEBUG
466  if (sb->buf_size > sb->buf_size_max) {
467  sb->buf_size_max = sb->buf_size;
468  }
469 #endif
470  return 0;
471 }
472 
473 /** \internal
474  * \brief try to double the buffer size
475  * \retval 0 ok
476  * \retval -1 failed, buffer unchanged
477  */
478 static int WARN_UNUSED Grow(StreamingBuffer *sb)
479 {
480  uint32_t grow = sb->buf_size * 2;
481  void *ptr = REALLOC(sb->cfg, sb->buf, sb->buf_size, grow);
482  if (ptr == NULL)
483  return -1;
484 
485  /* for safe printing and general caution, lets memset the
486  * new data to 0 */
487  size_t diff = grow - sb->buf_size;
488  void *new_mem = ((char *)ptr) + sb->buf_size;
489  memset(new_mem, 0, diff);
490 
491  sb->buf = ptr;
492  sb->buf_size = grow;
493  SCLogDebug("grown buffer to %u", grow);
494 #ifdef DEBUG
495  if (sb->buf_size > sb->buf_size_max) {
496  sb->buf_size_max = sb->buf_size;
497  }
498 #endif
499  return 0;
500 }
501 
502 /**
503  * \brief slide to absolute offset
504  * \todo if sliding beyond window, we could perhaps reset?
505  */
507 {
508  if (offset > sb->stream_offset &&
509  offset <= sb->stream_offset + sb->buf_offset)
510  {
511  uint32_t slide = offset - sb->stream_offset;
512  uint32_t size = sb->buf_offset - slide;
513  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide, size);
514  memmove(sb->buf, sb->buf+slide, size);
515  sb->stream_offset += slide;
516  sb->buf_offset = size;
517  SBBPrune(sb);
518  }
519 }
520 
521 void StreamingBufferSlide(StreamingBuffer *sb, uint32_t slide)
522 {
523  uint32_t size = sb->buf_offset - slide;
524  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide, size);
525  memmove(sb->buf, sb->buf+slide, size);
526  sb->stream_offset += slide;
527  sb->buf_offset = size;
528  SBBPrune(sb);
529 }
530 
531 #define DATA_FITS(sb, len) \
532  ((sb)->buf_offset + (len) <= (sb)->buf_size)
533 
534 StreamingBufferSegment *StreamingBufferAppendRaw(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
535 {
536  if (sb->buf == NULL) {
537  if (InitBuffer(sb) == -1)
538  return NULL;
539  }
540 
541  if (!DATA_FITS(sb, data_len)) {
542  if (sb->buf_size == 0) {
543  if (GrowToSize(sb, data_len) != 0)
544  return NULL;
545  } else {
546  while (!DATA_FITS(sb, data_len)) {
547  if (Grow(sb) != 0) {
548  return NULL;
549  }
550  }
551  }
552  }
553  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
554 
555  StreamingBufferSegment *seg = CALLOC(sb->cfg, 1, sizeof(StreamingBufferSegment));
556  if (seg != NULL) {
557  memcpy(sb->buf + sb->buf_offset, data, data_len);
558  seg->stream_offset = sb->stream_offset + sb->buf_offset;
559  seg->segment_len = data_len;
560  uint32_t rel_offset = sb->buf_offset;
561  sb->buf_offset += data_len;
562 
563  if (!RB_EMPTY(&sb->sbb_tree)) {
564  SBBUpdate(sb, rel_offset, data_len);
565  }
566  return seg;
567  }
568  return NULL;
569 }
570 
571 int StreamingBufferAppend(StreamingBuffer *sb, StreamingBufferSegment *seg,
572  const uint8_t *data, uint32_t data_len)
573 {
574  BUG_ON(seg == NULL);
575 
576  if (sb->buf == NULL) {
577  if (InitBuffer(sb) == -1)
578  return -1;
579  }
580 
581  if (!DATA_FITS(sb, data_len)) {
582  if (sb->buf_size == 0) {
583  if (GrowToSize(sb, data_len) != 0)
584  return -1;
585  } else {
586  while (!DATA_FITS(sb, data_len)) {
587  if (Grow(sb) != 0) {
588  return -1;
589  }
590  }
591  }
592  }
593  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
594 
595  memcpy(sb->buf + sb->buf_offset, data, data_len);
596  seg->stream_offset = sb->stream_offset + sb->buf_offset;
597  seg->segment_len = data_len;
598  uint32_t rel_offset = sb->buf_offset;
599  sb->buf_offset += data_len;
600 
601  if (!RB_EMPTY(&sb->sbb_tree)) {
602  SBBUpdate(sb, rel_offset, data_len);
603  }
604  return 0;
605 }
606 
607 /**
608  * \brief add data w/o tracking a segment
609  */
611  const uint8_t *data, uint32_t data_len)
612 {
613  if (sb->buf == NULL) {
614  if (InitBuffer(sb) == -1)
615  return -1;
616  }
617 
618  if (!DATA_FITS(sb, data_len)) {
619  if (sb->buf_size == 0) {
620  if (GrowToSize(sb, data_len) != 0)
621  return -1;
622  } else {
623  while (!DATA_FITS(sb, data_len)) {
624  if (Grow(sb) != 0) {
625  return -1;
626  }
627  }
628  }
629  }
630  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
631 
632  memcpy(sb->buf + sb->buf_offset, data, data_len);
633  uint32_t rel_offset = sb->buf_offset;
634  sb->buf_offset += data_len;
635 
636  if (!RB_EMPTY(&sb->sbb_tree)) {
637  SBBUpdate(sb, rel_offset, data_len);
638  }
639  return 0;
640 }
641 
642 #define DATA_FITS_AT_OFFSET(sb, len, offset) \
643  ((offset) + (len) <= (sb)->buf_size)
644 
645 /**
646  * \param offset offset relative to StreamingBuffer::stream_offset
647  *
648  * \return 0 in case of success
649  * \return -1 on memory allocation errors
650  * \return negative value on other errors
651  */
652 int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg,
653  const uint8_t *data, uint32_t data_len,
654  uint64_t offset)
655 {
656  BUG_ON(seg == NULL);
657  DEBUG_VALIDATE_BUG_ON(offset < sb->stream_offset);
658  if (offset < sb->stream_offset)
659  return -2;
660 
661  if (sb->buf == NULL) {
662  if (InitBuffer(sb) == -1)
663  return -1;
664  }
665 
666  uint32_t rel_offset = offset - sb->stream_offset;
667  if (!DATA_FITS_AT_OFFSET(sb, data_len, rel_offset)) {
668  if (GrowToSize(sb, (rel_offset + data_len)) != 0)
669  return -1;
670  }
671  DEBUG_VALIDATE_BUG_ON(!DATA_FITS_AT_OFFSET(sb, data_len, rel_offset));
672 
673  memcpy(sb->buf + rel_offset, data, data_len);
674  seg->stream_offset = offset;
675  seg->segment_len = data_len;
676 
677  SCLogDebug("rel_offset %u sb->stream_offset %"PRIu64", buf_offset %u",
678  rel_offset, sb->stream_offset, sb->buf_offset);
679 
680  if (RB_EMPTY(&sb->sbb_tree)) {
681  SCLogDebug("empty sbb list");
682 
683  if (sb->stream_offset == offset) {
684  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
685  /* empty list, data is exactly what is expected (append),
686  * so do nothing */
687  } else if ((rel_offset + data_len) <= sb->buf_offset) {
688  SCLogDebug("empty sbb list: block is within existing region");
689  } else {
690  if (sb->buf_offset && rel_offset == sb->buf_offset) {
691  // nothing to do
692  } else if (rel_offset < sb->buf_offset) {
693  // nothing to do
694  } else if (sb->buf_offset) {
695  /* existing data, but there is a gap between us */
696  SBBInit(sb, rel_offset, data_len);
697  } else {
698  /* gap before data in empty list */
699  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
700  SBBInitLeadingGap(sb, offset, data_len);
701  }
702  }
703  } else {
704  /* already have blocks, so append new block based on new data */
705  SBBUpdate(sb, rel_offset, data_len);
706  }
707 
708  if (rel_offset + data_len > sb->buf_offset)
709  sb->buf_offset = rel_offset + data_len;
710 
711  return 0;
712 }
713 
715  const StreamingBufferSegment *seg)
716 {
717  if (seg->stream_offset < sb->stream_offset) {
718  if (seg->stream_offset + seg->segment_len <= sb->stream_offset) {
719  return 1;
720  }
721  }
722  return 0;
723 }
724 
725 /** \brief get the data for one SBB */
727  const StreamingBufferBlock *sbb,
728  const uint8_t **data, uint32_t *data_len)
729 {
730  if (sbb->offset >= sb->stream_offset) {
731  uint64_t offset = sbb->offset - sb->stream_offset;
732  *data = sb->buf + offset;
733  if (offset + sbb->len > sb->buf_offset)
734  *data_len = sb->buf_offset - offset;
735  else
736  *data_len = sbb->len;
737  return;
738  } else {
739  uint64_t offset = sb->stream_offset - sbb->offset;
740  if (offset < sbb->len) {
741  *data = sb->buf;
742  *data_len = sbb->len - offset;
743  return;
744  }
745  }
746  *data = NULL;
747  *data_len = 0;
748  return;
749 }
750 
751 /** \brief get the data for one SBB */
753  const StreamingBufferBlock *sbb,
754  const uint8_t **data, uint32_t *data_len,
755  uint64_t offset)
756 {
757  if (offset >= sbb->offset && offset < (sbb->offset + sbb->len)) {
758  uint32_t sbblen = sbb->len - (offset - sbb->offset);
759 
760  if (offset >= sb->stream_offset) {
761  uint64_t data_offset = offset - sb->stream_offset;
762  *data = sb->buf + data_offset;
763  if (data_offset + sbblen > sb->buf_size)
764  *data_len = sb->buf_size - data_offset;
765  else
766  *data_len = sbblen;
767  BUG_ON(*data_len > sbblen);
768  return;
769  } else {
770  uint64_t data_offset = sb->stream_offset - sbb->offset;
771  if (data_offset < sbblen) {
772  *data = sb->buf;
773  *data_len = sbblen - data_offset;
774  BUG_ON(*data_len > sbblen);
775  return;
776  }
777  }
778  }
779 
780  *data = NULL;
781  *data_len = 0;
782  return;
783 }
784 
786  const StreamingBufferSegment *seg,
787  const uint8_t **data, uint32_t *data_len)
788 {
789  if (likely(sb->buf)) {
790  if (seg->stream_offset >= sb->stream_offset) {
791  uint64_t offset = seg->stream_offset - sb->stream_offset;
792  *data = sb->buf + offset;
793  if (offset + seg->segment_len > sb->buf_size)
794  *data_len = sb->buf_size - offset;
795  else
796  *data_len = seg->segment_len;
797  return;
798  } else {
799  uint64_t offset = sb->stream_offset - seg->stream_offset;
800  if (offset < seg->segment_len) {
801  *data = sb->buf;
802  *data_len = seg->segment_len - offset;
803  return;
804  }
805  }
806  }
807  *data = NULL;
808  *data_len = 0;
809  return;
810 }
811 
812 /**
813  * \retval 1 data is the same
814  * \retval 0 data is different
815  */
817  const StreamingBufferSegment *seg,
818  const uint8_t *rawdata, uint32_t rawdata_len)
819 {
820  const uint8_t *segdata = NULL;
821  uint32_t segdata_len = 0;
822  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
823  if (segdata && segdata_len &&
824  segdata_len == rawdata_len &&
825  memcmp(segdata, rawdata, segdata_len) == 0)
826  {
827  return 1;
828  }
829  return 0;
830 }
831 
833  const uint8_t **data, uint32_t *data_len,
834  uint64_t *stream_offset)
835 {
836  if (sb != NULL && sb->buf != NULL) {
837  *data = sb->buf;
838  *data_len = sb->buf_offset;
840  return 1;
841  } else {
842  *data = NULL;
843  *data_len = 0;
844  *stream_offset = 0;
845  return 0;
846  }
847 }
848 
850  const uint8_t **data, uint32_t *data_len,
851  uint64_t offset)
852 {
853  if (sb != NULL && sb->buf != NULL &&
854  offset >= sb->stream_offset &&
855  offset < (sb->stream_offset + sb->buf_offset))
856  {
857  uint32_t skip = offset - sb->stream_offset;
858  *data = sb->buf + skip;
859  *data_len = sb->buf_offset - skip;
860  return 1;
861  } else {
862  *data = NULL;
863  *data_len = 0;
864  return 0;
865  }
866 }
867 
868 /**
869  * \retval 1 data is the same
870  * \retval 0 data is different
871  */
873  const uint8_t *rawdata, uint32_t rawdata_len)
874 {
875  const uint8_t *sbdata = NULL;
876  uint32_t sbdata_len = 0;
877  uint64_t offset = 0;
878  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
879  if (offset == 0 &&
880  sbdata && sbdata_len &&
881  sbdata_len == rawdata_len &&
882  memcmp(sbdata, rawdata, sbdata_len) == 0)
883  {
884  return 1;
885  }
886  SCLogDebug("sbdata_len %u, offset %"PRIu64, sbdata_len, offset);
887  printf("got:\n");
888  PrintRawDataFp(stdout, sbdata,sbdata_len);
889  printf("wanted:\n");
890  PrintRawDataFp(stdout, rawdata,rawdata_len);
891  return 0;
892 }
893 
894 #ifdef UNITTESTS
895 static void Dump(StreamingBuffer *sb)
896 {
897  PrintRawDataFp(stdout, sb->buf, sb->buf_offset);
898 }
899 
900 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
901 {
902  const uint8_t *data = NULL;
903  uint32_t data_len = 0;
904  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
905  if (data && data_len) {
906  PrintRawDataFp(stdout, data, data_len);
907  }
908 }
909 
910 static int StreamingBufferTest02(void)
911 {
912  StreamingBufferConfig cfg = { 8, 24, NULL, NULL, NULL, NULL };
914  FAIL_IF(sb == NULL);
915 
916  StreamingBufferSegment seg1;
917  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
918  StreamingBufferSegment seg2;
919  FAIL_IF(StreamingBufferAppend(sb, &seg2, (const uint8_t *)"01234567", 8) != 0);
920  FAIL_IF(sb->stream_offset != 0);
921  FAIL_IF(sb->buf_offset != 16);
922  FAIL_IF(seg1.stream_offset != 0);
923  FAIL_IF(seg2.stream_offset != 8);
926  Dump(sb);
927  DumpSegment(sb, &seg1);
928  DumpSegment(sb, &seg2);
929  FAIL_IF_NOT_NULL(sb->head);
930  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
931 
932  StreamingBufferSlide(sb, 6);
933  FAIL_IF_NOT_NULL(sb->head);
934  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
935 
936  StreamingBufferSegment seg3;
937  FAIL_IF(StreamingBufferAppend(sb, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
938  FAIL_IF(sb->stream_offset != 6);
939  FAIL_IF(sb->buf_offset != 16);
940  FAIL_IF(seg3.stream_offset != 16);
944  Dump(sb);
945  DumpSegment(sb, &seg1);
946  DumpSegment(sb, &seg2);
947  DumpSegment(sb, &seg3);
948  FAIL_IF_NOT_NULL(sb->head);
949  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
950 
951  StreamingBufferSlide(sb, 6);
955  Dump(sb);
956  DumpSegment(sb, &seg1);
957  DumpSegment(sb, &seg2);
958  DumpSegment(sb, &seg3);
959  FAIL_IF_NOT_NULL(sb->head);
960  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
961 
963  PASS;
964 }
965 
966 static int StreamingBufferTest03(void)
967 {
968  StreamingBufferConfig cfg = { 8, 24, NULL, NULL, NULL, NULL };
970  FAIL_IF(sb == NULL);
971 
972  StreamingBufferSegment seg1;
973  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
974  StreamingBufferSegment seg2;
975  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
976  FAIL_IF(sb->stream_offset != 0);
977  FAIL_IF(sb->buf_offset != 22);
978  FAIL_IF(seg1.stream_offset != 0);
979  FAIL_IF(seg2.stream_offset != 14);
982  Dump(sb);
983  DumpSegment(sb, &seg1);
984  DumpSegment(sb, &seg2);
985  FAIL_IF_NULL(sb->head);
986  FAIL_IF_NOT(sb->sbb_size == 16);
987  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
988 
989  StreamingBufferSegment seg3;
990  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
991  FAIL_IF(sb->stream_offset != 0);
992  FAIL_IF(sb->buf_offset != 22);
993  FAIL_IF(seg3.stream_offset != 8);
997  Dump(sb);
998  DumpSegment(sb, &seg1);
999  DumpSegment(sb, &seg2);
1000  DumpSegment(sb, &seg3);
1001  FAIL_IF_NULL(sb->head);
1002  FAIL_IF_NOT(sb->sbb_size == 22);
1003  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1004 
1005  StreamingBufferSlide(sb, 10);
1009  Dump(sb);
1010  DumpSegment(sb, &seg1);
1011  DumpSegment(sb, &seg2);
1012  DumpSegment(sb, &seg3);
1013  FAIL_IF_NULL(sb->head);
1014  FAIL_IF_NOT(sb->sbb_size == 12);
1015  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1016 
1017  StreamingBufferFree(sb);
1018  PASS;
1019 }
1020 
1021 static int StreamingBufferTest04(void)
1022 {
1023  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL, NULL };
1025  FAIL_IF(sb == NULL);
1026 
1027  StreamingBufferSegment seg1;
1028  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1029  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
1030  StreamingBufferSegment seg2;
1031  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1032  FAIL_IF(sb->stream_offset != 0);
1033  FAIL_IF(sb->buf_offset != 22);
1034  FAIL_IF(seg1.stream_offset != 0);
1035  FAIL_IF(seg2.stream_offset != 14);
1038  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
1039  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1040  FAIL_IF(sbb1 != sb->head);
1041  FAIL_IF_NULL(sbb1);
1042  FAIL_IF(sbb1->offset != 0);
1043  FAIL_IF(sbb1->len != 8);
1044  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
1045  FAIL_IF_NULL(sbb2);
1046  FAIL_IF(sbb2 == sb->head);
1047  FAIL_IF(sbb2->offset != 14);
1048  FAIL_IF(sbb2->len != 8);
1049  Dump(sb);
1050  DumpSegment(sb, &seg1);
1051  DumpSegment(sb, &seg2);
1052  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1053 
1054  StreamingBufferSegment seg3;
1055  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1056  FAIL_IF(sb->stream_offset != 0);
1057  FAIL_IF(sb->buf_offset != 22);
1058  FAIL_IF(seg3.stream_offset != 8);
1062  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1063  FAIL_IF_NULL(sbb1);
1064  FAIL_IF(sbb1 != sb->head);
1065  FAIL_IF(sbb1->offset != 0);
1066  FAIL_IF(sbb1->len != 22);
1067  FAIL_IF(SBB_RB_NEXT(sbb1));
1068  Dump(sb);
1069  DumpSegment(sb, &seg1);
1070  DumpSegment(sb, &seg2);
1071  DumpSegment(sb, &seg3);
1072  FAIL_IF_NULL(sb->head);
1073  FAIL_IF_NOT(sb->sbb_size == 22);
1074  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1075 
1076  /* far ahead of curve: */
1077  StreamingBufferSegment seg4;
1078  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
1079  FAIL_IF(sb->stream_offset != 0);
1080  FAIL_IF(sb->buf_offset != 127);
1081  FAIL_IF(sb->buf_size != 128);
1082  FAIL_IF(seg4.stream_offset != 124);
1087  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1088  FAIL_IF_NULL(sbb1);
1089  FAIL_IF(sbb1 != sb->head);
1090  FAIL_IF(sbb1->offset != 0);
1091  FAIL_IF(sbb1->len != 22);
1092  FAIL_IF(!SBB_RB_NEXT(sbb1));
1093  Dump(sb);
1094  DumpSegment(sb, &seg1);
1095  DumpSegment(sb, &seg2);
1096  DumpSegment(sb, &seg3);
1097  DumpSegment(sb, &seg4);
1098  FAIL_IF_NULL(sb->head);
1099  FAIL_IF_NOT(sb->sbb_size == 25);
1100  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1101 
1102  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
1103  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
1104  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
1105  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
1106 
1107  StreamingBufferFree(sb);
1108  PASS;
1109 }
1110 
1111 /** \test lots of gaps in block list */
1112 static int StreamingBufferTest06(void)
1113 {
1114  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL, NULL };
1116  FAIL_IF(sb == NULL);
1117 
1118  StreamingBufferSegment seg1;
1119  FAIL_IF(StreamingBufferAppend(sb, &seg1, (const uint8_t *)"A", 1) != 0);
1120  StreamingBufferSegment seg2;
1121  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"C", 1, 2) != 0);
1122  Dump(sb);
1123  FAIL_IF_NULL(sb->head);
1124  FAIL_IF_NOT(sb->sbb_size == 2);
1125  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1126 
1127  StreamingBufferSegment seg3;
1128  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1129  Dump(sb);
1130  FAIL_IF_NULL(sb->head);
1131  FAIL_IF_NOT(sb->sbb_size == 3);
1132  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1133 
1134  StreamingBufferSegment seg4;
1135  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1136  Dump(sb);
1137  FAIL_IF_NULL(sb->head);
1138  FAIL_IF_NOT(sb->sbb_size == 4);
1139  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1140 
1141  StreamingBufferSegment seg5;
1142  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1143  Dump(sb);
1144  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1145  FAIL_IF_NULL(sbb1);
1146  FAIL_IF(sbb1->offset != 0);
1147  FAIL_IF(sbb1->len != 10);
1148  FAIL_IF(SBB_RB_NEXT(sbb1));
1149  FAIL_IF_NULL(sb->head);
1150  FAIL_IF_NOT(sb->sbb_size == 10);
1151  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1152 
1153  StreamingBufferSegment seg6;
1154  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1155  Dump(sb);
1156  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1157  FAIL_IF_NULL(sbb1);
1158  FAIL_IF(sbb1->offset != 0);
1159  FAIL_IF(sbb1->len != 10);
1160  FAIL_IF(SBB_RB_NEXT(sbb1));
1161  FAIL_IF_NULL(sb->head);
1162  FAIL_IF_NOT(sb->sbb_size == 10);
1163  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1164 
1165  StreamingBufferFree(sb);
1166  PASS;
1167 }
1168 
1169 /** \test lots of gaps in block list */
1170 static int StreamingBufferTest07(void)
1171 {
1172  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL, NULL };
1174  FAIL_IF(sb == NULL);
1175 
1176  StreamingBufferSegment seg1;
1177  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1178  StreamingBufferSegment seg2;
1179  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1180  Dump(sb);
1181  FAIL_IF_NULL(sb->head);
1182  FAIL_IF_NOT(sb->sbb_size == 2);
1183  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1184 
1185  StreamingBufferSegment seg3;
1186  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1187  Dump(sb);
1188  FAIL_IF_NULL(sb->head);
1189  FAIL_IF_NOT(sb->sbb_size == 3);
1190  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1191 
1192  StreamingBufferSegment seg4;
1193  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1194  Dump(sb);
1195  FAIL_IF_NULL(sb->head);
1196  FAIL_IF_NOT(sb->sbb_size == 4);
1197  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1198 
1199  StreamingBufferSegment seg5;
1200  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1201  Dump(sb);
1202  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1203  FAIL_IF_NULL(sbb1);
1204  FAIL_IF(sbb1->offset != 0);
1205  FAIL_IF(sbb1->len != 10);
1206  FAIL_IF(SBB_RB_NEXT(sbb1));
1207  FAIL_IF_NULL(sb->head);
1208  FAIL_IF_NOT(sb->sbb_size == 10);
1209  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1210 
1211  StreamingBufferSegment seg6;
1212  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1213  Dump(sb);
1214  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1215  FAIL_IF_NULL(sbb1);
1216  FAIL_IF(sbb1->offset != 0);
1217  FAIL_IF(sbb1->len != 10);
1218  FAIL_IF(SBB_RB_NEXT(sbb1));
1219  FAIL_IF_NULL(sb->head);
1220  FAIL_IF_NOT(sb->sbb_size == 10);
1221  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1222 
1223  StreamingBufferFree(sb);
1224  PASS;
1225 }
1226 
1227 /** \test lots of gaps in block list */
1228 static int StreamingBufferTest08(void)
1229 {
1230  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL, NULL };
1232  FAIL_IF(sb == NULL);
1233 
1234  StreamingBufferSegment seg1;
1235  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1236  StreamingBufferSegment seg2;
1237  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1238  Dump(sb);
1239  FAIL_IF_NULL(sb->head);
1240  FAIL_IF_NOT(sb->sbb_size == 2);
1241  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1242 
1243  StreamingBufferSegment seg3;
1244  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1245  Dump(sb);
1246  FAIL_IF_NULL(sb->head);
1247  FAIL_IF_NOT(sb->sbb_size == 3);
1248  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1249 
1250  StreamingBufferSegment seg4;
1251  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1252  Dump(sb);
1253  FAIL_IF_NULL(sb->head);
1254  FAIL_IF_NOT(sb->sbb_size == 4);
1255  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1256 
1257  StreamingBufferSegment seg5;
1258  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1259  Dump(sb);
1260  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1261  FAIL_IF_NULL(sbb1);
1262  FAIL_IF(sbb1->offset != 0);
1263  FAIL_IF(sbb1->len != 10);
1264  FAIL_IF(SBB_RB_NEXT(sbb1));
1265  FAIL_IF_NULL(sb->head);
1266  FAIL_IF_NOT(sb->sbb_size == 10);
1267  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1268 
1269  StreamingBufferSegment seg6;
1270  FAIL_IF(StreamingBufferAppend(sb, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
1271  Dump(sb);
1272  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1273  FAIL_IF_NULL(sbb1);
1274  FAIL_IF(sbb1->offset != 0);
1275  FAIL_IF(sbb1->len != 20);
1276  FAIL_IF(SBB_RB_NEXT(sbb1));
1277  FAIL_IF_NULL(sb->head);
1278  FAIL_IF_NOT(sb->sbb_size == 20);
1279  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1280 
1281  StreamingBufferFree(sb);
1282  PASS;
1283 }
1284 
1285 /** \test lots of gaps in block list */
1286 static int StreamingBufferTest09(void)
1287 {
1288  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL, NULL };
1290  FAIL_IF(sb == NULL);
1291 
1292  StreamingBufferSegment seg1;
1293  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1294  StreamingBufferSegment seg2;
1295  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1296  Dump(sb);
1297  FAIL_IF_NULL(sb->head);
1298  FAIL_IF_NOT(sb->sbb_size == 2);
1299  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1300 
1301  StreamingBufferSegment seg3;
1302  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"H", 1, 7) != 0);
1303  Dump(sb);
1304  FAIL_IF_NULL(sb->head);
1305  FAIL_IF_NOT(sb->sbb_size == 3);
1306  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1307 
1308  StreamingBufferSegment seg4;
1309  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"F", 1, 5) != 0);
1310  Dump(sb);
1311  FAIL_IF_NULL(sb->head);
1312  FAIL_IF_NOT(sb->sbb_size == 4);
1313  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1314 
1315  StreamingBufferSegment seg5;
1316  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1317  Dump(sb);
1318  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1319  FAIL_IF_NULL(sbb1);
1320  FAIL_IF(sbb1->offset != 0);
1321  FAIL_IF(sbb1->len != 10);
1322  FAIL_IF(SBB_RB_NEXT(sbb1));
1323  FAIL_IF_NULL(sb->head);
1324  FAIL_IF_NOT(sb->sbb_size == 10);
1325  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1326 
1327  StreamingBufferSegment seg6;
1328  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1329  Dump(sb);
1330  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1331  FAIL_IF_NULL(sbb1);
1332  FAIL_IF(sbb1->offset != 0);
1333  FAIL_IF(sbb1->len != 10);
1334  FAIL_IF(SBB_RB_NEXT(sbb1));
1335  FAIL_IF_NULL(sb->head);
1336  FAIL_IF_NOT(sb->sbb_size == 10);
1337  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1338 
1339  StreamingBufferFree(sb);
1340  PASS;
1341 }
1342 
1343 /** \test lots of gaps in block list */
1344 static int StreamingBufferTest10(void)
1345 {
1346  StreamingBufferConfig cfg = { 8, 16, NULL, NULL, NULL, NULL };
1348  FAIL_IF(sb == NULL);
1349 
1350  StreamingBufferSegment seg1;
1351  FAIL_IF(StreamingBufferInsertAt(sb, &seg1, (const uint8_t *)"A", 1, 0) != 0);
1352  Dump(sb);
1353  StreamingBufferSegment seg2;
1354  FAIL_IF(StreamingBufferInsertAt(sb, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1355  Dump(sb);
1356  StreamingBufferSegment seg3;
1357  FAIL_IF(StreamingBufferInsertAt(sb, &seg3, (const uint8_t *)"H", 1, 7) != 0);
1358  Dump(sb);
1359  FAIL_IF_NULL(sb->head);
1360  FAIL_IF_NOT(sb->sbb_size == 3);
1361 
1362  StreamingBufferSegment seg4;
1363  FAIL_IF(StreamingBufferInsertAt(sb, &seg4, (const uint8_t *)"B", 1, 1) != 0);
1364  Dump(sb);
1365  StreamingBufferSegment seg5;
1366  FAIL_IF(StreamingBufferInsertAt(sb, &seg5, (const uint8_t *)"C", 1, 2) != 0);
1367  Dump(sb);
1368  StreamingBufferSegment seg6;
1369  FAIL_IF(StreamingBufferInsertAt(sb, &seg6, (const uint8_t *)"G", 1, 6) != 0);
1370  Dump(sb);
1371  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1372  FAIL_IF_NULL(sb->head);
1373  FAIL_IF_NOT(sb->sbb_size == 6);
1374 
1375  StreamingBufferSegment seg7;
1376  FAIL_IF(StreamingBufferInsertAt(sb, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1377  Dump(sb);
1378  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1379  FAIL_IF_NULL(sbb1);
1380  FAIL_IF(sbb1->offset != 0);
1381  FAIL_IF(sbb1->len != 10);
1382  FAIL_IF(SBB_RB_NEXT(sbb1));
1383  FAIL_IF_NULL(sb->head);
1384  FAIL_IF_NOT(sb->sbb_size == 10);
1385 
1386  StreamingBufferSegment seg8;
1387  FAIL_IF(StreamingBufferInsertAt(sb, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1388  Dump(sb);
1389  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1390  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1391  FAIL_IF_NULL(sbb1);
1392  FAIL_IF(sbb1->offset != 0);
1393  FAIL_IF(sbb1->len != 10);
1394  FAIL_IF(SBB_RB_NEXT(sbb1));
1395  FAIL_IF_NULL(sb->head);
1396  FAIL_IF_NOT(sb->sbb_size == 10);
1397 
1398  StreamingBufferFree(sb);
1399  PASS;
1400 }
1401 
1402 #endif
1403 
1405 {
1406 #ifdef UNITTESTS
1407  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
1408  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
1409  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
1410  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
1411  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
1412  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
1413  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
1414  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
1415 #endif
1416 }
FREE
#define FREE(cfg, ptr, s)
Definition: util-streaming-buffer.c:40
StreamingBuffer_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:96
len
uint8_t len
Definition: app-layer-dnp3.h:2
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBuffer_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:100
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb)
Definition: util-streaming-buffer.c:145
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:103
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
Definition: util-streaming-buffer.c:610
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
StreamingBuffer_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:98
RB_LEFT
#define RB_LEFT(elm, field)
Definition: tree.h:322
StreamingBuffer_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:99
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:832
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:0
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:849
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:816
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
util-unittest.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb)
Definition: util-streaming-buffer.c:132
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
FAIL_IF_NOT_NULL
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
RB_ROOT
#define RB_ROOT(head)
Definition: tree.h:326
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:112
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:714
REALLOC
#define REALLOC(cfg, ptr, orig_s, s)
Definition: util-streaming-buffer.c:38
res
PoolThreadReserved res
Definition: stream-tcp-private.h:0
CALLOC
#define CALLOC(cfg, n, s)
Definition: util-streaming-buffer.c:36
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:104
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:872
RB_GENERATE
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:79
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:102
StreamingBuffer_
Definition: util-streaming-buffer.h:94
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:752
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
suricata-common.h
util-streaming-buffer.h
StreamingBuffer_::cfg
const StreamingBufferConfig * cfg
Definition: util-streaming-buffer.h:95
util-validate.h
StreamingBufferConfig_
Definition: util-streaming-buffer.h:64
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:506
RB_RIGHT
#define RB_RIGHT(elm, field)
Definition: tree.h:323
head
Flow * head
Definition: flow-hash.h:1
StreamingBufferAppendRaw
StreamingBufferSegment * StreamingBufferAppendRaw(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:534
StreamingBufferSlide
void StreamingBufferSlide(StreamingBuffer *sb, uint32_t slide)
Definition: util-streaming-buffer.c:521
DATA_FITS_AT_OFFSET
#define DATA_FITS_AT_OFFSET(sb, len, offset)
Definition: util-streaming-buffer.c:642
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:652
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:47
likely
#define likely(expr)
Definition: util-optimize.h:32
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:571
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:785
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:84
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:82
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:726
DATA_FITS
#define DATA_FITS(sb, len)
Definition: util-streaming-buffer.c:531
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:81
WARN_UNUSED
#define WARN_UNUSED
Definition: suricata-common.h:384
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:1404