suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2023 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-unittest.h"
21 #include "util-print.h"
22 #include "util-validate.h"
23 #include "util-debug.h"
24 
25 static void ListRegions(StreamingBuffer *sb);
26 
27 #define DUMP_REGIONS 0 // set to 1 to dump a visual representation of the regions list and sbb tree.
28 
29 /**
30  * \file
31  *
32  * \author Victor Julien <victor@inliniac.net>
33  *
34  * \brief Streaming Buffer API
35  */
36 
37 /* memory handling wrappers. If config doesn't define it's own set of
38  * functions, use the defaults */
39 #define CALLOC(cfg, n, s) \
40  (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : SCCalloc((n), (s))
41 #define REALLOC(cfg, ptr, orig_s, s) \
42  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : SCRealloc((ptr), (s))
43 #define FREE(cfg, ptr, s) \
44  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
45 
46 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg);
47 
48 RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare);
49 
50 int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
51 {
52  SCLogDebug("a %" PRIu64 " len %u, b %" PRIu64 " len %u", a->offset, a->len, b->offset, b->len);
53 
54  if (a->offset > b->offset)
55  SCReturnInt(1);
56  else if (a->offset < b->offset)
57  SCReturnInt(-1);
58  else {
59  if (a->len == 0 || b->len == 0 || a->len == b->len)
60  SCReturnInt(0);
61  else if (a->len > b->len)
62  SCReturnInt(1);
63  else
64  SCReturnInt(-1);
65  }
66 }
67 
68 /* inclusive compare function that also considers the right edge,
69  * not just the offset. */
70 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
71  const uint64_t lre = lookup->offset + lookup->len;
72  const uint64_t tre = intree->offset + intree->len;
73  if (lre <= intree->offset) // entirely before
74  return -1;
75  else if (lookup->offset < tre && lre <= tre) // (some) overlap
76  return 0;
77  else
78  return 1; // entirely after
79 }
80 
81 StreamingBufferBlock *SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
82 {
83  SCLogDebug("looking up %" PRIu64, elm->offset);
84 
85  struct StreamingBufferBlock *tmp = RB_ROOT(head);
86  struct StreamingBufferBlock *res = NULL;
87  while (tmp) {
88  SCLogDebug("compare with %" PRIu64 "/%u", tmp->offset, tmp->len);
89  const int comp = InclusiveCompare(elm, tmp);
90  SCLogDebug("compare result: %d", comp);
91  if (comp < 0) {
92  res = tmp;
93  tmp = RB_LEFT(tmp, rb);
94  } else if (comp > 0) {
95  tmp = RB_RIGHT(tmp, rb);
96  } else {
97  return tmp;
98  }
99  }
100  return res;
101 }
102 
103 static inline StreamingBufferRegion *InitBufferRegion(
104  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint32_t min_size)
105 {
106  if (sb->regions == USHRT_MAX || (cfg->max_regions != 0 && sb->regions >= cfg->max_regions)) {
107  SCLogDebug("max regions reached");
108  return NULL;
109  }
110 
111  StreamingBufferRegion *aux_r = CALLOC(cfg, 1, sizeof(*aux_r));
112  if (aux_r == NULL)
113  return NULL;
114 
115  aux_r->buf = CALLOC(cfg, 1, MAX(cfg->buf_size, min_size));
116  if (aux_r->buf == NULL) {
117  FREE(cfg, aux_r, sizeof(*aux_r));
118  return NULL;
119  }
120  aux_r->buf_size = MAX(cfg->buf_size, min_size);
121  sb->regions++;
122  sb->max_regions = MAX(sb->regions, sb->max_regions);
123  return aux_r;
124 }
125 
126 static inline int InitBuffer(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
127 {
128  sb->region.buf = CALLOC(cfg, 1, cfg->buf_size);
129  if (sb->region.buf == NULL) {
130  return -1;
131  }
132  sb->region.buf_size = cfg->buf_size;
133  return 0;
134 }
135 
136 StreamingBuffer *StreamingBufferInit(const StreamingBufferConfig *cfg)
137 {
138  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
139  if (sb != NULL) {
140  sb->region.buf_size = cfg->buf_size;
141  sb->regions = sb->max_regions = 1;
142 
143  if (cfg->buf_size > 0) {
144  if (InitBuffer(sb, cfg) == 0) {
145  return sb;
146  }
147  FREE(cfg, sb, sizeof(StreamingBuffer));
148  /* implied buf_size == 0 */
149  } else {
150  return sb;
151  }
152  }
153  return NULL;
154 }
155 
156 void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
157 {
158  if (sb != NULL) {
159  SCLogDebug("sb->region.buf_size %u max %u", sb->region.buf_size, sb->buf_size_max);
160 
161  SBBFree(sb, cfg);
162  ListRegions(sb);
163  if (sb->region.buf != NULL) {
164  FREE(cfg, sb->region.buf, sb->region.buf_size);
165  sb->region.buf = NULL;
166  }
167 
168  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
169  StreamingBufferRegion *next = r->next;
170  FREE(cfg, r->buf, r->buf_size);
171  FREE(cfg, r, sizeof(*r));
172  r = next;
173  }
174  sb->region.next = NULL;
175  sb->regions = sb->max_regions = 1;
176  }
177 }
178 
179 void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
180 {
181  if (sb != NULL) {
182  StreamingBufferClear(sb, cfg);
183  FREE(cfg, sb, sizeof(StreamingBuffer));
184  }
185 }
186 
187 #ifdef DEBUG
188 static void SBBPrintList(StreamingBuffer *sb)
189 {
190  StreamingBufferBlock *sbb = NULL;
191  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
192  SCLogDebug("sbb: offset %" PRIu64 ", len %u", sbb->offset, sbb->len);
193  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
194  if (next) {
195  if ((sbb->offset + sbb->len) != next->offset) {
196  SCLogDebug("gap: offset %" PRIu64 ", len %" PRIu64, (sbb->offset + sbb->len),
197  next->offset - (sbb->offset + sbb->len));
198  }
199  }
200  }
201 }
202 #endif
203 
204 /* setup with gap between 2 blocks
205  *
206  * [block][gap][block]
207  **/
208 static void SBBInit(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
209  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t data_len)
210 {
211  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
212  DEBUG_VALIDATE_BUG_ON(region->buf_offset > region->stream_offset + rel_offset);
213 
214  /* need to set up 2: existing data block and new data block */
215  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
216  if (sbb == NULL) {
217  return;
218  }
219  sbb->offset = sb->region.stream_offset;
220  sbb->len = sb->region.buf_offset;
221 
222  StreamingBufferBlock *sbb2 = CALLOC(cfg, 1, sizeof(*sbb2));
223  if (sbb2 == NULL) {
224  FREE(cfg, sbb, sizeof(*sbb));
225  return;
226  }
227  sbb2->offset = region->stream_offset + rel_offset;
228  sbb2->len = data_len;
229 
230  sb->head = sbb;
231  sb->sbb_size = sbb->len + sbb2->len;
232  SBB_RB_INSERT(&sb->sbb_tree, sbb);
233  SBB_RB_INSERT(&sb->sbb_tree, sbb2);
234 
235  SCLogDebug("sbb1 %" PRIu64 ", len %u, sbb2 %" PRIu64 ", len %u", sbb->offset, sbb->len,
236  sbb2->offset, sbb2->len);
237 #ifdef DEBUG
238  SBBPrintList(sb);
239 #endif
240  BUG_ON(sbb2->offset < sbb->len);
241 }
242 
243 /* setup with leading gap
244  *
245  * [gap][block]
246  **/
247 static void SBBInitLeadingGap(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
248  StreamingBufferRegion *region, uint64_t offset, uint32_t data_len)
249 {
250  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
251 
252  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
253  if (sbb == NULL)
254  return;
255  sbb->offset = offset;
256  sbb->len = data_len;
257 
258  sb->head = sbb;
259  sb->sbb_size = sbb->len;
260  SBB_RB_INSERT(&sb->sbb_tree, sbb);
261 
262  SCLogDebug("sbb %" PRIu64 ", len %u", sbb->offset, sbb->len);
263 #ifdef DEBUG
264  SBBPrintList(sb);
265 #endif
266 }
267 
268 static inline void ConsolidateFwd(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
269  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
270 {
271  uint64_t sa_re = sa->offset + sa->len;
272  StreamingBufferBlock *tr, *s = sa;
273  RB_FOREACH_FROM(tr, SBB, s) {
274  if (sa == tr)
275  continue;
276 
277  const uint64_t tr_re = tr->offset + tr->len;
278  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u re %" PRIu64, tr, tr->offset, tr->len, tr_re);
279 
280  if (sa_re < tr->offset) {
281  SCLogDebug("entirely before: %" PRIu64 " < %" PRIu64, sa_re, tr->offset);
282  break; // entirely before
283  }
284 
285  /* new block (sa) entirely eclipsed by in tree block (tr)
286  sa: [ ]
287  tr: [ ]
288  sa: [ ]
289  tr: [ ]
290  sa: [ ]
291  tr: [ ]
292  */
293  if (sa->offset >= tr->offset && sa_re <= tr_re) {
294  sb->sbb_size -= sa->len;
295  sa->len = tr->len;
296  sa->offset = tr->offset;
297  sa_re = sa->offset + sa->len;
298  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
299  tr->offset, tr->len);
300  SBB_RB_REMOVE(tree, tr);
301  FREE(cfg, tr, sizeof(StreamingBufferBlock));
302  /* new block (sa) entire eclipses in tree block (tr)
303  sa: [ ]
304  tr: [ ]
305  sa: [ ]
306  tr: [ ]
307  sa: [ ]
308  tr: [ ]
309  */
310  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
311  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
312  tr->offset, tr->len);
313  SBB_RB_REMOVE(tree, tr);
314  sb->sbb_size -= tr->len;
315  FREE(cfg, tr, sizeof(StreamingBufferBlock));
316 
317  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
318  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
319  region->buf_size);
320  if (sa->offset == region->stream_offset &&
321  sa_re > (region->stream_offset + region->buf_offset)) {
322  BUG_ON(sa_re < region->stream_offset);
323  region->buf_offset = sa_re - region->stream_offset;
324  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
325  " bo %u sz %u BUF_OFFSET UPDATED",
326  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
327  region->buf_size);
328  }
329  /* new block (sa) extended by in tree block (tr)
330  sa: [ ]
331  tr: [ ]
332  sa: [ ]
333  tr: [ ]
334  */
335  } else if (sa->offset < tr->offset && // starts before
336  sa_re >= tr->offset && sa_re < tr_re) // ends inside
337  {
338  // merge. sb->sbb_size includes both so we need to adjust that too.
339  uint32_t combined_len = sa->len + tr->len;
340  sa->len = tr_re - sa->offset;
341  sa_re = sa->offset + sa->len;
342  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
343  SBB_RB_REMOVE(tree, tr);
344  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
345  FREE(cfg, tr, sizeof(StreamingBufferBlock));
346  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u RESULT", sa, sa->offset, sa->len);
347  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
348  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
349  region->buf_size);
350  if (sa->offset == region->stream_offset &&
351  sa_re > (region->stream_offset + region->buf_offset)) {
352  BUG_ON(sa_re < region->stream_offset);
353  region->buf_offset = sa_re - region->stream_offset;
354  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
355  " bo %u sz %u BUF_OFFSET UPDATED",
356  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
357  region->buf_size);
358  }
359  }
360  }
361 }
362 
363 static inline void ConsolidateBackward(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
364  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
365 {
366  uint64_t sa_re = sa->offset + sa->len;
367  StreamingBufferBlock *tr, *s = sa;
368  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
369  if (sa == tr)
370  continue;
371  const uint64_t tr_re = tr->offset + tr->len;
372  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u", tr, tr->offset, tr->len);
373 
374  if (sa->offset > tr_re)
375  break; // entirely after
376 
377  /* new block (sa) entirely eclipsed by in tree block (tr)
378  sa: [ ]
379  tr: [ ]
380  sa: [ ]
381  tr: [ ]
382  sa: [ ]
383  tr: [ ]
384  */
385  if (sa->offset >= tr->offset && sa_re <= tr_re) {
386  sb->sbb_size -= sa->len; // sa entirely eclipsed so remove double accounting
387  sa->len = tr->len;
388  sa->offset = tr->offset;
389  sa_re = sa->offset + sa->len;
390  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
391  tr->offset, tr->len);
392  if (sb->head == tr)
393  sb->head = sa;
394  SBB_RB_REMOVE(tree, tr);
395  FREE(cfg, tr, sizeof(StreamingBufferBlock));
396  /* new block (sa) entire eclipses in tree block (tr)
397  sa: [ ]
398  tr: [ ]
399  sa: [ ]
400  tr: [ ]
401  sa: [ ]
402  tr: [ ]
403  */
404  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
405  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
406  tr->offset, tr->len);
407  if (sb->head == tr)
408  sb->head = sa;
409  SBB_RB_REMOVE(tree, tr);
410  sb->sbb_size -= tr->len; // tr entirely eclipsed so remove double accounting
411  FREE(cfg, tr, sizeof(StreamingBufferBlock));
412 
413  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
414  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
415  region->buf_size);
416 
417  if (sa->offset == region->stream_offset &&
418  sa_re > (region->stream_offset + region->buf_offset)) {
419  BUG_ON(sa_re < region->stream_offset);
420  region->buf_offset = sa_re - region->stream_offset;
421  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
422  " bo %u sz %u BUF_OFFSET UPDATED",
423  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
424  region->buf_size);
425  }
426 
427  /* new block (sa) extends in tree block (tr)
428  sa: [ ]
429  tr: [ ]
430  sa: [ ]
431  tr: [ ]
432  */
433  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
434  // merge. sb->sbb_size includes both so we need to adjust that too.
435  uint32_t combined_len = sa->len + tr->len;
436  sa->len = sa_re - tr->offset;
437  sa->offset = tr->offset;
438  sa_re = sa->offset + sa->len;
439  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
440  if (sb->head == tr)
441  sb->head = sa;
442  SBB_RB_REMOVE(tree, tr);
443  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
444  FREE(cfg, tr, sizeof(StreamingBufferBlock));
445 
446  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
447  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
448  region->buf_size);
449  if (sa->offset == region->stream_offset &&
450  sa_re > (region->stream_offset + region->buf_offset)) {
451  BUG_ON(sa_re < region->stream_offset);
452  region->buf_offset = sa_re - region->stream_offset;
453  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
454  " bo %u sz %u BUF_OFFSET UPDATED",
455  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
456  region->buf_size);
457  }
458  }
459  }
460 }
461 
462 /** \internal
463  * \param region the region that holds the new data
464  */
465 static int SBBUpdate(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
466  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t len)
467 {
468  struct SBB *tree = &sb->sbb_tree;
469  SCLogDebug("* inserting: %u/%u", rel_offset, len);
470 
471  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
472  if (sbb == NULL)
473  return -1;
474  sbb->offset = region->stream_offset + rel_offset;
475  sbb->len = len;
476 
477  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
478  if (res) {
479  // exact overlap
480  SCLogDebug("* insert failed: exact match in tree with %p %" PRIu64 "/%u", res, res->offset,
481  res->len);
482  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
483  return 0;
484  }
485  sb->sbb_size += len; // may adjust based on consolidation below
486 
487  /* handle backwards and forwards overlaps */
488  if (SBB_RB_PREV(sbb) == NULL) {
489  sb->head = sbb;
490  } else {
491  ConsolidateBackward(sb, cfg, region, tree, sbb);
492  }
493  ConsolidateFwd(sb, cfg, region, tree, sbb);
494 #ifdef DEBUG
495  SBBPrintList(sb);
496 #endif
497  if (sbb->offset == sb->region.stream_offset) {
498  SCLogDebug("insert at head");
499  sb->region.buf_offset = sbb->len;
500  }
501  return 0;
502 }
503 
504 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
505 {
506  StreamingBufferBlock *sbb = NULL, *safe = NULL;
507  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
508  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
509  sb->sbb_size -= sbb->len;
510  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
511  }
512  sb->head = NULL;
513 }
514 
515 static void SBBPrune(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
516 {
517  SCLogDebug("pruning %p to %" PRIu64, sb, sb->region.stream_offset);
518  StreamingBufferBlock *sbb = NULL, *safe = NULL;
519  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
520  /* completely beyond window, we're done */
521  if (sbb->offset >= sb->region.stream_offset) {
522  sb->head = sbb;
523  if (sbb->offset == sb->region.stream_offset) {
524  SCLogDebug("set buf_offset?");
525  if (sbb->offset == sb->region.stream_offset) {
526  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
527  BUG_ON(sbb->len > sb->region.buf_size);
528  sb->region.buf_offset = sbb->len;
529  }
530  }
531  break;
532  }
533 
534  /* partly before, partly beyond. Adjust */
535  if (sbb->offset < sb->region.stream_offset &&
536  sbb->offset + sbb->len > sb->region.stream_offset) {
537  uint32_t shrink_by = sb->region.stream_offset - sbb->offset;
538  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
539  if (sbb->len >= shrink_by) {
540  sbb->len -= shrink_by;
541  sbb->offset += shrink_by;
542  sb->sbb_size -= shrink_by;
543  SCLogDebug("shrunk by %u", shrink_by);
544  DEBUG_VALIDATE_BUG_ON(sbb->offset != sb->region.stream_offset);
545  }
546  sb->head = sbb;
547  if (sbb->offset == sb->region.stream_offset) {
548  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
549  BUG_ON(sbb->len > sb->region.buf_size);
550  sb->region.buf_offset = sbb->len;
551  }
552  break;
553  }
554 
555  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
556  /* either we set it again for the next sbb, or there isn't any */
557  sb->head = NULL;
558  sb->sbb_size -= sbb->len;
559  SCLogDebug("sb %p removed %p %" PRIu64 ", %u", sb, sbb, sbb->offset, sbb->len);
560  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
561  }
562 #ifdef DEBUG
563  SBBPrintList(sb);
564 #endif
565 }
566 
567 static thread_local bool g2s_warn_once = false;
568 
569 static inline int WARN_UNUSED GrowRegionToSize(StreamingBuffer *sb,
570  const StreamingBufferConfig *cfg, StreamingBufferRegion *region, const uint32_t size)
571 {
572  DEBUG_VALIDATE_BUG_ON(region->buf_size > BIT_U32(30));
573  if (size > BIT_U32(30)) { // 1GiB
574  if (!g2s_warn_once) {
575  SCLogWarning("StreamingBuffer::GrowRegionToSize() tried to alloc %u bytes, exceeds "
576  "limit of %lu",
577  size, BIT_U32(30));
578  g2s_warn_once = true;
579  }
580  return -1;
581  }
582 
583  /* try to grow in multiples of cfg->buf_size */
584  uint32_t x = cfg->buf_size ? size % cfg->buf_size : 0;
585  uint32_t base = size - x;
586  uint32_t grow = base + cfg->buf_size;
587 
588  void *ptr = REALLOC(cfg, region->buf, region->buf_size, grow);
589  if (ptr == NULL) {
590  return -1;
591  }
592  /* for safe printing and general caution, lets memset the
593  * new data to 0 */
594  size_t diff = grow - region->buf_size;
595  void *new_mem = ((char *)ptr) + region->buf_size;
596  memset(new_mem, 0, diff);
597 
598  region->buf = ptr;
599  region->buf_size = grow;
600  SCLogDebug("grown buffer to %u", grow);
601 #ifdef DEBUG
602  if (region->buf_size > sb->buf_size_max) {
603  sb->buf_size_max = region->buf_size;
604  }
605 #endif
606  return 0;
607 }
608 
609 static int WARN_UNUSED GrowToSize(
610  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint32_t size)
611 {
612  return GrowRegionToSize(sb, cfg, &sb->region, size);
613 }
614 
615 static inline bool RegionBeforeOffset(const StreamingBufferRegion *r, const uint64_t o)
616 {
617  return (r->stream_offset + r->buf_size <= o);
618 }
619 
620 static inline bool RegionContainsOffset(const StreamingBufferRegion *r, const uint64_t o)
621 {
622  return (o >= r->stream_offset && (r->stream_offset + r->buf_size) >= o);
623 }
624 
625 /** \internal
626  * \brief slide to offset for regions
627  *
628  *
629  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
630  * ^
631  * - main reset to 0
632  *
633  *
634  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
635  * ^
636  * - main shift
637  *
638  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
639  * ^
640  * - main reset
641  * - move aux into main
642  * - free aux
643  * - shift
644  *
645  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
646  * ^
647  * - main reset
648  * - move aux into main
649  * - free aux
650  * - no shift
651  */
652 static inline void StreamingBufferSlideToOffsetWithRegions(
653  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint64_t slide_offset)
654 {
655  ListRegions(sb);
656  BUG_ON(slide_offset == sb->region.stream_offset);
657 
658  SCLogDebug("slide_offset %" PRIu64, slide_offset);
659  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
660  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
661 
662  StreamingBufferRegion *to_shift = NULL;
663  const bool main_is_oow = RegionBeforeOffset(&sb->region, slide_offset);
664  if (main_is_oow) {
665  SCLogDebug("main_is_oow");
666  if (sb->region.buf != NULL) {
667  SCLogDebug("clearing main");
668  FREE(cfg, sb->region.buf, sb->region.buf_size);
669  sb->region.buf = NULL;
670  sb->region.buf_size = 0;
671  sb->region.buf_offset = 0;
672  sb->region.stream_offset = slide_offset;
673  } else {
674  sb->region.stream_offset = slide_offset;
675  }
676 
677  /* remove regions that are out of window & select the region to
678  * become the new main */
679  StreamingBufferRegion *prev = &sb->region;
680  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
681  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64, r, r->stream_offset,
682  r->stream_offset + r->buf_offset);
683  StreamingBufferRegion *next = r->next;
684  if (RegionBeforeOffset(r, slide_offset)) {
685  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> before", r, r->stream_offset,
686  r->stream_offset + r->buf_offset);
687  BUG_ON(r == &sb->region);
688  prev->next = next;
689 
690  FREE(cfg, r->buf, r->buf_size);
691  FREE(cfg, r, sizeof(*r));
692  sb->regions--;
693  BUG_ON(sb->regions == 0);
694  } else if (RegionContainsOffset(r, slide_offset)) {
695  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> within", r, r->stream_offset,
696  r->stream_offset + r->buf_offset);
697  /* remove from list, we will xfer contents to main below */
698  prev->next = next;
699  to_shift = r;
700  break;
701  } else {
702  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> post", r, r->stream_offset,
703  r->stream_offset + r->buf_offset);
704  /* implied beyond slide offset */
705  BUG_ON(r->stream_offset < slide_offset);
706  break;
707  }
708  r = next;
709  }
710  SCLogDebug("to_shift %p", to_shift);
711  } else {
712  to_shift = &sb->region;
713  SCLogDebug("shift start region %p", to_shift);
714  }
715 
716  // this region is main, or will xfer its buffer to main
717  if (to_shift) {
718  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", to_shift->stream_offset,
719  to_shift->buf, to_shift->buf_size, to_shift->buf_offset);
720  if (to_shift != &sb->region) {
721  BUG_ON(sb->region.buf != NULL);
722 
723  sb->region.buf = to_shift->buf;
724  sb->region.stream_offset = to_shift->stream_offset;
725  sb->region.buf_offset = to_shift->buf_offset;
726  sb->region.buf_size = to_shift->buf_size;
727  sb->region.next = to_shift->next;
728 
729  FREE(cfg, to_shift, sizeof(*to_shift));
730  to_shift = &sb->region;
731  sb->regions--;
732  BUG_ON(sb->regions == 0);
733  }
734 
735  // Do the shift. If new region is exactly at the slide offset we can skip this.
736  BUG_ON(to_shift->stream_offset > slide_offset);
737  const uint32_t s = slide_offset - to_shift->stream_offset;
738  if (s > 0) {
739  const uint32_t new_size = to_shift->buf_size - s;
740  SCLogDebug("s %u new_size %u", s, new_size);
741  memmove(to_shift->buf, to_shift->buf + s, new_size);
742  void *ptr = REALLOC(cfg, to_shift->buf, to_shift->buf_size, new_size);
743  BUG_ON(ptr == NULL); // TODO
744  to_shift->buf = ptr;
745  to_shift->buf_size = new_size;
746  if (s < to_shift->buf_offset)
747  to_shift->buf_offset -= s;
748  else
749  to_shift->buf_offset = 0;
750  to_shift->stream_offset = slide_offset;
751  }
752  }
753 
754  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
755  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
756  SCLogDebug("end of slide");
757 }
758 
759 /**
760  * \brief slide to absolute offset
761  * \todo if sliding beyond window, we could perhaps reset?
762  */
763 void StreamingBufferSlideToOffset(
764  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
765 {
766  SCLogDebug("sliding to offset %" PRIu64, offset);
767  ListRegions(sb);
768 #ifdef DEBUG
769  SBBPrintList(sb);
770 #endif
771 
772  if (sb->region.next) {
773  StreamingBufferSlideToOffsetWithRegions(sb, cfg, offset);
774  SBBPrune(sb, cfg);
775  SCLogDebug("post SBBPrune");
776  ListRegions(sb);
777 #ifdef DEBUG
778  SBBPrintList(sb);
779 #endif
780  BUG_ON(sb->region.buf != NULL && sb->region.buf_size == 0);
781  BUG_ON(sb->region.buf_offset > sb->region.buf_size);
782  BUG_ON(offset > sb->region.stream_offset);
783  BUG_ON(sb->head && sb->head->offset == sb->region.stream_offset &&
784  sb->head->len > sb->region.buf_offset);
785  BUG_ON(sb->region.stream_offset < offset);
786  return;
787  }
788 
789  if (offset > sb->region.stream_offset) {
790  const uint32_t slide = offset - sb->region.stream_offset;
791  if (sb->head != NULL) {
792  /* have sbb's, so can't rely on buf_offset for the slide */
793  if (slide < sb->region.buf_size) {
794  const uint32_t size = sb->region.buf_size - slide;
795  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
796  size);
797  memmove(sb->region.buf, sb->region.buf + slide, size);
798  if (sb->region.buf_offset > slide) {
799  sb->region.buf_offset -= slide;
800  } else {
801  sb->region.buf_offset = 0;
802  }
803  } else {
804  sb->region.buf_offset = 0;
805  }
806  sb->region.stream_offset = offset;
807  } else {
808  /* no sbb's, so we can use buf_offset */
809  if (offset <= sb->region.stream_offset + sb->region.buf_offset) {
810  const uint32_t size = sb->region.buf_offset - slide;
811  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
812  size);
813  memmove(sb->region.buf, sb->region.buf + slide, size);
814  sb->region.stream_offset = offset;
815  sb->region.buf_offset = size;
816  } else {
817  /* moved past all data */
818  sb->region.stream_offset = offset;
819  sb->region.buf_offset = 0;
820  }
821  }
822  SBBPrune(sb, cfg);
823  }
824 #ifdef DEBUG
825  SBBPrintList(sb);
826 #endif
827  BUG_ON(sb->region.stream_offset < offset);
828 }
829 
830 #define DATA_FITS(sb, len) ((sb)->region.buf_offset + (len) <= (sb)->region.buf_size)
831 
832 StreamingBufferSegment *StreamingBufferAppendRaw(StreamingBuffer *sb,
833  const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len)
834 {
835  if (sb->region.buf == NULL) {
836  if (InitBuffer(sb, cfg) == -1)
837  return NULL;
838  }
839 
840  if (!DATA_FITS(sb, data_len)) {
841  if (sb->region.buf_size == 0) {
842  if (GrowToSize(sb, cfg, data_len) != 0)
843  return NULL;
844  } else {
845  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != 0)
846  return NULL;
847  }
848  }
849  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
850 
851  StreamingBufferSegment *seg = CALLOC(cfg, 1, sizeof(StreamingBufferSegment));
852  if (seg != NULL) {
853  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
854  seg->stream_offset = sb->region.stream_offset + sb->region.buf_offset;
855  seg->segment_len = data_len;
856  uint32_t rel_offset = sb->region.buf_offset;
857  sb->region.buf_offset += data_len;
858 
859  if (!RB_EMPTY(&sb->sbb_tree)) {
860  SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
861  }
862  return seg;
863  }
864  return NULL;
865 }
866 
867 int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
868  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
869 {
870  BUG_ON(seg == NULL);
871 
872  if (sb->region.buf == NULL) {
873  if (InitBuffer(sb, cfg) == -1)
874  return -1;
875  }
876 
877  if (!DATA_FITS(sb, data_len)) {
878  if (sb->region.buf_size == 0) {
879  if (GrowToSize(sb, cfg, data_len) != 0)
880  return -1;
881  } else {
882  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != 0)
883  return -1;
884  }
885  }
886  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
887 
888  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
889  seg->stream_offset = sb->region.stream_offset + sb->region.buf_offset;
890  seg->segment_len = data_len;
891  uint32_t rel_offset = sb->region.buf_offset;
892  sb->region.buf_offset += data_len;
893 
894  if (!RB_EMPTY(&sb->sbb_tree)) {
895  SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
896  }
897  return 0;
898 }
899 
900 /**
901  * \brief add data w/o tracking a segment
902  */
903 int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
904  const uint8_t *data, uint32_t data_len)
905 {
906  if (sb->region.buf == NULL) {
907  if (InitBuffer(sb, cfg) == -1)
908  return -1;
909  }
910 
911  if (!DATA_FITS(sb, data_len)) {
912  if (sb->region.buf_size == 0) {
913  if (GrowToSize(sb, cfg, data_len) != 0)
914  return -1;
915  } else {
916  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != 0)
917  return -1;
918  }
919  }
920  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
921 
922  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
923  uint32_t rel_offset = sb->region.buf_offset;
924  sb->region.buf_offset += data_len;
925 
926  if (!RB_EMPTY(&sb->sbb_tree)) {
927  SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
928  }
929  return 0;
930 }
931 
932 #define DATA_FITS_AT_OFFSET(region, len, offset) ((offset) + (len) <= (region)->buf_size)
933 
934 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
935 static void Validate(const StreamingBuffer *sb)
936 {
937  bool bail = false;
938  uint32_t cnt = 0;
939  for (const StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
940  cnt++;
941  if (r->next) {
942  bail |= ((r->stream_offset + r->buf_size) > r->next->stream_offset);
943  }
944 
945  bail |= (r->buf != NULL && r->buf_size == 0);
946  bail |= (r->buf_offset > r->buf_size);
947  }
948  // leading gap, so buf_offset should be 0?
949  if (sb->head && sb->head->offset > sb->region.stream_offset) {
950  SCLogDebug("leading gap of size %" PRIu64, sb->head->offset - sb->region.stream_offset);
951  BUG_ON(sb->region.buf_offset != 0);
952  }
953 
954  if (sb->head && sb->head->offset == sb->region.stream_offset) {
955  BUG_ON(sb->head->len > sb->region.buf_offset);
956  BUG_ON(sb->head->len < sb->region.buf_offset);
957  }
958  BUG_ON(sb->regions != cnt);
959  BUG_ON(bail);
960 }
961 #endif
962 
963 static void ListRegions(StreamingBuffer *sb)
964 {
965  if (sb->region.buf == NULL && sb->region.buf_offset == 0 && sb->region.next == NULL)
966  return;
967 #ifdef DEBUG
968 #if DUMP_REGIONS == 1
969  uint32_t cnt = 0;
970  for (StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
971  cnt++;
972  char gap[64] = "";
973  if (r->next) {
974  snprintf(gap, sizeof(gap), "[ gap:%" PRIu64 " ]",
975  r->next->stream_offset - (r->stream_offset + r->buf_size));
976  }
977 
978  printf("[ %s offset:%" PRIu64 " size:%u offset:%u ]%s", r == &sb->region ? "main" : "aux",
979  r->stream_offset, r->buf_size, r->buf_offset, gap);
980  }
981  printf("(max %u, cnt %u, sb->regions %u)\n", sb->max_regions, cnt, sb->regions);
982  bool at_least_one = false;
983  uint64_t last_re = sb->region.stream_offset;
984  StreamingBufferBlock *sbb = NULL;
985  RB_FOREACH(sbb, SBB, &sb->sbb_tree)
986  {
987  if (last_re != sbb->offset) {
988  printf("[ gap:%" PRIu64 " ]", sbb->offset - last_re);
989  }
990  printf("[ sbb offset:%" PRIu64 " len:%u ]", sbb->offset, sbb->len);
991  at_least_one = true;
992  last_re = sbb->offset + sbb->len;
993  }
994  if (at_least_one)
995  printf("\n");
996 #endif
997 #endif
998 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
999  Validate(sb);
1000 #endif
1001 }
1002 
1003 /** \interal
1004  * \brief does data region intersect with list region 'r'
1005  * Takes the max gap into account.
1006  */
1007 static inline bool RegionsIntersect(const StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1008  const StreamingBufferRegion *r, const uint64_t offset, const uint32_t len)
1009 {
1010  const uint64_t re = offset + len;
1011 
1012  /* create the data range for the region, adding the max gap */
1013  const uint64_t reg_o =
1014  r->stream_offset > cfg->region_gap ? (r->stream_offset - cfg->region_gap) : 0;
1015  const uint64_t reg_re = r->stream_offset + r->buf_size + cfg->region_gap;
1016  SCLogDebug("r %p: %" PRIu64 "/%" PRIu64 " - adjusted %" PRIu64 "/%" PRIu64, r, r->stream_offset,
1017  r->stream_offset + r->buf_size, reg_o, reg_re);
1018  /* check if data range intersects with region range */
1019  if (offset >= reg_o && offset <= reg_re) {
1020  SCLogDebug("r %p is in-scope", r);
1021  return true;
1022  }
1023  if (re >= reg_o && re <= reg_re) {
1024  SCLogDebug("r %p is in-scope: %" PRIu64 " >= %" PRIu64 " && %" PRIu64 " <= %" PRIu64, r, re,
1025  reg_o, re, reg_re);
1026  return true;
1027  }
1028  SCLogDebug("r %p is out of scope: %" PRIu64 "/%" PRIu64, r, offset, re);
1029  return false;
1030 }
1031 
1032 /** \internal
1033  * \brief find the first region for merging.
1034  */
1035 static StreamingBufferRegion *FindFirstRegionForOffset(const StreamingBuffer *sb,
1036  const StreamingBufferConfig *cfg, StreamingBufferRegion *r, const uint64_t offset,
1037  const uint32_t len)
1038 {
1039  const uint64_t data_re = offset + len;
1040  SCLogDebug("looking for first region matching %" PRIu64 "/%" PRIu64, offset, data_re);
1041 
1042  for (; r != NULL; r = r->next) {
1043  if (RegionsIntersect(sb, cfg, r, offset, data_re) == true)
1044  return r;
1045  }
1046  return NULL;
1047 }
1048 
1049 static StreamingBufferRegion *FindLargestRegionForOffset(const StreamingBuffer *sb,
1050  const StreamingBufferConfig *cfg, StreamingBufferRegion *r, const uint64_t offset,
1051  const uint32_t len)
1052 {
1053  const uint64_t data_re = offset + len;
1054  SCLogDebug("starting at %p/%" PRIu64 ", offset %" PRIu64 ", data_re %" PRIu64, r,
1055  r->stream_offset, offset, data_re);
1056  StreamingBufferRegion *candidate = r;
1057  for (; r != NULL; r = r->next) {
1058 #ifdef DEBUG
1059  const uint64_t reg_re = r->stream_offset + r->buf_size;
1060  SCLogDebug("checking: %p/%" PRIu64 "/%" PRIu64 ", offset %" PRIu64 "/%" PRIu64, r,
1061  r->stream_offset, reg_re, offset, data_re);
1062 #endif
1063  if (!RegionsIntersect(sb, cfg, r, offset, data_re))
1064  return candidate;
1065 
1066  if (r->buf_size > candidate->buf_size) {
1067  SCLogDebug("candidate %p as size %u > %u", candidate, r->buf_size, candidate->buf_size);
1068  candidate = r;
1069  }
1070  }
1071  return candidate;
1072 }
1073 
1074 static StreamingBufferRegion *FindRightEdge(const StreamingBuffer *sb,
1075  const StreamingBufferConfig *cfg, StreamingBufferRegion *r, const uint64_t offset,
1076  const uint32_t len)
1077 {
1078  const uint64_t data_re = offset + len;
1079  StreamingBufferRegion *candidate = r;
1080  for (; r != NULL; r = r->next) {
1081  if (!RegionsIntersect(sb, cfg, r, offset, data_re)) {
1082  SCLogDebug("r %p is out of scope: %" PRIu64 "/%u", r, offset, len);
1083  return candidate;
1084  }
1085  candidate = r;
1086  }
1087  return candidate;
1088 }
1089 
1090 /** \internal
1091  * \brief process insert by consolidating the affected regions into one
1092  */
1093 static StreamingBufferRegion *BufferInsertAtRegionConsolidate(StreamingBuffer *sb,
1094  const StreamingBufferConfig *cfg, StreamingBufferRegion *dst,
1095  StreamingBufferRegion *src_start, StreamingBufferRegion *src_end,
1096  const uint64_t data_offset, const uint32_t data_len)
1097 {
1098  const uint64_t data_re = data_offset + data_len;
1099  SCLogDebug("sb %p dst %p src_start %p src_end %p data_offset %" PRIu64
1100  "/data_len %u/data_re %" PRIu64,
1101  sb, dst, src_start, src_end, data_offset, data_len, data_re);
1102 
1103  // 1. determine size for dst.
1104  const uint64_t dst_offset = MIN(src_start->stream_offset, data_offset);
1105  DEBUG_VALIDATE_BUG_ON(dst_offset < sb->region.stream_offset);
1106  const uint64_t dst_re = MAX((src_end->stream_offset + src_end->buf_size), data_re);
1107  const uint32_t dst_size = dst_re - dst_offset;
1108  SCLogDebug("dst_offset %" PRIu64 ", dst_re %" PRIu64 ", dst_size %u", dst_offset, dst_re,
1109  dst_size);
1110 
1111  // 2. resize dst
1112  const uint32_t old_size = dst->buf_size;
1113  const uint32_t dst_copy_offset = dst->stream_offset - dst_offset;
1114 #ifdef DEBUG
1115  const uint32_t old_offset = dst->buf_offset;
1116  SCLogDebug("old_size %u, old_offset %u, dst_copy_offset %u", old_size, old_offset,
1117  dst_copy_offset);
1118 #endif
1119  if (GrowRegionToSize(sb, cfg, dst, dst_size) != 0)
1120  return NULL;
1121  SCLogDebug("resized to %u", dst_size);
1122  if (dst_copy_offset != 0)
1123  memmove(dst->buf + dst_copy_offset, dst->buf, old_size);
1124  dst->stream_offset = dst_offset;
1125 
1126  uint32_t new_offset = src_start->buf_offset;
1127  if (data_offset == src_start->stream_offset + src_start->buf_offset) {
1128  new_offset += data_len;
1129  }
1130 
1131  bool start_is_main = false;
1132  StreamingBufferRegion *prev = NULL;
1133  if (src_start == &sb->region) {
1134  DEBUG_VALIDATE_BUG_ON(src_start->stream_offset != dst_offset);
1135 
1136  start_is_main = true;
1137  SCLogDebug("src_start is main region");
1138  if (src_start != dst)
1139  memcpy(dst->buf, src_start->buf, src_start->buf_offset);
1140  if (src_start == src_end) {
1141  SCLogDebug("src_start == src_end == main, we're done");
1142  BUG_ON(src_start != dst);
1143  return src_start;
1144  }
1145  prev = src_start;
1146  src_start = src_start->next; // skip in the loop below
1147  }
1148 
1149  // 3. copy all regions from src_start to dst_start into the new region
1150  for (StreamingBufferRegion *r = src_start; r != NULL;) {
1151  SCLogDebug("r %p %" PRIu64 ", offset %u, len %u, %s, last %s", r, r->stream_offset,
1152  r->buf_offset, r->buf_size, r == &sb->region ? "main" : "aux",
1153  BOOL2STR(r == src_end));
1154  // skip dst
1155  if (r == dst) {
1156  SCLogDebug("skipping r %p as it is 'dst'", r);
1157  if (r == src_end)
1158  break;
1159  prev = r;
1160  r = r->next;
1161  continue;
1162  }
1163  const uint32_t target_offset = r->stream_offset - dst_offset;
1164  SCLogDebug("r %p: target_offset %u", r, target_offset);
1165  memcpy(dst->buf + target_offset, r->buf, r->buf_size);
1166 
1167  StreamingBufferRegion *next = r->next;
1168  FREE(cfg, r->buf, r->buf_size);
1169  FREE(cfg, r, sizeof(*r));
1170  sb->regions--;
1171  BUG_ON(sb->regions == 0);
1172  if (prev != NULL) {
1173  SCLogDebug("setting prev %p next to %p (was %p)", prev, next, prev->next);
1174  prev->next = next;
1175  } else {
1176  SCLogDebug("no prev yet");
1177  }
1178 
1179  if (r == src_end)
1180  break;
1181  r = next;
1182  }
1183 
1184  /* special handling of main region being the start, but not the
1185  * region we expand. In this case we'll have main and dst. We will
1186  * move the buffer from dst into main and free dst. */
1187  if (start_is_main && dst != &sb->region) {
1188  BUG_ON(sb->region.next != dst);
1189  SCLogDebug("start_is_main && dst != main region");
1190  FREE(cfg, sb->region.buf, sb->region.buf_size);
1191  sb->region.buf = dst->buf;
1192  sb->region.buf_size = dst->buf_size;
1193  sb->region.buf_offset = new_offset;
1194  SCLogDebug("sb->region.buf_offset set to %u", sb->region.buf_offset);
1195  sb->region.next = dst->next;
1196  FREE(cfg, dst, sizeof(*dst));
1197  dst = &sb->region;
1198  sb->regions--;
1199  BUG_ON(sb->regions == 0);
1200  } else {
1201  SCLogDebug("dst: %p next %p", dst, dst->next);
1202  }
1203 
1204  SCLogDebug("returning dst %p stream_offset %" PRIu64 " buf_offset %u buf_size %u", dst,
1205  dst->stream_offset, dst->buf_offset, dst->buf_size);
1206  return dst;
1207 }
1208 
1209 static StreamingBufferRegion *BufferInsertAtRegionDo(StreamingBuffer *sb,
1210  const StreamingBufferConfig *cfg, const uint64_t offset, const uint32_t len)
1211 {
1212  SCLogDebug("offset %" PRIu64 ", len %u", offset, len);
1213  StreamingBufferRegion *start = FindFirstRegionForOffset(sb, cfg, &sb->region, offset, len);
1214  if (start) {
1215  SCLogDebug("start region %p/%" PRIu64 "/%u", start, start->stream_offset, start->buf_size);
1216  StreamingBufferRegion *big = FindLargestRegionForOffset(sb, cfg, start, offset, len);
1217  DEBUG_VALIDATE_BUG_ON(big == NULL);
1218  if (big == NULL)
1219  return NULL;
1220  SCLogDebug("big region %p/%" PRIu64 "/%u", big, big->stream_offset, big->buf_size);
1221  StreamingBufferRegion *end = FindRightEdge(sb, cfg, big, offset, len);
1222  DEBUG_VALIDATE_BUG_ON(end == NULL);
1223  if (end == NULL)
1224  return NULL;
1225  SCLogDebug("end region %p/%" PRIu64 "/%u", end, end->stream_offset, end->buf_size);
1226  StreamingBufferRegion *ret =
1227  BufferInsertAtRegionConsolidate(sb, cfg, big, start, end, offset, len);
1228  return ret;
1229  } else {
1230  /* if there was no region we can use we add a new region and insert it */
1231  StreamingBufferRegion *append = &sb->region;
1232  for (StreamingBufferRegion *r = append; r != NULL; r = r->next) {
1233  if (r->stream_offset > offset) {
1234  break;
1235  } else {
1236  append = r;
1237  }
1238  }
1239 
1240  SCLogDebug("no matching region found, append to %p (%s)", append,
1241  append == &sb->region ? "main" : "aux");
1242  StreamingBufferRegion *add = InitBufferRegion(sb, cfg, len);
1243  if (add == NULL)
1244  return NULL;
1245  add->stream_offset = offset;
1246  add->next = append->next;
1247  append->next = add;
1248  SCLogDebug("new region %p offset %" PRIu64, add, add->stream_offset);
1249  return add;
1250  }
1251 }
1252 
1253 /** \internal
1254  * \brief return the region to put the new data in
1255  *
1256  * Will find an existing region, expand it if needed. If no existing region exists or is
1257  * a good fit, it will try to set up a new region. If the region then overlaps or gets
1258  * too close to the next, merge them.
1259  */
1260 static StreamingBufferRegion *BufferInsertAtRegion(StreamingBuffer *sb,
1261  const StreamingBufferConfig *cfg, const uint8_t *data, const uint32_t data_len,
1262  const uint64_t data_offset)
1263 {
1264  SCLogDebug("data_offset %" PRIu64 ", data_len %u, re %" PRIu64, data_offset, data_len,
1265  data_offset + data_len);
1266  ListRegions(sb);
1267 
1268  if (RegionsIntersect(sb, cfg, &sb->region, data_offset, data_len)) {
1269  SCLogDebug("data_offset %" PRIu64 ", data_len %u intersects with main region (next %p)",
1270  data_offset, data_len, sb->region.next);
1271  if (sb->region.next == NULL ||
1272  !RegionsIntersect(sb, cfg, sb->region.next, data_offset, data_len)) {
1273  SCLogDebug(
1274  "data_offset %" PRIu64
1275  ", data_len %u intersects with main region, no next or way before next region",
1276  data_offset, data_len);
1277  if (sb->region.buf == NULL)
1278  if (InitBuffer(sb, cfg) == -1) // TODO init with size
1279  return NULL;
1280  return &sb->region;
1281  }
1282  } else if (sb->region.next == NULL) {
1283  StreamingBufferRegion *aux_r = sb->region.next = InitBufferRegion(sb, cfg, data_len);
1284  if (aux_r == NULL)
1285  return NULL;
1286  aux_r->stream_offset = data_offset;
1287  DEBUG_VALIDATE_BUG_ON(data_len > aux_r->buf_size);
1288  SCLogDebug("created new region %p with offset %" PRIu64 ", size %u", aux_r,
1289  aux_r->stream_offset, aux_r->buf_size);
1290  return aux_r;
1291  }
1292  StreamingBufferRegion *blob = BufferInsertAtRegionDo(sb, cfg, data_offset, data_len);
1293  SCLogDebug("blob %p (%s)", blob, blob == &sb->region ? "main" : "aux");
1294  return blob;
1295 }
1296 
1297 /**
1298  * \param offset offset relative to StreamingBuffer::stream_offset
1299  *
1300  * \return 0 in case of success
1301  * \return -1 on memory allocation errors
1302  * \return negative value on other errors
1303  */
1304 int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1305  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
1306 {
1307  BUG_ON(seg == NULL);
1308  DEBUG_VALIDATE_BUG_ON(offset < sb->region.stream_offset);
1309  if (offset < sb->region.stream_offset)
1310  return -2;
1311 
1312  StreamingBufferRegion *region = BufferInsertAtRegion(sb, cfg, data, data_len, offset);
1313  if (region == NULL) {
1314  return -1;
1315  }
1316 
1317  const bool region_is_main = region == &sb->region;
1318 
1319  SCLogDebug("inserting %" PRIu64 "/%u using %s region %p", offset, data_len,
1320  region == &sb->region ? "main" : "aux", region);
1321 
1322  uint32_t rel_offset = offset - region->stream_offset;
1323  if (!DATA_FITS_AT_OFFSET(region, data_len, rel_offset)) {
1324  if (GrowToSize(sb, cfg, (rel_offset + data_len)) != 0)
1325  return -1;
1326  }
1327  DEBUG_VALIDATE_BUG_ON(!DATA_FITS_AT_OFFSET(region, data_len, rel_offset));
1328 
1329  SCLogDebug("offset %" PRIu64 " data_len %u, rel_offset %u into region offset %" PRIu64
1330  ", buf_offset %u, buf_size %u",
1331  offset, data_len, rel_offset, region->stream_offset, region->buf_offset,
1332  region->buf_size);
1333  memcpy(region->buf + rel_offset, data, data_len);
1334  seg->stream_offset = offset;
1335  seg->segment_len = data_len;
1336 
1337  SCLogDebug("rel_offset %u region->stream_offset %" PRIu64 ", buf_offset %u", rel_offset,
1338  region->stream_offset, sb->region.buf_offset);
1339 
1340  if (RB_EMPTY(&sb->sbb_tree)) {
1341  SCLogDebug("empty sbb list");
1342 
1343  if (region_is_main) {
1344  if (sb->region.stream_offset == offset) {
1345  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
1346  /* empty list, data is exactly what is expected (append),
1347  * so do nothing.
1348  * Update buf_offset if needed, but in case of overlaps it might be beyond us. */
1349  sb->region.buf_offset = MAX(sb->region.buf_offset, rel_offset + data_len);
1350  } else if ((rel_offset + data_len) <= sb->region.buf_offset) {
1351  SCLogDebug("empty sbb list: block is within existing main data region");
1352  } else {
1353  if (sb->region.buf_offset && rel_offset == sb->region.buf_offset) {
1354  SCLogDebug("exactly at expected offset");
1355  // nothing to do
1356  sb->region.buf_offset = rel_offset + data_len;
1357 
1358  } else if (rel_offset < sb->region.buf_offset) {
1359  // nothing to do
1360 
1361  SCLogDebug("before expected offset: %u < sb->region.buf_offset %u", rel_offset,
1362  sb->region.buf_offset);
1363  if (rel_offset + data_len > sb->region.buf_offset) {
1364  SCLogDebug("before expected offset, ends after: %u < sb->region.buf_offset "
1365  "%u, %u > %u",
1366  rel_offset, sb->region.buf_offset, rel_offset + data_len,
1367  sb->region.buf_offset);
1368  sb->region.buf_offset = rel_offset + data_len;
1369  }
1370 
1371  } else if (sb->region.buf_offset) {
1372  SCLogDebug("beyond expected offset: SBBInit");
1373  /* existing data, but there is a gap between us */
1374  SBBInit(sb, cfg, region, rel_offset, data_len);
1375  } else {
1376  /* gap before data in empty list */
1377  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1378  SBBInitLeadingGap(sb, cfg, region, offset, data_len);
1379  }
1380  }
1381  } else {
1382  if (sb->region.buf_offset) {
1383  /* existing data, but there is a gap between us */
1384  SCLogDebug("empty sbb list, no data in main: use SBBInit");
1385  SBBInit(sb, cfg, region, rel_offset, data_len);
1386  } else {
1387  /* gap before data in empty list */
1388  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1389  SBBInitLeadingGap(sb, cfg, region, offset, data_len);
1390  }
1391  if (rel_offset == region->buf_offset) {
1392  SCLogDebug("pre region->buf_offset %u", region->buf_offset);
1393  region->buf_offset = rel_offset + data_len;
1394  SCLogDebug("post region->buf_offset %u", region->buf_offset);
1395  }
1396  }
1397  } else {
1398  SCLogDebug("updating sbb tree");
1399  /* already have blocks, so append new block based on new data */
1400  SBBUpdate(sb, cfg, region, rel_offset, data_len);
1401  }
1402  BUG_ON(!region_is_main && sb->head == NULL);
1403 
1404  ListRegions(sb);
1405  if (RB_EMPTY(&sb->sbb_tree)) {
1406  BUG_ON(offset + data_len > sb->region.stream_offset + sb->region.buf_offset);
1407  }
1408 
1409  return 0;
1410 }
1411 
1412 int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb,
1413  const StreamingBufferSegment *seg)
1414 {
1415  if (seg->stream_offset < sb->region.stream_offset) {
1416  if (seg->stream_offset + seg->segment_len <= sb->region.stream_offset) {
1417  return 1;
1418  }
1419  }
1420  return 0;
1421 }
1422 
1423 static inline const StreamingBufferRegion *GetRegionForOffset(
1424  const StreamingBuffer *sb, const uint64_t offset)
1425 {
1426  if (sb == NULL)
1427  return NULL;
1428  if (sb->region.next == NULL) {
1429  return &sb->region;
1430  }
1431  if (offset >= sb->region.stream_offset &&
1432  offset < (sb->region.stream_offset + sb->region.buf_size)) {
1433  return &sb->region;
1434  }
1435  for (const StreamingBufferRegion *r = sb->region.next; r != NULL; r = r->next) {
1436  if (offset >= r->stream_offset && offset < (r->stream_offset + r->buf_size)) {
1437  return r;
1438  }
1439  }
1440  return NULL;
1441 }
1442 
1443 /** \brief get the data for one SBB */
1444 void StreamingBufferSBBGetData(const StreamingBuffer *sb,
1445  const StreamingBufferBlock *sbb,
1446  const uint8_t **data, uint32_t *data_len)
1447 {
1448  ListRegions((StreamingBuffer *)sb);
1449  const StreamingBufferRegion *region = GetRegionForOffset(sb, sbb->offset);
1450  SCLogDebug("first find our region (offset %" PRIu64 ") -> %p", sbb->offset, region);
1451  if (region) {
1452  SCLogDebug("region %p found %" PRIu64 "/%u/%u", region, region->stream_offset,
1453  region->buf_size, region->buf_offset);
1454  if (sbb->offset >= region->stream_offset) {
1455  SCLogDebug("1");
1456  uint64_t offset = sbb->offset - region->stream_offset;
1457  *data = region->buf + offset;
1458  BUG_ON(offset + sbb->len > region->buf_size);
1459  *data_len = sbb->len;
1460  return;
1461  } else {
1462  SCLogDebug("2");
1463  uint64_t offset = region->stream_offset - sbb->offset;
1464  if (offset < sbb->len) {
1465  *data = region->buf;
1466  *data_len = sbb->len - offset;
1467  return;
1468  }
1469  SCLogDebug("3");
1470  }
1471  }
1472  *data = NULL;
1473  *data_len = 0;
1474  return;
1475 }
1476 
1477 /** \brief get the data for one SBB */
1478 void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb,
1479  const StreamingBufferBlock *sbb,
1480  const uint8_t **data, uint32_t *data_len,
1481  uint64_t offset)
1482 {
1483  /* validate that we are looking for a offset within the sbb */
1484  DEBUG_VALIDATE_BUG_ON(!(offset >= sbb->offset && offset < (sbb->offset + sbb->len)));
1485  if (!(offset >= sbb->offset && offset < (sbb->offset + sbb->len))) {
1486  *data = NULL;
1487  *data_len = 0;
1488  return;
1489  }
1490 
1491  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1492  if (region) {
1493  uint32_t sbblen = sbb->len - (offset - sbb->offset);
1494 
1495  if (offset >= region->stream_offset) {
1496  uint64_t data_offset = offset - region->stream_offset;
1497  *data = region->buf + data_offset;
1498  if (data_offset + sbblen > region->buf_size)
1499  *data_len = region->buf_size - data_offset;
1500  else
1501  *data_len = sbblen;
1502  BUG_ON(*data_len > sbblen);
1503  return;
1504  } else {
1505  uint64_t data_offset = region->stream_offset - sbb->offset;
1506  if (data_offset < sbblen) {
1507  *data = region->buf;
1508  *data_len = sbblen - data_offset;
1509  BUG_ON(*data_len > sbblen);
1510  return;
1511  }
1512  }
1513  }
1514 
1515  *data = NULL;
1516  *data_len = 0;
1517  return;
1518 }
1519 
1520 void StreamingBufferSegmentGetData(const StreamingBuffer *sb,
1521  const StreamingBufferSegment *seg,
1522  const uint8_t **data, uint32_t *data_len)
1523 {
1524  const StreamingBufferRegion *region = GetRegionForOffset(sb, seg->stream_offset);
1525  if (region) {
1526  if (seg->stream_offset >= region->stream_offset) {
1527  uint64_t offset = seg->stream_offset - region->stream_offset;
1528  *data = region->buf + offset;
1529  if (offset + seg->segment_len > region->buf_size)
1530  *data_len = region->buf_size - offset;
1531  else
1532  *data_len = seg->segment_len;
1533  SCLogDebug("*data_len %u", *data_len);
1534  return;
1535  } else {
1536  uint64_t offset = region->stream_offset - seg->stream_offset;
1537  if (offset < seg->segment_len) {
1538  *data = region->buf;
1539  *data_len = seg->segment_len - offset;
1540  SCLogDebug("*data_len %u", *data_len);
1541  return;
1542  }
1543  }
1544  }
1545  *data = NULL;
1546  *data_len = 0;
1547  return;
1548 }
1549 
1550 /**
1551  * \retval 1 data is the same
1552  * \retval 0 data is different
1553  */
1554 int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb,
1555  const StreamingBufferSegment *seg,
1556  const uint8_t *rawdata, uint32_t rawdata_len)
1557 {
1558  const uint8_t *segdata = NULL;
1559  uint32_t segdata_len = 0;
1560  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
1561  if (segdata && segdata_len &&
1562  segdata_len == rawdata_len &&
1563  memcmp(segdata, rawdata, segdata_len) == 0)
1564  {
1565  return 1;
1566  }
1567  return 0;
1568 }
1569 
1570 int StreamingBufferGetData(const StreamingBuffer *sb,
1571  const uint8_t **data, uint32_t *data_len,
1572  uint64_t *stream_offset)
1573 {
1574  if (sb != NULL && sb->region.buf != NULL) {
1575  *data = sb->region.buf;
1576  *data_len = sb->region.buf_offset;
1577  *stream_offset = sb->region.stream_offset;
1578  return 1;
1579  } else {
1580  *data = NULL;
1581  *data_len = 0;
1582  *stream_offset = 0;
1583  return 0;
1584  }
1585 }
1586 
1587 int StreamingBufferGetDataAtOffset (const StreamingBuffer *sb,
1588  const uint8_t **data, uint32_t *data_len,
1589  uint64_t offset)
1590 {
1591  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1592  if (region != NULL && region->buf != NULL && offset >= region->stream_offset &&
1593  offset < (region->stream_offset + region->buf_offset)) {
1594  uint32_t skip = offset - region->stream_offset;
1595  *data = region->buf + skip;
1596  *data_len = region->buf_offset - skip;
1597  return 1;
1598  } else {
1599  *data = NULL;
1600  *data_len = 0;
1601  return 0;
1602  }
1603 }
1604 
1605 /**
1606  * \retval 1 data is the same
1607  * \retval 0 data is different
1608  */
1609 int StreamingBufferCompareRawData(const StreamingBuffer *sb,
1610  const uint8_t *rawdata, uint32_t rawdata_len)
1611 {
1612  const uint8_t *sbdata = NULL;
1613  uint32_t sbdata_len = 0;
1614  uint64_t offset = 0;
1615  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
1616  if (offset == 0 &&
1617  sbdata && sbdata_len &&
1618  sbdata_len == rawdata_len &&
1619  memcmp(sbdata, rawdata, sbdata_len) == 0)
1620  {
1621  return 1;
1622  }
1623  SCLogDebug("sbdata_len %u, offset %" PRIu64, sbdata_len, offset);
1624  printf("got:\n");
1625  PrintRawDataFp(stdout, sbdata,sbdata_len);
1626  printf("wanted:\n");
1627  PrintRawDataFp(stdout, rawdata,rawdata_len);
1628  return 0;
1629 }
1630 
1631 #ifdef UNITTESTS
1632 static void Dump(StreamingBuffer *sb)
1633 {
1634  PrintRawDataFp(stdout, sb->region.buf, sb->region.buf_offset);
1635 }
1636 
1637 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
1638 {
1639  const uint8_t *data = NULL;
1640  uint32_t data_len = 0;
1641  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
1642  if (data && data_len) {
1643  PrintRawDataFp(stdout, data, data_len);
1644  }
1645 }
1646 
1647 static int StreamingBufferTest02(void)
1648 {
1649  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1650  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1651  FAIL_IF(sb == NULL);
1652 
1653  StreamingBufferSegment seg1;
1654  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1655  StreamingBufferSegment seg2;
1656  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8) != 0);
1657  FAIL_IF(sb->region.stream_offset != 0);
1658  FAIL_IF(sb->region.buf_offset != 16);
1659  FAIL_IF(seg1.stream_offset != 0);
1660  FAIL_IF(seg2.stream_offset != 8);
1661  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1662  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1663  Dump(sb);
1664  DumpSegment(sb, &seg1);
1665  DumpSegment(sb, &seg2);
1666  FAIL_IF_NOT_NULL(sb->head);
1667  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1668 
1669  StreamingBufferSlideToOffset(sb, &cfg, 6);
1670  FAIL_IF_NOT_NULL(sb->head);
1671  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1672 
1673  StreamingBufferSegment seg3;
1674  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
1675  FAIL_IF(sb->region.stream_offset != 6);
1676  FAIL_IF(sb->region.buf_offset != 16);
1677  FAIL_IF(seg3.stream_offset != 16);
1678  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1679  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1680  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1681  Dump(sb);
1682  DumpSegment(sb, &seg1);
1683  DumpSegment(sb, &seg2);
1684  DumpSegment(sb, &seg3);
1685  FAIL_IF_NOT_NULL(sb->head);
1686  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1687 
1688  StreamingBufferSlideToOffset(sb, &cfg, 12);
1689  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1690  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1691  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1692  Dump(sb);
1693  DumpSegment(sb, &seg1);
1694  DumpSegment(sb, &seg2);
1695  DumpSegment(sb, &seg3);
1696  FAIL_IF_NOT_NULL(sb->head);
1697  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1698 
1699  StreamingBufferFree(sb, &cfg);
1700  PASS;
1701 }
1702 
1703 static int StreamingBufferTest03(void)
1704 {
1705  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1706  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1707  FAIL_IF(sb == NULL);
1708 
1709  StreamingBufferSegment seg1;
1710  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1711  StreamingBufferSegment seg2;
1712  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1713  FAIL_IF(sb->region.stream_offset != 0);
1714  FAIL_IF(sb->region.buf_offset != 8);
1715  FAIL_IF(seg1.stream_offset != 0);
1716  FAIL_IF(seg2.stream_offset != 14);
1717  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1718  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1719  Dump(sb);
1720  DumpSegment(sb, &seg1);
1721  DumpSegment(sb, &seg2);
1722  FAIL_IF_NULL(sb->head);
1723  FAIL_IF_NOT(sb->sbb_size == 16);
1724  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1725 
1726  StreamingBufferSegment seg3;
1727  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1728  FAIL_IF(sb->region.stream_offset != 0);
1729  FAIL_IF(sb->region.buf_offset != 22);
1730  FAIL_IF(seg3.stream_offset != 8);
1731  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1732  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1733  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1734  Dump(sb);
1735  DumpSegment(sb, &seg1);
1736  DumpSegment(sb, &seg2);
1737  DumpSegment(sb, &seg3);
1738  FAIL_IF_NULL(sb->head);
1739  FAIL_IF_NOT(sb->sbb_size == 22);
1740  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1741 
1742  StreamingBufferSlideToOffset(sb, &cfg, 10);
1743  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1744  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1745  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1746  Dump(sb);
1747  DumpSegment(sb, &seg1);
1748  DumpSegment(sb, &seg2);
1749  DumpSegment(sb, &seg3);
1750  FAIL_IF_NULL(sb->head);
1751  FAIL_IF_NOT(sb->sbb_size == 12);
1752  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1753 
1754  StreamingBufferFree(sb, &cfg);
1755  PASS;
1756 }
1757 
1758 static int StreamingBufferTest04(void)
1759 {
1760  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1761  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1762  FAIL_IF(sb == NULL);
1763 
1764  StreamingBufferSegment seg1;
1765  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1766  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
1767  StreamingBufferSegment seg2;
1768  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1769  FAIL_IF(sb->region.stream_offset != 0);
1770  FAIL_IF(sb->region.buf_offset != 8);
1771  FAIL_IF(seg1.stream_offset != 0);
1772  FAIL_IF(seg2.stream_offset != 14);
1773  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1774  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1775  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
1776  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1777  FAIL_IF(sbb1 != sb->head);
1778  FAIL_IF_NULL(sbb1);
1779  FAIL_IF(sbb1->offset != 0);
1780  FAIL_IF(sbb1->len != 8);
1781  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
1782  FAIL_IF_NULL(sbb2);
1783  FAIL_IF(sbb2 == sb->head);
1784  FAIL_IF(sbb2->offset != 14);
1785  FAIL_IF(sbb2->len != 8);
1786  Dump(sb);
1787  DumpSegment(sb, &seg1);
1788  DumpSegment(sb, &seg2);
1789  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1790 
1791  StreamingBufferSegment seg3;
1792  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1793  FAIL_IF(sb->region.stream_offset != 0);
1794  FAIL_IF(sb->region.buf_offset != 22);
1795  FAIL_IF(seg3.stream_offset != 8);
1796  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1797  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1798  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1799  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1800  FAIL_IF_NULL(sbb1);
1801  FAIL_IF(sbb1 != sb->head);
1802  FAIL_IF(sbb1->offset != 0);
1803  FAIL_IF(sbb1->len != 22);
1804  FAIL_IF(SBB_RB_NEXT(sbb1));
1805  Dump(sb);
1806  DumpSegment(sb, &seg1);
1807  DumpSegment(sb, &seg2);
1808  DumpSegment(sb, &seg3);
1809  FAIL_IF_NULL(sb->head);
1810  FAIL_IF_NOT(sb->sbb_size == 22);
1811  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1812 
1813  /* far ahead of curve: */
1814  StreamingBufferSegment seg4;
1815  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
1816  FAIL_IF(sb->region.stream_offset != 0);
1817  FAIL_IF(sb->region.buf_offset != 22);
1818  FAIL_IF(sb->region.buf_size != 128);
1819  FAIL_IF(seg4.stream_offset != 124);
1820  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1821  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1822  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1823  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg4));
1824  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1825  FAIL_IF_NULL(sbb1);
1826  FAIL_IF(sbb1 != sb->head);
1827  FAIL_IF(sbb1->offset != 0);
1828  FAIL_IF(sbb1->len != 22);
1829  FAIL_IF(!SBB_RB_NEXT(sbb1));
1830  Dump(sb);
1831  DumpSegment(sb, &seg1);
1832  DumpSegment(sb, &seg2);
1833  DumpSegment(sb, &seg3);
1834  DumpSegment(sb, &seg4);
1835  FAIL_IF_NULL(sb->head);
1836  FAIL_IF_NOT(sb->sbb_size == 25);
1837  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1838 
1839  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
1840  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
1841  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
1842  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
1843 
1844  StreamingBufferFree(sb, &cfg);
1845  PASS;
1846 }
1847 
1848 /** \test lots of gaps in block list */
1849 static int StreamingBufferTest06(void)
1850 {
1851  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1852  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1853  FAIL_IF(sb == NULL);
1854 
1855  StreamingBufferSegment seg1;
1856  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"A", 1) != 0);
1857  StreamingBufferSegment seg2;
1858  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"C", 1, 2) != 0);
1859  Dump(sb);
1860  FAIL_IF_NULL(sb->head);
1861  FAIL_IF_NOT(sb->sbb_size == 2);
1862  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1863 
1864  StreamingBufferSegment seg3;
1865  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1866  Dump(sb);
1867  FAIL_IF_NULL(sb->head);
1868  FAIL_IF_NOT(sb->sbb_size == 3);
1869  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1870 
1871  StreamingBufferSegment seg4;
1872  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1873  Dump(sb);
1874  FAIL_IF_NULL(sb->head);
1875  FAIL_IF_NOT(sb->sbb_size == 4);
1876  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1877 
1878  StreamingBufferSegment seg5;
1879  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1880  Dump(sb);
1881  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1882  FAIL_IF_NULL(sbb1);
1883  FAIL_IF(sbb1->offset != 0);
1884  FAIL_IF(sbb1->len != 10);
1885  FAIL_IF(SBB_RB_NEXT(sbb1));
1886  FAIL_IF_NULL(sb->head);
1887  FAIL_IF_NOT(sb->sbb_size == 10);
1888  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1889 
1890  StreamingBufferSegment seg6;
1891  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1892  Dump(sb);
1893  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1894  FAIL_IF_NULL(sbb1);
1895  FAIL_IF(sbb1->offset != 0);
1896  FAIL_IF(sbb1->len != 10);
1897  FAIL_IF(SBB_RB_NEXT(sbb1));
1898  FAIL_IF_NULL(sb->head);
1899  FAIL_IF_NOT(sb->sbb_size == 10);
1900  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1901 
1902  StreamingBufferFree(sb, &cfg);
1903  PASS;
1904 }
1905 
1906 /** \test lots of gaps in block list */
1907 static int StreamingBufferTest07(void)
1908 {
1909  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1910  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1911  FAIL_IF(sb == NULL);
1912 
1913  StreamingBufferSegment seg1;
1914  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1915  StreamingBufferSegment seg2;
1916  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1917  Dump(sb);
1918  FAIL_IF_NULL(sb->head);
1919  FAIL_IF_NOT(sb->sbb_size == 2);
1920  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1921 
1922  StreamingBufferSegment seg3;
1923  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1924  Dump(sb);
1925  FAIL_IF_NULL(sb->head);
1926  FAIL_IF_NOT(sb->sbb_size == 3);
1927  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1928 
1929  StreamingBufferSegment seg4;
1930  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1931  Dump(sb);
1932  FAIL_IF_NULL(sb->head);
1933  FAIL_IF_NOT(sb->sbb_size == 4);
1934  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1935 
1936  StreamingBufferSegment seg5;
1937  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1938  Dump(sb);
1939  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1940  FAIL_IF_NULL(sbb1);
1941  FAIL_IF(sbb1->offset != 0);
1942  FAIL_IF(sbb1->len != 10);
1943  FAIL_IF(SBB_RB_NEXT(sbb1));
1944  FAIL_IF_NULL(sb->head);
1945  FAIL_IF_NOT(sb->sbb_size == 10);
1946  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1947 
1948  StreamingBufferSegment seg6;
1949  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
1950  Dump(sb);
1951  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1952  FAIL_IF_NULL(sbb1);
1953  FAIL_IF(sbb1->offset != 0);
1954  FAIL_IF(sbb1->len != 10);
1955  FAIL_IF(SBB_RB_NEXT(sbb1));
1956  FAIL_IF_NULL(sb->head);
1957  FAIL_IF_NOT(sb->sbb_size == 10);
1958  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1959 
1960  StreamingBufferFree(sb, &cfg);
1961  PASS;
1962 }
1963 
1964 /** \test lots of gaps in block list */
1965 static int StreamingBufferTest08(void)
1966 {
1967  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1968  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1969  FAIL_IF(sb == NULL);
1970 
1971  StreamingBufferSegment seg1;
1972  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
1973  StreamingBufferSegment seg2;
1974  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
1975  Dump(sb);
1976  FAIL_IF_NULL(sb->head);
1977  FAIL_IF_NOT(sb->sbb_size == 2);
1978  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1979 
1980  StreamingBufferSegment seg3;
1981  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
1982  Dump(sb);
1983  FAIL_IF_NULL(sb->head);
1984  FAIL_IF_NOT(sb->sbb_size == 3);
1985  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1986 
1987  StreamingBufferSegment seg4;
1988  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
1989  Dump(sb);
1990  FAIL_IF_NULL(sb->head);
1991  FAIL_IF_NOT(sb->sbb_size == 4);
1992  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1993 
1994  StreamingBufferSegment seg5;
1995  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
1996  Dump(sb);
1997  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1998  FAIL_IF_NULL(sbb1);
1999  FAIL_IF(sbb1->offset != 0);
2000  FAIL_IF(sbb1->len != 10);
2001  FAIL_IF(SBB_RB_NEXT(sbb1));
2002  FAIL_IF_NULL(sb->head);
2003  FAIL_IF_NOT(sb->sbb_size == 10);
2004  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2005 
2006  StreamingBufferSegment seg6;
2007  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
2008  Dump(sb);
2009  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2010  FAIL_IF_NULL(sbb1);
2011  FAIL_IF(sbb1->offset != 0);
2012  FAIL_IF(sbb1->len != 20);
2013  FAIL_IF(SBB_RB_NEXT(sbb1));
2014  FAIL_IF_NULL(sb->head);
2015  FAIL_IF_NOT(sb->sbb_size == 20);
2016  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2017 
2018  StreamingBufferFree(sb, &cfg);
2019  PASS;
2020 }
2021 
2022 /** \test lots of gaps in block list */
2023 static int StreamingBufferTest09(void)
2024 {
2025  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2026  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2027  FAIL_IF(sb == NULL);
2028 
2029  StreamingBufferSegment seg1;
2030  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2031  StreamingBufferSegment seg2;
2032  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2033  Dump(sb);
2034  FAIL_IF_NULL(sb->head);
2035  FAIL_IF_NOT(sb->sbb_size == 2);
2036  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2037 
2038  StreamingBufferSegment seg3;
2039  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2040  Dump(sb);
2041  FAIL_IF_NULL(sb->head);
2042  FAIL_IF_NOT(sb->sbb_size == 3);
2043  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2044 
2045  StreamingBufferSegment seg4;
2046  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"F", 1, 5) != 0);
2047  Dump(sb);
2048  FAIL_IF_NULL(sb->head);
2049  FAIL_IF_NOT(sb->sbb_size == 4);
2050  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2051 
2052  StreamingBufferSegment seg5;
2053  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2054  Dump(sb);
2055  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2056  FAIL_IF_NULL(sbb1);
2057  FAIL_IF(sbb1->offset != 0);
2058  FAIL_IF(sbb1->len != 10);
2059  FAIL_IF(SBB_RB_NEXT(sbb1));
2060  FAIL_IF_NULL(sb->head);
2061  FAIL_IF_NOT(sb->sbb_size == 10);
2062  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2063 
2064  StreamingBufferSegment seg6;
2065  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2066  Dump(sb);
2067  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2068  FAIL_IF_NULL(sbb1);
2069  FAIL_IF(sbb1->offset != 0);
2070  FAIL_IF(sbb1->len != 10);
2071  FAIL_IF(SBB_RB_NEXT(sbb1));
2072  FAIL_IF_NULL(sb->head);
2073  FAIL_IF_NOT(sb->sbb_size == 10);
2074  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2075 
2076  StreamingBufferFree(sb, &cfg);
2077  PASS;
2078 }
2079 
2080 /** \test lots of gaps in block list */
2081 static int StreamingBufferTest10(void)
2082 {
2083  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2084  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2085  FAIL_IF(sb == NULL);
2086 
2087  StreamingBufferSegment seg1;
2088  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"A", 1, 0) != 0);
2089  Dump(sb);
2090  StreamingBufferSegment seg2;
2091  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2092  Dump(sb);
2093  StreamingBufferSegment seg3;
2094  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2095  Dump(sb);
2096  FAIL_IF_NULL(sb->head);
2097  FAIL_IF_NOT(sb->sbb_size == 3);
2098 
2099  StreamingBufferSegment seg4;
2100  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"B", 1, 1) != 0);
2101  Dump(sb);
2102  StreamingBufferSegment seg5;
2103  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"C", 1, 2) != 0);
2104  Dump(sb);
2105  StreamingBufferSegment seg6;
2106  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"G", 1, 6) != 0);
2107  Dump(sb);
2108  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2109  FAIL_IF_NULL(sb->head);
2110  FAIL_IF_NOT(sb->sbb_size == 6);
2111 
2112  StreamingBufferSegment seg7;
2113  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2114  Dump(sb);
2115  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2116  FAIL_IF_NULL(sbb1);
2117  FAIL_IF(sbb1->offset != 0);
2118  FAIL_IF(sbb1->len != 10);
2119  FAIL_IF(SBB_RB_NEXT(sbb1));
2120  FAIL_IF_NULL(sb->head);
2121  FAIL_IF_NOT(sb->sbb_size == 10);
2122 
2123  StreamingBufferSegment seg8;
2124  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2125  Dump(sb);
2126  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2127  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2128  FAIL_IF_NULL(sbb1);
2129  FAIL_IF(sbb1->offset != 0);
2130  FAIL_IF(sbb1->len != 10);
2131  FAIL_IF(SBB_RB_NEXT(sbb1));
2132  FAIL_IF_NULL(sb->head);
2133  FAIL_IF_NOT(sb->sbb_size == 10);
2134 
2135  StreamingBufferFree(sb, &cfg);
2136  PASS;
2137 }
2138 
2139 #endif
2140 
2141 void StreamingBufferRegisterTests(void)
2142 {
2143 #ifdef UNITTESTS
2144  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
2145  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
2146  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
2147  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
2148  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
2149  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
2150  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
2151  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
2152 #endif
2153 }
FREE
#define FREE(cfg, ptr, s)
Definition: util-streaming-buffer.c:43
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:763
len
uint8_t len
Definition: app-layer-dnp3.h:2
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:67
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:112
StreamingBufferRegion_::next
struct StreamingBufferRegion_ * next
Definition: util-streaming-buffer.h:90
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:179
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
RB_LEFT
#define RB_LEFT(elm, field)
Definition: tree.h:322
StreamingBuffer_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:115
StreamingBuffer_::regions
uint16_t regions
Definition: util-streaming-buffer.h:114
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1570
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
MIN
#define MIN(x, y)
Definition: suricata-common.h:380
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:867
segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:0
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1587
MAX
#define MAX(x, y)
Definition: suricata-common.h:384
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1554
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:68
util-unittest.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
FAIL_IF_NOT_NULL
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
util-debug.h
StreamingBufferRegion_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:87
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
RB_ROOT
#define RB_ROOT(head)
Definition: tree.h:326
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:136
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:1412
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:389
REALLOC
#define REALLOC(cfg, ptr, orig_s, s)
Definition: util-streaming-buffer.c:41
CALLOC
#define CALLOC(cfg, n, s)
Definition: util-streaming-buffer.c:39
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:156
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:143
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:113
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:64
StreamingBufferRegion_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:86
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1609
RB_GENERATE
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:88
StreamingBufferAppendRaw
StreamingBufferSegment * StreamingBufferAppendRaw(StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:832
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:81
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:111
StreamingBuffer_
Definition: util-streaming-buffer.h:109
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1478
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
Definition: util-streaming-buffer.c:903
suricata-common.h
util-streaming-buffer.h
util-validate.h
StreamingBufferConfig_
Definition: util-streaming-buffer.h:66
HtpBodyChunk_::next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:177
RB_RIGHT
#define RB_RIGHT(elm, field)
Definition: tree.h:323
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1304
head
Flow * head
Definition: flow-hash.h:1
DATA_FITS_AT_OFFSET
#define DATA_FITS_AT_OFFSET(region, len, offset)
Definition: util-streaming-buffer.c:932
StreamingBufferRegion_
Definition: util-streaming-buffer.h:85
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:50
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:110
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:1520
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:69
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:99
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:97
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1444
DATA_FITS
#define DATA_FITS(sb, len)
Definition: util-streaming-buffer.c:830
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:111
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:96
WARN_UNUSED
#define WARN_UNUSED
Definition: suricata-common.h:392
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:89
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:2141