suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2023 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-unittest.h"
21 #include "util-print.h"
22 #include "util-validate.h"
23 #include "util-debug.h"
24 #include "util-error.h"
25 
26 static void ListRegions(StreamingBuffer *sb);
27 
28 #define DUMP_REGIONS 0 // set to 1 to dump a visual representation of the regions list and sbb tree.
29 
30 /**
31  * \file
32  *
33  * \author Victor Julien <victor@inliniac.net>
34  *
35  * \brief Streaming Buffer API
36  */
37 
38 static void *ReallocFunc(void *ptr, const size_t size)
39 {
40  void *ptrmem = SCRealloc(ptr, size);
41  if (unlikely(ptrmem == NULL)) {
42  sc_errno = SC_ENOMEM;
43  }
44  return ptrmem;
45 }
46 
47 static void *CallocFunc(const size_t nm, const size_t sz)
48 {
49  void *ptrmem = SCCalloc(nm, sz);
50  if (unlikely(ptrmem == NULL)) {
51  sc_errno = SC_ENOMEM;
52  }
53  return ptrmem;
54 }
55 
56 /* memory handling wrappers. If config doesn't define it's own set of
57  * functions, use the defaults */
58 // TODO the default allocators don't set `sc_errno` yet.
59 #define CALLOC(cfg, n, s) (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : CallocFunc((n), (s))
60 #define REALLOC(cfg, ptr, orig_s, s) \
61  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : ReallocFunc((ptr), (s))
62 #define FREE(cfg, ptr, s) \
63  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
64 
65 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg);
66 
67 RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare);
68 
69 int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
70 {
71  SCLogDebug("a %" PRIu64 " len %u, b %" PRIu64 " len %u", a->offset, a->len, b->offset, b->len);
72 
73  if (a->offset > b->offset)
74  SCReturnInt(1);
75  else if (a->offset < b->offset)
76  SCReturnInt(-1);
77  else {
78  if (a->len == 0 || b->len == 0 || a->len == b->len)
79  SCReturnInt(0);
80  else if (a->len > b->len)
81  SCReturnInt(1);
82  else
83  SCReturnInt(-1);
84  }
85 }
86 
87 /* inclusive compare function that also considers the right edge,
88  * not just the offset. */
89 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
90  const uint64_t lre = lookup->offset + lookup->len;
91  const uint64_t tre = intree->offset + intree->len;
92  if (lre <= intree->offset) // entirely before
93  return -1;
94  else if (lookup->offset < tre && lre <= tre) // (some) overlap
95  return 0;
96  else
97  return 1; // entirely after
98 }
99 
100 StreamingBufferBlock *SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
101 {
102  SCLogDebug("looking up %" PRIu64, elm->offset);
103 
104  struct StreamingBufferBlock *tmp = RB_ROOT(head);
105  struct StreamingBufferBlock *res = NULL;
106  while (tmp) {
107  SCLogDebug("compare with %" PRIu64 "/%u", tmp->offset, tmp->len);
108  const int comp = InclusiveCompare(elm, tmp);
109  SCLogDebug("compare result: %d", comp);
110  if (comp < 0) {
111  res = tmp;
112  tmp = RB_LEFT(tmp, rb);
113  } else if (comp > 0) {
114  tmp = RB_RIGHT(tmp, rb);
115  } else {
116  return tmp;
117  }
118  }
119  return res;
120 }
121 
122 /** \interal
123  * \brief does data region intersect with list region 'r'
124  * Takes the max gap into account.
125  */
126 static inline bool RegionsIntersect(const StreamingBufferConfig *cfg,
127  const StreamingBufferRegion *r, const uint64_t offset, const uint64_t re)
128 {
129  /* create the data range for the region, adding the max gap */
130  const uint64_t reg_o =
131  r->stream_offset > cfg->region_gap ? (r->stream_offset - cfg->region_gap) : 0;
132  const uint64_t reg_re = r->stream_offset + r->buf_size + cfg->region_gap;
133  SCLogDebug("r %p: %" PRIu64 "/%" PRIu64 " - adjusted %" PRIu64 "/%" PRIu64, r, r->stream_offset,
134  r->stream_offset + r->buf_size, reg_o, reg_re);
135  /* check if data range intersects with region range */
136  if (offset >= reg_o && offset <= reg_re) {
137  SCLogDebug("r %p is in-scope", r);
138  return true;
139  }
140  if (re >= reg_o && re <= reg_re) {
141  SCLogDebug("r %p is in-scope: %" PRIu64 " >= %" PRIu64 " && %" PRIu64 " <= %" PRIu64, r, re,
142  reg_o, re, reg_re);
143  return true;
144  }
145  SCLogDebug("r %p is out of scope: %" PRIu64 "/%" PRIu64, r, offset, re);
146  return false;
147 }
148 
149 /** \internal
150  * \brief find the first region for merging.
151  */
152 static StreamingBufferRegion *FindFirstRegionForOffset(const StreamingBuffer *sb,
153  const StreamingBufferConfig *cfg, StreamingBufferRegion *r, const uint64_t offset,
154  const uint32_t len, StreamingBufferRegion **prev)
155 {
156  const uint64_t data_re = offset + len;
157  SCLogDebug("looking for first region matching %" PRIu64 "/%" PRIu64, offset, data_re);
158 
159  StreamingBufferRegion *p = NULL;
160  for (; r != NULL; r = r->next) {
161  if (RegionsIntersect(cfg, r, offset, data_re) == true) {
162  *prev = p;
163  return r;
164  }
165  p = r;
166  }
167  *prev = NULL;
168  return NULL;
169 }
170 
171 static StreamingBufferRegion *FindLargestRegionForOffset(const StreamingBuffer *sb,
172  const StreamingBufferConfig *cfg, StreamingBufferRegion *r, const uint64_t offset,
173  const uint32_t len)
174 {
175  const uint64_t data_re = offset + len;
176  SCLogDebug("starting at %p/%" PRIu64 ", offset %" PRIu64 ", data_re %" PRIu64, r,
177  r->stream_offset, offset, data_re);
178  StreamingBufferRegion *candidate = r;
179  for (; r != NULL; r = r->next) {
180 #ifdef DEBUG
181  const uint64_t reg_re = r->stream_offset + r->buf_size;
182  SCLogDebug("checking: %p/%" PRIu64 "/%" PRIu64 ", offset %" PRIu64 "/%" PRIu64, r,
183  r->stream_offset, reg_re, offset, data_re);
184 #endif
185  if (!RegionsIntersect(cfg, r, offset, data_re))
186  return candidate;
187 
188  if (r->buf_size > candidate->buf_size) {
189  SCLogDebug("candidate %p as size %u > %u", candidate, r->buf_size, candidate->buf_size);
190  candidate = r;
191  }
192  }
193  return candidate;
194 }
195 
196 static StreamingBufferRegion *FindRightEdge(const StreamingBuffer *sb,
197  const StreamingBufferConfig *cfg, StreamingBufferRegion *r, const uint64_t offset,
198  const uint32_t len)
199 {
200  const uint64_t data_re = offset + len;
201  StreamingBufferRegion *candidate = r;
202  for (; r != NULL; r = r->next) {
203  if (!RegionsIntersect(cfg, r, offset, data_re)) {
204  SCLogDebug(
205  "r %p is out of scope: %" PRIu64 "/%u/%" PRIu64, r, offset, len, offset + len);
206  return candidate;
207  }
208  candidate = r;
209  }
210  return candidate;
211 }
212 
213 /** \note uses sc_errno to give error details */
214 static inline StreamingBufferRegion *InitBufferRegion(
215  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint32_t min_size)
216 {
217  if (sb->regions == USHRT_MAX || (cfg->max_regions != 0 && sb->regions >= cfg->max_regions)) {
218  SCLogDebug("max regions reached");
219  sc_errno = SC_ELIMIT;
220  return NULL;
221  }
222 
223  StreamingBufferRegion *aux_r = CALLOC(cfg, 1, sizeof(*aux_r));
224  if (aux_r == NULL) {
225  return NULL;
226  }
227 
228  aux_r->buf = CALLOC(cfg, 1, MAX(cfg->buf_size, min_size));
229  if (aux_r->buf == NULL) {
230  FREE(cfg, aux_r, sizeof(*aux_r));
231  return NULL;
232  }
233  aux_r->buf_size = MAX(cfg->buf_size, min_size);
234  sb->regions++;
235  sb->max_regions = MAX(sb->regions, sb->max_regions);
236  return aux_r;
237 }
238 
239 static inline int InitBuffer(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
240 {
241  sb->region.buf = CALLOC(cfg, 1, cfg->buf_size);
242  if (sb->region.buf == NULL) {
243  return sc_errno;
244  }
245  sb->region.buf_size = cfg->buf_size;
246  return SC_OK;
247 }
248 
249 StreamingBuffer *StreamingBufferInit(const StreamingBufferConfig *cfg)
250 {
251  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
252  if (sb == NULL) {
253  return NULL;
254  }
255 
256  sb->region.buf_size = cfg->buf_size;
257  sb->regions = sb->max_regions = 1;
258 
259  if (cfg->buf_size == 0) {
260  return sb;
261  }
262 
263  int r = InitBuffer(sb, cfg);
264  if (r != SC_OK) {
265  FREE(cfg, sb, sizeof(StreamingBuffer));
266  sc_errno = r;
267  return NULL;
268  }
269  return sb;
270 }
271 
272 void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
273 {
274  if (sb != NULL) {
275  SCLogDebug("sb->region.buf_size %u max %u", sb->region.buf_size, sb->buf_size_max);
276 
277  SBBFree(sb, cfg);
278  ListRegions(sb);
279  if (sb->region.buf != NULL) {
280  FREE(cfg, sb->region.buf, sb->region.buf_size);
281  sb->region.buf = NULL;
282  }
283 
284  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
285  StreamingBufferRegion *next = r->next;
286  FREE(cfg, r->buf, r->buf_size);
287  FREE(cfg, r, sizeof(*r));
288  r = next;
289  }
290  sb->region.next = NULL;
291  sb->regions = sb->max_regions = 1;
292  }
293 }
294 
295 void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
296 {
297  if (sb != NULL) {
298  StreamingBufferClear(sb, cfg);
299  FREE(cfg, sb, sizeof(StreamingBuffer));
300  }
301 }
302 
303 #ifdef DEBUG
304 static void SBBPrintList(StreamingBuffer *sb)
305 {
306  StreamingBufferBlock *sbb = NULL;
307  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
308  SCLogDebug("sbb: offset %" PRIu64 ", len %u", sbb->offset, sbb->len);
309  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
310  if (next) {
311  if ((sbb->offset + sbb->len) != next->offset) {
312  SCLogDebug("gap: offset %" PRIu64 ", len %" PRIu64, (sbb->offset + sbb->len),
313  next->offset - (sbb->offset + sbb->len));
314  }
315  }
316  }
317 }
318 #endif
319 
320 /* setup with gap between 2 blocks
321  *
322  * [block][gap][block]
323  **/
324 static int WARN_UNUSED SBBInit(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
325  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t data_len)
326 {
327  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
328  DEBUG_VALIDATE_BUG_ON(region->buf_offset > region->stream_offset + rel_offset);
329 
330  /* need to set up 2: existing data block and new data block */
331  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
332  if (sbb == NULL) {
333  return sc_errno;
334  }
335  sbb->offset = sb->region.stream_offset;
336  sbb->len = sb->region.buf_offset;
337  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
338  sb->head = sbb;
339  sb->sbb_size = sbb->len;
340 
341  StreamingBufferBlock *sbb2 = CALLOC(cfg, 1, sizeof(*sbb2));
342  if (sbb2 == NULL) {
343  return sc_errno;
344  }
345  sbb2->offset = region->stream_offset + rel_offset;
346  sbb2->len = data_len;
347  DEBUG_VALIDATE_BUG_ON(sbb2->offset < sbb->len);
348  SCLogDebug("sbb1 %" PRIu64 ", len %u, sbb2 %" PRIu64 ", len %u", sbb->offset, sbb->len,
349  sbb2->offset, sbb2->len);
350 
351  sb->sbb_size += sbb2->len;
352  if (SBB_RB_INSERT(&sb->sbb_tree, sbb2) != NULL) {
353  FREE(cfg, sbb2, sizeof(*sbb2));
354  return SC_EINVAL;
355  }
356 
357 #ifdef DEBUG
358  SBBPrintList(sb);
359 #endif
360  return SC_OK;
361 }
362 
363 /* setup with leading gap
364  *
365  * [gap][block]
366  **/
367 static int WARN_UNUSED SBBInitLeadingGap(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
368  StreamingBufferRegion *region, uint64_t offset, uint32_t data_len)
369 {
370  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
371 
372  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
373  if (sbb == NULL) {
374  return sc_errno;
375  }
376  sbb->offset = offset;
377  sbb->len = data_len;
378 
379  sb->head = sbb;
380  sb->sbb_size = sbb->len;
381  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
382 
383  SCLogDebug("sbb %" PRIu64 ", len %u", sbb->offset, sbb->len);
384 #ifdef DEBUG
385  SBBPrintList(sb);
386 #endif
387  return SC_OK;
388 }
389 
390 static inline void ConsolidateFwd(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
391  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
392 {
393  uint64_t sa_re = sa->offset + sa->len;
394  StreamingBufferBlock *tr, *s = sa;
395  RB_FOREACH_FROM(tr, SBB, s) {
396  if (sa == tr)
397  continue;
398 
399  const uint64_t tr_re = tr->offset + tr->len;
400  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u re %" PRIu64, tr, tr->offset, tr->len, tr_re);
401 
402  if (sa_re < tr->offset) {
403  SCLogDebug("entirely before: %" PRIu64 " < %" PRIu64, sa_re, tr->offset);
404  break; // entirely before
405  }
406 
407  /* new block (sa) entirely eclipsed by in tree block (tr)
408  sa: [ ]
409  tr: [ ]
410  sa: [ ]
411  tr: [ ]
412  sa: [ ]
413  tr: [ ]
414  */
415  if (sa->offset >= tr->offset && sa_re <= tr_re) {
416  sb->sbb_size -= sa->len;
417  sa->len = tr->len;
418  sa->offset = tr->offset;
419  sa_re = sa->offset + sa->len;
420  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
421  tr->offset, tr->len);
422  SBB_RB_REMOVE(tree, tr);
423  FREE(cfg, tr, sizeof(StreamingBufferBlock));
424  /* new block (sa) entire eclipses in tree block (tr)
425  sa: [ ]
426  tr: [ ]
427  sa: [ ]
428  tr: [ ]
429  sa: [ ]
430  tr: [ ]
431  */
432  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
433  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
434  tr->offset, tr->len);
435  SBB_RB_REMOVE(tree, tr);
436  sb->sbb_size -= tr->len;
437  FREE(cfg, tr, sizeof(StreamingBufferBlock));
438 
439  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
440  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
441  region->buf_size);
442  if (sa->offset == region->stream_offset &&
443  sa_re > (region->stream_offset + region->buf_offset)) {
444  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
445  region->buf_offset = sa_re - region->stream_offset;
446  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
447  " bo %u sz %u BUF_OFFSET UPDATED",
448  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
449  region->buf_size);
450  }
451  /* new block (sa) extended by in tree block (tr)
452  sa: [ ]
453  tr: [ ]
454  sa: [ ]
455  tr: [ ]
456  */
457  } else if (sa->offset < tr->offset && // starts before
458  sa_re >= tr->offset && sa_re < tr_re) // ends inside
459  {
460  // merge. sb->sbb_size includes both so we need to adjust that too.
461  uint32_t combined_len = sa->len + tr->len;
462  sa->len = tr_re - sa->offset;
463  sa_re = sa->offset + sa->len;
464  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
465  SBB_RB_REMOVE(tree, tr);
466  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
467  FREE(cfg, tr, sizeof(StreamingBufferBlock));
468  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u RESULT", sa, sa->offset, sa->len);
469  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
470  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
471  region->buf_size);
472  if (sa->offset == region->stream_offset &&
473  sa_re > (region->stream_offset + region->buf_offset)) {
474  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
475  region->buf_offset = sa_re - region->stream_offset;
476  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
477  " bo %u sz %u BUF_OFFSET UPDATED",
478  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
479  region->buf_size);
480  }
481  }
482  }
483 }
484 
485 static inline void ConsolidateBackward(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
486  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
487 {
488  uint64_t sa_re = sa->offset + sa->len;
489  StreamingBufferBlock *tr, *s = sa;
490  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
491  if (sa == tr)
492  continue;
493  const uint64_t tr_re = tr->offset + tr->len;
494  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u", tr, tr->offset, tr->len);
495 
496  if (sa->offset > tr_re)
497  break; // entirely after
498 
499  /* new block (sa) entirely eclipsed by in tree block (tr)
500  sa: [ ]
501  tr: [ ]
502  sa: [ ]
503  tr: [ ]
504  sa: [ ]
505  tr: [ ]
506  */
507  if (sa->offset >= tr->offset && sa_re <= tr_re) {
508  sb->sbb_size -= sa->len; // sa entirely eclipsed so remove double accounting
509  sa->len = tr->len;
510  sa->offset = tr->offset;
511  sa_re = sa->offset + sa->len;
512  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
513  tr->offset, tr->len);
514  if (sb->head == tr)
515  sb->head = sa;
516  SBB_RB_REMOVE(tree, tr);
517  FREE(cfg, tr, sizeof(StreamingBufferBlock));
518  /* new block (sa) entire eclipses in tree block (tr)
519  sa: [ ]
520  tr: [ ]
521  sa: [ ]
522  tr: [ ]
523  sa: [ ]
524  tr: [ ]
525  */
526  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
527  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
528  tr->offset, tr->len);
529  if (sb->head == tr)
530  sb->head = sa;
531  SBB_RB_REMOVE(tree, tr);
532  sb->sbb_size -= tr->len; // tr entirely eclipsed so remove double accounting
533  FREE(cfg, tr, sizeof(StreamingBufferBlock));
534 
535  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
536  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
537  region->buf_size);
538 
539  if (sa->offset == region->stream_offset &&
540  sa_re > (region->stream_offset + region->buf_offset)) {
541  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
542  region->buf_offset = sa_re - region->stream_offset;
543  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
544  " bo %u sz %u BUF_OFFSET UPDATED",
545  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
546  region->buf_size);
547  }
548 
549  /* new block (sa) extends in tree block (tr)
550  sa: [ ]
551  tr: [ ]
552  sa: [ ]
553  tr: [ ]
554  */
555  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
556  // merge. sb->sbb_size includes both so we need to adjust that too.
557  uint32_t combined_len = sa->len + tr->len;
558  sa->len = sa_re - tr->offset;
559  sa->offset = tr->offset;
560  sa_re = sa->offset + sa->len;
561  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
562  if (sb->head == tr)
563  sb->head = sa;
564  SBB_RB_REMOVE(tree, tr);
565  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
566  FREE(cfg, tr, sizeof(StreamingBufferBlock));
567 
568  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
569  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
570  region->buf_size);
571  if (sa->offset == region->stream_offset &&
572  sa_re > (region->stream_offset + region->buf_offset)) {
573  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
574  region->buf_offset = sa_re - region->stream_offset;
575  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
576  " bo %u sz %u BUF_OFFSET UPDATED",
577  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
578  region->buf_size);
579  }
580  }
581  }
582 }
583 
584 /** \internal
585  * \param region the region that holds the new data
586  */
587 static int SBBUpdate(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
588  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t len)
589 {
590  struct SBB *tree = &sb->sbb_tree;
591  SCLogDebug("* inserting: %u/%u", rel_offset, len);
592 
593  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
594  if (sbb == NULL) {
595  return sc_errno;
596  }
597  sbb->offset = region->stream_offset + rel_offset;
598  sbb->len = len;
599 
600  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
601  if (res) {
602  // exact overlap
603  SCLogDebug("* insert failed: exact match in tree with %p %" PRIu64 "/%u", res, res->offset,
604  res->len);
605  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
606  return SC_OK;
607  }
608  sb->sbb_size += len; // may adjust based on consolidation below
609 
610  /* handle backwards and forwards overlaps */
611  if (SBB_RB_PREV(sbb) == NULL) {
612  sb->head = sbb;
613  } else {
614  ConsolidateBackward(sb, cfg, region, tree, sbb);
615  }
616  ConsolidateFwd(sb, cfg, region, tree, sbb);
617 #ifdef DEBUG
618  SBBPrintList(sb);
619 #endif
620  if (sbb->offset == region->stream_offset) {
621  SCLogDebug("insert at region head");
622  region->buf_offset = sbb->len;
623  }
624  return SC_OK;
625 }
626 
627 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
628 {
629  StreamingBufferBlock *sbb = NULL, *safe = NULL;
630  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
631  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
632  sb->sbb_size -= sbb->len;
633  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
634  }
635  sb->head = NULL;
636 }
637 
638 static void SBBPrune(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
639 {
640  SCLogDebug("pruning %p to %" PRIu64, sb, sb->region.stream_offset);
641  StreamingBufferBlock *sbb = NULL, *safe = NULL;
642  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
643  /* completely beyond window, we're done */
644  if (sbb->offset >= sb->region.stream_offset) {
645  sb->head = sbb;
646  if (sbb->offset == sb->region.stream_offset) {
647  SCLogDebug("set buf_offset?");
648  if (sbb->offset == sb->region.stream_offset) {
649  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
650  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
651  sb->region.buf_offset = sbb->len;
652  }
653  }
654  break;
655  }
656 
657  /* partly before, partly beyond. Adjust */
658  if (sbb->offset < sb->region.stream_offset &&
659  sbb->offset + sbb->len > sb->region.stream_offset) {
660  uint32_t shrink_by = sb->region.stream_offset - sbb->offset;
661  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
662  if (sbb->len >= shrink_by) {
663  sbb->len -= shrink_by;
664  sbb->offset += shrink_by;
665  sb->sbb_size -= shrink_by;
666  SCLogDebug("shrunk by %u", shrink_by);
667  DEBUG_VALIDATE_BUG_ON(sbb->offset != sb->region.stream_offset);
668  }
669  sb->head = sbb;
670  if (sbb->offset == sb->region.stream_offset) {
671  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
672  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
673  sb->region.buf_offset = sbb->len;
674  }
675  break;
676  }
677 
678  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
679  /* either we set it again for the next sbb, or there isn't any */
680  sb->head = NULL;
681  sb->sbb_size -= sbb->len;
682  SCLogDebug("sb %p removed %p %" PRIu64 ", %u", sb, sbb, sbb->offset, sbb->len);
683  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
684  }
685 #ifdef DEBUG
686  SBBPrintList(sb);
687 #endif
688 }
689 
690 static inline uint32_t ToNextMultipleOf(const uint32_t in, const uint32_t m)
691 {
692  uint32_t r = in;
693  if (m > 0) {
694  const uint32_t x = in % m;
695  if (x != 0) {
696  r = (in - x) + m;
697  }
698  }
699  return r;
700 }
701 
702 static thread_local bool g2s_warn_once = false;
703 
704 static inline int WARN_UNUSED GrowRegionToSize(StreamingBuffer *sb,
705  const StreamingBufferConfig *cfg, StreamingBufferRegion *region, const uint32_t size)
706 {
707  DEBUG_VALIDATE_BUG_ON(region->buf_size > BIT_U32(30));
708  if (size > BIT_U32(30)) { // 1GiB
709  if (!g2s_warn_once) {
710  SCLogWarning("StreamingBuffer::GrowRegionToSize() tried to alloc %u bytes, exceeds "
711  "limit of %lu",
712  size, BIT_U32(30));
713  g2s_warn_once = true;
714  }
715  return SC_ELIMIT;
716  }
717 
718  /* try to grow in multiples of cfg->buf_size */
719  const uint32_t grow = ToNextMultipleOf(size, cfg->buf_size);
720  SCLogDebug("grow %u", grow);
721 
722  void *ptr = REALLOC(cfg, region->buf, region->buf_size, grow);
723  if (ptr == NULL) {
724  return sc_errno;
725  }
726  /* for safe printing and general caution, lets memset the
727  * new data to 0 */
728  size_t diff = grow - region->buf_size;
729  void *new_mem = ((char *)ptr) + region->buf_size;
730  memset(new_mem, 0, diff);
731 
732  region->buf = ptr;
733  region->buf_size = grow;
734  SCLogDebug("grown buffer to %u", grow);
735 #ifdef DEBUG
736  if (region->buf_size > sb->buf_size_max) {
737  sb->buf_size_max = region->buf_size;
738  }
739 #endif
740  return SC_OK;
741 }
742 
743 static int WARN_UNUSED GrowToSize(
744  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint32_t size)
745 {
746  return GrowRegionToSize(sb, cfg, &sb->region, size);
747 }
748 
749 static inline bool RegionBeforeOffset(const StreamingBufferRegion *r, const uint64_t o)
750 {
751  return (r->stream_offset + r->buf_size <= o);
752 }
753 
754 static inline bool RegionContainsOffset(const StreamingBufferRegion *r, const uint64_t o)
755 {
756  return (o >= r->stream_offset && (r->stream_offset + r->buf_size) >= o);
757 }
758 
759 /** \internal
760  * \brief slide to offset for regions
761  *
762  *
763  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
764  * ^
765  * - main reset to 0
766  *
767  *
768  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
769  * ^
770  * - main shift
771  *
772  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
773  * ^
774  * - main reset
775  * - move aux into main
776  * - free aux
777  * - shift
778  *
779  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
780  * ^
781  * - main reset
782  * - move aux into main
783  * - free aux
784  * - no shift
785  */
786 static inline void StreamingBufferSlideToOffsetWithRegions(
787  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint64_t slide_offset)
788 {
789  ListRegions(sb);
790  DEBUG_VALIDATE_BUG_ON(slide_offset == sb->region.stream_offset);
791 
792  SCLogDebug("slide_offset %" PRIu64, slide_offset);
793  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
794  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
795 
796  StreamingBufferRegion *to_shift = NULL;
797  const bool main_is_oow = RegionBeforeOffset(&sb->region, slide_offset);
798  if (main_is_oow) {
799  SCLogDebug("main_is_oow");
800  if (sb->region.buf != NULL) {
801  SCLogDebug("clearing main");
802  FREE(cfg, sb->region.buf, sb->region.buf_size);
803  sb->region.buf = NULL;
804  sb->region.buf_size = 0;
805  sb->region.buf_offset = 0;
806  sb->region.stream_offset = slide_offset;
807  } else {
808  sb->region.stream_offset = slide_offset;
809  }
810 
811  /* remove regions that are out of window & select the region to
812  * become the new main */
813  StreamingBufferRegion *prev = &sb->region;
814  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
815  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64, r, r->stream_offset,
816  r->stream_offset + r->buf_offset);
817  StreamingBufferRegion *next = r->next;
818  if (RegionBeforeOffset(r, slide_offset)) {
819  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> before", r, r->stream_offset,
820  r->stream_offset + r->buf_offset);
821  DEBUG_VALIDATE_BUG_ON(r == &sb->region);
822  prev->next = next;
823 
824  FREE(cfg, r->buf, r->buf_size);
825  FREE(cfg, r, sizeof(*r));
826  sb->regions--;
827  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
828  } else if (RegionContainsOffset(r, slide_offset)) {
829  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> within", r, r->stream_offset,
830  r->stream_offset + r->buf_offset);
831  /* remove from list, we will xfer contents to main below */
832  prev->next = next;
833  to_shift = r;
834  break;
835  } else {
836  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> post", r, r->stream_offset,
837  r->stream_offset + r->buf_offset);
838  /* implied beyond slide offset */
839  DEBUG_VALIDATE_BUG_ON(r->stream_offset < slide_offset);
840  break;
841  }
842  r = next;
843  }
844  SCLogDebug("to_shift %p", to_shift);
845 
846  // this region is main, or will xfer its buffer to main
847  if (to_shift && to_shift != &sb->region) {
848  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", to_shift->stream_offset,
849  to_shift->buf, to_shift->buf_size, to_shift->buf_offset);
850  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL);
851 
852  sb->region.buf = to_shift->buf;
853  sb->region.stream_offset = to_shift->stream_offset;
854  sb->region.buf_offset = to_shift->buf_offset;
855  sb->region.buf_size = to_shift->buf_size;
856  sb->region.next = to_shift->next;
857 
858  BUG_ON(to_shift == &sb->region);
859  FREE(cfg, to_shift, sizeof(*to_shift));
860  to_shift = &sb->region;
861  sb->regions--;
862  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
863  }
864 
865  } else {
866  to_shift = &sb->region;
867  SCLogDebug("shift start region %p", to_shift);
868  }
869 
870  // this region is main, or will xfer its buffer to main
871  if (to_shift) {
872  // Do the shift. If new region is exactly at the slide offset we can skip this.
873  DEBUG_VALIDATE_BUG_ON(to_shift->stream_offset > slide_offset);
874  const uint32_t s = slide_offset - to_shift->stream_offset;
875  if (s > 0) {
876  const uint32_t new_data_size = to_shift->buf_size - s;
877  uint32_t new_mem_size = ToNextMultipleOf(new_data_size, cfg->buf_size);
878 
879  /* see if after the slide we'd overlap with the next region. If so, we need
880  * to consolidate them into one. Error handling is a bit peculiar. We need
881  * to grow a region, move data from another region into it, then free the
882  * other region. This could fail if we're close to the memcap. In this case
883  * we relax our rounding up logic so we only shrink and don't merge the 2
884  * regions after all. */
885  if (to_shift->next && slide_offset + new_mem_size >= to_shift->next->stream_offset) {
886  StreamingBufferRegion *start = to_shift;
887  StreamingBufferRegion *next = start->next;
888  const uint64_t next_re = next->stream_offset + next->buf_size;
889  const uint32_t mem_size = ToNextMultipleOf(next_re - slide_offset, cfg->buf_size);
890 
891  /* using next as the new main */
892  if (start->buf_size < next->buf_size) {
893  SCLogDebug("replace main with the next bigger region");
894 
895  const uint32_t next_data_offset = next->stream_offset - slide_offset;
896  const uint32_t prev_buf_size = next->buf_size;
897  const uint32_t start_data_offset = slide_offset - start->stream_offset;
898  DEBUG_VALIDATE_BUG_ON(start_data_offset > start->buf_size);
899  if (start_data_offset > start->buf_size) {
900  new_mem_size = new_data_size;
901  goto just_main;
902  }
903  /* expand "next" to include relevant part of "start" */
904  if (GrowRegionToSize(sb, cfg, next, mem_size) != 0) {
905  new_mem_size = new_data_size;
906  goto just_main;
907  }
908  SCLogDebug("region now sized %u", mem_size);
909 
910  // slide "next":
911  // pre-grow: [nextnextnext]
912  // post-grow [nextnextnextXXX]
913  // post-move [XXXnextnextnext]
914  memmove(next->buf + next_data_offset, next->buf, prev_buf_size);
915 
916  // move portion of "start" into "next"
917  //
918  // start: [ooooookkkkk] (o: old, k: keep)
919  // pre-next [xxxxxnextnextnext]
920  // post-next [kkkkknextnextnext]
921  const uint32_t start_data_size = start->buf_size - start_data_offset;
922  memcpy(next->buf, start->buf + start_data_offset, start_data_size);
923 
924  // free "start"s buffer, we will use the one from "next"
925  FREE(cfg, start->buf, start->buf_size);
926 
927  // update "main" to use "next"
928  start->stream_offset = slide_offset;
929  start->buf = next->buf;
930  start->buf_size = next->buf_size;
931  start->next = next->next;
932 
933  // free "next"
934  FREE(cfg, next, sizeof(*next));
935  sb->regions--;
936  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
937  goto done;
938  } else {
939  /* using "main", expand to include "next" */
940  if (GrowRegionToSize(sb, cfg, start, mem_size) != 0) {
941  new_mem_size = new_data_size;
942  goto just_main;
943  }
944  SCLogDebug("start->buf now size %u", mem_size);
945 
946  // slide "start"
947  // pre: [xxxxxxxAAA]
948  // post: [AAAxxxxxxx]
949  SCLogDebug("s %u new_data_size %u", s, new_data_size);
950  memmove(start->buf, start->buf + s, new_data_size);
951 
952  // copy in "next"
953  // pre: [AAAxxxxxxx]
954  // [BBBBBBB]
955  // post: [AAABBBBBBB]
956  SCLogDebug("copy next->buf %p/%u to start->buf offset %u", next->buf,
957  next->buf_size, new_data_size);
958  memcpy(start->buf + new_data_size, next->buf, next->buf_size);
959 
960  start->stream_offset = slide_offset;
961  start->next = next->next;
962 
963  // free "next"
964  FREE(cfg, next->buf, next->buf_size);
965  FREE(cfg, next, sizeof(*next));
966  sb->regions--;
967  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
968  goto done;
969  }
970  }
971 
972  just_main:
973  SCLogDebug("s %u new_data_size %u", s, new_data_size);
974  memmove(to_shift->buf, to_shift->buf + s, new_data_size);
975  /* shrink memory region. If this fails we keep the old */
976  void *ptr = REALLOC(cfg, to_shift->buf, to_shift->buf_size, new_mem_size);
977  if (ptr != NULL) {
978  to_shift->buf = ptr;
979  to_shift->buf_size = new_mem_size;
980  SCLogDebug("new buf_size %u", to_shift->buf_size);
981  }
982  if (s < to_shift->buf_offset)
983  to_shift->buf_offset -= s;
984  else
985  to_shift->buf_offset = 0;
986  to_shift->stream_offset = slide_offset;
987  }
988  }
989 
990 done:
991  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
992  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
993  SCLogDebug("end of slide");
994 }
995 
996 /**
997  * \brief slide to absolute offset
998  * \todo if sliding beyond window, we could perhaps reset?
999  */
1000 void StreamingBufferSlideToOffset(
1001  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
1002 {
1003  SCLogDebug("sliding to offset %" PRIu64, offset);
1004  ListRegions(sb);
1005 #ifdef DEBUG
1006  SBBPrintList(sb);
1007 #endif
1008 
1009  if (sb->region.next) {
1010  StreamingBufferSlideToOffsetWithRegions(sb, cfg, offset);
1011  SBBPrune(sb, cfg);
1012  SCLogDebug("post SBBPrune");
1013  ListRegions(sb);
1014 #ifdef DEBUG
1015  SBBPrintList(sb);
1016 #endif
1017  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL && sb->region.buf_size == 0);
1018  DEBUG_VALIDATE_BUG_ON(sb->region.buf_offset > sb->region.buf_size);
1019  DEBUG_VALIDATE_BUG_ON(offset > sb->region.stream_offset);
1020  DEBUG_VALIDATE_BUG_ON(sb->head && sb->head->offset == sb->region.stream_offset &&
1021  sb->head->len > sb->region.buf_offset);
1022  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1023  return;
1024  }
1025 
1026  if (offset > sb->region.stream_offset) {
1027  const uint32_t slide = offset - sb->region.stream_offset;
1028  if (sb->head != NULL) {
1029  /* have sbb's, so can't rely on buf_offset for the slide */
1030  if (slide < sb->region.buf_size) {
1031  const uint32_t size = sb->region.buf_size - slide;
1032  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1033  size);
1034  memmove(sb->region.buf, sb->region.buf + slide, size);
1035  if (sb->region.buf_offset > slide) {
1036  sb->region.buf_offset -= slide;
1037  } else {
1038  sb->region.buf_offset = 0;
1039  }
1040  } else {
1041  sb->region.buf_offset = 0;
1042  }
1043  sb->region.stream_offset = offset;
1044  } else {
1045  /* no sbb's, so we can use buf_offset */
1046  if (offset <= sb->region.stream_offset + sb->region.buf_offset) {
1047  const uint32_t size = sb->region.buf_offset - slide;
1048  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1049  size);
1050  memmove(sb->region.buf, sb->region.buf + slide, size);
1051  sb->region.stream_offset = offset;
1052  sb->region.buf_offset = size;
1053  } else {
1054  /* moved past all data */
1055  sb->region.stream_offset = offset;
1056  sb->region.buf_offset = 0;
1057  }
1058  }
1059  SBBPrune(sb, cfg);
1060  }
1061 #ifdef DEBUG
1062  SBBPrintList(sb);
1063 #endif
1064  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1065 }
1066 
1067 #define DATA_FITS(sb, len) ((sb)->region.buf_offset + (len) <= (sb)->region.buf_size)
1068 
1069 int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1070  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
1071 {
1072  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1073 
1074  if (sb->region.buf == NULL) {
1075  if (InitBuffer(sb, cfg) == -1)
1076  return -1;
1077  }
1078 
1079  if (!DATA_FITS(sb, data_len)) {
1080  if (sb->region.buf_size == 0) {
1081  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1082  return -1;
1083  } else {
1084  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1085  return -1;
1086  }
1087  }
1088  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
1089 
1090  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1091  seg->stream_offset = sb->region.stream_offset + sb->region.buf_offset;
1092  seg->segment_len = data_len;
1093  uint32_t rel_offset = sb->region.buf_offset;
1094  sb->region.buf_offset += data_len;
1095 
1096  if (!RB_EMPTY(&sb->sbb_tree)) {
1097  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1098  } else {
1099  return 0;
1100  }
1101 }
1102 
1103 /**
1104  * \brief add data w/o tracking a segment
1105  */
1106 int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1107  const uint8_t *data, uint32_t data_len)
1108 {
1109  if (sb->region.buf == NULL) {
1110  if (InitBuffer(sb, cfg) == -1)
1111  return -1;
1112  }
1113 
1114  if (!DATA_FITS(sb, data_len)) {
1115  if (sb->region.buf_size == 0) {
1116  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1117  return -1;
1118  } else {
1119  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1120  return -1;
1121  }
1122  }
1123  DEBUG_VALIDATE_BUG_ON(!DATA_FITS(sb, data_len));
1124 
1125  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1126  uint32_t rel_offset = sb->region.buf_offset;
1127  sb->region.buf_offset += data_len;
1128 
1129  if (!RB_EMPTY(&sb->sbb_tree)) {
1130  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1131  } else {
1132  return 0;
1133  }
1134 }
1135 
1136 #define DATA_FITS_AT_OFFSET(region, len, offset) ((offset) + (len) <= (region)->buf_size)
1137 
1138 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1139 static void Validate(const StreamingBuffer *sb)
1140 {
1141  bool bail = false;
1142  uint32_t cnt = 0;
1143  for (const StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1144  cnt++;
1145  if (r->next) {
1146  bail |= ((r->stream_offset + r->buf_size) > r->next->stream_offset);
1147  }
1148 
1149  bail |= (r->buf != NULL && r->buf_size == 0);
1150  bail |= (r->buf_offset > r->buf_size);
1151  }
1152  // leading gap, so buf_offset should be 0?
1153  if (sb->head && sb->head->offset > sb->region.stream_offset) {
1154  SCLogDebug("leading gap of size %" PRIu64, sb->head->offset - sb->region.stream_offset);
1155  BUG_ON(sb->region.buf_offset != 0);
1156  }
1157 
1158  if (sb->head && sb->head->offset == sb->region.stream_offset) {
1159  BUG_ON(sb->head->len > sb->region.buf_offset);
1160  BUG_ON(sb->head->len < sb->region.buf_offset);
1161  }
1162  BUG_ON(sb->regions != cnt);
1163  BUG_ON(bail);
1164 }
1165 #endif
1166 
1167 static void ListRegions(StreamingBuffer *sb)
1168 {
1169  if (sb->region.buf == NULL && sb->region.buf_offset == 0 && sb->region.next == NULL)
1170  return;
1171 #if defined(DEBUG) && DUMP_REGIONS == 1
1172  uint32_t cnt = 0;
1173  for (StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1174  cnt++;
1175  char gap[64] = "";
1176  if (r->next) {
1177  snprintf(gap, sizeof(gap), "[ gap:%" PRIu64 " ]",
1178  r->next->stream_offset - (r->stream_offset + r->buf_size));
1179  }
1180 
1181  printf("[ %s offset:%" PRIu64 " size:%u offset:%u ]%s", r == &sb->region ? "main" : "aux",
1182  r->stream_offset, r->buf_size, r->buf_offset, gap);
1183  }
1184  printf("(max %u, cnt %u, sb->regions %u)\n", sb->max_regions, cnt, sb->regions);
1185  bool at_least_one = false;
1186  uint64_t last_re = sb->region.stream_offset;
1187  StreamingBufferBlock *sbb = NULL;
1188  RB_FOREACH(sbb, SBB, &sb->sbb_tree)
1189  {
1190  if (last_re != sbb->offset) {
1191  printf("[ gap:%" PRIu64 " ]", sbb->offset - last_re);
1192  }
1193  printf("[ sbb offset:%" PRIu64 " len:%u ]", sbb->offset, sbb->len);
1194  at_least_one = true;
1195  last_re = sbb->offset + sbb->len;
1196  }
1197  if (at_least_one)
1198  printf("\n");
1199 #endif
1200 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1201  StreamingBufferBlock *sbb2 = NULL;
1202  RB_FOREACH(sbb2, SBB, &sb->sbb_tree)
1203  {
1204  const uint8_t *_data = NULL;
1205  uint32_t _data_len = 0;
1206  (void)StreamingBufferSBBGetData(sb, sbb2, &_data, &_data_len);
1207  }
1208  Validate(sb);
1209 #endif
1210 }
1211 
1212 /** \internal
1213  * \brief process insert by consolidating the affected regions into one
1214  * \note sets sc_errno
1215  */
1216 static StreamingBufferRegion *BufferInsertAtRegionConsolidate(StreamingBuffer *sb,
1217  const StreamingBufferConfig *cfg, StreamingBufferRegion *dst,
1218  StreamingBufferRegion *src_start, StreamingBufferRegion *src_end,
1219  const uint64_t data_offset, const uint32_t data_len, StreamingBufferRegion *prev,
1220  uint32_t dst_buf_size)
1221 {
1222  int retval;
1223 #ifdef DEBUG
1224  const uint64_t data_re = data_offset + data_len;
1225  SCLogDebug("sb %p dst %p src_start %p src_end %p data_offset %" PRIu64
1226  "/data_len %u/data_re %" PRIu64,
1227  sb, dst, src_start, src_end, data_offset, data_len, data_re);
1228 #endif
1229 
1230  // 1. determine size and offset for dst.
1231  const uint64_t dst_offset = MIN(src_start->stream_offset, data_offset);
1232  DEBUG_VALIDATE_BUG_ON(dst_offset < sb->region.stream_offset);
1233  const uint32_t dst_size = dst_buf_size;
1234  SCLogDebug("dst_size %u", dst_size);
1235 
1236  // 2. resize dst
1237  const uint32_t old_size = dst->buf_size;
1238  const uint32_t dst_copy_offset = dst->stream_offset - dst_offset;
1239 #ifdef DEBUG
1240  const uint32_t old_offset = dst->buf_offset;
1241  SCLogDebug("old_size %u, old_offset %u, dst_copy_offset %u", old_size, old_offset,
1242  dst_copy_offset);
1243 #endif
1244  if ((retval = GrowRegionToSize(sb, cfg, dst, dst_size)) != SC_OK) {
1245  sc_errno = retval;
1246  return NULL;
1247  }
1248  SCLogDebug("resized to %u -> %u", dst_size, dst->buf_size);
1249  /* validate that the size is exactly what we asked for */
1250  DEBUG_VALIDATE_BUG_ON(dst_size != dst->buf_size);
1251  if (dst_copy_offset != 0)
1252  memmove(dst->buf + dst_copy_offset, dst->buf, old_size);
1253  if (dst_offset != dst->stream_offset) {
1254  dst->stream_offset = dst_offset;
1255  // buf_offset no longer valid, reset.
1256  dst->buf_offset = 0;
1257  }
1258 
1259  uint32_t new_offset = src_start->buf_offset;
1260  if (data_offset == src_start->stream_offset + src_start->buf_offset) {
1261  new_offset += data_len;
1262  }
1263 
1264  bool start_is_main = false;
1265  if (src_start == &sb->region) {
1266  DEBUG_VALIDATE_BUG_ON(src_start->stream_offset != dst_offset);
1267 
1268  start_is_main = true;
1269  SCLogDebug("src_start is main region");
1270  if (src_start != dst)
1271  memcpy(dst->buf, src_start->buf, src_start->buf_offset);
1272  if (src_start == src_end) {
1273  SCLogDebug("src_start == src_end == main, we're done");
1274  DEBUG_VALIDATE_BUG_ON(src_start != dst);
1275  return src_start;
1276  }
1277  prev = src_start;
1278  src_start = src_start->next; // skip in the loop below
1279  }
1280 
1281  // 3. copy all regions from src_start to dst_start into the new region
1282  for (StreamingBufferRegion *r = src_start; r != NULL;) {
1283  SCLogDebug("r %p %" PRIu64 ", offset %u, len %u, %s, last %s", r, r->stream_offset,
1284  r->buf_offset, r->buf_size, r == &sb->region ? "main" : "aux",
1285  BOOL2STR(r == src_end));
1286  // skip dst
1287  if (r == dst) {
1288  SCLogDebug("skipping r %p as it is 'dst'", r);
1289  if (r == src_end)
1290  break;
1291  prev = r;
1292  r = r->next;
1293  continue;
1294  }
1295  const uint32_t target_offset = r->stream_offset - dst_offset;
1296  SCLogDebug("r %p: target_offset %u", r, target_offset);
1297  DEBUG_VALIDATE_BUG_ON(target_offset > dst->buf_size);
1298  DEBUG_VALIDATE_BUG_ON(target_offset + r->buf_size > dst->buf_size);
1299  memcpy(dst->buf + target_offset, r->buf, r->buf_size);
1300 
1301  StreamingBufferRegion *next = r->next;
1302  FREE(cfg, r->buf, r->buf_size);
1303  FREE(cfg, r, sizeof(*r));
1304  sb->regions--;
1305  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1306 
1307  DEBUG_VALIDATE_BUG_ON(prev == NULL && src_start != &sb->region);
1308  if (prev != NULL) {
1309  SCLogDebug("setting prev %p next to %p (was %p)", prev, next, prev->next);
1310  prev->next = next;
1311  } else {
1312  SCLogDebug("no prev yet");
1313  }
1314 
1315  if (r == src_end)
1316  break;
1317  r = next;
1318  }
1319 
1320  /* special handling of main region being the start, but not the
1321  * region we expand. In this case we'll have main and dst. We will
1322  * move the buffer from dst into main and free dst. */
1323  if (start_is_main && dst != &sb->region) {
1324  DEBUG_VALIDATE_BUG_ON(sb->region.next != dst);
1325  SCLogDebug("start_is_main && dst != main region");
1326  FREE(cfg, sb->region.buf, sb->region.buf_size);
1327  sb->region.buf = dst->buf;
1328  sb->region.buf_size = dst->buf_size;
1329  sb->region.buf_offset = new_offset;
1330  SCLogDebug("sb->region.buf_offset set to %u", sb->region.buf_offset);
1331  sb->region.next = dst->next;
1332  FREE(cfg, dst, sizeof(*dst));
1333  dst = &sb->region;
1334  sb->regions--;
1335  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1336  } else {
1337  SCLogDebug("dst: %p next %p", dst, dst->next);
1338  }
1339 
1340  SCLogDebug("returning dst %p stream_offset %" PRIu64 " buf_offset %u buf_size %u", dst,
1341  dst->stream_offset, dst->buf_offset, dst->buf_size);
1342  return dst;
1343 }
1344 
1345 /** \note sets sc_errno */
1346 static StreamingBufferRegion *BufferInsertAtRegionDo(StreamingBuffer *sb,
1347  const StreamingBufferConfig *cfg, const uint64_t offset, const uint32_t len)
1348 {
1349  SCLogDebug("offset %" PRIu64 ", len %u", offset, len);
1350  StreamingBufferRegion *start_prev = NULL;
1351  StreamingBufferRegion *start =
1352  FindFirstRegionForOffset(sb, cfg, &sb->region, offset, len, &start_prev);
1353  if (start) {
1354  const uint64_t insert_re = offset + len;
1355  const uint64_t insert_start_offset = MIN(start->stream_offset, offset);
1356  uint64_t insert_adjusted_re = insert_re;
1357 
1358  SCLogDebug("start region %p/%" PRIu64 "/%u", start, start->stream_offset, start->buf_size);
1359  StreamingBufferRegion *big = FindLargestRegionForOffset(sb, cfg, start, offset, len);
1360  DEBUG_VALIDATE_BUG_ON(big == NULL);
1361  if (big == NULL) {
1362  sc_errno = SC_EINVAL;
1363  return NULL;
1364  }
1365  SCLogDebug("big region %p/%" PRIu64 "/%u", big, big->stream_offset, big->buf_size);
1366  StreamingBufferRegion *end = FindRightEdge(sb, cfg, big, offset, len);
1367  DEBUG_VALIDATE_BUG_ON(end == NULL);
1368  if (end == NULL) {
1369  sc_errno = SC_EINVAL;
1370  return NULL;
1371  }
1372  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1373 
1374  uint32_t new_buf_size =
1375  ToNextMultipleOf(insert_adjusted_re - insert_start_offset, cfg->buf_size);
1376  SCLogDebug("new_buf_size %u", new_buf_size);
1377 
1378  /* see if our new buf size + cfg->buf_size margin leads to an overlap with the next region.
1379  * If so, include that in the consolidation. */
1380  if (end->next != NULL && new_buf_size + insert_start_offset > end->next->stream_offset) {
1381  SCLogDebug("adjusted end from %p to %p", end, end->next);
1382  end = end->next;
1383  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1384  new_buf_size =
1385  ToNextMultipleOf(insert_adjusted_re - insert_start_offset, cfg->buf_size);
1386  SCLogDebug("new_buf_size %u", new_buf_size);
1387  }
1388 
1389  SCLogDebug("end region %p/%" PRIu64 "/%u", end, end->stream_offset, end->buf_size);
1390  /* sets sc_errno */
1391  StreamingBufferRegion *ret = BufferInsertAtRegionConsolidate(
1392  sb, cfg, big, start, end, offset, len, start_prev, new_buf_size);
1393  return ret;
1394  } else {
1395  /* if there was no region we can use we add a new region and insert it */
1396  StreamingBufferRegion *append = &sb->region;
1397  for (StreamingBufferRegion *r = append; r != NULL; r = r->next) {
1398  if (r->stream_offset > offset) {
1399  break;
1400  } else {
1401  append = r;
1402  }
1403  }
1404 
1405  SCLogDebug("no matching region found, append to %p (%s)", append,
1406  append == &sb->region ? "main" : "aux");
1407  /* sets sc_errno */
1408  StreamingBufferRegion *add = InitBufferRegion(sb, cfg, len);
1409  if (add == NULL)
1410  return NULL;
1411  add->stream_offset = offset;
1412  add->next = append->next;
1413  append->next = add;
1414  SCLogDebug("new region %p offset %" PRIu64, add, add->stream_offset);
1415  return add;
1416  }
1417 }
1418 
1419 /** \internal
1420  * \brief return the region to put the new data in
1421  *
1422  * Will find an existing region, expand it if needed. If no existing region exists or is
1423  * a good fit, it will try to set up a new region. If the region then overlaps or gets
1424  * too close to the next, merge them.
1425  *
1426  * \note sets sc_errno
1427  */
1428 static StreamingBufferRegion *BufferInsertAtRegion(StreamingBuffer *sb,
1429  const StreamingBufferConfig *cfg, const uint8_t *data, const uint32_t data_len,
1430  const uint64_t data_offset)
1431 {
1432  SCLogDebug("data_offset %" PRIu64 ", data_len %u, re %" PRIu64, data_offset, data_len,
1433  data_offset + data_len);
1434  ListRegions(sb);
1435 
1436  if (RegionsIntersect(cfg, &sb->region, data_offset, data_offset + data_len)) {
1437  SCLogDebug("data_offset %" PRIu64 ", data_len %u intersects with main region (next %p)",
1438  data_offset, data_len, sb->region.next);
1439  if (sb->region.next == NULL ||
1440  !RegionsIntersect(cfg, sb->region.next, data_offset, data_offset + data_len)) {
1441  SCLogDebug(
1442  "data_offset %" PRIu64
1443  ", data_len %u intersects with main region, no next or way before next region",
1444  data_offset, data_len);
1445  if (sb->region.buf == NULL) {
1446  int r;
1447  if ((r = InitBuffer(sb, cfg)) != SC_OK) { // TODO init with size
1448  sc_errno = r;
1449  return NULL;
1450  }
1451  }
1452  return &sb->region;
1453  }
1454  } else if (sb->region.next == NULL) {
1455  /* InitBufferRegion sets sc_errno */
1456  StreamingBufferRegion *aux_r = sb->region.next = InitBufferRegion(sb, cfg, data_len);
1457  if (aux_r == NULL)
1458  return NULL;
1459  aux_r->stream_offset = data_offset;
1460  DEBUG_VALIDATE_BUG_ON(data_len > aux_r->buf_size);
1461  SCLogDebug("created new region %p with offset %" PRIu64 ", size %u", aux_r,
1462  aux_r->stream_offset, aux_r->buf_size);
1463  return aux_r;
1464  }
1465  /* BufferInsertAtRegionDo sets sc_errno */
1466  StreamingBufferRegion *blob = BufferInsertAtRegionDo(sb, cfg, data_offset, data_len);
1467  SCLogDebug("blob %p (%s)", blob, blob == &sb->region ? "main" : "aux");
1468  return blob;
1469 }
1470 
1471 /**
1472  * \param offset offset relative to StreamingBuffer::stream_offset
1473  *
1474  * \return SC_OK in case of success
1475  * \return SC_E* errors otherwise
1476  */
1477 int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1478  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
1479 {
1480  int r;
1481 
1482  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1483  DEBUG_VALIDATE_BUG_ON(offset < sb->region.stream_offset);
1484  if (offset < sb->region.stream_offset) {
1485  return SC_EINVAL;
1486  }
1487 
1488  StreamingBufferRegion *region = BufferInsertAtRegion(sb, cfg, data, data_len, offset);
1489  if (region == NULL) {
1490  return sc_errno;
1491  }
1492 
1493  const bool region_is_main = region == &sb->region;
1494 
1495  SCLogDebug("inserting %" PRIu64 "/%u using %s region %p", offset, data_len,
1496  region == &sb->region ? "main" : "aux", region);
1497 
1498  uint32_t rel_offset = offset - region->stream_offset;
1499  if (!DATA_FITS_AT_OFFSET(region, data_len, rel_offset)) {
1500  if ((r = GrowToSize(sb, cfg, (rel_offset + data_len))) != SC_OK)
1501  return r;
1502  }
1503  DEBUG_VALIDATE_BUG_ON(!DATA_FITS_AT_OFFSET(region, data_len, rel_offset));
1504 
1505  SCLogDebug("offset %" PRIu64 " data_len %u, rel_offset %u into region offset %" PRIu64
1506  ", buf_offset %u, buf_size %u",
1507  offset, data_len, rel_offset, region->stream_offset, region->buf_offset,
1508  region->buf_size);
1509  memcpy(region->buf + rel_offset, data, data_len);
1510  seg->stream_offset = offset;
1511  seg->segment_len = data_len;
1512 
1513  SCLogDebug("rel_offset %u region->stream_offset %" PRIu64 ", buf_offset %u", rel_offset,
1514  region->stream_offset, region->buf_offset);
1515 
1516  if (RB_EMPTY(&sb->sbb_tree)) {
1517  SCLogDebug("empty sbb list");
1518 
1519  if (region_is_main) {
1520  if (sb->region.stream_offset == offset) {
1521  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
1522  /* empty list, data is exactly what is expected (append),
1523  * so do nothing.
1524  * Update buf_offset if needed, but in case of overlaps it might be beyond us. */
1525  sb->region.buf_offset = MAX(sb->region.buf_offset, rel_offset + data_len);
1526  } else if ((rel_offset + data_len) <= sb->region.buf_offset) {
1527  SCLogDebug("empty sbb list: block is within existing main data region");
1528  } else {
1529  if (sb->region.buf_offset && rel_offset == sb->region.buf_offset) {
1530  SCLogDebug("exactly at expected offset");
1531  // nothing to do
1532  sb->region.buf_offset = rel_offset + data_len;
1533 
1534  } else if (rel_offset < sb->region.buf_offset) {
1535  // nothing to do
1536 
1537  SCLogDebug("before expected offset: %u < sb->region.buf_offset %u", rel_offset,
1538  sb->region.buf_offset);
1539  if (rel_offset + data_len > sb->region.buf_offset) {
1540  SCLogDebug("before expected offset, ends after: %u < sb->region.buf_offset "
1541  "%u, %u > %u",
1542  rel_offset, sb->region.buf_offset, rel_offset + data_len,
1543  sb->region.buf_offset);
1544  sb->region.buf_offset = rel_offset + data_len;
1545  }
1546 
1547  } else if (sb->region.buf_offset) {
1548  SCLogDebug("beyond expected offset: SBBInit");
1549  /* existing data, but there is a gap between us */
1550  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1551  return r;
1552  } else {
1553  /* gap before data in empty list */
1554  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1555  if ((r = SBBInitLeadingGap(sb, cfg, region, offset, data_len)) != SC_OK)
1556  return r;
1557  }
1558  }
1559  } else {
1560  if (sb->region.buf_offset) {
1561  /* existing data, but there is a gap between us */
1562  SCLogDebug("empty sbb list, no data in main: use SBBInit");
1563  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1564  return r;
1565  } else {
1566  /* gap before data in empty list */
1567  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1568  if ((r = SBBInitLeadingGap(sb, cfg, region, offset, data_len)) != SC_OK)
1569  return r;
1570  }
1571  if (rel_offset == region->buf_offset) {
1572  SCLogDebug("pre region->buf_offset %u", region->buf_offset);
1573  region->buf_offset = rel_offset + data_len;
1574  SCLogDebug("post region->buf_offset %u", region->buf_offset);
1575  }
1576  }
1577  } else {
1578  SCLogDebug("updating sbb tree");
1579  /* already have blocks, so append new block based on new data */
1580  if ((r = SBBUpdate(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1581  return r;
1582  }
1583  DEBUG_VALIDATE_BUG_ON(!region_is_main && sb->head == NULL);
1584 
1585  ListRegions(sb);
1586  if (RB_EMPTY(&sb->sbb_tree)) {
1587  DEBUG_VALIDATE_BUG_ON(offset + data_len > sb->region.stream_offset + sb->region.buf_offset);
1588  }
1589 
1590  return SC_OK;
1591 }
1592 
1593 int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb,
1594  const StreamingBufferSegment *seg)
1595 {
1596  if (seg->stream_offset < sb->region.stream_offset) {
1597  if (seg->stream_offset + seg->segment_len <= sb->region.stream_offset) {
1598  return 1;
1599  }
1600  }
1601  return 0;
1602 }
1603 
1604 static inline const StreamingBufferRegion *GetRegionForOffset(
1605  const StreamingBuffer *sb, const uint64_t offset)
1606 {
1607  if (sb == NULL)
1608  return NULL;
1609  if (sb->region.next == NULL) {
1610  return &sb->region;
1611  }
1612  if (offset >= sb->region.stream_offset &&
1613  offset < (sb->region.stream_offset + sb->region.buf_size)) {
1614  return &sb->region;
1615  }
1616  for (const StreamingBufferRegion *r = sb->region.next; r != NULL; r = r->next) {
1617  if (offset >= r->stream_offset && offset < (r->stream_offset + r->buf_size)) {
1618  return r;
1619  }
1620  }
1621  return NULL;
1622 }
1623 
1624 /** \brief get the data for one SBB */
1625 void StreamingBufferSBBGetData(const StreamingBuffer *sb,
1626  const StreamingBufferBlock *sbb,
1627  const uint8_t **data, uint32_t *data_len)
1628 {
1629  const StreamingBufferRegion *region = GetRegionForOffset(sb, sbb->offset);
1630  SCLogDebug("first find our region (offset %" PRIu64 ") -> %p", sbb->offset, region);
1631  if (region) {
1632  SCLogDebug("region %p found %" PRIu64 "/%u/%u", region, region->stream_offset,
1633  region->buf_size, region->buf_offset);
1634  DEBUG_VALIDATE_BUG_ON(
1635  region->stream_offset == sbb->offset && region->buf_offset > sbb->len);
1636  // buf_offset should match first sbb len if it has the same offset
1637 
1638  if (sbb->offset >= region->stream_offset) {
1639  SCLogDebug("1");
1640  uint64_t offset = sbb->offset - region->stream_offset;
1641  *data = region->buf + offset;
1642  DEBUG_VALIDATE_BUG_ON(offset + sbb->len > region->buf_size);
1643  *data_len = sbb->len;
1644  return;
1645  } else {
1646  SCLogDebug("2");
1647  uint64_t offset = region->stream_offset - sbb->offset;
1648  if (offset < sbb->len) {
1649  *data = region->buf;
1650  *data_len = sbb->len - offset;
1651  return;
1652  }
1653  SCLogDebug("3");
1654  }
1655  }
1656  *data = NULL;
1657  *data_len = 0;
1658  return;
1659 }
1660 
1661 /** \brief get the data for one SBB */
1662 void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb,
1663  const StreamingBufferBlock *sbb,
1664  const uint8_t **data, uint32_t *data_len,
1665  uint64_t offset)
1666 {
1667  /* validate that we are looking for a offset within the sbb */
1668  DEBUG_VALIDATE_BUG_ON(!(offset >= sbb->offset && offset < (sbb->offset + sbb->len)));
1669  if (!(offset >= sbb->offset && offset < (sbb->offset + sbb->len))) {
1670  *data = NULL;
1671  *data_len = 0;
1672  return;
1673  }
1674 
1675  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1676  if (region) {
1677  uint32_t sbblen = sbb->len - (offset - sbb->offset);
1678 
1679  if (offset >= region->stream_offset) {
1680  uint64_t data_offset = offset - region->stream_offset;
1681  *data = region->buf + data_offset;
1682  if (data_offset + sbblen > region->buf_size)
1683  *data_len = region->buf_size - data_offset;
1684  else
1685  *data_len = sbblen;
1686  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1687  return;
1688  } else {
1689  uint64_t data_offset = region->stream_offset - sbb->offset;
1690  if (data_offset < sbblen) {
1691  *data = region->buf;
1692  *data_len = sbblen - data_offset;
1693  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1694  return;
1695  }
1696  }
1697  }
1698 
1699  *data = NULL;
1700  *data_len = 0;
1701  return;
1702 }
1703 
1704 void StreamingBufferSegmentGetData(const StreamingBuffer *sb,
1705  const StreamingBufferSegment *seg,
1706  const uint8_t **data, uint32_t *data_len)
1707 {
1708  const StreamingBufferRegion *region = GetRegionForOffset(sb, seg->stream_offset);
1709  if (region) {
1710  if (seg->stream_offset >= region->stream_offset) {
1711  uint64_t offset = seg->stream_offset - region->stream_offset;
1712  *data = region->buf + offset;
1713  if (offset + seg->segment_len > region->buf_size)
1714  *data_len = region->buf_size - offset;
1715  else
1716  *data_len = seg->segment_len;
1717  SCLogDebug("*data_len %u", *data_len);
1718  return;
1719  } else {
1720  uint64_t offset = region->stream_offset - seg->stream_offset;
1721  if (offset < seg->segment_len) {
1722  *data = region->buf;
1723  *data_len = seg->segment_len - offset;
1724  SCLogDebug("*data_len %u", *data_len);
1725  return;
1726  }
1727  }
1728  }
1729  *data = NULL;
1730  *data_len = 0;
1731  return;
1732 }
1733 
1734 /**
1735  * \retval 1 data is the same
1736  * \retval 0 data is different
1737  */
1738 int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb,
1739  const StreamingBufferSegment *seg,
1740  const uint8_t *rawdata, uint32_t rawdata_len)
1741 {
1742  const uint8_t *segdata = NULL;
1743  uint32_t segdata_len = 0;
1744  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
1745  if (segdata && segdata_len &&
1746  segdata_len == rawdata_len &&
1747  memcmp(segdata, rawdata, segdata_len) == 0)
1748  {
1749  return 1;
1750  }
1751  return 0;
1752 }
1753 
1754 int StreamingBufferGetData(const StreamingBuffer *sb,
1755  const uint8_t **data, uint32_t *data_len,
1756  uint64_t *stream_offset)
1757 {
1758  if (sb != NULL && sb->region.buf != NULL) {
1759  *data = sb->region.buf;
1760  *data_len = sb->region.buf_offset;
1761  *stream_offset = sb->region.stream_offset;
1762  return 1;
1763  } else {
1764  *data = NULL;
1765  *data_len = 0;
1766  *stream_offset = 0;
1767  return 0;
1768  }
1769 }
1770 
1771 int StreamingBufferGetDataAtOffset (const StreamingBuffer *sb,
1772  const uint8_t **data, uint32_t *data_len,
1773  uint64_t offset)
1774 {
1775  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1776  if (region != NULL && region->buf != NULL && offset >= region->stream_offset &&
1777  offset < (region->stream_offset + region->buf_offset)) {
1778  uint32_t skip = offset - region->stream_offset;
1779  *data = region->buf + skip;
1780  *data_len = region->buf_offset - skip;
1781  return 1;
1782  } else {
1783  *data = NULL;
1784  *data_len = 0;
1785  return 0;
1786  }
1787 }
1788 
1789 /**
1790  * \retval 1 data is the same
1791  * \retval 0 data is different
1792  */
1793 int StreamingBufferCompareRawData(const StreamingBuffer *sb,
1794  const uint8_t *rawdata, uint32_t rawdata_len)
1795 {
1796  const uint8_t *sbdata = NULL;
1797  uint32_t sbdata_len = 0;
1798  uint64_t offset = 0;
1799  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
1800  if (offset == 0 &&
1801  sbdata && sbdata_len &&
1802  sbdata_len == rawdata_len &&
1803  memcmp(sbdata, rawdata, sbdata_len) == 0)
1804  {
1805  return 1;
1806  }
1807  SCLogDebug("sbdata_len %u, offset %" PRIu64, sbdata_len, offset);
1808  printf("got:\n");
1809  PrintRawDataFp(stdout, sbdata,sbdata_len);
1810  printf("wanted:\n");
1811  PrintRawDataFp(stdout, rawdata,rawdata_len);
1812  return 0;
1813 }
1814 
1815 #ifdef UNITTESTS
1816 static void Dump(StreamingBuffer *sb)
1817 {
1818  PrintRawDataFp(stdout, sb->region.buf, sb->region.buf_offset);
1819 }
1820 
1821 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
1822 {
1823  const uint8_t *data = NULL;
1824  uint32_t data_len = 0;
1825  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
1826  if (data && data_len) {
1827  PrintRawDataFp(stdout, data, data_len);
1828  }
1829 }
1830 
1831 static int StreamingBufferTest02(void)
1832 {
1833  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1834  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1835  FAIL_IF(sb == NULL);
1836 
1837  StreamingBufferSegment seg1;
1838  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1839  StreamingBufferSegment seg2;
1840  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8) != 0);
1841  FAIL_IF(sb->region.stream_offset != 0);
1842  FAIL_IF(sb->region.buf_offset != 16);
1843  FAIL_IF(seg1.stream_offset != 0);
1844  FAIL_IF(seg2.stream_offset != 8);
1845  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1846  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1847  Dump(sb);
1848  DumpSegment(sb, &seg1);
1849  DumpSegment(sb, &seg2);
1850  FAIL_IF_NOT_NULL(sb->head);
1851  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1852 
1853  StreamingBufferSlideToOffset(sb, &cfg, 6);
1854  FAIL_IF_NOT_NULL(sb->head);
1855  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1856 
1857  StreamingBufferSegment seg3;
1858  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
1859  FAIL_IF(sb->region.stream_offset != 6);
1860  FAIL_IF(sb->region.buf_offset != 16);
1861  FAIL_IF(seg3.stream_offset != 16);
1862  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1863  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1864  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1865  Dump(sb);
1866  DumpSegment(sb, &seg1);
1867  DumpSegment(sb, &seg2);
1868  DumpSegment(sb, &seg3);
1869  FAIL_IF_NOT_NULL(sb->head);
1870  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1871 
1872  StreamingBufferSlideToOffset(sb, &cfg, 12);
1873  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1874  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1875  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1876  Dump(sb);
1877  DumpSegment(sb, &seg1);
1878  DumpSegment(sb, &seg2);
1879  DumpSegment(sb, &seg3);
1880  FAIL_IF_NOT_NULL(sb->head);
1881  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1882 
1883  StreamingBufferFree(sb, &cfg);
1884  PASS;
1885 }
1886 
1887 static int StreamingBufferTest03(void)
1888 {
1889  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1890  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1891  FAIL_IF(sb == NULL);
1892 
1893  StreamingBufferSegment seg1;
1894  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1895  StreamingBufferSegment seg2;
1896  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1897  FAIL_IF(sb->region.stream_offset != 0);
1898  FAIL_IF(sb->region.buf_offset != 8);
1899  FAIL_IF(seg1.stream_offset != 0);
1900  FAIL_IF(seg2.stream_offset != 14);
1901  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1902  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1903  Dump(sb);
1904  DumpSegment(sb, &seg1);
1905  DumpSegment(sb, &seg2);
1906  FAIL_IF_NULL(sb->head);
1907  FAIL_IF_NOT(sb->sbb_size == 16);
1908  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1909 
1910  StreamingBufferSegment seg3;
1911  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1912  FAIL_IF(sb->region.stream_offset != 0);
1913  FAIL_IF(sb->region.buf_offset != 22);
1914  FAIL_IF(seg3.stream_offset != 8);
1915  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1916  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1917  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1918  Dump(sb);
1919  DumpSegment(sb, &seg1);
1920  DumpSegment(sb, &seg2);
1921  DumpSegment(sb, &seg3);
1922  FAIL_IF_NULL(sb->head);
1923  FAIL_IF_NOT(sb->sbb_size == 22);
1924  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1925 
1926  StreamingBufferSlideToOffset(sb, &cfg, 10);
1927  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1928  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1929  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1930  Dump(sb);
1931  DumpSegment(sb, &seg1);
1932  DumpSegment(sb, &seg2);
1933  DumpSegment(sb, &seg3);
1934  FAIL_IF_NULL(sb->head);
1935  FAIL_IF_NOT(sb->sbb_size == 12);
1936  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1937 
1938  StreamingBufferFree(sb, &cfg);
1939  PASS;
1940 }
1941 
1942 static int StreamingBufferTest04(void)
1943 {
1944  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1945  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1946  FAIL_IF(sb == NULL);
1947 
1948  StreamingBufferSegment seg1;
1949  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1950  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
1951  StreamingBufferSegment seg2;
1952  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1953  FAIL_IF(sb->region.stream_offset != 0);
1954  FAIL_IF(sb->region.buf_offset != 8);
1955  FAIL_IF(seg1.stream_offset != 0);
1956  FAIL_IF(seg2.stream_offset != 14);
1957  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1958  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1959  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
1960  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1961  FAIL_IF(sbb1 != sb->head);
1962  FAIL_IF_NULL(sbb1);
1963  FAIL_IF(sbb1->offset != 0);
1964  FAIL_IF(sbb1->len != 8);
1965  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
1966  FAIL_IF_NULL(sbb2);
1967  FAIL_IF(sbb2 == sb->head);
1968  FAIL_IF(sbb2->offset != 14);
1969  FAIL_IF(sbb2->len != 8);
1970  Dump(sb);
1971  DumpSegment(sb, &seg1);
1972  DumpSegment(sb, &seg2);
1973  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1974 
1975  StreamingBufferSegment seg3;
1976  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1977  FAIL_IF(sb->region.stream_offset != 0);
1978  FAIL_IF(sb->region.buf_offset != 22);
1979  FAIL_IF(seg3.stream_offset != 8);
1980  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1981  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1982  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1983  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1984  FAIL_IF_NULL(sbb1);
1985  FAIL_IF(sbb1 != sb->head);
1986  FAIL_IF(sbb1->offset != 0);
1987  FAIL_IF(sbb1->len != 22);
1988  FAIL_IF(SBB_RB_NEXT(sbb1));
1989  Dump(sb);
1990  DumpSegment(sb, &seg1);
1991  DumpSegment(sb, &seg2);
1992  DumpSegment(sb, &seg3);
1993  FAIL_IF_NULL(sb->head);
1994  FAIL_IF_NOT(sb->sbb_size == 22);
1995  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1996 
1997  /* far ahead of curve: */
1998  StreamingBufferSegment seg4;
1999  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
2000  FAIL_IF(sb->region.stream_offset != 0);
2001  FAIL_IF(sb->region.buf_offset != 22);
2002  FAIL_IF(sb->region.buf_size != 128);
2003  FAIL_IF(seg4.stream_offset != 124);
2004  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2005  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2006  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
2007  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg4));
2008  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2009  FAIL_IF_NULL(sbb1);
2010  FAIL_IF(sbb1 != sb->head);
2011  FAIL_IF(sbb1->offset != 0);
2012  FAIL_IF(sbb1->len != 22);
2013  FAIL_IF(!SBB_RB_NEXT(sbb1));
2014  Dump(sb);
2015  DumpSegment(sb, &seg1);
2016  DumpSegment(sb, &seg2);
2017  DumpSegment(sb, &seg3);
2018  DumpSegment(sb, &seg4);
2019  FAIL_IF_NULL(sb->head);
2020  FAIL_IF_NOT(sb->sbb_size == 25);
2021  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2022 
2023  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
2024  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
2025  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
2026  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
2027 
2028  StreamingBufferFree(sb, &cfg);
2029  PASS;
2030 }
2031 
2032 /** \test lots of gaps in block list */
2033 static int StreamingBufferTest06(void)
2034 {
2035  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2036  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2037  FAIL_IF(sb == NULL);
2038 
2039  StreamingBufferSegment seg1;
2040  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"A", 1) != 0);
2041  StreamingBufferSegment seg2;
2042  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"C", 1, 2) != 0);
2043  Dump(sb);
2044  FAIL_IF_NULL(sb->head);
2045  FAIL_IF_NOT(sb->sbb_size == 2);
2046  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2047 
2048  StreamingBufferSegment seg3;
2049  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2050  Dump(sb);
2051  FAIL_IF_NULL(sb->head);
2052  FAIL_IF_NOT(sb->sbb_size == 3);
2053  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2054 
2055  StreamingBufferSegment seg4;
2056  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2057  Dump(sb);
2058  FAIL_IF_NULL(sb->head);
2059  FAIL_IF_NOT(sb->sbb_size == 4);
2060  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2061 
2062  StreamingBufferSegment seg5;
2063  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2064  Dump(sb);
2065  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2066  FAIL_IF_NULL(sbb1);
2067  FAIL_IF(sbb1->offset != 0);
2068  FAIL_IF(sbb1->len != 10);
2069  FAIL_IF(SBB_RB_NEXT(sbb1));
2070  FAIL_IF_NULL(sb->head);
2071  FAIL_IF_NOT(sb->sbb_size == 10);
2072  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2073 
2074  StreamingBufferSegment seg6;
2075  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2076  Dump(sb);
2077  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2078  FAIL_IF_NULL(sbb1);
2079  FAIL_IF(sbb1->offset != 0);
2080  FAIL_IF(sbb1->len != 10);
2081  FAIL_IF(SBB_RB_NEXT(sbb1));
2082  FAIL_IF_NULL(sb->head);
2083  FAIL_IF_NOT(sb->sbb_size == 10);
2084  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2085 
2086  StreamingBufferFree(sb, &cfg);
2087  PASS;
2088 }
2089 
2090 /** \test lots of gaps in block list */
2091 static int StreamingBufferTest07(void)
2092 {
2093  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2094  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2095  FAIL_IF(sb == NULL);
2096 
2097  StreamingBufferSegment seg1;
2098  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2099  StreamingBufferSegment seg2;
2100  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2101  Dump(sb);
2102  FAIL_IF_NULL(sb->head);
2103  FAIL_IF_NOT(sb->sbb_size == 2);
2104  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2105 
2106  StreamingBufferSegment seg3;
2107  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2108  Dump(sb);
2109  FAIL_IF_NULL(sb->head);
2110  FAIL_IF_NOT(sb->sbb_size == 3);
2111  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2112 
2113  StreamingBufferSegment seg4;
2114  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2115  Dump(sb);
2116  FAIL_IF_NULL(sb->head);
2117  FAIL_IF_NOT(sb->sbb_size == 4);
2118  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2119 
2120  StreamingBufferSegment seg5;
2121  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2122  Dump(sb);
2123  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2124  FAIL_IF_NULL(sbb1);
2125  FAIL_IF(sbb1->offset != 0);
2126  FAIL_IF(sbb1->len != 10);
2127  FAIL_IF(SBB_RB_NEXT(sbb1));
2128  FAIL_IF_NULL(sb->head);
2129  FAIL_IF_NOT(sb->sbb_size == 10);
2130  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2131 
2132  StreamingBufferSegment seg6;
2133  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2134  Dump(sb);
2135  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2136  FAIL_IF_NULL(sbb1);
2137  FAIL_IF(sbb1->offset != 0);
2138  FAIL_IF(sbb1->len != 10);
2139  FAIL_IF(SBB_RB_NEXT(sbb1));
2140  FAIL_IF_NULL(sb->head);
2141  FAIL_IF_NOT(sb->sbb_size == 10);
2142  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2143 
2144  StreamingBufferFree(sb, &cfg);
2145  PASS;
2146 }
2147 
2148 /** \test lots of gaps in block list */
2149 static int StreamingBufferTest08(void)
2150 {
2151  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2152  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2153  FAIL_IF(sb == NULL);
2154 
2155  StreamingBufferSegment seg1;
2156  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2157  StreamingBufferSegment seg2;
2158  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2159  Dump(sb);
2160  FAIL_IF_NULL(sb->head);
2161  FAIL_IF_NOT(sb->sbb_size == 2);
2162  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2163 
2164  StreamingBufferSegment seg3;
2165  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2166  Dump(sb);
2167  FAIL_IF_NULL(sb->head);
2168  FAIL_IF_NOT(sb->sbb_size == 3);
2169  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2170 
2171  StreamingBufferSegment seg4;
2172  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2173  Dump(sb);
2174  FAIL_IF_NULL(sb->head);
2175  FAIL_IF_NOT(sb->sbb_size == 4);
2176  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2177 
2178  StreamingBufferSegment seg5;
2179  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2180  Dump(sb);
2181  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2182  FAIL_IF_NULL(sbb1);
2183  FAIL_IF(sbb1->offset != 0);
2184  FAIL_IF(sbb1->len != 10);
2185  FAIL_IF(SBB_RB_NEXT(sbb1));
2186  FAIL_IF_NULL(sb->head);
2187  FAIL_IF_NOT(sb->sbb_size == 10);
2188  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2189 
2190  StreamingBufferSegment seg6;
2191  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
2192  Dump(sb);
2193  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2194  FAIL_IF_NULL(sbb1);
2195  FAIL_IF(sbb1->offset != 0);
2196  FAIL_IF(sbb1->len != 20);
2197  FAIL_IF(SBB_RB_NEXT(sbb1));
2198  FAIL_IF_NULL(sb->head);
2199  FAIL_IF_NOT(sb->sbb_size == 20);
2200  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2201 
2202  StreamingBufferFree(sb, &cfg);
2203  PASS;
2204 }
2205 
2206 /** \test lots of gaps in block list */
2207 static int StreamingBufferTest09(void)
2208 {
2209  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2210  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2211  FAIL_IF(sb == NULL);
2212 
2213  StreamingBufferSegment seg1;
2214  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2215  StreamingBufferSegment seg2;
2216  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2217  Dump(sb);
2218  FAIL_IF_NULL(sb->head);
2219  FAIL_IF_NOT(sb->sbb_size == 2);
2220  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2221 
2222  StreamingBufferSegment seg3;
2223  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2224  Dump(sb);
2225  FAIL_IF_NULL(sb->head);
2226  FAIL_IF_NOT(sb->sbb_size == 3);
2227  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2228 
2229  StreamingBufferSegment seg4;
2230  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"F", 1, 5) != 0);
2231  Dump(sb);
2232  FAIL_IF_NULL(sb->head);
2233  FAIL_IF_NOT(sb->sbb_size == 4);
2234  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2235 
2236  StreamingBufferSegment seg5;
2237  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2238  Dump(sb);
2239  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2240  FAIL_IF_NULL(sbb1);
2241  FAIL_IF(sbb1->offset != 0);
2242  FAIL_IF(sbb1->len != 10);
2243  FAIL_IF(SBB_RB_NEXT(sbb1));
2244  FAIL_IF_NULL(sb->head);
2245  FAIL_IF_NOT(sb->sbb_size == 10);
2246  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2247 
2248  StreamingBufferSegment seg6;
2249  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2250  Dump(sb);
2251  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2252  FAIL_IF_NULL(sbb1);
2253  FAIL_IF(sbb1->offset != 0);
2254  FAIL_IF(sbb1->len != 10);
2255  FAIL_IF(SBB_RB_NEXT(sbb1));
2256  FAIL_IF_NULL(sb->head);
2257  FAIL_IF_NOT(sb->sbb_size == 10);
2258  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2259 
2260  StreamingBufferFree(sb, &cfg);
2261  PASS;
2262 }
2263 
2264 /** \test lots of gaps in block list */
2265 static int StreamingBufferTest10(void)
2266 {
2267  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2268  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2269  FAIL_IF(sb == NULL);
2270 
2271  StreamingBufferSegment seg1;
2272  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"A", 1, 0) != 0);
2273  Dump(sb);
2274  StreamingBufferSegment seg2;
2275  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2276  Dump(sb);
2277  StreamingBufferSegment seg3;
2278  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2279  Dump(sb);
2280  FAIL_IF_NULL(sb->head);
2281  FAIL_IF_NOT(sb->sbb_size == 3);
2282 
2283  StreamingBufferSegment seg4;
2284  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"B", 1, 1) != 0);
2285  Dump(sb);
2286  StreamingBufferSegment seg5;
2287  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"C", 1, 2) != 0);
2288  Dump(sb);
2289  StreamingBufferSegment seg6;
2290  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"G", 1, 6) != 0);
2291  Dump(sb);
2292  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2293  FAIL_IF_NULL(sb->head);
2294  FAIL_IF_NOT(sb->sbb_size == 6);
2295 
2296  StreamingBufferSegment seg7;
2297  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2298  Dump(sb);
2299  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2300  FAIL_IF_NULL(sbb1);
2301  FAIL_IF(sbb1->offset != 0);
2302  FAIL_IF(sbb1->len != 10);
2303  FAIL_IF(SBB_RB_NEXT(sbb1));
2304  FAIL_IF_NULL(sb->head);
2305  FAIL_IF_NOT(sb->sbb_size == 10);
2306 
2307  StreamingBufferSegment seg8;
2308  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2309  Dump(sb);
2310  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2311  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2312  FAIL_IF_NULL(sbb1);
2313  FAIL_IF(sbb1->offset != 0);
2314  FAIL_IF(sbb1->len != 10);
2315  FAIL_IF(SBB_RB_NEXT(sbb1));
2316  FAIL_IF_NULL(sb->head);
2317  FAIL_IF_NOT(sb->sbb_size == 10);
2318 
2319  StreamingBufferFree(sb, &cfg);
2320  PASS;
2321 }
2322 
2323 #endif
2324 
2325 void StreamingBufferRegisterTests(void)
2326 {
2327 #ifdef UNITTESTS
2328  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
2329  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
2330  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
2331  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
2332  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
2333  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
2334  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
2335  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
2336 #endif
2337 }
FREE
#define FREE(cfg, ptr, s)
Definition: util-streaming-buffer.c:62
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:1000
len
uint8_t len
Definition: app-layer-dnp3.h:2
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
StreamingBufferRegion_::next
struct StreamingBufferRegion_ * next
Definition: util-streaming-buffer.h:89
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:295
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
RB_LEFT
#define RB_LEFT(elm, field)
Definition: tree.h:322
SC_EINVAL
@ SC_EINVAL
Definition: util-error.h:30
StreamingBuffer_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:114
StreamingBuffer_::regions
uint16_t regions
Definition: util-streaming-buffer.h:113
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1754
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
MIN
#define MIN(x, y)
Definition: suricata-common.h:391
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:1069
segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:0
m
SCMutex m
Definition: flow-hash.h:6
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1771
MAX
#define MAX(x, y)
Definition: suricata-common.h:395
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1738
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
util-unittest.h
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
FAIL_IF_NOT_NULL
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
util-debug.h
StreamingBufferRegion_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:86
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
util-error.h
RB_ROOT
#define RB_ROOT(head)
Definition: tree.h:326
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:249
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:1593
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:400
REALLOC
#define REALLOC(cfg, ptr, orig_s, s)
Definition: util-streaming-buffer.c:60
CALLOC
#define CALLOC(cfg, n, s)
Definition: util-streaming-buffer.c:59
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:272
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:114
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
StreamingBufferRegion_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:85
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1793
SC_OK
@ SC_OK
Definition: util-error.h:27
RB_GENERATE
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:100
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamingBuffer_
Definition: util-streaming-buffer.h:108
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1662
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
Definition: util-streaming-buffer.c:1106
suricata-common.h
util-streaming-buffer.h
util-validate.h
StreamingBufferConfig_
Definition: util-streaming-buffer.h:65
HtpBodyChunk_::next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:179
RB_RIGHT
#define RB_RIGHT(elm, field)
Definition: tree.h:323
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1477
head
Flow * head
Definition: flow-hash.h:1
DATA_FITS_AT_OFFSET
#define DATA_FITS_AT_OFFSET(region, len, offset)
Definition: util-streaming-buffer.c:1136
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamingBufferRegion_
Definition: util-streaming-buffer.h:84
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:69
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:1704
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1625
DATA_FITS
#define DATA_FITS(sb, len)
Definition: util-streaming-buffer.c:1067
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:103
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
WARN_UNUSED
#define WARN_UNUSED
Definition: suricata-common.h:403
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:2325