suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2023 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-unittest.h"
21 #include "util-print.h"
22 #include "util-validate.h"
23 #include "util-debug.h"
24 #include "util-error.h"
25 
26 #include "app-layer-htp-mem.h"
27 #include "conf-yaml-loader.h"
28 #include "app-layer-htp.h"
29 
30 static void ListRegions(StreamingBuffer *sb);
31 
32 #define DUMP_REGIONS 0 // set to 1 to dump a visual representation of the regions list and sbb tree.
33 
34 /**
35  * \file
36  *
37  * \author Victor Julien <victor@inliniac.net>
38  *
39  * \brief Streaming Buffer API
40  */
41 
42 static void *ReallocFunc(void *ptr, const size_t size)
43 {
44  void *ptrmem = SCRealloc(ptr, size);
45  if (unlikely(ptrmem == NULL)) {
46  sc_errno = SC_ENOMEM;
47  }
48  return ptrmem;
49 }
50 
51 static void *CallocFunc(const size_t nm, const size_t sz)
52 {
53  void *ptrmem = SCCalloc(nm, sz);
54  if (unlikely(ptrmem == NULL)) {
55  sc_errno = SC_ENOMEM;
56  }
57  return ptrmem;
58 }
59 
60 /* memory handling wrappers. If config doesn't define it's own set of
61  * functions, use the defaults */
62 // TODO the default allocators don't set `sc_errno` yet.
63 #define CALLOC(cfg, n, s) (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : CallocFunc((n), (s))
64 #define REALLOC(cfg, ptr, orig_s, s) \
65  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : ReallocFunc((ptr), (s))
66 #define FREE(cfg, ptr, s) \
67  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
68 
69 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg);
70 
71 RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare);
72 
73 int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
74 {
75  SCLogDebug("a %" PRIu64 " len %u, b %" PRIu64 " len %u", a->offset, a->len, b->offset, b->len);
76 
77  if (a->offset > b->offset)
78  SCReturnInt(1);
79  else if (a->offset < b->offset)
80  SCReturnInt(-1);
81  else {
82  if (a->len == 0 || b->len == 0 || a->len == b->len)
83  SCReturnInt(0);
84  else if (a->len > b->len)
85  SCReturnInt(1);
86  else
87  SCReturnInt(-1);
88  }
89 }
90 
91 /* inclusive compare function that also considers the right edge,
92  * not just the offset. */
93 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
94  const uint64_t lre = lookup->offset + lookup->len;
95  const uint64_t tre = intree->offset + intree->len;
96  if (lre <= intree->offset) // entirely before
97  return -1;
98  else if (lookup->offset < tre && lre <= tre) // (some) overlap
99  return 0;
100  else
101  return 1; // entirely after
102 }
103 
104 StreamingBufferBlock *SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
105 {
106  SCLogDebug("looking up %" PRIu64, elm->offset);
107 
108  struct StreamingBufferBlock *tmp = RB_ROOT(head);
109  struct StreamingBufferBlock *res = NULL;
110  while (tmp) {
111  SCLogDebug("compare with %" PRIu64 "/%u", tmp->offset, tmp->len);
112  const int comp = InclusiveCompare(elm, tmp);
113  SCLogDebug("compare result: %d", comp);
114  if (comp < 0) {
115  res = tmp;
116  tmp = RB_LEFT(tmp, rb);
117  } else if (comp > 0) {
118  tmp = RB_RIGHT(tmp, rb);
119  } else {
120  return tmp;
121  }
122  }
123  return res;
124 }
125 
126 /** \interal
127  * \brief does data region intersect with list region 'r'
128  * Takes the max gap into account.
129  */
130 static inline bool RegionsIntersect(const StreamingBufferConfig *cfg,
131  const StreamingBufferRegion *r, const uint64_t offset, const uint64_t re)
132 {
133  /* create the data range for the region, adding the max gap */
134  const uint64_t reg_o =
135  r->stream_offset > cfg->region_gap ? (r->stream_offset - cfg->region_gap) : 0;
136  const uint64_t reg_re = r->stream_offset + r->buf_size + cfg->region_gap;
137  SCLogDebug("r %p: %" PRIu64 "/%" PRIu64 " - adjusted %" PRIu64 "/%" PRIu64, r, r->stream_offset,
138  r->stream_offset + r->buf_size, reg_o, reg_re);
139  /* check if data range intersects with region range */
140  /* [offset:re] and [reg_o:reg_re] do not intersect if and only if
141  * re < reg_o or if reg_re < offset (one segment is strictly before the other)
142  * trusting that offset<=re and reg_o<=reg_re
143  */
144  if (re < reg_o || reg_re < offset) {
145  return false;
146  }
147  return true;
148 }
149 
150 /** \internal
151  * \brief find the first region for merging.
152  */
153 static StreamingBufferRegion *FindFirstRegionForOffset(const StreamingBufferConfig *cfg,
154  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len,
155  StreamingBufferRegion **prev)
156 {
157  const uint64_t data_re = offset + len;
158  SCLogDebug("looking for first region matching %" PRIu64 "/%" PRIu64, offset, data_re);
159 
160  StreamingBufferRegion *p = NULL;
161  for (; r != NULL; r = r->next) {
162  if (RegionsIntersect(cfg, r, offset, data_re) == true) {
163  *prev = p;
164  return r;
165  }
166  p = r;
167  }
168  *prev = NULL;
169  return NULL;
170 }
171 
172 static StreamingBufferRegion *FindLargestRegionForOffset(const StreamingBufferConfig *cfg,
173  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len)
174 {
175  const uint64_t data_re = offset + len;
176  SCLogDebug("starting at %p/%" PRIu64 ", offset %" PRIu64 ", data_re %" PRIu64, r,
177  r->stream_offset, offset, data_re);
178  StreamingBufferRegion *candidate = r;
179  for (; r != NULL; r = r->next) {
180 #ifdef DEBUG
181  const uint64_t reg_re = r->stream_offset + r->buf_size;
182  SCLogDebug("checking: %p/%" PRIu64 "/%" PRIu64 ", offset %" PRIu64 "/%" PRIu64, r,
183  r->stream_offset, reg_re, offset, data_re);
184 #endif
185  if (!RegionsIntersect(cfg, r, offset, data_re))
186  return candidate;
187 
188  if (r->buf_size > candidate->buf_size) {
189  SCLogDebug("candidate %p as size %u > %u", candidate, r->buf_size, candidate->buf_size);
190  candidate = r;
191  }
192  }
193  return candidate;
194 }
195 
196 static StreamingBufferRegion *FindRightEdge(const StreamingBufferConfig *cfg,
197  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len)
198 {
199  const uint64_t data_re = offset + len;
200  StreamingBufferRegion *candidate = r;
201  for (; r != NULL; r = r->next) {
202  if (!RegionsIntersect(cfg, r, offset, data_re)) {
203  SCLogDebug(
204  "r %p is out of scope: %" PRIu64 "/%u/%" PRIu64, r, offset, len, offset + len);
205  return candidate;
206  }
207  candidate = r;
208  }
209  return candidate;
210 }
211 
212 /** \note uses sc_errno to give error details */
213 static inline StreamingBufferRegion *InitBufferRegion(
214  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint32_t min_size)
215 {
216  if (sb->regions == USHRT_MAX || (cfg->max_regions != 0 && sb->regions >= cfg->max_regions)) {
217  SCLogDebug("max regions reached");
218  sc_errno = SC_ELIMIT;
219  return NULL;
220  }
221 
222  StreamingBufferRegion *aux_r = CALLOC(cfg, 1, sizeof(*aux_r));
223  if (aux_r == NULL) {
224  return NULL;
225  }
226 
227  aux_r->buf = CALLOC(cfg, 1, MAX(cfg->buf_size, min_size));
228  if (aux_r->buf == NULL) {
229  FREE(cfg, aux_r, sizeof(*aux_r));
230  return NULL;
231  }
232  aux_r->buf_size = MAX(cfg->buf_size, min_size);
233  sb->regions++;
234  sb->max_regions = MAX(sb->regions, sb->max_regions);
235  return aux_r;
236 }
237 
238 static inline int InitBuffer(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
239 {
240  sb->region.buf = CALLOC(cfg, 1, cfg->buf_size);
241  if (sb->region.buf == NULL) {
242  return sc_errno;
243  }
244  sb->region.buf_size = cfg->buf_size;
245  return SC_OK;
246 }
247 
248 StreamingBuffer *StreamingBufferInit(const StreamingBufferConfig *cfg)
249 {
250  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
251  if (sb == NULL) {
252  return NULL;
253  }
254 
255  sb->region.buf_size = cfg->buf_size;
256  sb->regions = sb->max_regions = 1;
257 
258  if (cfg->buf_size == 0) {
259  return sb;
260  }
261 
262  int r = InitBuffer(sb, cfg);
263  if (r != SC_OK) {
264  FREE(cfg, sb, sizeof(StreamingBuffer));
265  sc_errno = r;
266  return NULL;
267  }
268  return sb;
269 }
270 
271 void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
272 {
273  if (sb != NULL) {
274  SCLogDebug("sb->region.buf_size %u max %u", sb->region.buf_size, sb->buf_size_max);
275 
276  SBBFree(sb, cfg);
277  ListRegions(sb);
278  if (sb->region.buf != NULL) {
279  FREE(cfg, sb->region.buf, sb->region.buf_size);
280  sb->region.buf = NULL;
281  }
282 
283  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
284  StreamingBufferRegion *next = r->next;
285  FREE(cfg, r->buf, r->buf_size);
286  FREE(cfg, r, sizeof(*r));
287  r = next;
288  }
289  sb->region.next = NULL;
290  sb->regions = sb->max_regions = 1;
291  }
292 }
293 
294 void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
295 {
296  if (sb != NULL) {
297  StreamingBufferClear(sb, cfg);
298  FREE(cfg, sb, sizeof(StreamingBuffer));
299  }
300 }
301 
302 #ifdef DEBUG
303 static void SBBPrintList(StreamingBuffer *sb)
304 {
305  StreamingBufferBlock *sbb = NULL;
306  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
307  SCLogDebug("sbb: offset %" PRIu64 ", len %u", sbb->offset, sbb->len);
308  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
309  if (next) {
310  if ((sbb->offset + sbb->len) != next->offset) {
311  SCLogDebug("gap: offset %" PRIu64 ", len %" PRIu64, (sbb->offset + sbb->len),
312  next->offset - (sbb->offset + sbb->len));
313  }
314  }
315  }
316 }
317 #endif
318 
319 /* setup with gap between 2 blocks
320  *
321  * [block][gap][block]
322  **/
323 static int WARN_UNUSED SBBInit(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
324  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t data_len)
325 {
326  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
327  DEBUG_VALIDATE_BUG_ON(region->buf_offset > region->stream_offset + rel_offset);
328 
329  /* need to set up 2: existing data block and new data block */
330  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
331  if (sbb == NULL) {
332  return sc_errno;
333  }
334  sbb->offset = sb->region.stream_offset;
335  sbb->len = sb->region.buf_offset;
336  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
337  sb->head = sbb;
338  sb->sbb_size = sbb->len;
339 
340  StreamingBufferBlock *sbb2 = CALLOC(cfg, 1, sizeof(*sbb2));
341  if (sbb2 == NULL) {
342  return sc_errno;
343  }
344  sbb2->offset = region->stream_offset + rel_offset;
345  sbb2->len = data_len;
346  DEBUG_VALIDATE_BUG_ON(sbb2->offset < sbb->len);
347  SCLogDebug("sbb1 %" PRIu64 ", len %u, sbb2 %" PRIu64 ", len %u", sbb->offset, sbb->len,
348  sbb2->offset, sbb2->len);
349 
350  sb->sbb_size += sbb2->len;
351  if (SBB_RB_INSERT(&sb->sbb_tree, sbb2) != NULL) {
352  FREE(cfg, sbb2, sizeof(*sbb2));
353  return SC_EINVAL;
354  }
355 
356 #ifdef DEBUG
357  SBBPrintList(sb);
358 #endif
359  return SC_OK;
360 }
361 
362 /* setup with leading gap
363  *
364  * [gap][block]
365  **/
366 static int WARN_UNUSED SBBInitLeadingGap(
367  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset, uint32_t data_len)
368 {
369  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
370 
371  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
372  if (sbb == NULL) {
373  return sc_errno;
374  }
375  sbb->offset = offset;
376  sbb->len = data_len;
377 
378  sb->head = sbb;
379  sb->sbb_size = sbb->len;
380  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
381 
382  SCLogDebug("sbb %" PRIu64 ", len %u", sbb->offset, sbb->len);
383 #ifdef DEBUG
384  SBBPrintList(sb);
385 #endif
386  return SC_OK;
387 }
388 
389 static inline void ConsolidateFwd(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
390  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
391 {
392  uint64_t sa_re = sa->offset + sa->len;
393  StreamingBufferBlock *tr, *s = sa;
394  RB_FOREACH_FROM(tr, SBB, s) {
395  if (sa == tr)
396  continue;
397 
398  const uint64_t tr_re = tr->offset + tr->len;
399  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u re %" PRIu64, tr, tr->offset, tr->len, tr_re);
400 
401  if (sa_re < tr->offset) {
402  SCLogDebug("entirely before: %" PRIu64 " < %" PRIu64, sa_re, tr->offset);
403  break; // entirely before
404  }
405 
406  /* new block (sa) entirely eclipsed by in tree block (tr)
407  sa: [ ]
408  tr: [ ]
409  sa: [ ]
410  tr: [ ]
411  sa: [ ]
412  tr: [ ]
413  */
414  if (sa->offset >= tr->offset && sa_re <= tr_re) {
415  sb->sbb_size -= sa->len;
416  sa->len = tr->len;
417  sa->offset = tr->offset;
418  sa_re = sa->offset + sa->len;
419  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
420  tr->offset, tr->len);
421  SBB_RB_REMOVE(tree, tr);
422  FREE(cfg, tr, sizeof(StreamingBufferBlock));
423  /* new block (sa) entire eclipses in tree block (tr)
424  sa: [ ]
425  tr: [ ]
426  sa: [ ]
427  tr: [ ]
428  sa: [ ]
429  tr: [ ]
430  */
431  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
432  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
433  tr->offset, tr->len);
434  SBB_RB_REMOVE(tree, tr);
435  sb->sbb_size -= tr->len;
436  FREE(cfg, tr, sizeof(StreamingBufferBlock));
437 
438  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
439  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
440  region->buf_size);
441  if (sa->offset == region->stream_offset &&
442  sa_re > (region->stream_offset + region->buf_offset)) {
443  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
444  region->buf_offset = sa_re - region->stream_offset;
445  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
446  " bo %u sz %u BUF_OFFSET UPDATED",
447  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
448  region->buf_size);
449  }
450  /* new block (sa) extended by in tree block (tr)
451  sa: [ ]
452  tr: [ ]
453  sa: [ ]
454  tr: [ ]
455  */
456  } else if (sa->offset < tr->offset && // starts before
457  sa_re >= tr->offset && sa_re < tr_re) // ends inside
458  {
459  // merge. sb->sbb_size includes both so we need to adjust that too.
460  uint32_t combined_len = sa->len + tr->len;
461  sa->len = tr_re - sa->offset;
462  sa_re = sa->offset + sa->len;
463  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
464  SBB_RB_REMOVE(tree, tr);
465  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
466  FREE(cfg, tr, sizeof(StreamingBufferBlock));
467  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u RESULT", sa, sa->offset, sa->len);
468  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
469  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
470  region->buf_size);
471  if (sa->offset == region->stream_offset &&
472  sa_re > (region->stream_offset + region->buf_offset)) {
473  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
474  region->buf_offset = sa_re - region->stream_offset;
475  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
476  " bo %u sz %u BUF_OFFSET UPDATED",
477  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
478  region->buf_size);
479  }
480  }
481  }
482 }
483 
484 static inline void ConsolidateBackward(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
485  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
486 {
487  uint64_t sa_re = sa->offset + sa->len;
488  StreamingBufferBlock *tr, *s = sa;
489  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
490  if (sa == tr)
491  continue;
492  const uint64_t tr_re = tr->offset + tr->len;
493  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u", tr, tr->offset, tr->len);
494 
495  if (sa->offset > tr_re)
496  break; // entirely after
497 
498  /* new block (sa) entirely eclipsed by in tree block (tr)
499  sa: [ ]
500  tr: [ ]
501  sa: [ ]
502  tr: [ ]
503  sa: [ ]
504  tr: [ ]
505  */
506  if (sa->offset >= tr->offset && sa_re <= tr_re) {
507  sb->sbb_size -= sa->len; // sa entirely eclipsed so remove double accounting
508  sa->len = tr->len;
509  sa->offset = tr->offset;
510  sa_re = sa->offset + sa->len;
511  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
512  tr->offset, tr->len);
513  if (sb->head == tr)
514  sb->head = sa;
515  SBB_RB_REMOVE(tree, tr);
516  FREE(cfg, tr, sizeof(StreamingBufferBlock));
517  /* new block (sa) entire eclipses in tree block (tr)
518  sa: [ ]
519  tr: [ ]
520  sa: [ ]
521  tr: [ ]
522  sa: [ ]
523  tr: [ ]
524  */
525  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
526  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
527  tr->offset, tr->len);
528  if (sb->head == tr)
529  sb->head = sa;
530  SBB_RB_REMOVE(tree, tr);
531  sb->sbb_size -= tr->len; // tr entirely eclipsed so remove double accounting
532  FREE(cfg, tr, sizeof(StreamingBufferBlock));
533 
534  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
535  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
536  region->buf_size);
537 
538  if (sa->offset == region->stream_offset &&
539  sa_re > (region->stream_offset + region->buf_offset)) {
540  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
541  region->buf_offset = sa_re - region->stream_offset;
542  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
543  " bo %u sz %u BUF_OFFSET UPDATED",
544  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
545  region->buf_size);
546  }
547 
548  /* new block (sa) extends in tree block (tr)
549  sa: [ ]
550  tr: [ ]
551  sa: [ ]
552  tr: [ ]
553  */
554  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
555  // merge. sb->sbb_size includes both so we need to adjust that too.
556  uint32_t combined_len = sa->len + tr->len;
557  sa->len = sa_re - tr->offset;
558  sa->offset = tr->offset;
559  sa_re = sa->offset + sa->len;
560  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
561  if (sb->head == tr)
562  sb->head = sa;
563  SBB_RB_REMOVE(tree, tr);
564  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
565  FREE(cfg, tr, sizeof(StreamingBufferBlock));
566 
567  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
568  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
569  region->buf_size);
570  if (sa->offset == region->stream_offset &&
571  sa_re > (region->stream_offset + region->buf_offset)) {
572  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
573  region->buf_offset = sa_re - region->stream_offset;
574  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
575  " bo %u sz %u BUF_OFFSET UPDATED",
576  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
577  region->buf_size);
578  }
579  }
580  }
581 }
582 
583 /** \internal
584  * \param region the region that holds the new data
585  */
586 static int SBBUpdate(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
587  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t len)
588 {
589  struct SBB *tree = &sb->sbb_tree;
590  SCLogDebug("* inserting: %u/%u", rel_offset, len);
591 
592  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
593  if (sbb == NULL) {
594  return sc_errno;
595  }
596  sbb->offset = region->stream_offset + rel_offset;
597  sbb->len = len;
598 
599  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
600  if (res) {
601  // exact overlap
602  SCLogDebug("* insert failed: exact match in tree with %p %" PRIu64 "/%u", res, res->offset,
603  res->len);
604  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
605  return SC_OK;
606  }
607  sb->sbb_size += len; // may adjust based on consolidation below
608 
609  /* handle backwards and forwards overlaps */
610  if (SBB_RB_PREV(sbb) == NULL) {
611  sb->head = sbb;
612  } else {
613  ConsolidateBackward(sb, cfg, region, tree, sbb);
614  }
615  ConsolidateFwd(sb, cfg, region, tree, sbb);
616 #ifdef DEBUG
617  SBBPrintList(sb);
618 #endif
619  if (sbb->offset == region->stream_offset) {
620  SCLogDebug("insert at region head");
621  region->buf_offset = sbb->len;
622  }
623  return SC_OK;
624 }
625 
626 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
627 {
628  StreamingBufferBlock *sbb = NULL, *safe = NULL;
629  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
630  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
631  sb->sbb_size -= sbb->len;
632  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
633  }
634  sb->head = NULL;
635 }
636 
637 static void SBBPrune(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
638 {
639  SCLogDebug("pruning %p to %" PRIu64, sb, sb->region.stream_offset);
640  StreamingBufferBlock *sbb = NULL, *safe = NULL;
641  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
642  /* completely beyond window, we're done */
643  if (sbb->offset >= sb->region.stream_offset) {
644  sb->head = sbb;
645  if (sbb->offset == sb->region.stream_offset) {
646  SCLogDebug("set buf_offset?");
647  if (sbb->offset == sb->region.stream_offset) {
648  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
649  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
650  sb->region.buf_offset = sbb->len;
651  }
652  }
653  break;
654  }
655 
656  /* partly before, partly beyond. Adjust */
657  if (sbb->offset < sb->region.stream_offset &&
658  sbb->offset + sbb->len > sb->region.stream_offset) {
659  uint32_t shrink_by = sb->region.stream_offset - sbb->offset;
660  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
661  if (sbb->len >= shrink_by) {
662  sbb->len -= shrink_by;
663  sbb->offset += shrink_by;
664  sb->sbb_size -= shrink_by;
665  SCLogDebug("shrunk by %u", shrink_by);
666  DEBUG_VALIDATE_BUG_ON(sbb->offset != sb->region.stream_offset);
667  }
668  sb->head = sbb;
669  if (sbb->offset == sb->region.stream_offset) {
670  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
671  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
672  sb->region.buf_offset = sbb->len;
673  }
674  break;
675  }
676 
677  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
678  /* either we set it again for the next sbb, or there isn't any */
679  sb->head = NULL;
680  sb->sbb_size -= sbb->len;
681  SCLogDebug("sb %p removed %p %" PRIu64 ", %u", sb, sbb, sbb->offset, sbb->len);
682  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
683  }
684 #ifdef DEBUG
685  SBBPrintList(sb);
686 #endif
687 }
688 
689 static inline uint32_t ToNextMultipleOf(const uint32_t in, const uint32_t m)
690 {
691  uint32_t r = in;
692  if (m > 0) {
693  const uint32_t x = in % m;
694  if (x != 0) {
695  r = (in - x) + m;
696  }
697  }
698  return r;
699 }
700 
701 static thread_local bool g2s_warn_once = false;
702 
703 static inline int WARN_UNUSED GrowRegionToSize(StreamingBuffer *sb,
704  const StreamingBufferConfig *cfg, StreamingBufferRegion *region, const uint32_t size)
705 {
706  DEBUG_VALIDATE_BUG_ON(region->buf_size > BIT_U32(30));
707  if (size > BIT_U32(30)) { // 1GiB
708  if (!g2s_warn_once) {
709  SCLogWarning("StreamingBuffer::GrowRegionToSize() tried to alloc %u bytes, exceeds "
710  "limit of %" PRIu32,
711  size, BIT_U32(30));
712  g2s_warn_once = true;
713  }
714  return SC_ELIMIT;
715  }
716 
717  /* try to grow in multiples of cfg->buf_size */
718  const uint32_t grow = ToNextMultipleOf(size, cfg->buf_size);
719  SCLogDebug("grow %u", grow);
720  if (grow <= region->buf_size) {
721  // do not try to shrink, and do not memset with diff having unsigned underflow
722  return SC_OK;
723  }
724 
725  void *ptr = REALLOC(cfg, region->buf, region->buf_size, grow);
726  if (ptr == NULL) {
727  if (sc_errno == SC_OK)
728  sc_errno = SC_ENOMEM;
729  return sc_errno;
730  }
731  /* for safe printing and general caution, lets memset the
732  * new data to 0 */
733  size_t diff = grow - region->buf_size;
734  void *new_mem = ((char *)ptr) + region->buf_size;
735  memset(new_mem, 0, diff);
736 
737  region->buf = ptr;
738  region->buf_size = grow;
739  SCLogDebug("grown buffer to %u", grow);
740 #ifdef DEBUG
741  if (region->buf_size > sb->buf_size_max) {
742  sb->buf_size_max = region->buf_size;
743  }
744 #endif
745  return SC_OK;
746 }
747 
748 static int WARN_UNUSED GrowToSize(
749  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint32_t size)
750 {
751  return GrowRegionToSize(sb, cfg, &sb->region, size);
752 }
753 
754 static inline bool RegionBeforeOffset(const StreamingBufferRegion *r, const uint64_t o)
755 {
756  return (r->stream_offset + r->buf_size <= o);
757 }
758 
759 static inline bool RegionContainsOffset(const StreamingBufferRegion *r, const uint64_t o)
760 {
761  return (o >= r->stream_offset && (r->stream_offset + r->buf_size) >= o);
762 }
763 
764 /** \internal
765  * \brief slide to offset for regions
766  *
767  *
768  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
769  * ^
770  * - main reset to 0
771  *
772  *
773  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
774  * ^
775  * - main shift
776  *
777  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
778  * ^
779  * - main reset
780  * - move aux into main
781  * - free aux
782  * - shift
783  *
784  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
785  * ^
786  * - main reset
787  * - move aux into main
788  * - free aux
789  * - no shift
790  */
791 static inline void StreamingBufferSlideToOffsetWithRegions(
792  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint64_t slide_offset)
793 {
794  ListRegions(sb);
795  DEBUG_VALIDATE_BUG_ON(slide_offset == sb->region.stream_offset);
796 
797  SCLogDebug("slide_offset %" PRIu64, slide_offset);
798  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
799  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
800 
801  StreamingBufferRegion *to_shift = NULL;
802  const bool main_is_oow = RegionBeforeOffset(&sb->region, slide_offset);
803  if (main_is_oow) {
804  SCLogDebug("main_is_oow");
805  if (sb->region.buf != NULL) {
806  SCLogDebug("clearing main");
807  FREE(cfg, sb->region.buf, sb->region.buf_size);
808  sb->region.buf = NULL;
809  sb->region.buf_size = 0;
810  sb->region.buf_offset = 0;
811  sb->region.stream_offset = slide_offset;
812  } else {
813  sb->region.stream_offset = slide_offset;
814  }
815 
816  /* remove regions that are out of window & select the region to
817  * become the new main */
818  StreamingBufferRegion *prev = &sb->region;
819  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
820  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64, r, r->stream_offset,
821  r->stream_offset + r->buf_offset);
822  StreamingBufferRegion *next = r->next;
823  if (RegionBeforeOffset(r, slide_offset)) {
824  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> before", r, r->stream_offset,
825  r->stream_offset + r->buf_offset);
826  DEBUG_VALIDATE_BUG_ON(r == &sb->region);
827  prev->next = next;
828 
829  FREE(cfg, r->buf, r->buf_size);
830  FREE(cfg, r, sizeof(*r));
831  sb->regions--;
832  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
833  } else if (RegionContainsOffset(r, slide_offset)) {
834  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> within", r, r->stream_offset,
835  r->stream_offset + r->buf_offset);
836  /* remove from list, we will xfer contents to main below */
837  prev->next = next;
838  to_shift = r;
839  break;
840  } else {
841  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> post", r, r->stream_offset,
842  r->stream_offset + r->buf_offset);
843  /* implied beyond slide offset */
844  DEBUG_VALIDATE_BUG_ON(r->stream_offset < slide_offset);
845  break;
846  }
847  r = next;
848  }
849  SCLogDebug("to_shift %p", to_shift);
850 
851  // this region is main, or will xfer its buffer to main
852  if (to_shift && to_shift != &sb->region) {
853  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", to_shift->stream_offset,
854  to_shift->buf, to_shift->buf_size, to_shift->buf_offset);
855  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL);
856 
857  sb->region.buf = to_shift->buf;
858  sb->region.stream_offset = to_shift->stream_offset;
859  sb->region.buf_offset = to_shift->buf_offset;
860  sb->region.buf_size = to_shift->buf_size;
861  sb->region.next = to_shift->next;
862 
863  BUG_ON(to_shift == &sb->region);
864  FREE(cfg, to_shift, sizeof(*to_shift));
865  to_shift = &sb->region;
866  sb->regions--;
867  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
868  }
869 
870  } else {
871  to_shift = &sb->region;
872  SCLogDebug("shift start region %p", to_shift);
873  }
874 
875  // this region is main, or will xfer its buffer to main
876  if (to_shift) {
877  // Do the shift. If new region is exactly at the slide offset we can skip this.
878  DEBUG_VALIDATE_BUG_ON(to_shift->stream_offset > slide_offset);
879  const uint32_t s = slide_offset - to_shift->stream_offset;
880  if (s > 0) {
881  const uint32_t new_data_size = to_shift->buf_size - s;
882  uint32_t new_mem_size = ToNextMultipleOf(new_data_size, cfg->buf_size);
883 
884  /* see if after the slide we'd overlap with the next region. If so, we need
885  * to consolidate them into one. Error handling is a bit peculiar. We need
886  * to grow a region, move data from another region into it, then free the
887  * other region. This could fail if we're close to the memcap. In this case
888  * we relax our rounding up logic so we only shrink and don't merge the 2
889  * regions after all. */
890  if (to_shift->next && slide_offset + new_mem_size >= to_shift->next->stream_offset) {
891  StreamingBufferRegion *start = to_shift;
892  StreamingBufferRegion *next = start->next;
893  const uint64_t next_re = next->stream_offset + next->buf_size;
894  const uint32_t mem_size = ToNextMultipleOf(next_re - slide_offset, cfg->buf_size);
895 
896  /* using next as the new main */
897  if (start->buf_size < next->buf_size) {
898  SCLogDebug("replace main with the next bigger region");
899 
900  const uint32_t next_data_offset = next->stream_offset - slide_offset;
901  const uint32_t prev_buf_size = next->buf_size;
902  const uint32_t start_data_offset = slide_offset - start->stream_offset;
903  DEBUG_VALIDATE_BUG_ON(start_data_offset > start->buf_size);
904  if (start_data_offset > start->buf_size) {
905  new_mem_size = new_data_size;
906  goto just_main;
907  }
908  /* expand "next" to include relevant part of "start" */
909  if (GrowRegionToSize(sb, cfg, next, mem_size) != 0) {
910  new_mem_size = new_data_size;
911  goto just_main;
912  }
913  SCLogDebug("region now sized %u", mem_size);
914 
915  // slide "next":
916  // pre-grow: [nextnextnext]
917  // post-grow [nextnextnextXXX]
918  // post-move [XXXnextnextnext]
919  memmove(next->buf + next_data_offset, next->buf, prev_buf_size);
920 
921  // move portion of "start" into "next"
922  //
923  // start: [ooooookkkkk] (o: old, k: keep)
924  // pre-next [xxxxxnextnextnext]
925  // post-next [kkkkknextnextnext]
926  const uint32_t start_data_size = start->buf_size - start_data_offset;
927  memcpy(next->buf, start->buf + start_data_offset, start_data_size);
928 
929  // free "start"s buffer, we will use the one from "next"
930  FREE(cfg, start->buf, start->buf_size);
931 
932  // update "main" to use "next"
933  start->stream_offset = slide_offset;
934  start->buf = next->buf;
935  start->buf_size = next->buf_size;
936  start->next = next->next;
937 
938  // free "next"
939  FREE(cfg, next, sizeof(*next));
940  sb->regions--;
941  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
942  goto done;
943  } else {
944  /* using "main", expand to include "next" */
945  if (mem_size > start->buf_size) {
946  // Check that start->buf_size is actually not big enough
947  // As mem_size computation and earlier checks do not make it clear.
948  if (GrowRegionToSize(sb, cfg, start, mem_size) != 0) {
949  new_mem_size = new_data_size;
950  goto just_main;
951  }
952  }
953  SCLogDebug("start->buf now size %u", mem_size);
954 
955  // slide "start"
956  // pre: [xxxxxxxAAA]
957  // post: [AAAxxxxxxx]
958  SCLogDebug("s %u new_data_size %u", s, new_data_size);
959  memmove(start->buf, start->buf + s, new_data_size);
960 
961  // copy in "next"
962  // pre: [AAAxxxxxxx]
963  // [BBBBBBB]
964  // post: [AAABBBBBBB]
965  SCLogDebug("copy next->buf %p/%u to start->buf offset %u", next->buf,
966  next->buf_size, new_data_size);
967  memcpy(start->buf + new_data_size, next->buf, next->buf_size);
968 
969  start->stream_offset = slide_offset;
970  start->next = next->next;
971 
972  // free "next"
973  FREE(cfg, next->buf, next->buf_size);
974  FREE(cfg, next, sizeof(*next));
975  sb->regions--;
976  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
977  goto done;
978  }
979  }
980 
981  just_main:
982  SCLogDebug("s %u new_data_size %u", s, new_data_size);
983  memmove(to_shift->buf, to_shift->buf + s, new_data_size);
984  /* shrink memory region. If this fails we keep the old */
985  void *ptr = REALLOC(cfg, to_shift->buf, to_shift->buf_size, new_mem_size);
986  if (ptr != NULL) {
987  to_shift->buf = ptr;
988  to_shift->buf_size = new_mem_size;
989  SCLogDebug("new buf_size %u", to_shift->buf_size);
990  }
991  if (s < to_shift->buf_offset)
992  to_shift->buf_offset -= s;
993  else
994  to_shift->buf_offset = 0;
995  to_shift->stream_offset = slide_offset;
996  }
997  }
998 
999 done:
1000  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
1001  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
1002  SCLogDebug("end of slide");
1003 }
1004 
1005 /**
1006  * \brief slide to absolute offset
1007  * \todo if sliding beyond window, we could perhaps reset?
1008  */
1009 void StreamingBufferSlideToOffset(
1010  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
1011 {
1012  SCLogDebug("sliding to offset %" PRIu64, offset);
1013  ListRegions(sb);
1014 #ifdef DEBUG
1015  SBBPrintList(sb);
1016 #endif
1017 
1018  if (sb->region.next) {
1019  StreamingBufferSlideToOffsetWithRegions(sb, cfg, offset);
1020  SBBPrune(sb, cfg);
1021  SCLogDebug("post SBBPrune");
1022  ListRegions(sb);
1023 #ifdef DEBUG
1024  SBBPrintList(sb);
1025 #endif
1026  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL && sb->region.buf_size == 0);
1027  DEBUG_VALIDATE_BUG_ON(sb->region.buf_offset > sb->region.buf_size);
1028  DEBUG_VALIDATE_BUG_ON(offset > sb->region.stream_offset);
1029  DEBUG_VALIDATE_BUG_ON(sb->head && sb->head->offset == sb->region.stream_offset &&
1030  sb->head->len > sb->region.buf_offset);
1031  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1032  return;
1033  }
1034 
1035  if (offset > sb->region.stream_offset) {
1036  const uint32_t slide = offset - sb->region.stream_offset;
1037  if (sb->head != NULL) {
1038  /* have sbb's, so can't rely on buf_offset for the slide */
1039  if (slide < sb->region.buf_size) {
1040  const uint32_t size = sb->region.buf_size - slide;
1041  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1042  size);
1043  memmove(sb->region.buf, sb->region.buf + slide, size);
1044  if (sb->region.buf_offset > slide) {
1045  sb->region.buf_offset -= slide;
1046  } else {
1047  sb->region.buf_offset = 0;
1048  }
1049  } else {
1050  sb->region.buf_offset = 0;
1051  }
1052  sb->region.stream_offset = offset;
1053  } else {
1054  /* no sbb's, so we can use buf_offset */
1055  if (offset <= sb->region.stream_offset + sb->region.buf_offset) {
1056  const uint32_t size = sb->region.buf_offset - slide;
1057  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1058  size);
1059  memmove(sb->region.buf, sb->region.buf + slide, size);
1060  sb->region.stream_offset = offset;
1061  sb->region.buf_offset = size;
1062  } else {
1063  /* moved past all data */
1064  sb->region.stream_offset = offset;
1065  sb->region.buf_offset = 0;
1066  }
1067  }
1068  SBBPrune(sb, cfg);
1069  }
1070 #ifdef DEBUG
1071  SBBPrintList(sb);
1072 #endif
1073  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1074 }
1075 
1076 static int DataFits(const StreamingBuffer *sb, const uint32_t len)
1077 {
1078  uint64_t buf_offset64 = sb->region.buf_offset;
1079  uint64_t len64 = len;
1080  if (len64 + buf_offset64 > UINT32_MAX) {
1081  return -1;
1082  }
1083  return sb->region.buf_offset + len <= sb->region.buf_size;
1084 }
1085 
1086 int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1087  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
1088 {
1089  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1090  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1091 
1092  if (sb->region.buf == NULL) {
1093  if (InitBuffer(sb, cfg) == -1)
1094  return -1;
1095  }
1096 
1097  int r = DataFits(sb, data_len);
1098  if (r < 0) {
1099  DEBUG_VALIDATE_BUG_ON(1);
1100  return -1;
1101  } else if (r == 0) {
1102  if (sb->region.buf_size == 0) {
1103  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1104  return -1;
1105  } else {
1106  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1107  return -1;
1108  }
1109  }
1110  DEBUG_VALIDATE_BUG_ON(DataFits(sb, data_len) != 1);
1111  if (DataFits(sb, data_len) != 1)
1112  return -1;
1113 
1114  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1115  seg->stream_offset = sb->region.stream_offset + sb->region.buf_offset;
1116  seg->segment_len = data_len;
1117  uint32_t rel_offset = sb->region.buf_offset;
1118  sb->region.buf_offset += data_len;
1119 
1120  if (!RB_EMPTY(&sb->sbb_tree)) {
1121  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1122  } else {
1123  return 0;
1124  }
1125 }
1126 
1127 /**
1128  * \brief add data w/o tracking a segment
1129  */
1130 int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1131  const uint8_t *data, uint32_t data_len)
1132 {
1133  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1134 
1135  if (sb->region.buf == NULL) {
1136  if (InitBuffer(sb, cfg) == -1)
1137  return -1;
1138  }
1139 
1140  int r = DataFits(sb, data_len);
1141  if (r < 0) {
1142  DEBUG_VALIDATE_BUG_ON(1);
1143  return -1;
1144  } else if (r == 0) {
1145  if (sb->region.buf_size == 0) {
1146  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1147  return -1;
1148  } else {
1149  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1150  return -1;
1151  }
1152  }
1153  DEBUG_VALIDATE_BUG_ON(DataFits(sb, data_len) != 1);
1154 
1155  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1156  uint32_t rel_offset = sb->region.buf_offset;
1157  sb->region.buf_offset += data_len;
1158 
1159  if (!RB_EMPTY(&sb->sbb_tree)) {
1160  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1161  } else {
1162  return 0;
1163  }
1164 }
1165 
1166 static int DataFitsAtOffset(
1167  const StreamingBufferRegion *region, const uint32_t len, const uint32_t offset)
1168 {
1169  const uint64_t offset64 = offset;
1170  const uint64_t len64 = len;
1171  if (offset64 + len64 > UINT32_MAX)
1172  return -1;
1173  return (offset + len <= region->buf_size);
1174 }
1175 
1176 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1177 static void Validate(const StreamingBuffer *sb)
1178 {
1179  bool bail = false;
1180  uint32_t cnt = 0;
1181  for (const StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1182  cnt++;
1183  if (r->next) {
1184  bail |= ((r->stream_offset + r->buf_size) > r->next->stream_offset);
1185  }
1186 
1187  bail |= (r->buf != NULL && r->buf_size == 0);
1188  bail |= (r->buf_offset > r->buf_size);
1189  }
1190  // leading gap, so buf_offset should be 0?
1191  if (sb->head && sb->head->offset > sb->region.stream_offset) {
1192  SCLogDebug("leading gap of size %" PRIu64, sb->head->offset - sb->region.stream_offset);
1193  BUG_ON(sb->region.buf_offset != 0);
1194  }
1195 
1196  if (sb->head && sb->head->offset == sb->region.stream_offset) {
1197  BUG_ON(sb->head->len > sb->region.buf_offset);
1198  BUG_ON(sb->head->len < sb->region.buf_offset);
1199  }
1200  BUG_ON(sb->regions != cnt);
1201  BUG_ON(bail);
1202 }
1203 #endif
1204 
1205 static void ListRegions(StreamingBuffer *sb)
1206 {
1207  if (sb->region.buf == NULL && sb->region.buf_offset == 0 && sb->region.next == NULL)
1208  return;
1209 #if defined(DEBUG) && DUMP_REGIONS == 1
1210  uint32_t cnt = 0;
1211  for (StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1212  cnt++;
1213  char gap[64] = "";
1214  if (r->next) {
1215  snprintf(gap, sizeof(gap), "[ gap:%" PRIu64 " ]",
1216  r->next->stream_offset - (r->stream_offset + r->buf_size));
1217  }
1218 
1219  printf("[ %s offset:%" PRIu64 " size:%u offset:%u ]%s", r == &sb->region ? "main" : "aux",
1220  r->stream_offset, r->buf_size, r->buf_offset, gap);
1221  }
1222  printf("(max %u, cnt %u, sb->regions %u)\n", sb->max_regions, cnt, sb->regions);
1223  bool at_least_one = false;
1224  uint64_t last_re = sb->region.stream_offset;
1225  StreamingBufferBlock *sbb = NULL;
1226  RB_FOREACH(sbb, SBB, &sb->sbb_tree)
1227  {
1228  if (last_re != sbb->offset) {
1229  printf("[ gap:%" PRIu64 " ]", sbb->offset - last_re);
1230  }
1231  printf("[ sbb offset:%" PRIu64 " len:%u ]", sbb->offset, sbb->len);
1232  at_least_one = true;
1233  last_re = sbb->offset + sbb->len;
1234  }
1235  if (at_least_one)
1236  printf("\n");
1237 #endif
1238 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1239  StreamingBufferBlock *sbb2 = NULL;
1240  RB_FOREACH(sbb2, SBB, &sb->sbb_tree)
1241  {
1242  const uint8_t *_data = NULL;
1243  uint32_t _data_len = 0;
1244  (void)StreamingBufferSBBGetData(sb, sbb2, &_data, &_data_len);
1245  }
1246  Validate(sb);
1247 #endif
1248 }
1249 
1250 /** \internal
1251  * \brief process insert by consolidating the affected regions into one
1252  * \note sets sc_errno
1253  */
1254 static StreamingBufferRegion *BufferInsertAtRegionConsolidate(StreamingBuffer *sb,
1255  const StreamingBufferConfig *cfg, StreamingBufferRegion *dst,
1256  StreamingBufferRegion *src_start, StreamingBufferRegion *src_end,
1257  const uint64_t data_offset, const uint32_t data_len, StreamingBufferRegion *prev,
1258  uint32_t dst_buf_size)
1259 {
1260  int retval;
1261 #ifdef DEBUG
1262  const uint64_t data_re = data_offset + data_len;
1263  SCLogDebug("sb %p dst %p src_start %p src_end %p data_offset %" PRIu64
1264  "/data_len %u/data_re %" PRIu64,
1265  sb, dst, src_start, src_end, data_offset, data_len, data_re);
1266 #endif
1267 
1268  // 1. determine size and offset for dst.
1269  const uint64_t dst_offset = MIN(src_start->stream_offset, data_offset);
1270  DEBUG_VALIDATE_BUG_ON(dst_offset < sb->region.stream_offset);
1271  const uint32_t dst_size = dst_buf_size;
1272  SCLogDebug("dst_size %u", dst_size);
1273 
1274  // 2. resize dst
1275  const uint32_t old_size = dst->buf_size;
1276  const uint32_t dst_copy_offset = dst->stream_offset - dst_offset;
1277 #ifdef DEBUG
1278  const uint32_t old_offset = dst->buf_offset;
1279  SCLogDebug("old_size %u, old_offset %u, dst_copy_offset %u", old_size, old_offset,
1280  dst_copy_offset);
1281 #endif
1282  if ((retval = GrowRegionToSize(sb, cfg, dst, dst_size)) != SC_OK) {
1283  sc_errno = retval;
1284  return NULL;
1285  }
1286  SCLogDebug("resized to %u -> %u", dst_size, dst->buf_size);
1287  /* validate that the size is exactly what we asked for */
1288  DEBUG_VALIDATE_BUG_ON(dst_size != dst->buf_size);
1289  if (dst_copy_offset != 0)
1290  memmove(dst->buf + dst_copy_offset, dst->buf, old_size);
1291  if (dst_offset != dst->stream_offset) {
1292  dst->stream_offset = dst_offset;
1293  // buf_offset no longer valid, reset.
1294  dst->buf_offset = 0;
1295  }
1296 
1297  uint32_t new_offset = src_start->buf_offset;
1298  if (data_offset == src_start->stream_offset + src_start->buf_offset) {
1299  new_offset += data_len;
1300  }
1301 
1302  bool start_is_main = false;
1303  if (src_start == &sb->region) {
1304  DEBUG_VALIDATE_BUG_ON(src_start->stream_offset != dst_offset);
1305 
1306  start_is_main = true;
1307  SCLogDebug("src_start is main region");
1308  if (src_start != dst)
1309  memcpy(dst->buf, src_start->buf, src_start->buf_offset);
1310  if (src_start == src_end) {
1311  SCLogDebug("src_start == src_end == main, we're done");
1312  DEBUG_VALIDATE_BUG_ON(src_start != dst);
1313  return src_start;
1314  }
1315  prev = src_start;
1316  src_start = src_start->next; // skip in the loop below
1317  }
1318 
1319  // 3. copy all regions from src_start to dst_start into the new region
1320  for (StreamingBufferRegion *r = src_start; r != NULL;) {
1321  SCLogDebug("r %p %" PRIu64 ", offset %u, len %u, %s, last %s", r, r->stream_offset,
1322  r->buf_offset, r->buf_size, r == &sb->region ? "main" : "aux",
1323  BOOL2STR(r == src_end));
1324  // skip dst
1325  if (r == dst) {
1326  SCLogDebug("skipping r %p as it is 'dst'", r);
1327  if (r == src_end)
1328  break;
1329  prev = r;
1330  r = r->next;
1331  continue;
1332  }
1333  const uint32_t target_offset = r->stream_offset - dst_offset;
1334  SCLogDebug("r %p: target_offset %u", r, target_offset);
1335  DEBUG_VALIDATE_BUG_ON(target_offset > dst->buf_size);
1336  DEBUG_VALIDATE_BUG_ON(target_offset + r->buf_size > dst->buf_size);
1337  memcpy(dst->buf + target_offset, r->buf, r->buf_size);
1338 
1339  StreamingBufferRegion *next = r->next;
1340  FREE(cfg, r->buf, r->buf_size);
1341  FREE(cfg, r, sizeof(*r));
1342  sb->regions--;
1343  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1344 
1345  DEBUG_VALIDATE_BUG_ON(prev == NULL && src_start != &sb->region);
1346  if (prev != NULL) {
1347  SCLogDebug("setting prev %p next to %p (was %p)", prev, next, prev->next);
1348  prev->next = next;
1349  } else {
1350  SCLogDebug("no prev yet");
1351  }
1352 
1353  if (r == src_end)
1354  break;
1355  r = next;
1356  }
1357 
1358  /* special handling of main region being the start, but not the
1359  * region we expand. In this case we'll have main and dst. We will
1360  * move the buffer from dst into main and free dst. */
1361  if (start_is_main && dst != &sb->region) {
1362  DEBUG_VALIDATE_BUG_ON(sb->region.next != dst);
1363  SCLogDebug("start_is_main && dst != main region");
1364  FREE(cfg, sb->region.buf, sb->region.buf_size);
1365  sb->region.buf = dst->buf;
1366  sb->region.buf_size = dst->buf_size;
1367  sb->region.buf_offset = new_offset;
1368  SCLogDebug("sb->region.buf_offset set to %u", sb->region.buf_offset);
1369  sb->region.next = dst->next;
1370  FREE(cfg, dst, sizeof(*dst));
1371  dst = &sb->region;
1372  sb->regions--;
1373  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1374  } else {
1375  SCLogDebug("dst: %p next %p", dst, dst->next);
1376  }
1377 
1378  SCLogDebug("returning dst %p stream_offset %" PRIu64 " buf_offset %u buf_size %u", dst,
1379  dst->stream_offset, dst->buf_offset, dst->buf_size);
1380  return dst;
1381 }
1382 
1383 /** \note sets sc_errno */
1384 static StreamingBufferRegion *BufferInsertAtRegionDo(StreamingBuffer *sb,
1385  const StreamingBufferConfig *cfg, const uint64_t offset, const uint32_t len)
1386 {
1387  SCLogDebug("offset %" PRIu64 ", len %u", offset, len);
1388  StreamingBufferRegion *start_prev = NULL;
1389  StreamingBufferRegion *start =
1390  FindFirstRegionForOffset(cfg, &sb->region, offset, len, &start_prev);
1391  if (start) {
1392  const uint64_t insert_re = offset + len;
1393  const uint64_t insert_start_offset = MIN(start->stream_offset, offset);
1394  uint64_t insert_adjusted_re = insert_re;
1395 
1396  SCLogDebug("start region %p/%" PRIu64 "/%u", start, start->stream_offset, start->buf_size);
1397  StreamingBufferRegion *big = FindLargestRegionForOffset(cfg, start, offset, len);
1398  DEBUG_VALIDATE_BUG_ON(big == NULL);
1399  if (big == NULL) {
1400  sc_errno = SC_EINVAL;
1401  return NULL;
1402  }
1403  SCLogDebug("big region %p/%" PRIu64 "/%u", big, big->stream_offset, big->buf_size);
1404  StreamingBufferRegion *end = FindRightEdge(cfg, big, offset, len);
1405  DEBUG_VALIDATE_BUG_ON(end == NULL);
1406  if (end == NULL) {
1407  sc_errno = SC_EINVAL;
1408  return NULL;
1409  }
1410  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1411 
1412  uint32_t new_buf_size =
1413  ToNextMultipleOf(insert_adjusted_re - insert_start_offset, cfg->buf_size);
1414  SCLogDebug("new_buf_size %u", new_buf_size);
1415 
1416  /* see if our new buf size + cfg->buf_size margin leads to an overlap with the next region.
1417  * If so, include that in the consolidation. */
1418  if (end->next != NULL && new_buf_size + insert_start_offset > end->next->stream_offset) {
1419  SCLogDebug("adjusted end from %p to %p", end, end->next);
1420  end = end->next;
1421  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1422  new_buf_size =
1423  ToNextMultipleOf(insert_adjusted_re - insert_start_offset, cfg->buf_size);
1424  SCLogDebug("new_buf_size %u", new_buf_size);
1425  }
1426 
1427  SCLogDebug("end region %p/%" PRIu64 "/%u", end, end->stream_offset, end->buf_size);
1428  /* sets sc_errno */
1429  StreamingBufferRegion *ret = BufferInsertAtRegionConsolidate(
1430  sb, cfg, big, start, end, offset, len, start_prev, new_buf_size);
1431  return ret;
1432  } else {
1433  /* if there was no region we can use we add a new region and insert it */
1434  StreamingBufferRegion *append = &sb->region;
1435  for (StreamingBufferRegion *r = append; r != NULL; r = r->next) {
1436  if (r->stream_offset > offset) {
1437  break;
1438  } else {
1439  append = r;
1440  }
1441  }
1442 
1443  SCLogDebug("no matching region found, append to %p (%s)", append,
1444  append == &sb->region ? "main" : "aux");
1445  /* sets sc_errno */
1446  StreamingBufferRegion *add = InitBufferRegion(sb, cfg, len);
1447  if (add == NULL)
1448  return NULL;
1449  add->stream_offset = offset;
1450  add->next = append->next;
1451  append->next = add;
1452  SCLogDebug("new region %p offset %" PRIu64, add, add->stream_offset);
1453  return add;
1454  }
1455 }
1456 
1457 /** \internal
1458  * \brief return the region to put the new data in
1459  *
1460  * Will find an existing region, expand it if needed. If no existing region exists or is
1461  * a good fit, it will try to set up a new region. If the region then overlaps or gets
1462  * too close to the next, merge them.
1463  *
1464  * \note sets sc_errno
1465  */
1466 static StreamingBufferRegion *BufferInsertAtRegion(StreamingBuffer *sb,
1467  const StreamingBufferConfig *cfg, const uint32_t data_len, const uint64_t data_offset)
1468 {
1469  SCLogDebug("data_offset %" PRIu64 ", data_len %u, re %" PRIu64, data_offset, data_len,
1470  data_offset + data_len);
1471  ListRegions(sb);
1472 
1473  if (RegionsIntersect(cfg, &sb->region, data_offset, data_offset + data_len)) {
1474  SCLogDebug("data_offset %" PRIu64 ", data_len %u intersects with main region (next %p)",
1475  data_offset, data_len, sb->region.next);
1476  if (sb->region.next == NULL ||
1477  !RegionsIntersect(cfg, sb->region.next, data_offset, data_offset + data_len)) {
1478  SCLogDebug(
1479  "data_offset %" PRIu64
1480  ", data_len %u intersects with main region, no next or way before next region",
1481  data_offset, data_len);
1482  if (sb->region.buf == NULL) {
1483  int r;
1484  if ((r = InitBuffer(sb, cfg)) != SC_OK) { // TODO init with size
1485  sc_errno = r;
1486  return NULL;
1487  }
1488  }
1489  return &sb->region;
1490  }
1491  } else if (sb->region.next == NULL) {
1492  /* InitBufferRegion sets sc_errno */
1493  StreamingBufferRegion *aux_r = sb->region.next = InitBufferRegion(sb, cfg, data_len);
1494  if (aux_r == NULL)
1495  return NULL;
1496  aux_r->stream_offset = data_offset;
1497  DEBUG_VALIDATE_BUG_ON(data_len > aux_r->buf_size);
1498  SCLogDebug("created new region %p with offset %" PRIu64 ", size %u", aux_r,
1499  aux_r->stream_offset, aux_r->buf_size);
1500  return aux_r;
1501  }
1502  /* BufferInsertAtRegionDo sets sc_errno */
1503  StreamingBufferRegion *blob = BufferInsertAtRegionDo(sb, cfg, data_offset, data_len);
1504  SCLogDebug("blob %p (%s)", blob, blob == &sb->region ? "main" : "aux");
1505  return blob;
1506 }
1507 
1508 /**
1509  * \param offset offset relative to StreamingBuffer::stream_offset
1510  *
1511  * \return SC_OK in case of success
1512  * \return SC_E* errors otherwise
1513  */
1514 int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1515  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
1516 {
1517  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1518  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1519  DEBUG_VALIDATE_BUG_ON(offset < sb->region.stream_offset);
1520  if (offset < sb->region.stream_offset) {
1521  return SC_EINVAL;
1522  }
1523 
1524  StreamingBufferRegion *region = BufferInsertAtRegion(sb, cfg, data_len, offset);
1525  if (region == NULL) {
1526  return sc_errno;
1527  }
1528 
1529  const bool region_is_main = region == &sb->region;
1530 
1531  SCLogDebug("inserting %" PRIu64 "/%u using %s region %p", offset, data_len,
1532  region == &sb->region ? "main" : "aux", region);
1533 
1534  uint32_t rel_offset = offset - region->stream_offset;
1535  int r = DataFitsAtOffset(region, data_len, rel_offset);
1536  if (r < 0) {
1537  DEBUG_VALIDATE_BUG_ON(1);
1538  return SC_ELIMIT;
1539  } else if (r == 0) {
1540  if ((r = GrowToSize(sb, cfg, (rel_offset + data_len))) != SC_OK)
1541  return r;
1542  }
1543  DEBUG_VALIDATE_BUG_ON(DataFitsAtOffset(region, data_len, rel_offset) != 1);
1544 
1545  SCLogDebug("offset %" PRIu64 " data_len %u, rel_offset %u into region offset %" PRIu64
1546  ", buf_offset %u, buf_size %u",
1547  offset, data_len, rel_offset, region->stream_offset, region->buf_offset,
1548  region->buf_size);
1549  memcpy(region->buf + rel_offset, data, data_len);
1550  seg->stream_offset = offset;
1551  seg->segment_len = data_len;
1552 
1553  SCLogDebug("rel_offset %u region->stream_offset %" PRIu64 ", buf_offset %u", rel_offset,
1554  region->stream_offset, region->buf_offset);
1555 
1556  if (RB_EMPTY(&sb->sbb_tree)) {
1557  SCLogDebug("empty sbb list");
1558 
1559  if (region_is_main) {
1560  if (sb->region.stream_offset == offset) {
1561  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
1562  /* empty list, data is exactly what is expected (append),
1563  * so do nothing.
1564  * Update buf_offset if needed, but in case of overlaps it might be beyond us. */
1565  sb->region.buf_offset = MAX(sb->region.buf_offset, rel_offset + data_len);
1566  } else if ((rel_offset + data_len) <= sb->region.buf_offset) {
1567  SCLogDebug("empty sbb list: block is within existing main data region");
1568  } else {
1569  if (sb->region.buf_offset && rel_offset == sb->region.buf_offset) {
1570  SCLogDebug("exactly at expected offset");
1571  // nothing to do
1572  sb->region.buf_offset = rel_offset + data_len;
1573 
1574  } else if (rel_offset < sb->region.buf_offset) {
1575  // nothing to do
1576 
1577  SCLogDebug("before expected offset: %u < sb->region.buf_offset %u", rel_offset,
1578  sb->region.buf_offset);
1579  if (rel_offset + data_len > sb->region.buf_offset) {
1580  SCLogDebug("before expected offset, ends after: %u < sb->region.buf_offset "
1581  "%u, %u > %u",
1582  rel_offset, sb->region.buf_offset, rel_offset + data_len,
1583  sb->region.buf_offset);
1584  sb->region.buf_offset = rel_offset + data_len;
1585  }
1586 
1587  } else if (sb->region.buf_offset) {
1588  SCLogDebug("beyond expected offset: SBBInit");
1589  /* existing data, but there is a gap between us */
1590  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1591  return r;
1592  } else {
1593  /* gap before data in empty list */
1594  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1595  if ((r = SBBInitLeadingGap(sb, cfg, offset, data_len)) != SC_OK)
1596  return r;
1597  }
1598  }
1599  } else {
1600  if (sb->region.buf_offset) {
1601  /* existing data, but there is a gap between us */
1602  SCLogDebug("empty sbb list, no data in main: use SBBInit");
1603  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1604  return r;
1605  } else {
1606  /* gap before data in empty list */
1607  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1608  if ((r = SBBInitLeadingGap(sb, cfg, offset, data_len)) != SC_OK)
1609  return r;
1610  }
1611  if (rel_offset == region->buf_offset) {
1612  SCLogDebug("pre region->buf_offset %u", region->buf_offset);
1613  region->buf_offset = rel_offset + data_len;
1614  SCLogDebug("post region->buf_offset %u", region->buf_offset);
1615  }
1616  }
1617  } else {
1618  SCLogDebug("updating sbb tree");
1619  /* already have blocks, so append new block based on new data */
1620  if ((r = SBBUpdate(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1621  return r;
1622  }
1623  DEBUG_VALIDATE_BUG_ON(!region_is_main && sb->head == NULL);
1624 
1625  ListRegions(sb);
1626  if (RB_EMPTY(&sb->sbb_tree)) {
1627  DEBUG_VALIDATE_BUG_ON(offset + data_len > sb->region.stream_offset + sb->region.buf_offset);
1628  }
1629 
1630  return SC_OK;
1631 }
1632 
1633 int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb,
1634  const StreamingBufferSegment *seg)
1635 {
1636  if (seg->stream_offset < sb->region.stream_offset) {
1637  if (seg->stream_offset + seg->segment_len <= sb->region.stream_offset) {
1638  return 1;
1639  }
1640  }
1641  return 0;
1642 }
1643 
1644 static inline const StreamingBufferRegion *GetRegionForOffset(
1645  const StreamingBuffer *sb, const uint64_t offset)
1646 {
1647  if (sb == NULL)
1648  return NULL;
1649  if (sb->region.next == NULL) {
1650  return &sb->region;
1651  }
1652  if (offset >= sb->region.stream_offset &&
1653  offset < (sb->region.stream_offset + sb->region.buf_size)) {
1654  return &sb->region;
1655  }
1656  for (const StreamingBufferRegion *r = sb->region.next; r != NULL; r = r->next) {
1657  if (offset >= r->stream_offset && offset < (r->stream_offset + r->buf_size)) {
1658  return r;
1659  }
1660  }
1661  return NULL;
1662 }
1663 
1664 /** \brief get the data for one SBB */
1665 void StreamingBufferSBBGetData(const StreamingBuffer *sb,
1666  const StreamingBufferBlock *sbb,
1667  const uint8_t **data, uint32_t *data_len)
1668 {
1669  const StreamingBufferRegion *region = GetRegionForOffset(sb, sbb->offset);
1670  SCLogDebug("first find our region (offset %" PRIu64 ") -> %p", sbb->offset, region);
1671  if (region) {
1672  SCLogDebug("region %p found %" PRIu64 "/%u/%u", region, region->stream_offset,
1673  region->buf_size, region->buf_offset);
1674  DEBUG_VALIDATE_BUG_ON(
1675  region->stream_offset == sbb->offset && region->buf_offset > sbb->len);
1676  // buf_offset should match first sbb len if it has the same offset
1677 
1678  if (sbb->offset >= region->stream_offset) {
1679  SCLogDebug("1");
1680  uint64_t offset = sbb->offset - region->stream_offset;
1681  *data = region->buf + offset;
1682  DEBUG_VALIDATE_BUG_ON(offset + sbb->len > region->buf_size);
1683  *data_len = sbb->len;
1684  return;
1685  } else {
1686  SCLogDebug("2");
1687  uint64_t offset = region->stream_offset - sbb->offset;
1688  if (offset < sbb->len) {
1689  *data = region->buf;
1690  *data_len = sbb->len - offset;
1691  return;
1692  }
1693  SCLogDebug("3");
1694  }
1695  }
1696  *data = NULL;
1697  *data_len = 0;
1698 }
1699 
1700 /** \brief get the data for one SBB */
1701 void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb,
1702  const StreamingBufferBlock *sbb,
1703  const uint8_t **data, uint32_t *data_len,
1704  uint64_t offset)
1705 {
1706  /* validate that we are looking for a offset within the sbb */
1707  DEBUG_VALIDATE_BUG_ON(!(offset >= sbb->offset && offset < (sbb->offset + sbb->len)));
1708  if (!(offset >= sbb->offset && offset < (sbb->offset + sbb->len))) {
1709  *data = NULL;
1710  *data_len = 0;
1711  return;
1712  }
1713 
1714  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1715  if (region) {
1716  uint32_t sbblen = sbb->len - (offset - sbb->offset);
1717 
1718  if (offset >= region->stream_offset) {
1719  uint64_t data_offset = offset - region->stream_offset;
1720  *data = region->buf + data_offset;
1721  if (data_offset + sbblen > region->buf_size)
1722  *data_len = region->buf_size - data_offset;
1723  else
1724  *data_len = sbblen;
1725  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1726  return;
1727  } else {
1728  uint64_t data_offset = region->stream_offset - sbb->offset;
1729  if (data_offset < sbblen) {
1730  *data = region->buf;
1731  *data_len = sbblen - data_offset;
1732  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1733  return;
1734  }
1735  }
1736  }
1737 
1738  *data = NULL;
1739  *data_len = 0;
1740 }
1741 
1742 void StreamingBufferSegmentGetData(const StreamingBuffer *sb,
1743  const StreamingBufferSegment *seg,
1744  const uint8_t **data, uint32_t *data_len)
1745 {
1746  const StreamingBufferRegion *region = GetRegionForOffset(sb, seg->stream_offset);
1747  if (region) {
1748  if (seg->stream_offset >= region->stream_offset) {
1749  uint64_t offset = seg->stream_offset - region->stream_offset;
1750  *data = region->buf + offset;
1751  if (offset + seg->segment_len > region->buf_size)
1752  *data_len = region->buf_size - offset;
1753  else
1754  *data_len = seg->segment_len;
1755  SCLogDebug("*data_len %u", *data_len);
1756  return;
1757  } else {
1758  uint64_t offset = region->stream_offset - seg->stream_offset;
1759  if (offset < seg->segment_len) {
1760  *data = region->buf;
1761  *data_len = seg->segment_len - offset;
1762  SCLogDebug("*data_len %u", *data_len);
1763  return;
1764  }
1765  }
1766  }
1767  *data = NULL;
1768  *data_len = 0;
1769 }
1770 
1771 /**
1772  * \retval 1 data is the same
1773  * \retval 0 data is different
1774  */
1775 int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb,
1776  const StreamingBufferSegment *seg,
1777  const uint8_t *rawdata, uint32_t rawdata_len)
1778 {
1779  const uint8_t *segdata = NULL;
1780  uint32_t segdata_len = 0;
1781  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
1782  if (segdata && segdata_len &&
1783  segdata_len == rawdata_len &&
1784  memcmp(segdata, rawdata, segdata_len) == 0)
1785  {
1786  return 1;
1787  }
1788  return 0;
1789 }
1790 
1791 int StreamingBufferGetData(const StreamingBuffer *sb,
1792  const uint8_t **data, uint32_t *data_len,
1793  uint64_t *stream_offset)
1794 {
1795  if (sb != NULL && sb->region.buf != NULL) {
1796  *data = sb->region.buf;
1797  *data_len = sb->region.buf_offset;
1798  *stream_offset = sb->region.stream_offset;
1799  return 1;
1800  } else {
1801  *data = NULL;
1802  *data_len = 0;
1803  *stream_offset = 0;
1804  return 0;
1805  }
1806 }
1807 
1808 int StreamingBufferGetDataAtOffset (const StreamingBuffer *sb,
1809  const uint8_t **data, uint32_t *data_len,
1810  uint64_t offset)
1811 {
1812  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1813  if (region != NULL && region->buf != NULL && offset >= region->stream_offset &&
1814  offset < (region->stream_offset + region->buf_offset)) {
1815  uint32_t skip = offset - region->stream_offset;
1816  *data = region->buf + skip;
1817  *data_len = region->buf_offset - skip;
1818  return 1;
1819  } else {
1820  *data = NULL;
1821  *data_len = 0;
1822  return 0;
1823  }
1824 }
1825 
1826 /**
1827  * \retval 1 data is the same
1828  * \retval 0 data is different
1829  */
1830 int StreamingBufferCompareRawData(const StreamingBuffer *sb,
1831  const uint8_t *rawdata, uint32_t rawdata_len)
1832 {
1833  const uint8_t *sbdata = NULL;
1834  uint32_t sbdata_len = 0;
1835  uint64_t offset = 0;
1836  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
1837  if (offset == 0 &&
1838  sbdata && sbdata_len &&
1839  sbdata_len == rawdata_len &&
1840  memcmp(sbdata, rawdata, sbdata_len) == 0)
1841  {
1842  return 1;
1843  }
1844  SCLogDebug("sbdata_len %u, offset %" PRIu64, sbdata_len, offset);
1845  printf("got:\n");
1846  PrintRawDataFp(stdout, sbdata,sbdata_len);
1847  printf("wanted:\n");
1848  PrintRawDataFp(stdout, rawdata,rawdata_len);
1849  return 0;
1850 }
1851 
1852 #ifdef UNITTESTS
1853 static void Dump(StreamingBuffer *sb)
1854 {
1855  PrintRawDataFp(stdout, sb->region.buf, sb->region.buf_offset);
1856 }
1857 
1858 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
1859 {
1860  const uint8_t *data = NULL;
1861  uint32_t data_len = 0;
1862  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
1863  if (data && data_len) {
1864  PrintRawDataFp(stdout, data, data_len);
1865  }
1866 }
1867 
1868 static int StreamingBufferTest02(void)
1869 {
1870  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1871  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1872  FAIL_IF(sb == NULL);
1873 
1874  StreamingBufferSegment seg1;
1875  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1876  StreamingBufferSegment seg2;
1877  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8) != 0);
1878  FAIL_IF(sb->region.stream_offset != 0);
1879  FAIL_IF(sb->region.buf_offset != 16);
1880  FAIL_IF(seg1.stream_offset != 0);
1881  FAIL_IF(seg2.stream_offset != 8);
1882  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1883  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1884  Dump(sb);
1885  DumpSegment(sb, &seg1);
1886  DumpSegment(sb, &seg2);
1887  FAIL_IF_NOT_NULL(sb->head);
1888  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1889 
1890  StreamingBufferSlideToOffset(sb, &cfg, 6);
1891  FAIL_IF_NOT_NULL(sb->head);
1892  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1893 
1894  StreamingBufferSegment seg3;
1895  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
1896  FAIL_IF(sb->region.stream_offset != 6);
1897  FAIL_IF(sb->region.buf_offset != 16);
1898  FAIL_IF(seg3.stream_offset != 16);
1899  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1900  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1901  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1902  Dump(sb);
1903  DumpSegment(sb, &seg1);
1904  DumpSegment(sb, &seg2);
1905  DumpSegment(sb, &seg3);
1906  FAIL_IF_NOT_NULL(sb->head);
1907  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1908 
1909  StreamingBufferSlideToOffset(sb, &cfg, 12);
1910  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1911  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1912  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1913  Dump(sb);
1914  DumpSegment(sb, &seg1);
1915  DumpSegment(sb, &seg2);
1916  DumpSegment(sb, &seg3);
1917  FAIL_IF_NOT_NULL(sb->head);
1918  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1919 
1920  StreamingBufferFree(sb, &cfg);
1921  PASS;
1922 }
1923 
1924 static int StreamingBufferTest03(void)
1925 {
1926  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1927  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1928  FAIL_IF(sb == NULL);
1929 
1930  StreamingBufferSegment seg1;
1931  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1932  StreamingBufferSegment seg2;
1933  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1934  FAIL_IF(sb->region.stream_offset != 0);
1935  FAIL_IF(sb->region.buf_offset != 8);
1936  FAIL_IF(seg1.stream_offset != 0);
1937  FAIL_IF(seg2.stream_offset != 14);
1938  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1939  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1940  Dump(sb);
1941  DumpSegment(sb, &seg1);
1942  DumpSegment(sb, &seg2);
1943  FAIL_IF_NULL(sb->head);
1944  FAIL_IF_NOT(sb->sbb_size == 16);
1945  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1946 
1947  StreamingBufferSegment seg3;
1948  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1949  FAIL_IF(sb->region.stream_offset != 0);
1950  FAIL_IF(sb->region.buf_offset != 22);
1951  FAIL_IF(seg3.stream_offset != 8);
1952  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1953  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1954  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1955  Dump(sb);
1956  DumpSegment(sb, &seg1);
1957  DumpSegment(sb, &seg2);
1958  DumpSegment(sb, &seg3);
1959  FAIL_IF_NULL(sb->head);
1960  FAIL_IF_NOT(sb->sbb_size == 22);
1961  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1962 
1963  StreamingBufferSlideToOffset(sb, &cfg, 10);
1964  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1965  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1966  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1967  Dump(sb);
1968  DumpSegment(sb, &seg1);
1969  DumpSegment(sb, &seg2);
1970  DumpSegment(sb, &seg3);
1971  FAIL_IF_NULL(sb->head);
1972  FAIL_IF_NOT(sb->sbb_size == 12);
1973  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1974 
1975  StreamingBufferFree(sb, &cfg);
1976  PASS;
1977 }
1978 
1979 static int StreamingBufferTest04(void)
1980 {
1981  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1982  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1983  FAIL_IF(sb == NULL);
1984 
1985  StreamingBufferSegment seg1;
1986  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1987  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
1988  StreamingBufferSegment seg2;
1989  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1990  FAIL_IF(sb->region.stream_offset != 0);
1991  FAIL_IF(sb->region.buf_offset != 8);
1992  FAIL_IF(seg1.stream_offset != 0);
1993  FAIL_IF(seg2.stream_offset != 14);
1994  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1995  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1996  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
1997  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1998  FAIL_IF(sbb1 != sb->head);
1999  FAIL_IF_NULL(sbb1);
2000  FAIL_IF(sbb1->offset != 0);
2001  FAIL_IF(sbb1->len != 8);
2002  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
2003  FAIL_IF_NULL(sbb2);
2004  FAIL_IF(sbb2 == sb->head);
2005  FAIL_IF(sbb2->offset != 14);
2006  FAIL_IF(sbb2->len != 8);
2007  Dump(sb);
2008  DumpSegment(sb, &seg1);
2009  DumpSegment(sb, &seg2);
2010  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2011 
2012  StreamingBufferSegment seg3;
2013  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
2014  FAIL_IF(sb->region.stream_offset != 0);
2015  FAIL_IF(sb->region.buf_offset != 22);
2016  FAIL_IF(seg3.stream_offset != 8);
2017  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2018  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2019  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
2020  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2021  FAIL_IF_NULL(sbb1);
2022  FAIL_IF(sbb1 != sb->head);
2023  FAIL_IF(sbb1->offset != 0);
2024  FAIL_IF(sbb1->len != 22);
2025  FAIL_IF(SBB_RB_NEXT(sbb1));
2026  Dump(sb);
2027  DumpSegment(sb, &seg1);
2028  DumpSegment(sb, &seg2);
2029  DumpSegment(sb, &seg3);
2030  FAIL_IF_NULL(sb->head);
2031  FAIL_IF_NOT(sb->sbb_size == 22);
2032  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2033 
2034  /* far ahead of curve: */
2035  StreamingBufferSegment seg4;
2036  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
2037  FAIL_IF(sb->region.stream_offset != 0);
2038  FAIL_IF(sb->region.buf_offset != 22);
2039  FAIL_IF(sb->region.buf_size != 128);
2040  FAIL_IF(seg4.stream_offset != 124);
2041  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2042  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2043  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
2044  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg4));
2045  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2046  FAIL_IF_NULL(sbb1);
2047  FAIL_IF(sbb1 != sb->head);
2048  FAIL_IF(sbb1->offset != 0);
2049  FAIL_IF(sbb1->len != 22);
2050  FAIL_IF(!SBB_RB_NEXT(sbb1));
2051  Dump(sb);
2052  DumpSegment(sb, &seg1);
2053  DumpSegment(sb, &seg2);
2054  DumpSegment(sb, &seg3);
2055  DumpSegment(sb, &seg4);
2056  FAIL_IF_NULL(sb->head);
2057  FAIL_IF_NOT(sb->sbb_size == 25);
2058  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2059 
2060  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
2061  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
2062  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
2063  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
2064 
2065  StreamingBufferFree(sb, &cfg);
2066  PASS;
2067 }
2068 
2069 /** \test lots of gaps in block list */
2070 static int StreamingBufferTest06(void)
2071 {
2072  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2073  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2074  FAIL_IF(sb == NULL);
2075 
2076  StreamingBufferSegment seg1;
2077  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"A", 1) != 0);
2078  StreamingBufferSegment seg2;
2079  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"C", 1, 2) != 0);
2080  Dump(sb);
2081  FAIL_IF_NULL(sb->head);
2082  FAIL_IF_NOT(sb->sbb_size == 2);
2083  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2084 
2085  StreamingBufferSegment seg3;
2086  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2087  Dump(sb);
2088  FAIL_IF_NULL(sb->head);
2089  FAIL_IF_NOT(sb->sbb_size == 3);
2090  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2091 
2092  StreamingBufferSegment seg4;
2093  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2094  Dump(sb);
2095  FAIL_IF_NULL(sb->head);
2096  FAIL_IF_NOT(sb->sbb_size == 4);
2097  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2098 
2099  StreamingBufferSegment seg5;
2100  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2101  Dump(sb);
2102  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2103  FAIL_IF_NULL(sbb1);
2104  FAIL_IF(sbb1->offset != 0);
2105  FAIL_IF(sbb1->len != 10);
2106  FAIL_IF(SBB_RB_NEXT(sbb1));
2107  FAIL_IF_NULL(sb->head);
2108  FAIL_IF_NOT(sb->sbb_size == 10);
2109  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2110 
2111  StreamingBufferSegment seg6;
2112  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2113  Dump(sb);
2114  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2115  FAIL_IF_NULL(sbb1);
2116  FAIL_IF(sbb1->offset != 0);
2117  FAIL_IF(sbb1->len != 10);
2118  FAIL_IF(SBB_RB_NEXT(sbb1));
2119  FAIL_IF_NULL(sb->head);
2120  FAIL_IF_NOT(sb->sbb_size == 10);
2121  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2122 
2123  StreamingBufferFree(sb, &cfg);
2124  PASS;
2125 }
2126 
2127 /** \test lots of gaps in block list */
2128 static int StreamingBufferTest07(void)
2129 {
2130  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2131  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2132  FAIL_IF(sb == NULL);
2133 
2134  StreamingBufferSegment seg1;
2135  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2136  StreamingBufferSegment seg2;
2137  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2138  Dump(sb);
2139  FAIL_IF_NULL(sb->head);
2140  FAIL_IF_NOT(sb->sbb_size == 2);
2141  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2142 
2143  StreamingBufferSegment seg3;
2144  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2145  Dump(sb);
2146  FAIL_IF_NULL(sb->head);
2147  FAIL_IF_NOT(sb->sbb_size == 3);
2148  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2149 
2150  StreamingBufferSegment seg4;
2151  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2152  Dump(sb);
2153  FAIL_IF_NULL(sb->head);
2154  FAIL_IF_NOT(sb->sbb_size == 4);
2155  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2156 
2157  StreamingBufferSegment seg5;
2158  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2159  Dump(sb);
2160  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2161  FAIL_IF_NULL(sbb1);
2162  FAIL_IF(sbb1->offset != 0);
2163  FAIL_IF(sbb1->len != 10);
2164  FAIL_IF(SBB_RB_NEXT(sbb1));
2165  FAIL_IF_NULL(sb->head);
2166  FAIL_IF_NOT(sb->sbb_size == 10);
2167  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2168 
2169  StreamingBufferSegment seg6;
2170  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2171  Dump(sb);
2172  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2173  FAIL_IF_NULL(sbb1);
2174  FAIL_IF(sbb1->offset != 0);
2175  FAIL_IF(sbb1->len != 10);
2176  FAIL_IF(SBB_RB_NEXT(sbb1));
2177  FAIL_IF_NULL(sb->head);
2178  FAIL_IF_NOT(sb->sbb_size == 10);
2179  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2180 
2181  StreamingBufferFree(sb, &cfg);
2182  PASS;
2183 }
2184 
2185 /** \test lots of gaps in block list */
2186 static int StreamingBufferTest08(void)
2187 {
2188  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2189  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2190  FAIL_IF(sb == NULL);
2191 
2192  StreamingBufferSegment seg1;
2193  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2194  StreamingBufferSegment seg2;
2195  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2196  Dump(sb);
2197  FAIL_IF_NULL(sb->head);
2198  FAIL_IF_NOT(sb->sbb_size == 2);
2199  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2200 
2201  StreamingBufferSegment seg3;
2202  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2203  Dump(sb);
2204  FAIL_IF_NULL(sb->head);
2205  FAIL_IF_NOT(sb->sbb_size == 3);
2206  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2207 
2208  StreamingBufferSegment seg4;
2209  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2210  Dump(sb);
2211  FAIL_IF_NULL(sb->head);
2212  FAIL_IF_NOT(sb->sbb_size == 4);
2213  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2214 
2215  StreamingBufferSegment seg5;
2216  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2217  Dump(sb);
2218  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2219  FAIL_IF_NULL(sbb1);
2220  FAIL_IF(sbb1->offset != 0);
2221  FAIL_IF(sbb1->len != 10);
2222  FAIL_IF(SBB_RB_NEXT(sbb1));
2223  FAIL_IF_NULL(sb->head);
2224  FAIL_IF_NOT(sb->sbb_size == 10);
2225  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2226 
2227  StreamingBufferSegment seg6;
2228  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
2229  Dump(sb);
2230  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2231  FAIL_IF_NULL(sbb1);
2232  FAIL_IF(sbb1->offset != 0);
2233  FAIL_IF(sbb1->len != 20);
2234  FAIL_IF(SBB_RB_NEXT(sbb1));
2235  FAIL_IF_NULL(sb->head);
2236  FAIL_IF_NOT(sb->sbb_size == 20);
2237  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2238 
2239  StreamingBufferFree(sb, &cfg);
2240  PASS;
2241 }
2242 
2243 /** \test lots of gaps in block list */
2244 static int StreamingBufferTest09(void)
2245 {
2246  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2247  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2248  FAIL_IF(sb == NULL);
2249 
2250  StreamingBufferSegment seg1;
2251  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2252  StreamingBufferSegment seg2;
2253  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2254  Dump(sb);
2255  FAIL_IF_NULL(sb->head);
2256  FAIL_IF_NOT(sb->sbb_size == 2);
2257  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2258 
2259  StreamingBufferSegment seg3;
2260  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2261  Dump(sb);
2262  FAIL_IF_NULL(sb->head);
2263  FAIL_IF_NOT(sb->sbb_size == 3);
2264  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2265 
2266  StreamingBufferSegment seg4;
2267  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"F", 1, 5) != 0);
2268  Dump(sb);
2269  FAIL_IF_NULL(sb->head);
2270  FAIL_IF_NOT(sb->sbb_size == 4);
2271  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2272 
2273  StreamingBufferSegment seg5;
2274  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2275  Dump(sb);
2276  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2277  FAIL_IF_NULL(sbb1);
2278  FAIL_IF(sbb1->offset != 0);
2279  FAIL_IF(sbb1->len != 10);
2280  FAIL_IF(SBB_RB_NEXT(sbb1));
2281  FAIL_IF_NULL(sb->head);
2282  FAIL_IF_NOT(sb->sbb_size == 10);
2283  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2284 
2285  StreamingBufferSegment seg6;
2286  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2287  Dump(sb);
2288  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2289  FAIL_IF_NULL(sbb1);
2290  FAIL_IF(sbb1->offset != 0);
2291  FAIL_IF(sbb1->len != 10);
2292  FAIL_IF(SBB_RB_NEXT(sbb1));
2293  FAIL_IF_NULL(sb->head);
2294  FAIL_IF_NOT(sb->sbb_size == 10);
2295  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2296 
2297  StreamingBufferFree(sb, &cfg);
2298  PASS;
2299 }
2300 
2301 /** \test lots of gaps in block list */
2302 static int StreamingBufferTest10(void)
2303 {
2304  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2305  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2306  FAIL_IF(sb == NULL);
2307 
2308  StreamingBufferSegment seg1;
2309  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"A", 1, 0) != 0);
2310  Dump(sb);
2311  StreamingBufferSegment seg2;
2312  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2313  Dump(sb);
2314  StreamingBufferSegment seg3;
2315  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2316  Dump(sb);
2317  FAIL_IF_NULL(sb->head);
2318  FAIL_IF_NOT(sb->sbb_size == 3);
2319 
2320  StreamingBufferSegment seg4;
2321  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"B", 1, 1) != 0);
2322  Dump(sb);
2323  StreamingBufferSegment seg5;
2324  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"C", 1, 2) != 0);
2325  Dump(sb);
2326  StreamingBufferSegment seg6;
2327  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"G", 1, 6) != 0);
2328  Dump(sb);
2329  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2330  FAIL_IF_NULL(sb->head);
2331  FAIL_IF_NOT(sb->sbb_size == 6);
2332 
2333  StreamingBufferSegment seg7;
2334  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2335  Dump(sb);
2336  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2337  FAIL_IF_NULL(sbb1);
2338  FAIL_IF(sbb1->offset != 0);
2339  FAIL_IF(sbb1->len != 10);
2340  FAIL_IF(SBB_RB_NEXT(sbb1));
2341  FAIL_IF_NULL(sb->head);
2342  FAIL_IF_NOT(sb->sbb_size == 10);
2343 
2344  StreamingBufferSegment seg8;
2345  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2346  Dump(sb);
2347  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2348  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2349  FAIL_IF_NULL(sbb1);
2350  FAIL_IF(sbb1->offset != 0);
2351  FAIL_IF(sbb1->len != 10);
2352  FAIL_IF(SBB_RB_NEXT(sbb1));
2353  FAIL_IF_NULL(sb->head);
2354  FAIL_IF_NOT(sb->sbb_size == 10);
2355 
2356  StreamingBufferFree(sb, &cfg);
2357  PASS;
2358 }
2359 
2360 static int StreamingBufferTest11(void)
2361 {
2362  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2363  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2364  FAIL_IF(sb == NULL);
2365 
2366  StreamingBufferSegment seg1;
2367  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
2368  StreamingBufferSegment seg2;
2369  unsigned int data_len = 0xffffffff;
2370  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"unused", data_len) != -1);
2371  FAIL_IF(StreamingBufferInsertAt(
2372  sb, &cfg, &seg2, (const uint8_t *)"abcdefghij", data_len, 100000) != SC_ELIMIT);
2373  StreamingBufferFree(sb, &cfg);
2374  PASS;
2375 }
2376 
2377 static const char *dummy_conf_string = "%YAML 1.1\n"
2378  "---\n"
2379  "\n"
2380  "app-layer:\n"
2381  " protocols:\n"
2382  " http:\n"
2383  " enabled: yes\n"
2384  " memcap: 88\n"
2385  "\n";
2386 
2387 static int StreamingBufferTest12(void)
2388 {
2389  ConfCreateContextBackup();
2390  ConfInit();
2391  HtpConfigCreateBackup();
2392  ConfYamlLoadString((const char *)dummy_conf_string, strlen(dummy_conf_string));
2393  HTPConfigure();
2394 
2395  StreamingBufferConfig cfg = { 8, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, HTPCalloc, HTPRealloc,
2396  HTPFree };
2397  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2398  FAIL_IF(sb == NULL);
2399 
2400  StreamingBufferSegment seg1;
2401  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGHIJKLMNOP", 16) != 0);
2402 
2403  StreamingBufferSegment seg2;
2404  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2,
2405  (const uint8_t *)"ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ",
2406  52) != -1);
2407  FAIL_IF(sc_errno != SC_ELIMIT);
2408 
2409  StreamingBufferFree(sb, &cfg);
2410  HtpConfigRestoreBackup();
2411  ConfRestoreContextBackup();
2412 
2413  PASS;
2414 }
2415 #endif
2416 
2417 void StreamingBufferRegisterTests(void)
2418 {
2419 #ifdef UNITTESTS
2420  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
2421  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
2422  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
2423  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
2424  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
2425  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
2426  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
2427  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
2428  UtRegisterTest("StreamingBufferTest11 Bug 6903", StreamingBufferTest11);
2429  UtRegisterTest("StreamingBufferTest12 Bug 6782", StreamingBufferTest12);
2430 #endif
2431 }
FREE
#define FREE(cfg, ptr, s)
Definition: util-streaming-buffer.c:66
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:1009
len
uint8_t len
Definition: app-layer-dnp3.h:2
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
StreamingBufferRegion_::next
struct StreamingBufferRegion_ * next
Definition: util-streaming-buffer.h:89
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:294
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
RB_LEFT
#define RB_LEFT(elm, field)
Definition: tree.h:322
SC_EINVAL
@ SC_EINVAL
Definition: util-error.h:30
StreamingBuffer_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:114
StreamingBuffer_::regions
uint16_t regions
Definition: util-streaming-buffer.h:113
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1791
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
MIN
#define MIN(x, y)
Definition: suricata-common.h:391
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:1086
HTPRealloc
void * HTPRealloc(void *ptr, size_t orig_size, size_t size)
Definition: app-layer-htp-mem.c:175
segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:0
m
SCMutex m
Definition: flow-hash.h:6
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1808
MAX
#define MAX(x, y)
Definition: suricata-common.h:395
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1775
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
util-unittest.h
HTPConfigure
void HTPConfigure(void)
Definition: app-layer-htp.c:2572
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
app-layer-htp.h
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
FAIL_IF_NOT_NULL
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
util-debug.h
StreamingBufferRegion_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:86
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
util-error.h
RB_ROOT
#define RB_ROOT(head)
Definition: tree.h:326
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:248
HtpConfigCreateBackup
void HtpConfigCreateBackup(void)
Definition: app-layer-htp.c:2882
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:1633
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:400
REALLOC
#define REALLOC(cfg, ptr, orig_s, s)
Definition: util-streaming-buffer.c:64
CALLOC
#define CALLOC(cfg, n, s)
Definition: util-streaming-buffer.c:63
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:271
HTPCalloc
void * HTPCalloc(size_t n, size_t size)
Definition: app-layer-htp-mem.c:154
ConfYamlLoadString
int ConfYamlLoadString(const char *string, size_t len)
Load configuration from a YAML string.
Definition: conf-yaml-loader.c:523
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:111
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
conf-yaml-loader.h
StreamingBufferRegion_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:85
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1830
SC_OK
@ SC_OK
Definition: util-error.h:27
RB_GENERATE
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
ConfCreateContextBackup
void ConfCreateContextBackup(void)
Creates a backup of the conf_hash hash_table used by the conf API.
Definition: conf.c:669
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:104
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamingBuffer_
Definition: util-streaming-buffer.h:108
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1701
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
Definition: util-streaming-buffer.c:1130
suricata-common.h
util-streaming-buffer.h
ConfRestoreContextBackup
void ConfRestoreContextBackup(void)
Restores the backup of the hash_table present in backup_conf_hash back to conf_hash.
Definition: conf.c:679
util-validate.h
HtpConfigRestoreBackup
void HtpConfigRestoreBackup(void)
Definition: app-layer-htp.c:2887
StreamingBufferConfig_
Definition: util-streaming-buffer.h:65
ConfInit
void ConfInit(void)
Initialize the configuration system.
Definition: conf.c:120
HtpBodyChunk_::next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:178
RB_RIGHT
#define RB_RIGHT(elm, field)
Definition: tree.h:323
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1514
head
Flow * head
Definition: flow-hash.h:1
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamingBufferRegion_
Definition: util-streaming-buffer.h:84
app-layer-htp-mem.h
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:73
HTPFree
void HTPFree(void *ptr, size_t size)
Definition: app-layer-htp-mem.c:199
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:1742
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1665
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
WARN_UNUSED
#define WARN_UNUSED
Definition: suricata-common.h:403
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:2417