suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2023 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-print.h"
21 #include "util-validate.h"
22 #include "util-debug.h"
23 #include "util-error.h"
24 
25 #include "app-layer-htp-mem.h"
26 #include "conf-yaml-loader.h"
27 #include "app-layer-htp.h"
28 
29 static void ListRegions(StreamingBuffer *sb);
30 
31 #define DUMP_REGIONS 0 // set to 1 to dump a visual representation of the regions list and sbb tree.
32 
33 /**
34  * \file
35  *
36  * \author Victor Julien <victor@inliniac.net>
37  *
38  * \brief Streaming Buffer API
39  */
40 
41 static void *ReallocFunc(void *ptr, const size_t size)
42 {
43  void *ptrmem = SCRealloc(ptr, size);
44  if (unlikely(ptrmem == NULL)) {
45  sc_errno = SC_ENOMEM;
46  }
47  return ptrmem;
48 }
49 
50 static void *CallocFunc(const size_t nm, const size_t sz)
51 {
52  void *ptrmem = SCCalloc(nm, sz);
53  if (unlikely(ptrmem == NULL)) {
54  sc_errno = SC_ENOMEM;
55  }
56  return ptrmem;
57 }
58 
59 /* memory handling wrappers. If config doesn't define it's own set of
60  * functions, use the defaults */
61 // TODO the default allocators don't set `sc_errno` yet.
62 #define CALLOC(cfg, n, s) (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : CallocFunc((n), (s))
63 #define REALLOC(cfg, ptr, orig_s, s) \
64  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : ReallocFunc((ptr), (s))
65 #define FREE(cfg, ptr, s) \
66  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
67 
68 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg);
69 
70 RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare);
71 
72 int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
73 {
74  SCLogDebug("a %" PRIu64 " len %u, b %" PRIu64 " len %u", a->offset, a->len, b->offset, b->len);
75 
76  if (a->offset > b->offset)
77  SCReturnInt(1);
78  else if (a->offset < b->offset)
79  SCReturnInt(-1);
80  else {
81  if (a->len == 0 || b->len == 0 || a->len == b->len)
82  SCReturnInt(0);
83  else if (a->len > b->len)
84  SCReturnInt(1);
85  else
86  SCReturnInt(-1);
87  }
88 }
89 
90 /* inclusive compare function that also considers the right edge,
91  * not just the offset. */
92 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
93  const uint64_t lre = lookup->offset + lookup->len;
94  const uint64_t tre = intree->offset + intree->len;
95  if (lre <= intree->offset) // entirely before
96  return -1;
97  else if (lookup->offset < tre && lre <= tre) // (some) overlap
98  return 0;
99  else
100  return 1; // entirely after
101 }
102 
103 StreamingBufferBlock *SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
104 {
105  SCLogDebug("looking up %" PRIu64, elm->offset);
106 
107  struct StreamingBufferBlock *tmp = RB_ROOT(head);
108  struct StreamingBufferBlock *res = NULL;
109  while (tmp) {
110  SCLogDebug("compare with %" PRIu64 "/%u", tmp->offset, tmp->len);
111  const int comp = InclusiveCompare(elm, tmp);
112  SCLogDebug("compare result: %d", comp);
113  if (comp < 0) {
114  res = tmp;
115  tmp = RB_LEFT(tmp, rb);
116  } else if (comp > 0) {
117  tmp = RB_RIGHT(tmp, rb);
118  } else {
119  return tmp;
120  }
121  }
122  return res;
123 }
124 
125 /** \interal
126  * \brief does data region intersect with list region 'r'
127  * Takes the max gap into account.
128  */
129 static inline bool RegionsIntersect(const StreamingBufferConfig *cfg,
130  const StreamingBufferRegion *r, const uint64_t offset, const uint64_t re)
131 {
132  /* create the data range for the region, adding the max gap */
133  const uint64_t reg_o =
134  r->stream_offset > cfg->region_gap ? (r->stream_offset - cfg->region_gap) : 0;
135  const uint64_t reg_re = r->stream_offset + r->buf_size + cfg->region_gap;
136  SCLogDebug("r %p: %" PRIu64 "/%" PRIu64 " - adjusted %" PRIu64 "/%" PRIu64, r, r->stream_offset,
137  r->stream_offset + r->buf_size, reg_o, reg_re);
138  /* check if data range intersects with region range */
139  /* [offset:re] and [reg_o:reg_re] do not intersect if and only if
140  * re < reg_o or if reg_re < offset (one segment is strictly before the other)
141  * trusting that offset<=re and reg_o<=reg_re
142  */
143  return re >= reg_o && reg_re >= offset;
144 }
145 
146 /** \internal
147  * \brief find the first region for merging.
148  */
149 static StreamingBufferRegion *FindFirstRegionForOffset(const StreamingBufferConfig *cfg,
150  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len,
151  StreamingBufferRegion **prev)
152 {
153  const uint64_t data_re = offset + len;
154  SCLogDebug("looking for first region matching %" PRIu64 "/%" PRIu64, offset, data_re);
155 
156  StreamingBufferRegion *p = NULL;
157  for (; r != NULL; r = r->next) {
158  if (RegionsIntersect(cfg, r, offset, data_re)) {
159  *prev = p;
160  return r;
161  }
162  p = r;
163  }
164  *prev = NULL;
165  return NULL;
166 }
167 
168 static StreamingBufferRegion *FindLargestRegionForOffset(const StreamingBufferConfig *cfg,
169  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len)
170 {
171  const uint64_t data_re = offset + len;
172  SCLogDebug("starting at %p/%" PRIu64 ", offset %" PRIu64 ", data_re %" PRIu64, r,
173  r->stream_offset, offset, data_re);
174  StreamingBufferRegion *candidate = r;
175  for (; r != NULL; r = r->next) {
176 #ifdef DEBUG
177  const uint64_t reg_re = r->stream_offset + r->buf_size;
178  SCLogDebug("checking: %p/%" PRIu64 "/%" PRIu64 ", offset %" PRIu64 "/%" PRIu64, r,
179  r->stream_offset, reg_re, offset, data_re);
180 #endif
181  if (!RegionsIntersect(cfg, r, offset, data_re))
182  return candidate;
183 
184  if (r->buf_size > candidate->buf_size) {
185  SCLogDebug("candidate %p as size %u > %u", candidate, r->buf_size, candidate->buf_size);
186  candidate = r;
187  }
188  }
189  return candidate;
190 }
191 
192 static StreamingBufferRegion *FindRightEdge(const StreamingBufferConfig *cfg,
193  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len)
194 {
195  const uint64_t data_re = offset + len;
196  StreamingBufferRegion *candidate = r;
197  for (; r != NULL; r = r->next) {
198  if (!RegionsIntersect(cfg, r, offset, data_re)) {
199  SCLogDebug(
200  "r %p is out of scope: %" PRIu64 "/%u/%" PRIu64, r, offset, len, offset + len);
201  return candidate;
202  }
203  candidate = r;
204  }
205  return candidate;
206 }
207 
208 /** \note uses sc_errno to give error details */
209 static inline StreamingBufferRegion *InitBufferRegion(
210  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint32_t min_size)
211 {
212  if ((cfg->max_regions != 0 && sb->regions >= cfg->max_regions) || (sb->regions == UINT16_MAX)) {
213  SCLogDebug("max regions reached");
214  sc_errno = SC_ELIMIT;
215  return NULL;
216  }
217 
218  StreamingBufferRegion *aux_r = CALLOC(cfg, 1, sizeof(*aux_r));
219  if (aux_r == NULL) {
220  return NULL;
221  }
222 
223  aux_r->buf = CALLOC(cfg, 1, MAX(cfg->buf_size, min_size));
224  if (aux_r->buf == NULL) {
225  FREE(cfg, aux_r, sizeof(*aux_r));
226  return NULL;
227  }
228  aux_r->buf_size = MAX(cfg->buf_size, min_size);
229  sb->regions++;
230  sb->max_regions = MAX(sb->regions, sb->max_regions);
231  return aux_r;
232 }
233 
234 static inline int InitBuffer(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
235 {
236  sb->region.buf = CALLOC(cfg, 1, cfg->buf_size);
237  if (sb->region.buf == NULL) {
238  return sc_errno;
239  }
240  sb->region.buf_size = cfg->buf_size;
241  return SC_OK;
242 }
243 
244 StreamingBuffer *StreamingBufferInit(const StreamingBufferConfig *cfg)
245 {
246  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
247  if (sb == NULL) {
248  return NULL;
249  }
250 
251  sb->region.buf_size = cfg->buf_size;
252  sb->regions = sb->max_regions = 1;
253 
254  if (cfg->buf_size == 0) {
255  return sb;
256  }
257 
258  int r = InitBuffer(sb, cfg);
259  if (r != SC_OK) {
260  FREE(cfg, sb, sizeof(StreamingBuffer));
261  sc_errno = r;
262  return NULL;
263  }
264  return sb;
265 }
266 
267 void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
268 {
269  if (sb != NULL) {
270  SCLogDebug("sb->region.buf_size %u max %u", sb->region.buf_size, sb->buf_size_max);
271 
272  SBBFree(sb, cfg);
273  ListRegions(sb);
274  if (sb->region.buf != NULL) {
275  FREE(cfg, sb->region.buf, sb->region.buf_size);
276  sb->region.buf = NULL;
277  }
278 
279  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
280  StreamingBufferRegion *next = r->next;
281  FREE(cfg, r->buf, r->buf_size);
282  FREE(cfg, r, sizeof(*r));
283  r = next;
284  }
285  sb->region.next = NULL;
286  sb->regions = sb->max_regions = 1;
287  }
288 }
289 
290 void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
291 {
292  if (sb != NULL) {
293  StreamingBufferClear(sb, cfg);
294  FREE(cfg, sb, sizeof(StreamingBuffer));
295  }
296 }
297 
298 #ifdef DEBUG
299 static void SBBPrintList(StreamingBuffer *sb)
300 {
301  StreamingBufferBlock *sbb = NULL;
302  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
303  SCLogDebug("sbb: offset %" PRIu64 ", len %u", sbb->offset, sbb->len);
304  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
305  if (next) {
306  if ((sbb->offset + sbb->len) != next->offset) {
307  SCLogDebug("gap: offset %" PRIu64 ", len %" PRIu64, (sbb->offset + sbb->len),
308  next->offset - (sbb->offset + sbb->len));
309  }
310  }
311  }
312 }
313 #endif
314 
315 /* setup with gap between 2 blocks
316  *
317  * [block][gap][block]
318  **/
319 static int WARN_UNUSED SBBInit(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
320  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t data_len)
321 {
322  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
323  DEBUG_VALIDATE_BUG_ON(region->buf_offset > region->stream_offset + rel_offset);
324 
325  /* need to set up 2: existing data block and new data block */
326  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
327  if (sbb == NULL) {
328  return sc_errno;
329  }
330  sbb->offset = sb->region.stream_offset;
331  sbb->len = sb->region.buf_offset;
332  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
333  sb->head = sbb;
334  sb->sbb_size = sbb->len;
335 
336  StreamingBufferBlock *sbb2 = CALLOC(cfg, 1, sizeof(*sbb2));
337  if (sbb2 == NULL) {
338  return sc_errno;
339  }
340  sbb2->offset = region->stream_offset + rel_offset;
341  sbb2->len = data_len;
342  DEBUG_VALIDATE_BUG_ON(sbb2->offset < sbb->len);
343  SCLogDebug("sbb1 %" PRIu64 ", len %u, sbb2 %" PRIu64 ", len %u", sbb->offset, sbb->len,
344  sbb2->offset, sbb2->len);
345 
346  sb->sbb_size += sbb2->len;
347  if (SBB_RB_INSERT(&sb->sbb_tree, sbb2) != NULL) {
348  FREE(cfg, sbb2, sizeof(*sbb2));
349  return SC_EINVAL;
350  }
351 
352 #ifdef DEBUG
353  SBBPrintList(sb);
354 #endif
355  return SC_OK;
356 }
357 
358 /* setup with leading gap
359  *
360  * [gap][block]
361  **/
362 static int WARN_UNUSED SBBInitLeadingGap(
363  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset, uint32_t data_len)
364 {
365  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
366 
367  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
368  if (sbb == NULL) {
369  return sc_errno;
370  }
371  sbb->offset = offset;
372  sbb->len = data_len;
373 
374  sb->head = sbb;
375  sb->sbb_size = sbb->len;
376  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
377 
378  SCLogDebug("sbb %" PRIu64 ", len %u", sbb->offset, sbb->len);
379 #ifdef DEBUG
380  SBBPrintList(sb);
381 #endif
382  return SC_OK;
383 }
384 
385 static inline void ConsolidateFwd(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
386  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
387 {
388  uint64_t sa_re = sa->offset + sa->len;
389  StreamingBufferBlock *tr, *s = sa;
390  RB_FOREACH_FROM(tr, SBB, s) {
391  if (sa == tr)
392  continue;
393 
394  const uint64_t tr_re = tr->offset + tr->len;
395  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u re %" PRIu64, tr, tr->offset, tr->len, tr_re);
396 
397  if (sa_re < tr->offset) {
398  SCLogDebug("entirely before: %" PRIu64 " < %" PRIu64, sa_re, tr->offset);
399  break; // entirely before
400  }
401 
402  /* new block (sa) entirely eclipsed by in tree block (tr)
403  sa: [ ]
404  tr: [ ]
405  sa: [ ]
406  tr: [ ]
407  sa: [ ]
408  tr: [ ]
409  */
410  if (sa->offset >= tr->offset && sa_re <= tr_re) {
411  sb->sbb_size -= sa->len;
412  sa->len = tr->len;
413  sa->offset = tr->offset;
414  sa_re = sa->offset + sa->len;
415  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
416  tr->offset, tr->len);
417  SBB_RB_REMOVE(tree, tr);
418  FREE(cfg, tr, sizeof(StreamingBufferBlock));
419  /* new block (sa) entire eclipses in tree block (tr)
420  sa: [ ]
421  tr: [ ]
422  sa: [ ]
423  tr: [ ]
424  sa: [ ]
425  tr: [ ]
426  */
427  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
428  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
429  tr->offset, tr->len);
430  SBB_RB_REMOVE(tree, tr);
431  sb->sbb_size -= tr->len;
432  FREE(cfg, tr, sizeof(StreamingBufferBlock));
433 
434  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
435  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
436  region->buf_size);
437  if (sa->offset == region->stream_offset &&
438  sa_re > (region->stream_offset + region->buf_offset)) {
439  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
440  DEBUG_VALIDATE_BUG_ON(sa_re - region->stream_offset > UINT32_MAX);
441  region->buf_offset = (uint32_t)(sa_re - region->stream_offset);
442  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
443  " bo %u sz %u BUF_OFFSET UPDATED",
444  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
445  region->buf_size);
446  }
447  /* new block (sa) extended by in tree block (tr)
448  sa: [ ]
449  tr: [ ]
450  sa: [ ]
451  tr: [ ]
452  */
453  } else if (sa->offset < tr->offset && // starts before
454  sa_re >= tr->offset && sa_re < tr_re) // ends inside
455  {
456  // merge. sb->sbb_size includes both so we need to adjust that too.
457  DEBUG_VALIDATE_BUG_ON(sa->len + tr->len > UINT32_MAX);
458  uint32_t combined_len = (uint32_t)(sa->len + tr->len);
459  DEBUG_VALIDATE_BUG_ON(tr_re - sa->offset > UINT32_MAX);
460  sa->len = (uint32_t)(tr_re - sa->offset);
461  sa_re = sa->offset + sa->len;
462  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
463  SBB_RB_REMOVE(tree, tr);
464  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
465  FREE(cfg, tr, sizeof(StreamingBufferBlock));
466  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u RESULT", sa, sa->offset, sa->len);
467  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
468  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
469  region->buf_size);
470  if (sa->offset == region->stream_offset &&
471  sa_re > (region->stream_offset + region->buf_offset)) {
472  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
473  DEBUG_VALIDATE_BUG_ON(sa_re - region->stream_offset > UINT32_MAX);
474  region->buf_offset = (uint32_t)(sa_re - region->stream_offset);
475  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
476  " bo %u sz %u BUF_OFFSET UPDATED",
477  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
478  region->buf_size);
479  }
480  }
481  }
482 }
483 
484 static inline void ConsolidateBackward(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
485  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
486 {
487  uint64_t sa_re = sa->offset + sa->len;
488  StreamingBufferBlock *tr, *s = sa;
489  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
490  if (sa == tr)
491  continue;
492  const uint64_t tr_re = tr->offset + tr->len;
493  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u", tr, tr->offset, tr->len);
494 
495  if (sa->offset > tr_re)
496  break; // entirely after
497 
498  /* new block (sa) entirely eclipsed by in tree block (tr)
499  sa: [ ]
500  tr: [ ]
501  sa: [ ]
502  tr: [ ]
503  sa: [ ]
504  tr: [ ]
505  */
506  if (sa->offset >= tr->offset && sa_re <= tr_re) {
507  sb->sbb_size -= sa->len; // sa entirely eclipsed so remove double accounting
508  sa->len = tr->len;
509  sa->offset = tr->offset;
510  sa_re = sa->offset + sa->len;
511  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
512  tr->offset, tr->len);
513  if (sb->head == tr)
514  sb->head = sa;
515  SBB_RB_REMOVE(tree, tr);
516  FREE(cfg, tr, sizeof(StreamingBufferBlock));
517  /* new block (sa) entire eclipses in tree block (tr)
518  sa: [ ]
519  tr: [ ]
520  sa: [ ]
521  tr: [ ]
522  sa: [ ]
523  tr: [ ]
524  */
525  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
526  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
527  tr->offset, tr->len);
528  if (sb->head == tr)
529  sb->head = sa;
530  SBB_RB_REMOVE(tree, tr);
531  sb->sbb_size -= tr->len; // tr entirely eclipsed so remove double accounting
532  FREE(cfg, tr, sizeof(StreamingBufferBlock));
533 
534  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
535  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
536  region->buf_size);
537 
538  if (sa->offset == region->stream_offset &&
539  sa_re > (region->stream_offset + region->buf_offset)) {
540  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
541  DEBUG_VALIDATE_BUG_ON(sa_re - region->stream_offset > UINT32_MAX);
542  region->buf_offset = (uint32_t)(sa_re - region->stream_offset);
543  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
544  " bo %u sz %u BUF_OFFSET UPDATED",
545  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
546  region->buf_size);
547  }
548 
549  /* new block (sa) extends in tree block (tr)
550  sa: [ ]
551  tr: [ ]
552  sa: [ ]
553  tr: [ ]
554  */
555  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
556  // merge. sb->sbb_size includes both so we need to adjust that too.
557  DEBUG_VALIDATE_BUG_ON(sa->len + tr->len > UINT32_MAX);
558  uint32_t combined_len = (uint32_t)(sa->len + tr->len);
559  DEBUG_VALIDATE_BUG_ON(sa_re - tr->offset > UINT32_MAX);
560  sa->len = (uint32_t)(sa_re - tr->offset);
561  sa->offset = tr->offset;
562  sa_re = sa->offset + sa->len;
563  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
564  if (sb->head == tr)
565  sb->head = sa;
566  SBB_RB_REMOVE(tree, tr);
567  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
568  FREE(cfg, tr, sizeof(StreamingBufferBlock));
569 
570  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
571  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
572  region->buf_size);
573  if (sa->offset == region->stream_offset &&
574  sa_re > (region->stream_offset + region->buf_offset)) {
575  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
576  DEBUG_VALIDATE_BUG_ON(sa_re - region->stream_offset > UINT32_MAX);
577  region->buf_offset = (uint32_t)(sa_re - region->stream_offset);
578  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
579  " bo %u sz %u BUF_OFFSET UPDATED",
580  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
581  region->buf_size);
582  }
583  }
584  }
585 }
586 
587 /** \internal
588  * \param region the region that holds the new data
589  */
590 static int SBBUpdate(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
591  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t len)
592 {
593  struct SBB *tree = &sb->sbb_tree;
594  SCLogDebug("* inserting: %u/%u", rel_offset, len);
595 
596  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
597  if (sbb == NULL) {
598  return sc_errno;
599  }
600  sbb->offset = region->stream_offset + rel_offset;
601  sbb->len = len;
602 
603  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
604  if (res) {
605  // exact overlap
606  SCLogDebug("* insert failed: exact match in tree with %p %" PRIu64 "/%u", res, res->offset,
607  res->len);
608  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
609  return SC_OK;
610  }
611  sb->sbb_size += len; // may adjust based on consolidation below
612 
613  /* handle backwards and forwards overlaps */
614  if (SBB_RB_PREV(sbb) == NULL) {
615  sb->head = sbb;
616  } else {
617  ConsolidateBackward(sb, cfg, region, tree, sbb);
618  }
619  ConsolidateFwd(sb, cfg, region, tree, sbb);
620 #ifdef DEBUG
621  SBBPrintList(sb);
622 #endif
623  if (sbb->offset == region->stream_offset) {
624  SCLogDebug("insert at region head");
625  region->buf_offset = sbb->len;
626  }
627  return SC_OK;
628 }
629 
630 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
631 {
632  StreamingBufferBlock *sbb = NULL, *safe = NULL;
633  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
634  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
635  sb->sbb_size -= sbb->len;
636  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
637  }
638  sb->head = NULL;
639 }
640 
641 static void SBBPrune(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
642 {
643  SCLogDebug("pruning %p to %" PRIu64, sb, sb->region.stream_offset);
644  StreamingBufferBlock *sbb = NULL, *safe = NULL;
645  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
646  /* completely beyond window, we're done */
647  if (sbb->offset >= sb->region.stream_offset) {
648  sb->head = sbb;
649  if (sbb->offset == sb->region.stream_offset) {
650  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
651  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
652  sb->region.buf_offset = sbb->len;
653  }
654  break;
655  }
656 
657  /* partly before, partly beyond. Adjust */
658  if (sbb->offset < sb->region.stream_offset &&
659  sbb->offset + sbb->len > sb->region.stream_offset) {
660  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset - sbb->offset > UINT32_MAX);
661  uint32_t shrink_by = (uint32_t)(sb->region.stream_offset - sbb->offset);
662  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
663  if (sbb->len >= shrink_by) {
664  sbb->len -= shrink_by;
665  sbb->offset += shrink_by;
666  sb->sbb_size -= shrink_by;
667  SCLogDebug("shrunk by %u", shrink_by);
668  DEBUG_VALIDATE_BUG_ON(sbb->offset != sb->region.stream_offset);
669  }
670  sb->head = sbb;
671  if (sbb->offset == sb->region.stream_offset) {
672  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
673  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
674  sb->region.buf_offset = sbb->len;
675  }
676  break;
677  }
678 
679  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
680  /* either we set it again for the next sbb, or there isn't any */
681  sb->head = NULL;
682  sb->sbb_size -= sbb->len;
683  SCLogDebug("sb %p removed %p %" PRIu64 ", %u", sb, sbb, sbb->offset, sbb->len);
684  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
685  }
686 #ifdef DEBUG
687  SBBPrintList(sb);
688 #endif
689 }
690 
691 static inline uint32_t ToNextMultipleOf(const uint32_t in, const uint32_t m)
692 {
693  uint32_t r = in;
694  if (m > 0) {
695  const uint32_t x = in % m;
696  if (x != 0) {
697  r = (in - x) + m;
698  }
699  }
700  return r;
701 }
702 
703 static thread_local bool g2s_warn_once = false;
704 
705 static inline int WARN_UNUSED GrowRegionToSize(StreamingBuffer *sb,
706  const StreamingBufferConfig *cfg, StreamingBufferRegion *region, const uint32_t size)
707 {
708  DEBUG_VALIDATE_BUG_ON(region->buf_size > BIT_U32(30));
709  if (size > BIT_U32(30)) { // 1GiB
710  if (!g2s_warn_once) {
711  SCLogWarning("StreamingBuffer::GrowRegionToSize() tried to alloc %u bytes, exceeds "
712  "limit of %" PRIu32,
713  size, BIT_U32(30));
714  g2s_warn_once = true;
715  }
716  return SC_ELIMIT;
717  }
718 
719  /* try to grow in multiples of cfg->buf_size */
720  const uint32_t grow = ToNextMultipleOf(size, cfg->buf_size);
721  SCLogDebug("grow %u", grow);
722  if (grow <= region->buf_size) {
723  // do not try to shrink, and do not memset with diff having unsigned underflow
724  return SC_OK;
725  }
726 
727  void *ptr = REALLOC(cfg, region->buf, region->buf_size, grow);
728  if (ptr == NULL) {
729  if (sc_errno == SC_OK)
730  sc_errno = SC_ENOMEM;
731  return sc_errno;
732  }
733  /* for safe printing and general caution, lets memset the
734  * new data to 0 */
735  size_t diff = grow - region->buf_size;
736  void *new_mem = ((char *)ptr) + region->buf_size;
737  memset(new_mem, 0, diff);
738 
739  region->buf = ptr;
740  region->buf_size = grow;
741  SCLogDebug("grown buffer to %u", grow);
742 #ifdef DEBUG
743  if (region->buf_size > sb->buf_size_max) {
744  sb->buf_size_max = region->buf_size;
745  }
746 #endif
747  return SC_OK;
748 }
749 
750 static int WARN_UNUSED GrowToSize(
751  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint32_t size)
752 {
753  return GrowRegionToSize(sb, cfg, &sb->region, size);
754 }
755 
756 static inline bool RegionBeforeOffset(const StreamingBufferRegion *r, const uint64_t o)
757 {
758  return (r->stream_offset + r->buf_size <= o);
759 }
760 
761 static inline bool RegionContainsOffset(const StreamingBufferRegion *r, const uint64_t o)
762 {
763  return (o >= r->stream_offset && (r->stream_offset + r->buf_size) >= o);
764 }
765 
766 /** \internal
767  * \brief slide to offset for regions
768  *
769  *
770  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
771  * ^
772  * - main reset to 0
773  *
774  *
775  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
776  * ^
777  * - main shift
778  *
779  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
780  * ^
781  * - main reset
782  * - move aux into main
783  * - free aux
784  * - shift
785  *
786  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
787  * ^
788  * - main reset
789  * - move aux into main
790  * - free aux
791  * - no shift
792  */
793 static inline void StreamingBufferSlideToOffsetWithRegions(
794  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint64_t slide_offset)
795 {
796  ListRegions(sb);
797  DEBUG_VALIDATE_BUG_ON(slide_offset == sb->region.stream_offset);
798 
799  SCLogDebug("slide_offset %" PRIu64, slide_offset);
800  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
801  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
802 
803  StreamingBufferRegion *to_shift = NULL;
804  const bool main_is_oow = RegionBeforeOffset(&sb->region, slide_offset);
805  if (main_is_oow) {
806  SCLogDebug("main_is_oow");
807  if (sb->region.buf != NULL) {
808  SCLogDebug("clearing main");
809  FREE(cfg, sb->region.buf, sb->region.buf_size);
810  sb->region.buf = NULL;
811  sb->region.buf_size = 0;
812  sb->region.buf_offset = 0;
813  sb->region.stream_offset = slide_offset;
814  } else {
815  sb->region.stream_offset = slide_offset;
816  }
817 
818  /* remove regions that are out of window & select the region to
819  * become the new main */
820  StreamingBufferRegion *prev = &sb->region;
821  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
822  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64, r, r->stream_offset,
823  r->stream_offset + r->buf_offset);
824  StreamingBufferRegion *next = r->next;
825  if (RegionBeforeOffset(r, slide_offset)) {
826  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> before", r, r->stream_offset,
827  r->stream_offset + r->buf_offset);
828  DEBUG_VALIDATE_BUG_ON(r == &sb->region);
829  prev->next = next;
830 
831  FREE(cfg, r->buf, r->buf_size);
832  FREE(cfg, r, sizeof(*r));
833  sb->regions--;
834  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
835  } else if (RegionContainsOffset(r, slide_offset)) {
836  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> within", r, r->stream_offset,
837  r->stream_offset + r->buf_offset);
838  /* remove from list, we will xfer contents to main below */
839  prev->next = next;
840  to_shift = r;
841  break;
842  } else {
843  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> post", r, r->stream_offset,
844  r->stream_offset + r->buf_offset);
845  /* implied beyond slide offset */
846  DEBUG_VALIDATE_BUG_ON(r->stream_offset < slide_offset);
847  break;
848  }
849  r = next;
850  }
851  SCLogDebug("to_shift %p", to_shift);
852 
853  // this region is main, or will xfer its buffer to main
854  if (to_shift && to_shift != &sb->region) {
855  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", to_shift->stream_offset,
856  to_shift->buf, to_shift->buf_size, to_shift->buf_offset);
857  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL);
858 
859  sb->region.buf = to_shift->buf;
860  sb->region.stream_offset = to_shift->stream_offset;
861  sb->region.buf_offset = to_shift->buf_offset;
862  sb->region.buf_size = to_shift->buf_size;
863  sb->region.next = to_shift->next;
864 
865  BUG_ON(to_shift == &sb->region);
866  FREE(cfg, to_shift, sizeof(*to_shift));
867  to_shift = &sb->region;
868  sb->regions--;
869  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
870  }
871 
872  } else {
873  to_shift = &sb->region;
874  SCLogDebug("shift start region %p", to_shift);
875  }
876 
877  // this region is main, or will xfer its buffer to main
878  if (to_shift) {
879  // Do the shift. If new region is exactly at the slide offset we can skip this.
880  DEBUG_VALIDATE_BUG_ON(to_shift->stream_offset > slide_offset);
881  DEBUG_VALIDATE_BUG_ON(slide_offset - to_shift->stream_offset > UINT32_MAX);
882  const uint32_t s = (uint32_t)(slide_offset - to_shift->stream_offset);
883  if (s > 0) {
884  const uint32_t new_data_size = to_shift->buf_size - s;
885  uint32_t new_mem_size = ToNextMultipleOf(new_data_size, cfg->buf_size);
886 
887  /* see if after the slide we'd overlap with the next region. If so, we need
888  * to consolidate them into one. Error handling is a bit peculiar. We need
889  * to grow a region, move data from another region into it, then free the
890  * other region. This could fail if we're close to the memcap. In this case
891  * we relax our rounding up logic so we only shrink and don't merge the 2
892  * regions after all. */
893  if (to_shift->next && slide_offset + new_mem_size >= to_shift->next->stream_offset) {
894  StreamingBufferRegion *start = to_shift;
895  StreamingBufferRegion *next = start->next;
896  const uint64_t next_re = next->stream_offset + next->buf_size;
897  DEBUG_VALIDATE_BUG_ON(next_re - slide_offset > UINT32_MAX);
898  const uint32_t mem_size =
899  ToNextMultipleOf((uint32_t)(next_re - slide_offset), cfg->buf_size);
900 
901  /* using next as the new main */
902  if (start->buf_size < next->buf_size) {
903  SCLogDebug("replace main with the next bigger region");
904 
905  DEBUG_VALIDATE_BUG_ON(next->stream_offset - slide_offset > UINT32_MAX);
906  const uint32_t next_data_offset =
907  (uint32_t)(next->stream_offset - slide_offset);
908  const uint32_t prev_buf_size = next->buf_size;
909  DEBUG_VALIDATE_BUG_ON(slide_offset - start->stream_offset > UINT32_MAX);
910  const uint32_t start_data_offset =
911  (uint32_t)(slide_offset - start->stream_offset);
912  DEBUG_VALIDATE_BUG_ON(start_data_offset > start->buf_size);
913  if (start_data_offset > start->buf_size) {
914  new_mem_size = new_data_size;
915  goto just_main;
916  }
917  /* expand "next" to include relevant part of "start" */
918  if (GrowRegionToSize(sb, cfg, next, mem_size) != 0) {
919  new_mem_size = new_data_size;
920  goto just_main;
921  }
922  SCLogDebug("region now sized %u", mem_size);
923 
924  // slide "next":
925  // pre-grow: [nextnextnext]
926  // post-grow [nextnextnextXXX]
927  // post-move [XXXnextnextnext]
928  SCMemmove(next->buf + next_data_offset, next->buf_size - next_data_offset,
929  next->buf, prev_buf_size);
930 
931  // move portion of "start" into "next"
932  //
933  // start: [ooooookkkkk] (o: old, k: keep)
934  // pre-next [xxxxxnextnextnext]
935  // post-next [kkkkknextnextnext]
936  const uint32_t start_data_size = start->buf_size - start_data_offset;
937  memcpy(next->buf, start->buf + start_data_offset, start_data_size);
938 
939  // free "start"s buffer, we will use the one from "next"
940  FREE(cfg, start->buf, start->buf_size);
941 
942  // update "main" to use "next"
943  start->stream_offset = slide_offset;
944  start->buf = next->buf;
945  start->buf_size = next->buf_size;
946  start->next = next->next;
947 
948  // free "next"
949  FREE(cfg, next, sizeof(*next));
950  sb->regions--;
951  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
952  goto done;
953  } else {
954  /* using "main", expand to include "next" */
955  if (mem_size > start->buf_size) {
956  // Check that start->buf_size is actually not big enough
957  // As mem_size computation and earlier checks do not make it clear.
958  if (GrowRegionToSize(sb, cfg, start, mem_size) != 0) {
959  new_mem_size = new_data_size;
960  goto just_main;
961  }
962  }
963  SCLogDebug("start->buf now size %u", mem_size);
964 
965  // slide "start"
966  // pre: [xxxxxxxAAA]
967  // post: [AAAxxxxxxx]
968  SCLogDebug("s %u new_data_size %u", s, new_data_size);
969  SCMemmove(start->buf, start->buf_size, start->buf + s, new_data_size);
970 
971  // copy in "next"
972  // pre: [AAAxxxxxxx]
973  // [BBBBBBB]
974  // post: [AAABBBBBBB]
975  SCLogDebug("copy next->buf %p/%u to start->buf offset %u", next->buf,
976  next->buf_size, new_data_size);
977  memcpy(start->buf + new_data_size, next->buf, next->buf_size);
978 
979  start->stream_offset = slide_offset;
980  start->next = next->next;
981 
982  // free "next"
983  FREE(cfg, next->buf, next->buf_size);
984  FREE(cfg, next, sizeof(*next));
985  sb->regions--;
986  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
987  goto done;
988  }
989  }
990 
991  just_main:
992  SCLogDebug("s %u new_data_size %u", s, new_data_size);
993  SCMemmove(to_shift->buf, to_shift->buf_size, to_shift->buf + s, new_data_size);
994  /* shrink memory region. If this fails we keep the old */
995  void *ptr = REALLOC(cfg, to_shift->buf, to_shift->buf_size, new_mem_size);
996  if (ptr != NULL) {
997  to_shift->buf = ptr;
998  to_shift->buf_size = new_mem_size;
999  SCLogDebug("new buf_size %u", to_shift->buf_size);
1000  }
1001  if (s < to_shift->buf_offset)
1002  to_shift->buf_offset -= s;
1003  else
1004  to_shift->buf_offset = 0;
1005  to_shift->stream_offset = slide_offset;
1006  }
1007  }
1008 
1009 done:
1010  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
1011  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
1012  SCLogDebug("end of slide");
1013 }
1014 
1015 /**
1016  * \brief slide to absolute offset
1017  * \todo if sliding beyond window, we could perhaps reset?
1018  */
1019 void StreamingBufferSlideToOffset(
1020  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
1021 {
1022  SCLogDebug("sliding to offset %" PRIu64, offset);
1023  ListRegions(sb);
1024 #ifdef DEBUG
1025  SBBPrintList(sb);
1026 #endif
1027 
1028  if (sb->region.next) {
1029  StreamingBufferSlideToOffsetWithRegions(sb, cfg, offset);
1030  SBBPrune(sb, cfg);
1031  SCLogDebug("post SBBPrune");
1032  ListRegions(sb);
1033 #ifdef DEBUG
1034  SBBPrintList(sb);
1035 #endif
1036  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL && sb->region.buf_size == 0);
1037  DEBUG_VALIDATE_BUG_ON(sb->region.buf_offset > sb->region.buf_size);
1038  DEBUG_VALIDATE_BUG_ON(offset > sb->region.stream_offset);
1039  DEBUG_VALIDATE_BUG_ON(sb->head && sb->head->offset == sb->region.stream_offset &&
1040  sb->head->len > sb->region.buf_offset);
1041  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1042  return;
1043  }
1044 
1045  if (offset > sb->region.stream_offset) {
1046  DEBUG_VALIDATE_BUG_ON(offset - sb->region.stream_offset > UINT32_MAX);
1047  const uint32_t slide = (uint32_t)(offset - sb->region.stream_offset);
1048  if (sb->head != NULL) {
1049  /* have sbb's, so can't rely on buf_offset for the slide */
1050  if (slide < sb->region.buf_size) {
1051  const uint32_t size = sb->region.buf_size - slide;
1052  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1053  size);
1054  SCMemmove(sb->region.buf, sb->region.buf_size, sb->region.buf + slide, size);
1055  if (sb->region.buf_offset > slide) {
1056  sb->region.buf_offset -= slide;
1057  } else {
1058  sb->region.buf_offset = 0;
1059  }
1060  } else {
1061  sb->region.buf_offset = 0;
1062  }
1063  sb->region.stream_offset = offset;
1064  } else {
1065  /* no sbb's, so we can use buf_offset */
1066  if (offset <= sb->region.stream_offset + sb->region.buf_offset) {
1067  const uint32_t size = sb->region.buf_offset - slide;
1068  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1069  size);
1070  SCMemmove(sb->region.buf, sb->region.buf_size, sb->region.buf + slide, size);
1071  sb->region.stream_offset = offset;
1072  sb->region.buf_offset = size;
1073  } else {
1074  /* moved past all data */
1075  sb->region.stream_offset = offset;
1076  sb->region.buf_offset = 0;
1077  }
1078  }
1079  SBBPrune(sb, cfg);
1080  }
1081 #ifdef DEBUG
1082  SBBPrintList(sb);
1083 #endif
1084  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1085 }
1086 
1087 static int DataFits(const StreamingBuffer *sb, const uint32_t len)
1088 {
1089  uint64_t buf_offset64 = sb->region.buf_offset;
1090  uint64_t len64 = len;
1091  if (len64 + buf_offset64 > UINT32_MAX) {
1092  return -1;
1093  }
1094  return sb->region.buf_offset + len <= sb->region.buf_size;
1095 }
1096 
1097 int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1098  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
1099 {
1100  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1101  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1102 
1103  if (sb->region.buf == NULL) {
1104  if (InitBuffer(sb, cfg) == -1)
1105  return -1;
1106  }
1107 
1108  int r = DataFits(sb, data_len);
1109  if (r < 0) {
1110  DEBUG_VALIDATE_BUG_ON(1);
1111  return -1;
1112  } else if (r == 0) {
1113  if (sb->region.buf_size == 0) {
1114  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1115  return -1;
1116  } else {
1117  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1118  return -1;
1119  }
1120  }
1121  DEBUG_VALIDATE_BUG_ON(DataFits(sb, data_len) != 1);
1122  if (DataFits(sb, data_len) != 1)
1123  return -1;
1124 
1125  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1126  seg->stream_offset = sb->region.stream_offset + sb->region.buf_offset;
1127  seg->segment_len = data_len;
1128  uint32_t rel_offset = sb->region.buf_offset;
1129  sb->region.buf_offset += data_len;
1130 
1131  if (!RB_EMPTY(&sb->sbb_tree)) {
1132  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1133  } else {
1134  return 0;
1135  }
1136 }
1137 
1138 /**
1139  * \brief add data w/o tracking a segment
1140  */
1141 int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1142  const uint8_t *data, uint32_t data_len)
1143 {
1144  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1145 
1146  if (sb->region.buf == NULL) {
1147  if (InitBuffer(sb, cfg) == -1)
1148  return -1;
1149  }
1150 
1151  int r = DataFits(sb, data_len);
1152  if (r < 0) {
1153  DEBUG_VALIDATE_BUG_ON(1);
1154  return -1;
1155  } else if (r == 0) {
1156  if (sb->region.buf_size == 0) {
1157  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1158  return -1;
1159  } else {
1160  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1161  return -1;
1162  }
1163  }
1164  DEBUG_VALIDATE_BUG_ON(DataFits(sb, data_len) != 1);
1165 
1166  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1167  uint32_t rel_offset = sb->region.buf_offset;
1168  sb->region.buf_offset += data_len;
1169 
1170  if (!RB_EMPTY(&sb->sbb_tree)) {
1171  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1172  } else {
1173  return 0;
1174  }
1175 }
1176 
1177 static int DataFitsAtOffset(
1178  const StreamingBufferRegion *region, const uint32_t len, const uint32_t offset)
1179 {
1180  const uint64_t offset64 = offset;
1181  const uint64_t len64 = len;
1182  if (offset64 + len64 > UINT32_MAX)
1183  return -1;
1184  return (offset + len <= region->buf_size);
1185 }
1186 
1187 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1188 static void Validate(const StreamingBuffer *sb)
1189 {
1190  bool bail = false;
1191  uint32_t cnt = 0;
1192  for (const StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1193  cnt++;
1194  if (r->next) {
1195  bail |= ((r->stream_offset + r->buf_size) > r->next->stream_offset);
1196  }
1197 
1198  bail |= (r->buf != NULL && r->buf_size == 0);
1199  bail |= (r->buf_offset > r->buf_size);
1200  }
1201  // leading gap, so buf_offset should be 0?
1202  if (sb->head && sb->head->offset > sb->region.stream_offset) {
1203  SCLogDebug("leading gap of size %" PRIu64, sb->head->offset - sb->region.stream_offset);
1204  BUG_ON(sb->region.buf_offset != 0);
1205  }
1206 
1207  if (sb->head && sb->head->offset == sb->region.stream_offset) {
1208  BUG_ON(sb->head->len > sb->region.buf_offset);
1209  BUG_ON(sb->head->len < sb->region.buf_offset);
1210  }
1211  BUG_ON(sb->regions != cnt);
1212  BUG_ON(bail);
1213 }
1214 #endif
1215 
1216 static void ListRegions(StreamingBuffer *sb)
1217 {
1218  if (sb->region.buf == NULL && sb->region.buf_offset == 0 && sb->region.next == NULL)
1219  return;
1220 #if defined(DEBUG) && DUMP_REGIONS == 1
1221  uint32_t cnt = 0;
1222  for (StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1223  cnt++;
1224  char gap[64] = "";
1225  if (r->next) {
1226  snprintf(gap, sizeof(gap), "[ gap:%" PRIu64 " ]",
1227  r->next->stream_offset - (r->stream_offset + r->buf_size));
1228  }
1229 
1230  printf("[ %s offset:%" PRIu64 " size:%u offset:%u ]%s", r == &sb->region ? "main" : "aux",
1231  r->stream_offset, r->buf_size, r->buf_offset, gap);
1232  }
1233  printf("(max %u, cnt %u, sb->regions %u)\n", sb->max_regions, cnt, sb->regions);
1234  bool at_least_one = false;
1235  uint64_t last_re = sb->region.stream_offset;
1236  StreamingBufferBlock *sbb = NULL;
1237  RB_FOREACH(sbb, SBB, &sb->sbb_tree)
1238  {
1239  if (last_re != sbb->offset) {
1240  printf("[ gap:%" PRIu64 " ]", sbb->offset - last_re);
1241  }
1242  printf("[ sbb offset:%" PRIu64 " len:%u ]", sbb->offset, sbb->len);
1243  at_least_one = true;
1244  last_re = sbb->offset + sbb->len;
1245  }
1246  if (at_least_one)
1247  printf("\n");
1248 #endif
1249 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1250  StreamingBufferBlock *sbb2 = NULL;
1251  RB_FOREACH(sbb2, SBB, &sb->sbb_tree)
1252  {
1253  const uint8_t *_data = NULL;
1254  uint32_t _data_len = 0;
1255  (void)StreamingBufferSBBGetData(sb, sbb2, &_data, &_data_len);
1256  }
1257  Validate(sb);
1258 #endif
1259 }
1260 
1261 /** \internal
1262  * \brief process insert by consolidating the affected regions into one
1263  * \note sets sc_errno
1264  */
1265 static StreamingBufferRegion *BufferInsertAtRegionConsolidate(StreamingBuffer *sb,
1266  const StreamingBufferConfig *cfg, StreamingBufferRegion *dst,
1267  StreamingBufferRegion *src_start, StreamingBufferRegion *src_end,
1268  const uint64_t data_offset, const uint32_t data_len, StreamingBufferRegion *prev,
1269  uint32_t dst_buf_size)
1270 {
1271  int retval;
1272 #ifdef DEBUG
1273  const uint64_t data_re = data_offset + data_len;
1274  SCLogDebug("sb %p dst %p src_start %p src_end %p data_offset %" PRIu64
1275  "/data_len %u/data_re %" PRIu64,
1276  sb, dst, src_start, src_end, data_offset, data_len, data_re);
1277 #endif
1278 
1279  // 1. determine size and offset for dst.
1280  const uint64_t dst_offset = MIN(src_start->stream_offset, data_offset);
1281  DEBUG_VALIDATE_BUG_ON(dst_offset < sb->region.stream_offset);
1282  const uint32_t dst_size = dst_buf_size;
1283  SCLogDebug("dst_size %u", dst_size);
1284 
1285  // 2. resize dst
1286  const uint32_t old_size = dst->buf_size;
1287  DEBUG_VALIDATE_BUG_ON(dst->stream_offset - dst_offset > UINT32_MAX);
1288  const uint32_t dst_copy_offset = (uint32_t)(dst->stream_offset - dst_offset);
1289 #ifdef DEBUG
1290  const uint32_t old_offset = dst->buf_offset;
1291  SCLogDebug("old_size %u, old_offset %u, dst_copy_offset %u", old_size, old_offset,
1292  dst_copy_offset);
1293 #endif
1294  if ((retval = GrowRegionToSize(sb, cfg, dst, dst_size)) != SC_OK) {
1295  sc_errno = retval;
1296  return NULL;
1297  }
1298  SCLogDebug("resized to %u -> %u", dst_size, dst->buf_size);
1299  /* validate that the size is exactly what we asked for */
1300  DEBUG_VALIDATE_BUG_ON(dst_size != dst->buf_size);
1301  if (dst_copy_offset != 0)
1302  SCMemmove(dst->buf + dst_copy_offset, dst->buf_size - dst_copy_offset, dst->buf, old_size);
1303  if (dst_offset != dst->stream_offset) {
1304  dst->stream_offset = dst_offset;
1305  // buf_offset no longer valid, reset.
1306  dst->buf_offset = 0;
1307  }
1308 
1309  uint32_t new_offset = src_start->buf_offset;
1310  if (data_offset == src_start->stream_offset + src_start->buf_offset) {
1311  new_offset += data_len;
1312  }
1313 
1314  bool start_is_main = false;
1315  if (src_start == &sb->region) {
1316  DEBUG_VALIDATE_BUG_ON(src_start->stream_offset != dst_offset);
1317 
1318  start_is_main = true;
1319  SCLogDebug("src_start is main region");
1320  if (src_start != dst)
1321  memcpy(dst->buf, src_start->buf, src_start->buf_offset);
1322  if (src_start == src_end) {
1323  SCLogDebug("src_start == src_end == main, we're done");
1324  DEBUG_VALIDATE_BUG_ON(src_start != dst);
1325  return src_start;
1326  }
1327  prev = src_start;
1328  src_start = src_start->next; // skip in the loop below
1329  }
1330 
1331  // 3. copy all regions from src_start to dst_start into the new region
1332  for (StreamingBufferRegion *r = src_start; r != NULL;) {
1333  SCLogDebug("r %p %" PRIu64 ", offset %u, len %u, %s, last %s", r, r->stream_offset,
1334  r->buf_offset, r->buf_size, r == &sb->region ? "main" : "aux",
1335  BOOL2STR(r == src_end));
1336  // skip dst
1337  if (r == dst) {
1338  SCLogDebug("skipping r %p as it is 'dst'", r);
1339  if (r == src_end)
1340  break;
1341  prev = r;
1342  r = r->next;
1343  continue;
1344  }
1345  DEBUG_VALIDATE_BUG_ON(r->stream_offset - dst_offset > UINT32_MAX);
1346  const uint32_t target_offset = (uint32_t)(r->stream_offset - dst_offset);
1347  SCLogDebug("r %p: target_offset %u", r, target_offset);
1348  DEBUG_VALIDATE_BUG_ON(target_offset > dst->buf_size);
1349  DEBUG_VALIDATE_BUG_ON(target_offset + r->buf_size > dst->buf_size);
1350  memcpy(dst->buf + target_offset, r->buf, r->buf_size);
1351 
1352  StreamingBufferRegion *next = r->next;
1353  FREE(cfg, r->buf, r->buf_size);
1354  FREE(cfg, r, sizeof(*r));
1355  sb->regions--;
1356  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1357 
1358  DEBUG_VALIDATE_BUG_ON(prev == NULL && src_start != &sb->region);
1359  if (prev != NULL) {
1360  SCLogDebug("setting prev %p next to %p (was %p)", prev, next, prev->next);
1361  prev->next = next;
1362  } else {
1363  SCLogDebug("no prev yet");
1364  }
1365 
1366  if (r == src_end)
1367  break;
1368  r = next;
1369  }
1370 
1371  /* special handling of main region being the start, but not the
1372  * region we expand. In this case we'll have main and dst. We will
1373  * move the buffer from dst into main and free dst. */
1374  if (start_is_main && dst != &sb->region) {
1375  DEBUG_VALIDATE_BUG_ON(sb->region.next != dst);
1376  SCLogDebug("start_is_main && dst != main region");
1377  FREE(cfg, sb->region.buf, sb->region.buf_size);
1378  sb->region.buf = dst->buf;
1379  sb->region.buf_size = dst->buf_size;
1380  sb->region.buf_offset = new_offset;
1381  SCLogDebug("sb->region.buf_offset set to %u", sb->region.buf_offset);
1382  sb->region.next = dst->next;
1383  FREE(cfg, dst, sizeof(*dst));
1384  dst = &sb->region;
1385  sb->regions--;
1386  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1387  } else {
1388  SCLogDebug("dst: %p next %p", dst, dst->next);
1389  }
1390 
1391  SCLogDebug("returning dst %p stream_offset %" PRIu64 " buf_offset %u buf_size %u", dst,
1392  dst->stream_offset, dst->buf_offset, dst->buf_size);
1393  return dst;
1394 }
1395 
1396 /** \note sets sc_errno */
1397 static StreamingBufferRegion *BufferInsertAtRegionDo(StreamingBuffer *sb,
1398  const StreamingBufferConfig *cfg, const uint64_t offset, const uint32_t len)
1399 {
1400  SCLogDebug("offset %" PRIu64 ", len %u", offset, len);
1401  StreamingBufferRegion *start_prev = NULL;
1402  StreamingBufferRegion *start =
1403  FindFirstRegionForOffset(cfg, &sb->region, offset, len, &start_prev);
1404  if (start) {
1405  const uint64_t insert_re = offset + len;
1406  const uint64_t insert_start_offset = MIN(start->stream_offset, offset);
1407  uint64_t insert_adjusted_re = insert_re;
1408 
1409  SCLogDebug("start region %p/%" PRIu64 "/%u", start, start->stream_offset, start->buf_size);
1410  StreamingBufferRegion *big = FindLargestRegionForOffset(cfg, start, offset, len);
1411  DEBUG_VALIDATE_BUG_ON(big == NULL);
1412  if (big == NULL) {
1413  sc_errno = SC_EINVAL;
1414  return NULL;
1415  }
1416  SCLogDebug("big region %p/%" PRIu64 "/%u", big, big->stream_offset, big->buf_size);
1417  StreamingBufferRegion *end = FindRightEdge(cfg, big, offset, len);
1418  DEBUG_VALIDATE_BUG_ON(end == NULL);
1419  if (end == NULL) {
1420  sc_errno = SC_EINVAL;
1421  return NULL;
1422  }
1423  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1424 
1425  DEBUG_VALIDATE_BUG_ON(insert_adjusted_re - insert_start_offset > UINT32_MAX);
1426  uint32_t new_buf_size = ToNextMultipleOf(
1427  (uint32_t)(insert_adjusted_re - insert_start_offset), cfg->buf_size);
1428  SCLogDebug("new_buf_size %u", new_buf_size);
1429 
1430  /* see if our new buf size + cfg->buf_size margin leads to an overlap with the next region.
1431  * If so, include that in the consolidation. */
1432  if (end->next != NULL && new_buf_size + insert_start_offset > end->next->stream_offset) {
1433  SCLogDebug("adjusted end from %p to %p", end, end->next);
1434  end = end->next;
1435  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1436  DEBUG_VALIDATE_BUG_ON(insert_adjusted_re - insert_start_offset > UINT32_MAX);
1437  new_buf_size = ToNextMultipleOf(
1438  (uint32_t)(insert_adjusted_re - insert_start_offset), cfg->buf_size);
1439  SCLogDebug("new_buf_size %u", new_buf_size);
1440  }
1441 
1442  SCLogDebug("end region %p/%" PRIu64 "/%u", end, end->stream_offset, end->buf_size);
1443  /* sets sc_errno */
1444  StreamingBufferRegion *ret = BufferInsertAtRegionConsolidate(
1445  sb, cfg, big, start, end, offset, len, start_prev, new_buf_size);
1446  return ret;
1447  } else {
1448  /* if there was no region we can use we add a new region and insert it */
1449  StreamingBufferRegion *append = &sb->region;
1450  for (StreamingBufferRegion *r = append; r != NULL; r = r->next) {
1451  if (r->stream_offset > offset) {
1452  break;
1453  } else {
1454  append = r;
1455  }
1456  }
1457 
1458  SCLogDebug("no matching region found, append to %p (%s)", append,
1459  append == &sb->region ? "main" : "aux");
1460  /* sets sc_errno */
1461  StreamingBufferRegion *add = InitBufferRegion(sb, cfg, len);
1462  if (add == NULL)
1463  return NULL;
1464  add->stream_offset = offset;
1465  add->next = append->next;
1466  append->next = add;
1467  SCLogDebug("new region %p offset %" PRIu64, add, add->stream_offset);
1468  return add;
1469  }
1470 }
1471 
1472 /** \internal
1473  * \brief return the region to put the new data in
1474  *
1475  * Will find an existing region, expand it if needed. If no existing region exists or is
1476  * a good fit, it will try to set up a new region. If the region then overlaps or gets
1477  * too close to the next, merge them.
1478  *
1479  * \note sets sc_errno
1480  */
1481 static StreamingBufferRegion *BufferInsertAtRegion(StreamingBuffer *sb,
1482  const StreamingBufferConfig *cfg, const uint32_t data_len, const uint64_t data_offset)
1483 {
1484  SCLogDebug("data_offset %" PRIu64 ", data_len %u, re %" PRIu64, data_offset, data_len,
1485  data_offset + data_len);
1486  ListRegions(sb);
1487 
1488  if (RegionsIntersect(cfg, &sb->region, data_offset, data_offset + data_len)) {
1489  SCLogDebug("data_offset %" PRIu64 ", data_len %u intersects with main region (next %p)",
1490  data_offset, data_len, sb->region.next);
1491  if (sb->region.next == NULL ||
1492  !RegionsIntersect(cfg, sb->region.next, data_offset, data_offset + data_len)) {
1493  SCLogDebug(
1494  "data_offset %" PRIu64
1495  ", data_len %u intersects with main region, no next or way before next region",
1496  data_offset, data_len);
1497  if (sb->region.buf == NULL) {
1498  int r;
1499  if ((r = InitBuffer(sb, cfg)) != SC_OK) { // TODO init with size
1500  sc_errno = r;
1501  return NULL;
1502  }
1503  }
1504  return &sb->region;
1505  }
1506  } else if (sb->region.next == NULL) {
1507  /* InitBufferRegion sets sc_errno */
1508  StreamingBufferRegion *aux_r = sb->region.next = InitBufferRegion(sb, cfg, data_len);
1509  if (aux_r == NULL)
1510  return NULL;
1511  aux_r->stream_offset = data_offset;
1512  DEBUG_VALIDATE_BUG_ON(data_len > aux_r->buf_size);
1513  SCLogDebug("created new region %p with offset %" PRIu64 ", size %u", aux_r,
1514  aux_r->stream_offset, aux_r->buf_size);
1515  return aux_r;
1516  }
1517  /* BufferInsertAtRegionDo sets sc_errno */
1518  StreamingBufferRegion *blob = BufferInsertAtRegionDo(sb, cfg, data_offset, data_len);
1519  SCLogDebug("blob %p (%s)", blob, blob == &sb->region ? "main" : "aux");
1520  return blob;
1521 }
1522 
1523 /**
1524  * \param offset offset relative to StreamingBuffer::stream_offset
1525  *
1526  * \return SC_OK in case of success
1527  * \return SC_E* errors otherwise
1528  */
1529 int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1530  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
1531 {
1532  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1533  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1534  DEBUG_VALIDATE_BUG_ON(offset < sb->region.stream_offset);
1535  if (offset < sb->region.stream_offset) {
1536  return SC_EINVAL;
1537  }
1538 
1539  StreamingBufferRegion *region = BufferInsertAtRegion(sb, cfg, data_len, offset);
1540  if (region == NULL) {
1541  return sc_errno;
1542  }
1543 
1544  const bool region_is_main = region == &sb->region;
1545 
1546  SCLogDebug("inserting %" PRIu64 "/%u using %s region %p", offset, data_len,
1547  region == &sb->region ? "main" : "aux", region);
1548 
1549  DEBUG_VALIDATE_BUG_ON(offset - region->stream_offset > UINT32_MAX);
1550  uint32_t rel_offset = (uint32_t)(offset - region->stream_offset);
1551  int r = DataFitsAtOffset(region, data_len, rel_offset);
1552  if (r < 0) {
1553  DEBUG_VALIDATE_BUG_ON(1);
1554  return SC_ELIMIT;
1555  } else if (r == 0) {
1556  if ((r = GrowToSize(sb, cfg, (rel_offset + data_len))) != SC_OK)
1557  return r;
1558  }
1559  DEBUG_VALIDATE_BUG_ON(DataFitsAtOffset(region, data_len, rel_offset) != 1);
1560 
1561  SCLogDebug("offset %" PRIu64 " data_len %u, rel_offset %u into region offset %" PRIu64
1562  ", buf_offset %u, buf_size %u",
1563  offset, data_len, rel_offset, region->stream_offset, region->buf_offset,
1564  region->buf_size);
1565  memcpy(region->buf + rel_offset, data, data_len);
1566  seg->stream_offset = offset;
1567  seg->segment_len = data_len;
1568 
1569  SCLogDebug("rel_offset %u region->stream_offset %" PRIu64 ", buf_offset %u", rel_offset,
1570  region->stream_offset, region->buf_offset);
1571 
1572  if (RB_EMPTY(&sb->sbb_tree)) {
1573  SCLogDebug("empty sbb list");
1574 
1575  if (region_is_main) {
1576  if (sb->region.stream_offset == offset) {
1577  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
1578  /* empty list, data is exactly what is expected (append),
1579  * so do nothing.
1580  * Update buf_offset if needed, but in case of overlaps it might be beyond us. */
1581  sb->region.buf_offset = MAX(sb->region.buf_offset, rel_offset + data_len);
1582  } else if ((rel_offset + data_len) <= sb->region.buf_offset) {
1583  SCLogDebug("empty sbb list: block is within existing main data region");
1584  } else {
1585  if (sb->region.buf_offset && rel_offset == sb->region.buf_offset) {
1586  SCLogDebug("exactly at expected offset");
1587  // nothing to do
1588  sb->region.buf_offset = rel_offset + data_len;
1589 
1590  } else if (rel_offset < sb->region.buf_offset) {
1591  // nothing to do
1592 
1593  SCLogDebug("before expected offset: %u < sb->region.buf_offset %u", rel_offset,
1594  sb->region.buf_offset);
1595  if (rel_offset + data_len > sb->region.buf_offset) {
1596  SCLogDebug("before expected offset, ends after: %u < sb->region.buf_offset "
1597  "%u, %u > %u",
1598  rel_offset, sb->region.buf_offset, rel_offset + data_len,
1599  sb->region.buf_offset);
1600  sb->region.buf_offset = rel_offset + data_len;
1601  }
1602 
1603  } else if (sb->region.buf_offset) {
1604  SCLogDebug("beyond expected offset: SBBInit");
1605  /* existing data, but there is a gap between us */
1606  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1607  return r;
1608  } else {
1609  /* gap before data in empty list */
1610  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1611  if ((r = SBBInitLeadingGap(sb, cfg, offset, data_len)) != SC_OK)
1612  return r;
1613  }
1614  }
1615  } else {
1616  if (sb->region.buf_offset) {
1617  /* existing data, but there is a gap between us */
1618  SCLogDebug("empty sbb list, no data in main: use SBBInit");
1619  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1620  return r;
1621  } else {
1622  /* gap before data in empty list */
1623  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1624  if ((r = SBBInitLeadingGap(sb, cfg, offset, data_len)) != SC_OK)
1625  return r;
1626  }
1627  if (rel_offset == region->buf_offset) {
1628  SCLogDebug("pre region->buf_offset %u", region->buf_offset);
1629  region->buf_offset = rel_offset + data_len;
1630  SCLogDebug("post region->buf_offset %u", region->buf_offset);
1631  }
1632  }
1633  } else {
1634  SCLogDebug("updating sbb tree");
1635  /* already have blocks, so append new block based on new data */
1636  if ((r = SBBUpdate(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1637  return r;
1638  }
1639  DEBUG_VALIDATE_BUG_ON(!region_is_main && sb->head == NULL);
1640 
1641  ListRegions(sb);
1642  if (RB_EMPTY(&sb->sbb_tree)) {
1643  DEBUG_VALIDATE_BUG_ON(offset + data_len > sb->region.stream_offset + sb->region.buf_offset);
1644  }
1645 
1646  return SC_OK;
1647 }
1648 
1649 int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb,
1650  const StreamingBufferSegment *seg)
1651 {
1652  if (seg->stream_offset < sb->region.stream_offset) {
1653  if (seg->stream_offset + seg->segment_len <= sb->region.stream_offset) {
1654  return 1;
1655  }
1656  }
1657  return 0;
1658 }
1659 
1660 static inline const StreamingBufferRegion *GetRegionForOffset(
1661  const StreamingBuffer *sb, const uint64_t offset)
1662 {
1663  if (sb == NULL)
1664  return NULL;
1665  if (sb->region.next == NULL) {
1666  return &sb->region;
1667  }
1668  if (offset >= sb->region.stream_offset &&
1669  offset < (sb->region.stream_offset + sb->region.buf_size)) {
1670  return &sb->region;
1671  }
1672  for (const StreamingBufferRegion *r = sb->region.next; r != NULL; r = r->next) {
1673  if (offset >= r->stream_offset && offset < (r->stream_offset + r->buf_size)) {
1674  return r;
1675  }
1676  }
1677  return NULL;
1678 }
1679 
1680 /** \brief get the data for one SBB */
1681 void StreamingBufferSBBGetData(const StreamingBuffer *sb,
1682  const StreamingBufferBlock *sbb,
1683  const uint8_t **data, uint32_t *data_len)
1684 {
1685  const StreamingBufferRegion *region = GetRegionForOffset(sb, sbb->offset);
1686  SCLogDebug("first find our region (offset %" PRIu64 ") -> %p", sbb->offset, region);
1687  if (region) {
1688  SCLogDebug("region %p found %" PRIu64 "/%u/%u", region, region->stream_offset,
1689  region->buf_size, region->buf_offset);
1690  DEBUG_VALIDATE_BUG_ON(
1691  region->stream_offset == sbb->offset && region->buf_offset > sbb->len);
1692  // buf_offset should match first sbb len if it has the same offset
1693 
1694  if (sbb->offset >= region->stream_offset) {
1695  SCLogDebug("1");
1696  uint64_t offset = sbb->offset - region->stream_offset;
1697  *data = region->buf + offset;
1698  DEBUG_VALIDATE_BUG_ON(offset + sbb->len > region->buf_size);
1699  *data_len = sbb->len;
1700  return;
1701  } else {
1702  SCLogDebug("2");
1703  uint32_t offset = (uint32_t)(region->stream_offset - sbb->offset);
1704  if (offset < sbb->len) {
1705  *data = region->buf;
1706  *data_len = sbb->len - offset;
1707  return;
1708  }
1709  SCLogDebug("3");
1710  }
1711  }
1712  *data = NULL;
1713  *data_len = 0;
1714 }
1715 
1716 /** \brief get the data for one SBB */
1717 void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb,
1718  const StreamingBufferBlock *sbb,
1719  const uint8_t **data, uint32_t *data_len,
1720  uint64_t offset)
1721 {
1722  /* validate that we are looking for a offset within the sbb */
1723  DEBUG_VALIDATE_BUG_ON(!(offset >= sbb->offset && offset < (sbb->offset + sbb->len)));
1724  if (!(offset >= sbb->offset && offset < (sbb->offset + sbb->len))) {
1725  *data = NULL;
1726  *data_len = 0;
1727  return;
1728  }
1729 
1730  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1731  if (region) {
1732  DEBUG_VALIDATE_BUG_ON(sbb->len - (offset - sbb->offset) > UINT32_MAX);
1733  uint32_t sbblen = (uint32_t)(sbb->len - (offset - sbb->offset));
1734 
1735  if (offset >= region->stream_offset) {
1736  uint32_t data_offset = (uint32_t)(offset - region->stream_offset);
1737  *data = region->buf + data_offset;
1738  if (data_offset + sbblen > region->buf_size) {
1739  *data_len = region->buf_size - data_offset;
1740  } else {
1741  *data_len = sbblen;
1742  }
1743  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1744  return;
1745  } else {
1746  uint32_t data_offset = (uint32_t)(region->stream_offset - sbb->offset);
1747  if (data_offset < sbblen) {
1748  *data = region->buf;
1749  *data_len = sbblen - data_offset;
1750  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1751  return;
1752  }
1753  }
1754  }
1755 
1756  *data = NULL;
1757  *data_len = 0;
1758 }
1759 
1760 void StreamingBufferSegmentGetData(const StreamingBuffer *sb,
1761  const StreamingBufferSegment *seg,
1762  const uint8_t **data, uint32_t *data_len)
1763 {
1764  const StreamingBufferRegion *region = GetRegionForOffset(sb, seg->stream_offset);
1765  if (region) {
1766  if (seg->stream_offset >= region->stream_offset) {
1767  uint32_t offset = (uint32_t)(seg->stream_offset - region->stream_offset);
1768  *data = region->buf + offset;
1769  if (offset + seg->segment_len > region->buf_size) {
1770  *data_len = region->buf_size - offset;
1771  } else {
1772  *data_len = seg->segment_len;
1773  }
1774  SCLogDebug("*data_len %u", *data_len);
1775  return;
1776  } else {
1777  uint32_t offset = (uint32_t)(region->stream_offset - seg->stream_offset);
1778  if (offset < seg->segment_len) {
1779  *data = region->buf;
1780  *data_len = seg->segment_len - offset;
1781  SCLogDebug("*data_len %u", *data_len);
1782  return;
1783  }
1784  }
1785  }
1786  *data = NULL;
1787  *data_len = 0;
1788 }
1789 
1790 /**
1791  * \retval 1 data is the same
1792  * \retval 0 data is different
1793  */
1794 int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb,
1795  const StreamingBufferSegment *seg,
1796  const uint8_t *rawdata, uint32_t rawdata_len)
1797 {
1798  const uint8_t *segdata = NULL;
1799  uint32_t segdata_len = 0;
1800  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
1801  if (segdata && segdata_len &&
1802  segdata_len == rawdata_len &&
1803  memcmp(segdata, rawdata, segdata_len) == 0)
1804  {
1805  return 1;
1806  }
1807  return 0;
1808 }
1809 
1810 int StreamingBufferGetData(const StreamingBuffer *sb,
1811  const uint8_t **data, uint32_t *data_len,
1812  uint64_t *stream_offset)
1813 {
1814  if (sb != NULL && sb->region.buf != NULL) {
1815  *data = sb->region.buf;
1816  *data_len = sb->region.buf_offset;
1817  *stream_offset = sb->region.stream_offset;
1818  return 1;
1819  } else {
1820  *data = NULL;
1821  *data_len = 0;
1822  *stream_offset = 0;
1823  return 0;
1824  }
1825 }
1826 
1827 int StreamingBufferGetDataAtOffset (const StreamingBuffer *sb,
1828  const uint8_t **data, uint32_t *data_len,
1829  uint64_t offset)
1830 {
1831  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1832  if (region != NULL && region->buf != NULL && offset >= region->stream_offset &&
1833  offset < (region->stream_offset + region->buf_offset)) {
1834  DEBUG_VALIDATE_BUG_ON(offset - region->stream_offset > UINT32_MAX);
1835  uint32_t skip = (uint32_t)(offset - region->stream_offset);
1836  *data = region->buf + skip;
1837  *data_len = region->buf_offset - skip;
1838  return 1;
1839  } else {
1840  *data = NULL;
1841  *data_len = 0;
1842  return 0;
1843  }
1844 }
1845 
1846 /**
1847  * \retval 1 data is the same
1848  * \retval 0 data is different
1849  */
1850 int StreamingBufferCompareRawData(const StreamingBuffer *sb,
1851  const uint8_t *rawdata, uint32_t rawdata_len)
1852 {
1853  const uint8_t *sbdata = NULL;
1854  uint32_t sbdata_len = 0;
1855  uint64_t offset = 0;
1856  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
1857  if (offset == 0 &&
1858  sbdata && sbdata_len &&
1859  sbdata_len == rawdata_len &&
1860  memcmp(sbdata, rawdata, sbdata_len) == 0)
1861  {
1862  return 1;
1863  }
1864  SCLogDebug("sbdata_len %u, offset %" PRIu64, sbdata_len, offset);
1865  printf("got:\n");
1866  PrintRawDataFp(stdout, sbdata,sbdata_len);
1867  printf("wanted:\n");
1868  PrintRawDataFp(stdout, rawdata,rawdata_len);
1869  return 0;
1870 }
1871 
1872 #ifdef UNITTESTS
1873 static void Dump(StreamingBuffer *sb)
1874 {
1875  PrintRawDataFp(stdout, sb->region.buf, sb->region.buf_offset);
1876 }
1877 
1878 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
1879 {
1880  const uint8_t *data = NULL;
1881  uint32_t data_len = 0;
1882  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
1883  if (data && data_len) {
1884  PrintRawDataFp(stdout, data, data_len);
1885  }
1886 }
1887 
1888 static int StreamingBufferTest02(void)
1889 {
1890  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1891  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1892  FAIL_IF(sb == NULL);
1893 
1894  StreamingBufferSegment seg1;
1895  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1896  StreamingBufferSegment seg2;
1897  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8) != 0);
1898  FAIL_IF(sb->region.stream_offset != 0);
1899  FAIL_IF(sb->region.buf_offset != 16);
1900  FAIL_IF(seg1.stream_offset != 0);
1901  FAIL_IF(seg2.stream_offset != 8);
1902  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1903  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1904  Dump(sb);
1905  DumpSegment(sb, &seg1);
1906  DumpSegment(sb, &seg2);
1907  FAIL_IF_NOT_NULL(sb->head);
1908  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1909 
1910  StreamingBufferSlideToOffset(sb, &cfg, 6);
1911  FAIL_IF_NOT_NULL(sb->head);
1912  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1913 
1914  StreamingBufferSegment seg3;
1915  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
1916  FAIL_IF(sb->region.stream_offset != 6);
1917  FAIL_IF(sb->region.buf_offset != 16);
1918  FAIL_IF(seg3.stream_offset != 16);
1919  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1920  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1921  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1922  Dump(sb);
1923  DumpSegment(sb, &seg1);
1924  DumpSegment(sb, &seg2);
1925  DumpSegment(sb, &seg3);
1926  FAIL_IF_NOT_NULL(sb->head);
1927  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1928 
1929  StreamingBufferSlideToOffset(sb, &cfg, 12);
1930  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1931  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1932  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1933  Dump(sb);
1934  DumpSegment(sb, &seg1);
1935  DumpSegment(sb, &seg2);
1936  DumpSegment(sb, &seg3);
1937  FAIL_IF_NOT_NULL(sb->head);
1938  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1939 
1940  StreamingBufferFree(sb, &cfg);
1941  PASS;
1942 }
1943 
1944 static int StreamingBufferTest03(void)
1945 {
1946  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1947  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1948  FAIL_IF(sb == NULL);
1949 
1950  StreamingBufferSegment seg1;
1951  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1952  StreamingBufferSegment seg2;
1953  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1954  FAIL_IF(sb->region.stream_offset != 0);
1955  FAIL_IF(sb->region.buf_offset != 8);
1956  FAIL_IF(seg1.stream_offset != 0);
1957  FAIL_IF(seg2.stream_offset != 14);
1958  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1959  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1960  Dump(sb);
1961  DumpSegment(sb, &seg1);
1962  DumpSegment(sb, &seg2);
1963  FAIL_IF_NULL(sb->head);
1964  FAIL_IF_NOT(sb->sbb_size == 16);
1965  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1966 
1967  StreamingBufferSegment seg3;
1968  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1969  FAIL_IF(sb->region.stream_offset != 0);
1970  FAIL_IF(sb->region.buf_offset != 22);
1971  FAIL_IF(seg3.stream_offset != 8);
1972  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1973  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1974  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1975  Dump(sb);
1976  DumpSegment(sb, &seg1);
1977  DumpSegment(sb, &seg2);
1978  DumpSegment(sb, &seg3);
1979  FAIL_IF_NULL(sb->head);
1980  FAIL_IF_NOT(sb->sbb_size == 22);
1981  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1982 
1983  StreamingBufferSlideToOffset(sb, &cfg, 10);
1984  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1985  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1986  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1987  Dump(sb);
1988  DumpSegment(sb, &seg1);
1989  DumpSegment(sb, &seg2);
1990  DumpSegment(sb, &seg3);
1991  FAIL_IF_NULL(sb->head);
1992  FAIL_IF_NOT(sb->sbb_size == 12);
1993  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1994 
1995  StreamingBufferFree(sb, &cfg);
1996  PASS;
1997 }
1998 
1999 static int StreamingBufferTest04(void)
2000 {
2001  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2002  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2003  FAIL_IF(sb == NULL);
2004 
2005  StreamingBufferSegment seg1;
2006  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
2007  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
2008  StreamingBufferSegment seg2;
2009  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
2010  FAIL_IF(sb->region.stream_offset != 0);
2011  FAIL_IF(sb->region.buf_offset != 8);
2012  FAIL_IF(seg1.stream_offset != 0);
2013  FAIL_IF(seg2.stream_offset != 14);
2014  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2015  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2016  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
2017  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2018  FAIL_IF(sbb1 != sb->head);
2019  FAIL_IF_NULL(sbb1);
2020  FAIL_IF(sbb1->offset != 0);
2021  FAIL_IF(sbb1->len != 8);
2022  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
2023  FAIL_IF_NULL(sbb2);
2024  FAIL_IF(sbb2 == sb->head);
2025  FAIL_IF(sbb2->offset != 14);
2026  FAIL_IF(sbb2->len != 8);
2027  Dump(sb);
2028  DumpSegment(sb, &seg1);
2029  DumpSegment(sb, &seg2);
2030  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2031 
2032  StreamingBufferSegment seg3;
2033  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
2034  FAIL_IF(sb->region.stream_offset != 0);
2035  FAIL_IF(sb->region.buf_offset != 22);
2036  FAIL_IF(seg3.stream_offset != 8);
2037  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2038  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2039  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
2040  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2041  FAIL_IF_NULL(sbb1);
2042  FAIL_IF(sbb1 != sb->head);
2043  FAIL_IF(sbb1->offset != 0);
2044  FAIL_IF(sbb1->len != 22);
2045  FAIL_IF(SBB_RB_NEXT(sbb1));
2046  Dump(sb);
2047  DumpSegment(sb, &seg1);
2048  DumpSegment(sb, &seg2);
2049  DumpSegment(sb, &seg3);
2050  FAIL_IF_NULL(sb->head);
2051  FAIL_IF_NOT(sb->sbb_size == 22);
2052  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2053 
2054  /* far ahead of curve: */
2055  StreamingBufferSegment seg4;
2056  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
2057  FAIL_IF(sb->region.stream_offset != 0);
2058  FAIL_IF(sb->region.buf_offset != 22);
2059  FAIL_IF(sb->region.buf_size != 128);
2060  FAIL_IF(seg4.stream_offset != 124);
2061  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2062  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2063  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
2064  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg4));
2065  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2066  FAIL_IF_NULL(sbb1);
2067  FAIL_IF(sbb1 != sb->head);
2068  FAIL_IF(sbb1->offset != 0);
2069  FAIL_IF(sbb1->len != 22);
2070  FAIL_IF(!SBB_RB_NEXT(sbb1));
2071  Dump(sb);
2072  DumpSegment(sb, &seg1);
2073  DumpSegment(sb, &seg2);
2074  DumpSegment(sb, &seg3);
2075  DumpSegment(sb, &seg4);
2076  FAIL_IF_NULL(sb->head);
2077  FAIL_IF_NOT(sb->sbb_size == 25);
2078  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2079 
2080  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
2081  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
2082  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
2083  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
2084 
2085  StreamingBufferFree(sb, &cfg);
2086  PASS;
2087 }
2088 
2089 /** \test lots of gaps in block list */
2090 static int StreamingBufferTest06(void)
2091 {
2092  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2093  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2094  FAIL_IF(sb == NULL);
2095 
2096  StreamingBufferSegment seg1;
2097  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"A", 1) != 0);
2098  StreamingBufferSegment seg2;
2099  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"C", 1, 2) != 0);
2100  Dump(sb);
2101  FAIL_IF_NULL(sb->head);
2102  FAIL_IF_NOT(sb->sbb_size == 2);
2103  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2104 
2105  StreamingBufferSegment seg3;
2106  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2107  Dump(sb);
2108  FAIL_IF_NULL(sb->head);
2109  FAIL_IF_NOT(sb->sbb_size == 3);
2110  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2111 
2112  StreamingBufferSegment seg4;
2113  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2114  Dump(sb);
2115  FAIL_IF_NULL(sb->head);
2116  FAIL_IF_NOT(sb->sbb_size == 4);
2117  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2118 
2119  StreamingBufferSegment seg5;
2120  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2121  Dump(sb);
2122  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2123  FAIL_IF_NULL(sbb1);
2124  FAIL_IF(sbb1->offset != 0);
2125  FAIL_IF(sbb1->len != 10);
2126  FAIL_IF(SBB_RB_NEXT(sbb1));
2127  FAIL_IF_NULL(sb->head);
2128  FAIL_IF_NOT(sb->sbb_size == 10);
2129  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2130 
2131  StreamingBufferSegment seg6;
2132  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2133  Dump(sb);
2134  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2135  FAIL_IF_NULL(sbb1);
2136  FAIL_IF(sbb1->offset != 0);
2137  FAIL_IF(sbb1->len != 10);
2138  FAIL_IF(SBB_RB_NEXT(sbb1));
2139  FAIL_IF_NULL(sb->head);
2140  FAIL_IF_NOT(sb->sbb_size == 10);
2141  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2142 
2143  StreamingBufferFree(sb, &cfg);
2144  PASS;
2145 }
2146 
2147 /** \test lots of gaps in block list */
2148 static int StreamingBufferTest07(void)
2149 {
2150  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2151  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2152  FAIL_IF(sb == NULL);
2153 
2154  StreamingBufferSegment seg1;
2155  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2156  StreamingBufferSegment seg2;
2157  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2158  Dump(sb);
2159  FAIL_IF_NULL(sb->head);
2160  FAIL_IF_NOT(sb->sbb_size == 2);
2161  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2162 
2163  StreamingBufferSegment seg3;
2164  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2165  Dump(sb);
2166  FAIL_IF_NULL(sb->head);
2167  FAIL_IF_NOT(sb->sbb_size == 3);
2168  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2169 
2170  StreamingBufferSegment seg4;
2171  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2172  Dump(sb);
2173  FAIL_IF_NULL(sb->head);
2174  FAIL_IF_NOT(sb->sbb_size == 4);
2175  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2176 
2177  StreamingBufferSegment seg5;
2178  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2179  Dump(sb);
2180  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2181  FAIL_IF_NULL(sbb1);
2182  FAIL_IF(sbb1->offset != 0);
2183  FAIL_IF(sbb1->len != 10);
2184  FAIL_IF(SBB_RB_NEXT(sbb1));
2185  FAIL_IF_NULL(sb->head);
2186  FAIL_IF_NOT(sb->sbb_size == 10);
2187  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2188 
2189  StreamingBufferSegment seg6;
2190  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2191  Dump(sb);
2192  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2193  FAIL_IF_NULL(sbb1);
2194  FAIL_IF(sbb1->offset != 0);
2195  FAIL_IF(sbb1->len != 10);
2196  FAIL_IF(SBB_RB_NEXT(sbb1));
2197  FAIL_IF_NULL(sb->head);
2198  FAIL_IF_NOT(sb->sbb_size == 10);
2199  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2200 
2201  StreamingBufferFree(sb, &cfg);
2202  PASS;
2203 }
2204 
2205 /** \test lots of gaps in block list */
2206 static int StreamingBufferTest08(void)
2207 {
2208  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2209  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2210  FAIL_IF(sb == NULL);
2211 
2212  StreamingBufferSegment seg1;
2213  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2214  StreamingBufferSegment seg2;
2215  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2216  Dump(sb);
2217  FAIL_IF_NULL(sb->head);
2218  FAIL_IF_NOT(sb->sbb_size == 2);
2219  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2220 
2221  StreamingBufferSegment seg3;
2222  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2223  Dump(sb);
2224  FAIL_IF_NULL(sb->head);
2225  FAIL_IF_NOT(sb->sbb_size == 3);
2226  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2227 
2228  StreamingBufferSegment seg4;
2229  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2230  Dump(sb);
2231  FAIL_IF_NULL(sb->head);
2232  FAIL_IF_NOT(sb->sbb_size == 4);
2233  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2234 
2235  StreamingBufferSegment seg5;
2236  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2237  Dump(sb);
2238  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2239  FAIL_IF_NULL(sbb1);
2240  FAIL_IF(sbb1->offset != 0);
2241  FAIL_IF(sbb1->len != 10);
2242  FAIL_IF(SBB_RB_NEXT(sbb1));
2243  FAIL_IF_NULL(sb->head);
2244  FAIL_IF_NOT(sb->sbb_size == 10);
2245  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2246 
2247  StreamingBufferSegment seg6;
2248  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
2249  Dump(sb);
2250  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2251  FAIL_IF_NULL(sbb1);
2252  FAIL_IF(sbb1->offset != 0);
2253  FAIL_IF(sbb1->len != 20);
2254  FAIL_IF(SBB_RB_NEXT(sbb1));
2255  FAIL_IF_NULL(sb->head);
2256  FAIL_IF_NOT(sb->sbb_size == 20);
2257  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2258 
2259  StreamingBufferFree(sb, &cfg);
2260  PASS;
2261 }
2262 
2263 /** \test lots of gaps in block list */
2264 static int StreamingBufferTest09(void)
2265 {
2266  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2267  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2268  FAIL_IF(sb == NULL);
2269 
2270  StreamingBufferSegment seg1;
2271  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2272  StreamingBufferSegment seg2;
2273  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2274  Dump(sb);
2275  FAIL_IF_NULL(sb->head);
2276  FAIL_IF_NOT(sb->sbb_size == 2);
2277  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2278 
2279  StreamingBufferSegment seg3;
2280  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2281  Dump(sb);
2282  FAIL_IF_NULL(sb->head);
2283  FAIL_IF_NOT(sb->sbb_size == 3);
2284  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2285 
2286  StreamingBufferSegment seg4;
2287  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"F", 1, 5) != 0);
2288  Dump(sb);
2289  FAIL_IF_NULL(sb->head);
2290  FAIL_IF_NOT(sb->sbb_size == 4);
2291  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2292 
2293  StreamingBufferSegment seg5;
2294  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2295  Dump(sb);
2296  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2297  FAIL_IF_NULL(sbb1);
2298  FAIL_IF(sbb1->offset != 0);
2299  FAIL_IF(sbb1->len != 10);
2300  FAIL_IF(SBB_RB_NEXT(sbb1));
2301  FAIL_IF_NULL(sb->head);
2302  FAIL_IF_NOT(sb->sbb_size == 10);
2303  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2304 
2305  StreamingBufferSegment seg6;
2306  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2307  Dump(sb);
2308  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2309  FAIL_IF_NULL(sbb1);
2310  FAIL_IF(sbb1->offset != 0);
2311  FAIL_IF(sbb1->len != 10);
2312  FAIL_IF(SBB_RB_NEXT(sbb1));
2313  FAIL_IF_NULL(sb->head);
2314  FAIL_IF_NOT(sb->sbb_size == 10);
2315  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2316 
2317  StreamingBufferFree(sb, &cfg);
2318  PASS;
2319 }
2320 
2321 /** \test lots of gaps in block list */
2322 static int StreamingBufferTest10(void)
2323 {
2324  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2325  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2326  FAIL_IF(sb == NULL);
2327 
2328  StreamingBufferSegment seg1;
2329  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"A", 1, 0) != 0);
2330  Dump(sb);
2331  StreamingBufferSegment seg2;
2332  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2333  Dump(sb);
2334  StreamingBufferSegment seg3;
2335  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2336  Dump(sb);
2337  FAIL_IF_NULL(sb->head);
2338  FAIL_IF_NOT(sb->sbb_size == 3);
2339 
2340  StreamingBufferSegment seg4;
2341  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"B", 1, 1) != 0);
2342  Dump(sb);
2343  StreamingBufferSegment seg5;
2344  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"C", 1, 2) != 0);
2345  Dump(sb);
2346  StreamingBufferSegment seg6;
2347  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"G", 1, 6) != 0);
2348  Dump(sb);
2349  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2350  FAIL_IF_NULL(sb->head);
2351  FAIL_IF_NOT(sb->sbb_size == 6);
2352 
2353  StreamingBufferSegment seg7;
2354  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2355  Dump(sb);
2356  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2357  FAIL_IF_NULL(sbb1);
2358  FAIL_IF(sbb1->offset != 0);
2359  FAIL_IF(sbb1->len != 10);
2360  FAIL_IF(SBB_RB_NEXT(sbb1));
2361  FAIL_IF_NULL(sb->head);
2362  FAIL_IF_NOT(sb->sbb_size == 10);
2363 
2364  StreamingBufferSegment seg8;
2365  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2366  Dump(sb);
2367  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2368  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2369  FAIL_IF_NULL(sbb1);
2370  FAIL_IF(sbb1->offset != 0);
2371  FAIL_IF(sbb1->len != 10);
2372  FAIL_IF(SBB_RB_NEXT(sbb1));
2373  FAIL_IF_NULL(sb->head);
2374  FAIL_IF_NOT(sb->sbb_size == 10);
2375 
2376  StreamingBufferFree(sb, &cfg);
2377  PASS;
2378 }
2379 
2380 static int StreamingBufferTest11(void)
2381 {
2382  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2383  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2384  FAIL_IF(sb == NULL);
2385 
2386  StreamingBufferSegment seg1;
2387  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
2388  StreamingBufferSegment seg2;
2389  unsigned int data_len = 0xffffffff;
2390  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"unused", data_len) != -1);
2391  FAIL_IF(StreamingBufferInsertAt(
2392  sb, &cfg, &seg2, (const uint8_t *)"abcdefghij", data_len, 100000) != SC_ELIMIT);
2393  StreamingBufferFree(sb, &cfg);
2394  PASS;
2395 }
2396 
2397 static const char *dummy_conf_string = "%YAML 1.1\n"
2398  "---\n"
2399  "\n"
2400  "app-layer:\n"
2401  " protocols:\n"
2402  " http:\n"
2403  " enabled: yes\n"
2404  " memcap: 88\n"
2405  "\n";
2406 
2407 static int StreamingBufferTest12(void)
2408 {
2409  SCConfCreateContextBackup();
2410  SCConfInit();
2411  HtpConfigCreateBackup();
2412  SCConfYamlLoadString((const char *)dummy_conf_string, strlen(dummy_conf_string));
2413  HTPConfigure();
2414 
2415  StreamingBufferConfig cfg = { 8, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, HTPCalloc, HTPRealloc,
2416  HTPFree };
2417  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2418  FAIL_IF(sb == NULL);
2419 
2420  StreamingBufferSegment seg1;
2421  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGHIJKLMNOP", 16) != 0);
2422 
2423  StreamingBufferSegment seg2;
2424  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2,
2425  (const uint8_t *)"ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ",
2426  52) != -1);
2427  FAIL_IF(sc_errno != SC_ELIMIT);
2428 
2429  StreamingBufferFree(sb, &cfg);
2430  HTPFreeConfig();
2431  SCConfDeInit();
2432  HtpConfigRestoreBackup();
2433  SCConfRestoreContextBackup();
2434 
2435  PASS;
2436 }
2437 #endif
2438 
2439 void StreamingBufferRegisterTests(void)
2440 {
2441 #ifdef UNITTESTS
2442  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
2443  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
2444  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
2445  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
2446  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
2447  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
2448  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
2449  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
2450  UtRegisterTest("StreamingBufferTest11 Bug 6903", StreamingBufferTest11);
2451  UtRegisterTest("StreamingBufferTest12 Bug 6782", StreamingBufferTest12);
2452 #endif
2453 }
SCConfYamlLoadString
int SCConfYamlLoadString(const char *string, size_t len)
Load configuration from a YAML string.
Definition: conf-yaml-loader.c:535
FREE
#define FREE(cfg, ptr, s)
Definition: util-streaming-buffer.c:65
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:1019
len
uint8_t len
Definition: app-layer-dnp3.h:2
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
StreamingBufferRegion_::next
struct StreamingBufferRegion_ * next
Definition: util-streaming-buffer.h:89
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:101
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:282
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:290
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
RB_LEFT
#define RB_LEFT(elm, field)
Definition: tree.h:322
SC_EINVAL
@ SC_EINVAL
Definition: util-error.h:30
StreamingBuffer_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:114
StreamingBuffer_::regions
uint16_t regions
Definition: util-streaming-buffer.h:113
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1810
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
MIN
#define MIN(x, y)
Definition: suricata-common.h:413
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:1097
HTPRealloc
void * HTPRealloc(void *ptr, size_t orig_size, size_t size)
Definition: app-layer-htp-mem.c:175
p
Packet * p
Definition: fuzz_dataset.c:30
segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:0
m
SCMutex m
Definition: flow-hash.h:6
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1827
MAX
#define MAX(x, y)
Definition: suricata-common.h:417
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1794
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
HTPConfigure
void HTPConfigure(void)
Definition: app-layer-htp.c:2361
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
SCConfInit
void SCConfInit(void)
Initialize the configuration system.
Definition: conf.c:120
app-layer-htp.h
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
FAIL_IF_NOT_NULL
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
util-debug.h
StreamingBufferRegion_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:86
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
util-error.h
RB_ROOT
#define RB_ROOT(head)
Definition: tree.h:326
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:244
HtpConfigCreateBackup
void HtpConfigCreateBackup(void)
Definition: app-layer-htp.c:2706
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:1649
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:422
REALLOC
#define REALLOC(cfg, ptr, orig_s, s)
Definition: util-streaming-buffer.c:63
CALLOC
#define CALLOC(cfg, n, s)
Definition: util-streaming-buffer.c:62
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:542
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:267
HTPCalloc
void * HTPCalloc(size_t n, size_t size)
Definition: app-layer-htp-mem.c:154
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:262
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:112
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:322
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
conf-yaml-loader.h
StreamingBufferRegion_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:85
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1850
SC_OK
@ SC_OK
Definition: util-error.h:27
RB_GENERATE
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
SCConfCreateContextBackup
void SCConfCreateContextBackup(void)
Creates a backup of the conf_hash hash_table used by the conf API.
Definition: conf.c:740
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:103
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamingBuffer_
Definition: util-streaming-buffer.h:108
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1717
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
Definition: util-streaming-buffer.c:1141
suricata-common.h
util-streaming-buffer.h
SCConfDeInit
void SCConfDeInit(void)
De-initializes the configuration system.
Definition: conf.c:759
util-validate.h
HtpConfigRestoreBackup
void HtpConfigRestoreBackup(void)
Definition: app-layer-htp.c:2711
StreamingBufferConfig_
Definition: util-streaming-buffer.h:65
HtpBodyChunk_::next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:124
RB_RIGHT
#define RB_RIGHT(elm, field)
Definition: tree.h:323
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1529
head
Flow * head
Definition: flow-hash.h:1
SCConfRestoreContextBackup
void SCConfRestoreContextBackup(void)
Restores the backup of the hash_table present in backup_conf_hash back to conf_hash.
Definition: conf.c:750
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
WARN_UNUSED
#define WARN_UNUSED
Definition: bindgen.h:33
StreamingBufferRegion_
Definition: util-streaming-buffer.h:84
app-layer-htp-mem.h
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:72
HTPFree
void HTPFree(void *ptr, size_t size)
Definition: app-layer-htp-mem.c:199
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
HTPFreeConfig
void HTPFreeConfig(void)
Clears the HTTP server configuration memory used by HTP library.
Definition: app-layer-htp.c:1585
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:1760
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:288
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1681
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:109
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:2439