suricata
util-streaming-buffer.c
Go to the documentation of this file.
1 /* Copyright (C) 2015-2023 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "util-streaming-buffer.h"
20 #include "util-unittest.h"
21 #include "util-print.h"
22 #include "util-validate.h"
23 #include "util-debug.h"
24 #include "util-error.h"
25 
26 #include "app-layer-htp-mem.h"
27 #include "conf-yaml-loader.h"
28 #include "app-layer-htp.h"
29 
30 static void ListRegions(StreamingBuffer *sb);
31 
32 #define DUMP_REGIONS 0 // set to 1 to dump a visual representation of the regions list and sbb tree.
33 
34 /**
35  * \file
36  *
37  * \author Victor Julien <victor@inliniac.net>
38  *
39  * \brief Streaming Buffer API
40  */
41 
42 static void *ReallocFunc(void *ptr, const size_t size)
43 {
44  void *ptrmem = SCRealloc(ptr, size);
45  if (unlikely(ptrmem == NULL)) {
46  sc_errno = SC_ENOMEM;
47  }
48  return ptrmem;
49 }
50 
51 static void *CallocFunc(const size_t nm, const size_t sz)
52 {
53  void *ptrmem = SCCalloc(nm, sz);
54  if (unlikely(ptrmem == NULL)) {
55  sc_errno = SC_ENOMEM;
56  }
57  return ptrmem;
58 }
59 
60 /* memory handling wrappers. If config doesn't define it's own set of
61  * functions, use the defaults */
62 // TODO the default allocators don't set `sc_errno` yet.
63 #define CALLOC(cfg, n, s) (cfg)->Calloc ? (cfg)->Calloc((n), (s)) : CallocFunc((n), (s))
64 #define REALLOC(cfg, ptr, orig_s, s) \
65  (cfg)->Realloc ? (cfg)->Realloc((ptr), (orig_s), (s)) : ReallocFunc((ptr), (s))
66 #define FREE(cfg, ptr, s) \
67  (cfg)->Free ? (cfg)->Free((ptr), (s)) : SCFree((ptr))
68 
69 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg);
70 
71 RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare);
72 
73 int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
74 {
75  SCLogDebug("a %" PRIu64 " len %u, b %" PRIu64 " len %u", a->offset, a->len, b->offset, b->len);
76 
77  if (a->offset > b->offset)
78  SCReturnInt(1);
79  else if (a->offset < b->offset)
80  SCReturnInt(-1);
81  else {
82  if (a->len == 0 || b->len == 0 || a->len == b->len)
83  SCReturnInt(0);
84  else if (a->len > b->len)
85  SCReturnInt(1);
86  else
87  SCReturnInt(-1);
88  }
89 }
90 
91 /* inclusive compare function that also considers the right edge,
92  * not just the offset. */
93 static inline int InclusiveCompare(StreamingBufferBlock *lookup, StreamingBufferBlock *intree) {
94  const uint64_t lre = lookup->offset + lookup->len;
95  const uint64_t tre = intree->offset + intree->len;
96  if (lre <= intree->offset) // entirely before
97  return -1;
98  else if (lookup->offset < tre && lre <= tre) // (some) overlap
99  return 0;
100  else
101  return 1; // entirely after
102 }
103 
104 StreamingBufferBlock *SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
105 {
106  SCLogDebug("looking up %" PRIu64, elm->offset);
107 
108  struct StreamingBufferBlock *tmp = RB_ROOT(head);
109  struct StreamingBufferBlock *res = NULL;
110  while (tmp) {
111  SCLogDebug("compare with %" PRIu64 "/%u", tmp->offset, tmp->len);
112  const int comp = InclusiveCompare(elm, tmp);
113  SCLogDebug("compare result: %d", comp);
114  if (comp < 0) {
115  res = tmp;
116  tmp = RB_LEFT(tmp, rb);
117  } else if (comp > 0) {
118  tmp = RB_RIGHT(tmp, rb);
119  } else {
120  return tmp;
121  }
122  }
123  return res;
124 }
125 
126 /** \interal
127  * \brief does data region intersect with list region 'r'
128  * Takes the max gap into account.
129  */
130 static inline bool RegionsIntersect(const StreamingBufferConfig *cfg,
131  const StreamingBufferRegion *r, const uint64_t offset, const uint64_t re)
132 {
133  /* create the data range for the region, adding the max gap */
134  const uint64_t reg_o =
135  r->stream_offset > cfg->region_gap ? (r->stream_offset - cfg->region_gap) : 0;
136  const uint64_t reg_re = r->stream_offset + r->buf_size + cfg->region_gap;
137  SCLogDebug("r %p: %" PRIu64 "/%" PRIu64 " - adjusted %" PRIu64 "/%" PRIu64, r, r->stream_offset,
138  r->stream_offset + r->buf_size, reg_o, reg_re);
139  /* check if data range intersects with region range */
140  if (offset >= reg_o && offset <= reg_re) {
141  SCLogDebug("r %p is in-scope", r);
142  return true;
143  }
144  if (re >= reg_o && re <= reg_re) {
145  SCLogDebug("r %p is in-scope: %" PRIu64 " >= %" PRIu64 " && %" PRIu64 " <= %" PRIu64, r, re,
146  reg_o, re, reg_re);
147  return true;
148  }
149  SCLogDebug("r %p is out of scope: %" PRIu64 "/%" PRIu64, r, offset, re);
150  return false;
151 }
152 
153 /** \internal
154  * \brief find the first region for merging.
155  */
156 static StreamingBufferRegion *FindFirstRegionForOffset(const StreamingBufferConfig *cfg,
157  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len,
158  StreamingBufferRegion **prev)
159 {
160  const uint64_t data_re = offset + len;
161  SCLogDebug("looking for first region matching %" PRIu64 "/%" PRIu64, offset, data_re);
162 
163  StreamingBufferRegion *p = NULL;
164  for (; r != NULL; r = r->next) {
165  if (RegionsIntersect(cfg, r, offset, data_re) == true) {
166  *prev = p;
167  return r;
168  }
169  p = r;
170  }
171  *prev = NULL;
172  return NULL;
173 }
174 
175 static StreamingBufferRegion *FindLargestRegionForOffset(const StreamingBufferConfig *cfg,
176  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len)
177 {
178  const uint64_t data_re = offset + len;
179  SCLogDebug("starting at %p/%" PRIu64 ", offset %" PRIu64 ", data_re %" PRIu64, r,
180  r->stream_offset, offset, data_re);
181  StreamingBufferRegion *candidate = r;
182  for (; r != NULL; r = r->next) {
183 #ifdef DEBUG
184  const uint64_t reg_re = r->stream_offset + r->buf_size;
185  SCLogDebug("checking: %p/%" PRIu64 "/%" PRIu64 ", offset %" PRIu64 "/%" PRIu64, r,
186  r->stream_offset, reg_re, offset, data_re);
187 #endif
188  if (!RegionsIntersect(cfg, r, offset, data_re))
189  return candidate;
190 
191  if (r->buf_size > candidate->buf_size) {
192  SCLogDebug("candidate %p as size %u > %u", candidate, r->buf_size, candidate->buf_size);
193  candidate = r;
194  }
195  }
196  return candidate;
197 }
198 
199 static StreamingBufferRegion *FindRightEdge(const StreamingBufferConfig *cfg,
200  StreamingBufferRegion *r, const uint64_t offset, const uint32_t len)
201 {
202  const uint64_t data_re = offset + len;
203  StreamingBufferRegion *candidate = r;
204  for (; r != NULL; r = r->next) {
205  if (!RegionsIntersect(cfg, r, offset, data_re)) {
206  SCLogDebug(
207  "r %p is out of scope: %" PRIu64 "/%u/%" PRIu64, r, offset, len, offset + len);
208  return candidate;
209  }
210  candidate = r;
211  }
212  return candidate;
213 }
214 
215 /** \note uses sc_errno to give error details */
216 static inline StreamingBufferRegion *InitBufferRegion(
217  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint32_t min_size)
218 {
219  if (sb->regions == USHRT_MAX || (cfg->max_regions != 0 && sb->regions >= cfg->max_regions)) {
220  SCLogDebug("max regions reached");
221  sc_errno = SC_ELIMIT;
222  return NULL;
223  }
224 
225  StreamingBufferRegion *aux_r = CALLOC(cfg, 1, sizeof(*aux_r));
226  if (aux_r == NULL) {
227  return NULL;
228  }
229 
230  aux_r->buf = CALLOC(cfg, 1, MAX(cfg->buf_size, min_size));
231  if (aux_r->buf == NULL) {
232  FREE(cfg, aux_r, sizeof(*aux_r));
233  return NULL;
234  }
235  aux_r->buf_size = MAX(cfg->buf_size, min_size);
236  sb->regions++;
237  sb->max_regions = MAX(sb->regions, sb->max_regions);
238  return aux_r;
239 }
240 
241 static inline int InitBuffer(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
242 {
243  sb->region.buf = CALLOC(cfg, 1, cfg->buf_size);
244  if (sb->region.buf == NULL) {
245  return sc_errno;
246  }
247  sb->region.buf_size = cfg->buf_size;
248  return SC_OK;
249 }
250 
251 StreamingBuffer *StreamingBufferInit(const StreamingBufferConfig *cfg)
252 {
253  StreamingBuffer *sb = CALLOC(cfg, 1, sizeof(StreamingBuffer));
254  if (sb == NULL) {
255  return NULL;
256  }
257 
258  sb->region.buf_size = cfg->buf_size;
259  sb->regions = sb->max_regions = 1;
260 
261  if (cfg->buf_size == 0) {
262  return sb;
263  }
264 
265  int r = InitBuffer(sb, cfg);
266  if (r != SC_OK) {
267  FREE(cfg, sb, sizeof(StreamingBuffer));
268  sc_errno = r;
269  return NULL;
270  }
271  return sb;
272 }
273 
274 void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
275 {
276  if (sb != NULL) {
277  SCLogDebug("sb->region.buf_size %u max %u", sb->region.buf_size, sb->buf_size_max);
278 
279  SBBFree(sb, cfg);
280  ListRegions(sb);
281  if (sb->region.buf != NULL) {
282  FREE(cfg, sb->region.buf, sb->region.buf_size);
283  sb->region.buf = NULL;
284  }
285 
286  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
287  StreamingBufferRegion *next = r->next;
288  FREE(cfg, r->buf, r->buf_size);
289  FREE(cfg, r, sizeof(*r));
290  r = next;
291  }
292  sb->region.next = NULL;
293  sb->regions = sb->max_regions = 1;
294  }
295 }
296 
297 void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
298 {
299  if (sb != NULL) {
300  StreamingBufferClear(sb, cfg);
301  FREE(cfg, sb, sizeof(StreamingBuffer));
302  }
303 }
304 
305 #ifdef DEBUG
306 static void SBBPrintList(StreamingBuffer *sb)
307 {
308  StreamingBufferBlock *sbb = NULL;
309  RB_FOREACH(sbb, SBB, &sb->sbb_tree) {
310  SCLogDebug("sbb: offset %" PRIu64 ", len %u", sbb->offset, sbb->len);
311  StreamingBufferBlock *next = SBB_RB_NEXT(sbb);
312  if (next) {
313  if ((sbb->offset + sbb->len) != next->offset) {
314  SCLogDebug("gap: offset %" PRIu64 ", len %" PRIu64, (sbb->offset + sbb->len),
315  next->offset - (sbb->offset + sbb->len));
316  }
317  }
318  }
319 }
320 #endif
321 
322 /* setup with gap between 2 blocks
323  *
324  * [block][gap][block]
325  **/
326 static int WARN_UNUSED SBBInit(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
327  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t data_len)
328 {
329  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
330  DEBUG_VALIDATE_BUG_ON(region->buf_offset > region->stream_offset + rel_offset);
331 
332  /* need to set up 2: existing data block and new data block */
333  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
334  if (sbb == NULL) {
335  return sc_errno;
336  }
337  sbb->offset = sb->region.stream_offset;
338  sbb->len = sb->region.buf_offset;
339  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
340  sb->head = sbb;
341  sb->sbb_size = sbb->len;
342 
343  StreamingBufferBlock *sbb2 = CALLOC(cfg, 1, sizeof(*sbb2));
344  if (sbb2 == NULL) {
345  return sc_errno;
346  }
347  sbb2->offset = region->stream_offset + rel_offset;
348  sbb2->len = data_len;
349  DEBUG_VALIDATE_BUG_ON(sbb2->offset < sbb->len);
350  SCLogDebug("sbb1 %" PRIu64 ", len %u, sbb2 %" PRIu64 ", len %u", sbb->offset, sbb->len,
351  sbb2->offset, sbb2->len);
352 
353  sb->sbb_size += sbb2->len;
354  if (SBB_RB_INSERT(&sb->sbb_tree, sbb2) != NULL) {
355  FREE(cfg, sbb2, sizeof(*sbb2));
356  return SC_EINVAL;
357  }
358 
359 #ifdef DEBUG
360  SBBPrintList(sb);
361 #endif
362  return SC_OK;
363 }
364 
365 /* setup with leading gap
366  *
367  * [gap][block]
368  **/
369 static int WARN_UNUSED SBBInitLeadingGap(
370  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset, uint32_t data_len)
371 {
372  DEBUG_VALIDATE_BUG_ON(!RB_EMPTY(&sb->sbb_tree));
373 
374  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
375  if (sbb == NULL) {
376  return sc_errno;
377  }
378  sbb->offset = offset;
379  sbb->len = data_len;
380 
381  sb->head = sbb;
382  sb->sbb_size = sbb->len;
383  (void)SBB_RB_INSERT(&sb->sbb_tree, sbb);
384 
385  SCLogDebug("sbb %" PRIu64 ", len %u", sbb->offset, sbb->len);
386 #ifdef DEBUG
387  SBBPrintList(sb);
388 #endif
389  return SC_OK;
390 }
391 
392 static inline void ConsolidateFwd(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
393  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
394 {
395  uint64_t sa_re = sa->offset + sa->len;
396  StreamingBufferBlock *tr, *s = sa;
397  RB_FOREACH_FROM(tr, SBB, s) {
398  if (sa == tr)
399  continue;
400 
401  const uint64_t tr_re = tr->offset + tr->len;
402  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u re %" PRIu64, tr, tr->offset, tr->len, tr_re);
403 
404  if (sa_re < tr->offset) {
405  SCLogDebug("entirely before: %" PRIu64 " < %" PRIu64, sa_re, tr->offset);
406  break; // entirely before
407  }
408 
409  /* new block (sa) entirely eclipsed by in tree block (tr)
410  sa: [ ]
411  tr: [ ]
412  sa: [ ]
413  tr: [ ]
414  sa: [ ]
415  tr: [ ]
416  */
417  if (sa->offset >= tr->offset && sa_re <= tr_re) {
418  sb->sbb_size -= sa->len;
419  sa->len = tr->len;
420  sa->offset = tr->offset;
421  sa_re = sa->offset + sa->len;
422  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
423  tr->offset, tr->len);
424  SBB_RB_REMOVE(tree, tr);
425  FREE(cfg, tr, sizeof(StreamingBufferBlock));
426  /* new block (sa) entire eclipses in tree block (tr)
427  sa: [ ]
428  tr: [ ]
429  sa: [ ]
430  tr: [ ]
431  sa: [ ]
432  tr: [ ]
433  */
434  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
435  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
436  tr->offset, tr->len);
437  SBB_RB_REMOVE(tree, tr);
438  sb->sbb_size -= tr->len;
439  FREE(cfg, tr, sizeof(StreamingBufferBlock));
440 
441  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
442  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
443  region->buf_size);
444  if (sa->offset == region->stream_offset &&
445  sa_re > (region->stream_offset + region->buf_offset)) {
446  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
447  region->buf_offset = sa_re - region->stream_offset;
448  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
449  " bo %u sz %u BUF_OFFSET UPDATED",
450  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
451  region->buf_size);
452  }
453  /* new block (sa) extended by in tree block (tr)
454  sa: [ ]
455  tr: [ ]
456  sa: [ ]
457  tr: [ ]
458  */
459  } else if (sa->offset < tr->offset && // starts before
460  sa_re >= tr->offset && sa_re < tr_re) // ends inside
461  {
462  // merge. sb->sbb_size includes both so we need to adjust that too.
463  uint32_t combined_len = sa->len + tr->len;
464  sa->len = tr_re - sa->offset;
465  sa_re = sa->offset + sa->len;
466  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
467  SBB_RB_REMOVE(tree, tr);
468  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
469  FREE(cfg, tr, sizeof(StreamingBufferBlock));
470  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u RESULT", sa, sa->offset, sa->len);
471  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
472  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
473  region->buf_size);
474  if (sa->offset == region->stream_offset &&
475  sa_re > (region->stream_offset + region->buf_offset)) {
476  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
477  region->buf_offset = sa_re - region->stream_offset;
478  SCLogDebug("-> (fwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
479  " bo %u sz %u BUF_OFFSET UPDATED",
480  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
481  region->buf_size);
482  }
483  }
484  }
485 }
486 
487 static inline void ConsolidateBackward(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
488  StreamingBufferRegion *region, struct SBB *tree, StreamingBufferBlock *sa)
489 {
490  uint64_t sa_re = sa->offset + sa->len;
491  StreamingBufferBlock *tr, *s = sa;
492  RB_FOREACH_REVERSE_FROM(tr, SBB, s) {
493  if (sa == tr)
494  continue;
495  const uint64_t tr_re = tr->offset + tr->len;
496  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u", tr, tr->offset, tr->len);
497 
498  if (sa->offset > tr_re)
499  break; // entirely after
500 
501  /* new block (sa) entirely eclipsed by in tree block (tr)
502  sa: [ ]
503  tr: [ ]
504  sa: [ ]
505  tr: [ ]
506  sa: [ ]
507  tr: [ ]
508  */
509  if (sa->offset >= tr->offset && sa_re <= tr_re) {
510  sb->sbb_size -= sa->len; // sa entirely eclipsed so remove double accounting
511  sa->len = tr->len;
512  sa->offset = tr->offset;
513  sa_re = sa->offset + sa->len;
514  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (sa overlapped by tr)", tr,
515  tr->offset, tr->len);
516  if (sb->head == tr)
517  sb->head = sa;
518  SBB_RB_REMOVE(tree, tr);
519  FREE(cfg, tr, sizeof(StreamingBufferBlock));
520  /* new block (sa) entire eclipses in tree block (tr)
521  sa: [ ]
522  tr: [ ]
523  sa: [ ]
524  tr: [ ]
525  sa: [ ]
526  tr: [ ]
527  */
528  } else if (sa->offset <= tr->offset && sa_re >= tr_re) {
529  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED ECLIPSED (tr overlapped by sa)", tr,
530  tr->offset, tr->len);
531  if (sb->head == tr)
532  sb->head = sa;
533  SBB_RB_REMOVE(tree, tr);
534  sb->sbb_size -= tr->len; // tr entirely eclipsed so remove double accounting
535  FREE(cfg, tr, sizeof(StreamingBufferBlock));
536 
537  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
538  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
539  region->buf_size);
540 
541  if (sa->offset == region->stream_offset &&
542  sa_re > (region->stream_offset + region->buf_offset)) {
543  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
544  region->buf_offset = sa_re - region->stream_offset;
545  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
546  " bo %u sz %u BUF_OFFSET UPDATED",
547  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
548  region->buf_size);
549  }
550 
551  /* new block (sa) extends in tree block (tr)
552  sa: [ ]
553  tr: [ ]
554  sa: [ ]
555  tr: [ ]
556  */
557  } else if (sa->offset > tr->offset && sa_re > tr_re && sa->offset <= tr_re) {
558  // merge. sb->sbb_size includes both so we need to adjust that too.
559  uint32_t combined_len = sa->len + tr->len;
560  sa->len = sa_re - tr->offset;
561  sa->offset = tr->offset;
562  sa_re = sa->offset + sa->len;
563  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u REMOVED MERGED", tr, tr->offset, tr->len);
564  if (sb->head == tr)
565  sb->head = sa;
566  SBB_RB_REMOVE(tree, tr);
567  sb->sbb_size -= (combined_len - sa->len); // remove what we added twice
568  FREE(cfg, tr, sizeof(StreamingBufferBlock));
569 
570  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64 " bo %u sz %u", sa,
571  sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
572  region->buf_size);
573  if (sa->offset == region->stream_offset &&
574  sa_re > (region->stream_offset + region->buf_offset)) {
575  DEBUG_VALIDATE_BUG_ON(sa_re < region->stream_offset);
576  region->buf_offset = sa_re - region->stream_offset;
577  SCLogDebug("-> (bwd) tr %p %" PRIu64 "/%u region %p so %" PRIu64
578  " bo %u sz %u BUF_OFFSET UPDATED",
579  sa, sa->offset, sa->len, region, region->stream_offset, region->buf_offset,
580  region->buf_size);
581  }
582  }
583  }
584 }
585 
586 /** \internal
587  * \param region the region that holds the new data
588  */
589 static int SBBUpdate(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
590  StreamingBufferRegion *region, uint32_t rel_offset, uint32_t len)
591 {
592  struct SBB *tree = &sb->sbb_tree;
593  SCLogDebug("* inserting: %u/%u", rel_offset, len);
594 
595  StreamingBufferBlock *sbb = CALLOC(cfg, 1, sizeof(*sbb));
596  if (sbb == NULL) {
597  return sc_errno;
598  }
599  sbb->offset = region->stream_offset + rel_offset;
600  sbb->len = len;
601 
602  StreamingBufferBlock *res = SBB_RB_INSERT(tree, sbb);
603  if (res) {
604  // exact overlap
605  SCLogDebug("* insert failed: exact match in tree with %p %" PRIu64 "/%u", res, res->offset,
606  res->len);
607  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
608  return SC_OK;
609  }
610  sb->sbb_size += len; // may adjust based on consolidation below
611 
612  /* handle backwards and forwards overlaps */
613  if (SBB_RB_PREV(sbb) == NULL) {
614  sb->head = sbb;
615  } else {
616  ConsolidateBackward(sb, cfg, region, tree, sbb);
617  }
618  ConsolidateFwd(sb, cfg, region, tree, sbb);
619 #ifdef DEBUG
620  SBBPrintList(sb);
621 #endif
622  if (sbb->offset == region->stream_offset) {
623  SCLogDebug("insert at region head");
624  region->buf_offset = sbb->len;
625  }
626  return SC_OK;
627 }
628 
629 static void SBBFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
630 {
631  StreamingBufferBlock *sbb = NULL, *safe = NULL;
632  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
633  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
634  sb->sbb_size -= sbb->len;
635  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
636  }
637  sb->head = NULL;
638 }
639 
640 static void SBBPrune(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
641 {
642  SCLogDebug("pruning %p to %" PRIu64, sb, sb->region.stream_offset);
643  StreamingBufferBlock *sbb = NULL, *safe = NULL;
644  RB_FOREACH_SAFE(sbb, SBB, &sb->sbb_tree, safe) {
645  /* completely beyond window, we're done */
646  if (sbb->offset >= sb->region.stream_offset) {
647  sb->head = sbb;
648  if (sbb->offset == sb->region.stream_offset) {
649  SCLogDebug("set buf_offset?");
650  if (sbb->offset == sb->region.stream_offset) {
651  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
652  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
653  sb->region.buf_offset = sbb->len;
654  }
655  }
656  break;
657  }
658 
659  /* partly before, partly beyond. Adjust */
660  if (sbb->offset < sb->region.stream_offset &&
661  sbb->offset + sbb->len > sb->region.stream_offset) {
662  uint32_t shrink_by = sb->region.stream_offset - sbb->offset;
663  DEBUG_VALIDATE_BUG_ON(shrink_by > sbb->len);
664  if (sbb->len >= shrink_by) {
665  sbb->len -= shrink_by;
666  sbb->offset += shrink_by;
667  sb->sbb_size -= shrink_by;
668  SCLogDebug("shrunk by %u", shrink_by);
669  DEBUG_VALIDATE_BUG_ON(sbb->offset != sb->region.stream_offset);
670  }
671  sb->head = sbb;
672  if (sbb->offset == sb->region.stream_offset) {
673  SCLogDebug("set buf_offset to first sbb len %u", sbb->len);
674  DEBUG_VALIDATE_BUG_ON(sbb->len > sb->region.buf_size);
675  sb->region.buf_offset = sbb->len;
676  }
677  break;
678  }
679 
680  SBB_RB_REMOVE(&sb->sbb_tree, sbb);
681  /* either we set it again for the next sbb, or there isn't any */
682  sb->head = NULL;
683  sb->sbb_size -= sbb->len;
684  SCLogDebug("sb %p removed %p %" PRIu64 ", %u", sb, sbb, sbb->offset, sbb->len);
685  FREE(cfg, sbb, sizeof(StreamingBufferBlock));
686  }
687 #ifdef DEBUG
688  SBBPrintList(sb);
689 #endif
690 }
691 
692 static inline uint32_t ToNextMultipleOf(const uint32_t in, const uint32_t m)
693 {
694  uint32_t r = in;
695  if (m > 0) {
696  const uint32_t x = in % m;
697  if (x != 0) {
698  r = (in - x) + m;
699  }
700  }
701  return r;
702 }
703 
704 static thread_local bool g2s_warn_once = false;
705 
706 static inline int WARN_UNUSED GrowRegionToSize(StreamingBuffer *sb,
707  const StreamingBufferConfig *cfg, StreamingBufferRegion *region, const uint32_t size)
708 {
709  DEBUG_VALIDATE_BUG_ON(region->buf_size > BIT_U32(30));
710  if (size > BIT_U32(30)) { // 1GiB
711  if (!g2s_warn_once) {
712  SCLogWarning("StreamingBuffer::GrowRegionToSize() tried to alloc %u bytes, exceeds "
713  "limit of %" PRIu32,
714  size, BIT_U32(30));
715  g2s_warn_once = true;
716  }
717  return SC_ELIMIT;
718  }
719 
720  /* try to grow in multiples of cfg->buf_size */
721  const uint32_t grow = ToNextMultipleOf(size, cfg->buf_size);
722  SCLogDebug("grow %u", grow);
723 
724  void *ptr = REALLOC(cfg, region->buf, region->buf_size, grow);
725  if (ptr == NULL) {
726  if (sc_errno == SC_OK)
727  sc_errno = SC_ENOMEM;
728  return sc_errno;
729  }
730  /* for safe printing and general caution, lets memset the
731  * new data to 0 */
732  size_t diff = grow - region->buf_size;
733  void *new_mem = ((char *)ptr) + region->buf_size;
734  memset(new_mem, 0, diff);
735 
736  region->buf = ptr;
737  region->buf_size = grow;
738  SCLogDebug("grown buffer to %u", grow);
739 #ifdef DEBUG
740  if (region->buf_size > sb->buf_size_max) {
741  sb->buf_size_max = region->buf_size;
742  }
743 #endif
744  return SC_OK;
745 }
746 
747 static int WARN_UNUSED GrowToSize(
748  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint32_t size)
749 {
750  return GrowRegionToSize(sb, cfg, &sb->region, size);
751 }
752 
753 static inline bool RegionBeforeOffset(const StreamingBufferRegion *r, const uint64_t o)
754 {
755  return (r->stream_offset + r->buf_size <= o);
756 }
757 
758 static inline bool RegionContainsOffset(const StreamingBufferRegion *r, const uint64_t o)
759 {
760  return (o >= r->stream_offset && (r->stream_offset + r->buf_size) >= o);
761 }
762 
763 /** \internal
764  * \brief slide to offset for regions
765  *
766  *
767  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
768  * ^
769  * - main reset to 0
770  *
771  *
772  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
773  * ^
774  * - main shift
775  *
776  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
777  * ^
778  * - main reset
779  * - move aux into main
780  * - free aux
781  * - shift
782  *
783  * [ main ] [ gap ] [ aux ] [ gap ] [ aux ]
784  * ^
785  * - main reset
786  * - move aux into main
787  * - free aux
788  * - no shift
789  */
790 static inline void StreamingBufferSlideToOffsetWithRegions(
791  StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint64_t slide_offset)
792 {
793  ListRegions(sb);
794  DEBUG_VALIDATE_BUG_ON(slide_offset == sb->region.stream_offset);
795 
796  SCLogDebug("slide_offset %" PRIu64, slide_offset);
797  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
798  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
799 
800  StreamingBufferRegion *to_shift = NULL;
801  const bool main_is_oow = RegionBeforeOffset(&sb->region, slide_offset);
802  if (main_is_oow) {
803  SCLogDebug("main_is_oow");
804  if (sb->region.buf != NULL) {
805  SCLogDebug("clearing main");
806  FREE(cfg, sb->region.buf, sb->region.buf_size);
807  sb->region.buf = NULL;
808  sb->region.buf_size = 0;
809  sb->region.buf_offset = 0;
810  sb->region.stream_offset = slide_offset;
811  } else {
812  sb->region.stream_offset = slide_offset;
813  }
814 
815  /* remove regions that are out of window & select the region to
816  * become the new main */
817  StreamingBufferRegion *prev = &sb->region;
818  for (StreamingBufferRegion *r = sb->region.next; r != NULL;) {
819  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64, r, r->stream_offset,
820  r->stream_offset + r->buf_offset);
821  StreamingBufferRegion *next = r->next;
822  if (RegionBeforeOffset(r, slide_offset)) {
823  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> before", r, r->stream_offset,
824  r->stream_offset + r->buf_offset);
825  DEBUG_VALIDATE_BUG_ON(r == &sb->region);
826  prev->next = next;
827 
828  FREE(cfg, r->buf, r->buf_size);
829  FREE(cfg, r, sizeof(*r));
830  sb->regions--;
831  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
832  } else if (RegionContainsOffset(r, slide_offset)) {
833  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> within", r, r->stream_offset,
834  r->stream_offset + r->buf_offset);
835  /* remove from list, we will xfer contents to main below */
836  prev->next = next;
837  to_shift = r;
838  break;
839  } else {
840  SCLogDebug("r %p so %" PRIu64 ", re %" PRIu64 " -> post", r, r->stream_offset,
841  r->stream_offset + r->buf_offset);
842  /* implied beyond slide offset */
843  DEBUG_VALIDATE_BUG_ON(r->stream_offset < slide_offset);
844  break;
845  }
846  r = next;
847  }
848  SCLogDebug("to_shift %p", to_shift);
849 
850  // this region is main, or will xfer its buffer to main
851  if (to_shift && to_shift != &sb->region) {
852  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", to_shift->stream_offset,
853  to_shift->buf, to_shift->buf_size, to_shift->buf_offset);
854  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL);
855 
856  sb->region.buf = to_shift->buf;
857  sb->region.stream_offset = to_shift->stream_offset;
858  sb->region.buf_offset = to_shift->buf_offset;
859  sb->region.buf_size = to_shift->buf_size;
860  sb->region.next = to_shift->next;
861 
862  BUG_ON(to_shift == &sb->region);
863  FREE(cfg, to_shift, sizeof(*to_shift));
864  to_shift = &sb->region;
865  sb->regions--;
866  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
867  }
868 
869  } else {
870  to_shift = &sb->region;
871  SCLogDebug("shift start region %p", to_shift);
872  }
873 
874  // this region is main, or will xfer its buffer to main
875  if (to_shift) {
876  // Do the shift. If new region is exactly at the slide offset we can skip this.
877  DEBUG_VALIDATE_BUG_ON(to_shift->stream_offset > slide_offset);
878  const uint32_t s = slide_offset - to_shift->stream_offset;
879  if (s > 0) {
880  const uint32_t new_data_size = to_shift->buf_size - s;
881  uint32_t new_mem_size = ToNextMultipleOf(new_data_size, cfg->buf_size);
882 
883  /* see if after the slide we'd overlap with the next region. If so, we need
884  * to consolidate them into one. Error handling is a bit peculiar. We need
885  * to grow a region, move data from another region into it, then free the
886  * other region. This could fail if we're close to the memcap. In this case
887  * we relax our rounding up logic so we only shrink and don't merge the 2
888  * regions after all. */
889  if (to_shift->next && slide_offset + new_mem_size >= to_shift->next->stream_offset) {
890  StreamingBufferRegion *start = to_shift;
891  StreamingBufferRegion *next = start->next;
892  const uint64_t next_re = next->stream_offset + next->buf_size;
893  const uint32_t mem_size = ToNextMultipleOf(next_re - slide_offset, cfg->buf_size);
894 
895  /* using next as the new main */
896  if (start->buf_size < next->buf_size) {
897  SCLogDebug("replace main with the next bigger region");
898 
899  const uint32_t next_data_offset = next->stream_offset - slide_offset;
900  const uint32_t prev_buf_size = next->buf_size;
901  const uint32_t start_data_offset = slide_offset - start->stream_offset;
902  DEBUG_VALIDATE_BUG_ON(start_data_offset > start->buf_size);
903  if (start_data_offset > start->buf_size) {
904  new_mem_size = new_data_size;
905  goto just_main;
906  }
907  /* expand "next" to include relevant part of "start" */
908  if (GrowRegionToSize(sb, cfg, next, mem_size) != 0) {
909  new_mem_size = new_data_size;
910  goto just_main;
911  }
912  SCLogDebug("region now sized %u", mem_size);
913 
914  // slide "next":
915  // pre-grow: [nextnextnext]
916  // post-grow [nextnextnextXXX]
917  // post-move [XXXnextnextnext]
918  memmove(next->buf + next_data_offset, next->buf, prev_buf_size);
919 
920  // move portion of "start" into "next"
921  //
922  // start: [ooooookkkkk] (o: old, k: keep)
923  // pre-next [xxxxxnextnextnext]
924  // post-next [kkkkknextnextnext]
925  const uint32_t start_data_size = start->buf_size - start_data_offset;
926  memcpy(next->buf, start->buf + start_data_offset, start_data_size);
927 
928  // free "start"s buffer, we will use the one from "next"
929  FREE(cfg, start->buf, start->buf_size);
930 
931  // update "main" to use "next"
932  start->stream_offset = slide_offset;
933  start->buf = next->buf;
934  start->buf_size = next->buf_size;
935  start->next = next->next;
936 
937  // free "next"
938  FREE(cfg, next, sizeof(*next));
939  sb->regions--;
940  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
941  goto done;
942  } else {
943  /* using "main", expand to include "next" */
944  if (GrowRegionToSize(sb, cfg, start, mem_size) != 0) {
945  new_mem_size = new_data_size;
946  goto just_main;
947  }
948  SCLogDebug("start->buf now size %u", mem_size);
949 
950  // slide "start"
951  // pre: [xxxxxxxAAA]
952  // post: [AAAxxxxxxx]
953  SCLogDebug("s %u new_data_size %u", s, new_data_size);
954  memmove(start->buf, start->buf + s, new_data_size);
955 
956  // copy in "next"
957  // pre: [AAAxxxxxxx]
958  // [BBBBBBB]
959  // post: [AAABBBBBBB]
960  SCLogDebug("copy next->buf %p/%u to start->buf offset %u", next->buf,
961  next->buf_size, new_data_size);
962  memcpy(start->buf + new_data_size, next->buf, next->buf_size);
963 
964  start->stream_offset = slide_offset;
965  start->next = next->next;
966 
967  // free "next"
968  FREE(cfg, next->buf, next->buf_size);
969  FREE(cfg, next, sizeof(*next));
970  sb->regions--;
971  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
972  goto done;
973  }
974  }
975 
976  just_main:
977  SCLogDebug("s %u new_data_size %u", s, new_data_size);
978  memmove(to_shift->buf, to_shift->buf + s, new_data_size);
979  /* shrink memory region. If this fails we keep the old */
980  void *ptr = REALLOC(cfg, to_shift->buf, to_shift->buf_size, new_mem_size);
981  if (ptr != NULL) {
982  to_shift->buf = ptr;
983  to_shift->buf_size = new_mem_size;
984  SCLogDebug("new buf_size %u", to_shift->buf_size);
985  }
986  if (s < to_shift->buf_offset)
987  to_shift->buf_offset -= s;
988  else
989  to_shift->buf_offset = 0;
990  to_shift->stream_offset = slide_offset;
991  }
992  }
993 
994 done:
995  SCLogDebug("main: offset %" PRIu64 " buf %p size %u offset %u", sb->region.stream_offset,
996  sb->region.buf, sb->region.buf_size, sb->region.buf_offset);
997  SCLogDebug("end of slide");
998 }
999 
1000 /**
1001  * \brief slide to absolute offset
1002  * \todo if sliding beyond window, we could perhaps reset?
1003  */
1004 void StreamingBufferSlideToOffset(
1005  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
1006 {
1007  SCLogDebug("sliding to offset %" PRIu64, offset);
1008  ListRegions(sb);
1009 #ifdef DEBUG
1010  SBBPrintList(sb);
1011 #endif
1012 
1013  if (sb->region.next) {
1014  StreamingBufferSlideToOffsetWithRegions(sb, cfg, offset);
1015  SBBPrune(sb, cfg);
1016  SCLogDebug("post SBBPrune");
1017  ListRegions(sb);
1018 #ifdef DEBUG
1019  SBBPrintList(sb);
1020 #endif
1021  DEBUG_VALIDATE_BUG_ON(sb->region.buf != NULL && sb->region.buf_size == 0);
1022  DEBUG_VALIDATE_BUG_ON(sb->region.buf_offset > sb->region.buf_size);
1023  DEBUG_VALIDATE_BUG_ON(offset > sb->region.stream_offset);
1024  DEBUG_VALIDATE_BUG_ON(sb->head && sb->head->offset == sb->region.stream_offset &&
1025  sb->head->len > sb->region.buf_offset);
1026  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1027  return;
1028  }
1029 
1030  if (offset > sb->region.stream_offset) {
1031  const uint32_t slide = offset - sb->region.stream_offset;
1032  if (sb->head != NULL) {
1033  /* have sbb's, so can't rely on buf_offset for the slide */
1034  if (slide < sb->region.buf_size) {
1035  const uint32_t size = sb->region.buf_size - slide;
1036  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1037  size);
1038  memmove(sb->region.buf, sb->region.buf + slide, size);
1039  if (sb->region.buf_offset > slide) {
1040  sb->region.buf_offset -= slide;
1041  } else {
1042  sb->region.buf_offset = 0;
1043  }
1044  } else {
1045  sb->region.buf_offset = 0;
1046  }
1047  sb->region.stream_offset = offset;
1048  } else {
1049  /* no sbb's, so we can use buf_offset */
1050  if (offset <= sb->region.stream_offset + sb->region.buf_offset) {
1051  const uint32_t size = sb->region.buf_offset - slide;
1052  SCLogDebug("sliding %u forward, size of original buffer left after slide %u", slide,
1053  size);
1054  memmove(sb->region.buf, sb->region.buf + slide, size);
1055  sb->region.stream_offset = offset;
1056  sb->region.buf_offset = size;
1057  } else {
1058  /* moved past all data */
1059  sb->region.stream_offset = offset;
1060  sb->region.buf_offset = 0;
1061  }
1062  }
1063  SBBPrune(sb, cfg);
1064  }
1065 #ifdef DEBUG
1066  SBBPrintList(sb);
1067 #endif
1068  DEBUG_VALIDATE_BUG_ON(sb->region.stream_offset < offset);
1069 }
1070 
1071 static int DataFits(const StreamingBuffer *sb, const uint32_t len)
1072 {
1073  uint64_t buf_offset64 = sb->region.buf_offset;
1074  uint64_t len64 = len;
1075  if (len64 + buf_offset64 > UINT32_MAX) {
1076  return -1;
1077  }
1078  return sb->region.buf_offset + len <= sb->region.buf_size;
1079 }
1080 
1081 int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1082  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
1083 {
1084  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1085  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1086 
1087  if (sb->region.buf == NULL) {
1088  if (InitBuffer(sb, cfg) == -1)
1089  return -1;
1090  }
1091 
1092  int r = DataFits(sb, data_len);
1093  if (r < 0) {
1094  DEBUG_VALIDATE_BUG_ON(1);
1095  return -1;
1096  } else if (r == 0) {
1097  if (sb->region.buf_size == 0) {
1098  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1099  return -1;
1100  } else {
1101  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1102  return -1;
1103  }
1104  }
1105  DEBUG_VALIDATE_BUG_ON(DataFits(sb, data_len) != 1);
1106  if (DataFits(sb, data_len) != 1)
1107  return -1;
1108 
1109  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1110  seg->stream_offset = sb->region.stream_offset + sb->region.buf_offset;
1111  seg->segment_len = data_len;
1112  uint32_t rel_offset = sb->region.buf_offset;
1113  sb->region.buf_offset += data_len;
1114 
1115  if (!RB_EMPTY(&sb->sbb_tree)) {
1116  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1117  } else {
1118  return 0;
1119  }
1120 }
1121 
1122 /**
1123  * \brief add data w/o tracking a segment
1124  */
1125 int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1126  const uint8_t *data, uint32_t data_len)
1127 {
1128  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1129 
1130  if (sb->region.buf == NULL) {
1131  if (InitBuffer(sb, cfg) == -1)
1132  return -1;
1133  }
1134 
1135  int r = DataFits(sb, data_len);
1136  if (r < 0) {
1137  DEBUG_VALIDATE_BUG_ON(1);
1138  return -1;
1139  } else if (r == 0) {
1140  if (sb->region.buf_size == 0) {
1141  if (GrowToSize(sb, cfg, data_len) != SC_OK)
1142  return -1;
1143  } else {
1144  if (GrowToSize(sb, cfg, sb->region.buf_offset + data_len) != SC_OK)
1145  return -1;
1146  }
1147  }
1148  DEBUG_VALIDATE_BUG_ON(DataFits(sb, data_len) != 1);
1149 
1150  memcpy(sb->region.buf + sb->region.buf_offset, data, data_len);
1151  uint32_t rel_offset = sb->region.buf_offset;
1152  sb->region.buf_offset += data_len;
1153 
1154  if (!RB_EMPTY(&sb->sbb_tree)) {
1155  return SBBUpdate(sb, cfg, &sb->region, rel_offset, data_len);
1156  } else {
1157  return 0;
1158  }
1159 }
1160 
1161 static int DataFitsAtOffset(
1162  const StreamingBufferRegion *region, const uint32_t len, const uint32_t offset)
1163 {
1164  const uint64_t offset64 = offset;
1165  const uint64_t len64 = len;
1166  if (offset64 + len64 > UINT32_MAX)
1167  return -1;
1168  return (offset + len <= region->buf_size);
1169 }
1170 
1171 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1172 static void Validate(const StreamingBuffer *sb)
1173 {
1174  bool bail = false;
1175  uint32_t cnt = 0;
1176  for (const StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1177  cnt++;
1178  if (r->next) {
1179  bail |= ((r->stream_offset + r->buf_size) > r->next->stream_offset);
1180  }
1181 
1182  bail |= (r->buf != NULL && r->buf_size == 0);
1183  bail |= (r->buf_offset > r->buf_size);
1184  }
1185  // leading gap, so buf_offset should be 0?
1186  if (sb->head && sb->head->offset > sb->region.stream_offset) {
1187  SCLogDebug("leading gap of size %" PRIu64, sb->head->offset - sb->region.stream_offset);
1188  BUG_ON(sb->region.buf_offset != 0);
1189  }
1190 
1191  if (sb->head && sb->head->offset == sb->region.stream_offset) {
1192  BUG_ON(sb->head->len > sb->region.buf_offset);
1193  BUG_ON(sb->head->len < sb->region.buf_offset);
1194  }
1195  BUG_ON(sb->regions != cnt);
1196  BUG_ON(bail);
1197 }
1198 #endif
1199 
1200 static void ListRegions(StreamingBuffer *sb)
1201 {
1202  if (sb->region.buf == NULL && sb->region.buf_offset == 0 && sb->region.next == NULL)
1203  return;
1204 #if defined(DEBUG) && DUMP_REGIONS == 1
1205  uint32_t cnt = 0;
1206  for (StreamingBufferRegion *r = &sb->region; r != NULL; r = r->next) {
1207  cnt++;
1208  char gap[64] = "";
1209  if (r->next) {
1210  snprintf(gap, sizeof(gap), "[ gap:%" PRIu64 " ]",
1211  r->next->stream_offset - (r->stream_offset + r->buf_size));
1212  }
1213 
1214  printf("[ %s offset:%" PRIu64 " size:%u offset:%u ]%s", r == &sb->region ? "main" : "aux",
1215  r->stream_offset, r->buf_size, r->buf_offset, gap);
1216  }
1217  printf("(max %u, cnt %u, sb->regions %u)\n", sb->max_regions, cnt, sb->regions);
1218  bool at_least_one = false;
1219  uint64_t last_re = sb->region.stream_offset;
1220  StreamingBufferBlock *sbb = NULL;
1221  RB_FOREACH(sbb, SBB, &sb->sbb_tree)
1222  {
1223  if (last_re != sbb->offset) {
1224  printf("[ gap:%" PRIu64 " ]", sbb->offset - last_re);
1225  }
1226  printf("[ sbb offset:%" PRIu64 " len:%u ]", sbb->offset, sbb->len);
1227  at_least_one = true;
1228  last_re = sbb->offset + sbb->len;
1229  }
1230  if (at_least_one)
1231  printf("\n");
1232 #endif
1233 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
1234  StreamingBufferBlock *sbb2 = NULL;
1235  RB_FOREACH(sbb2, SBB, &sb->sbb_tree)
1236  {
1237  const uint8_t *_data = NULL;
1238  uint32_t _data_len = 0;
1239  (void)StreamingBufferSBBGetData(sb, sbb2, &_data, &_data_len);
1240  }
1241  Validate(sb);
1242 #endif
1243 }
1244 
1245 /** \internal
1246  * \brief process insert by consolidating the affected regions into one
1247  * \note sets sc_errno
1248  */
1249 static StreamingBufferRegion *BufferInsertAtRegionConsolidate(StreamingBuffer *sb,
1250  const StreamingBufferConfig *cfg, StreamingBufferRegion *dst,
1251  StreamingBufferRegion *src_start, StreamingBufferRegion *src_end,
1252  const uint64_t data_offset, const uint32_t data_len, StreamingBufferRegion *prev,
1253  uint32_t dst_buf_size)
1254 {
1255  int retval;
1256 #ifdef DEBUG
1257  const uint64_t data_re = data_offset + data_len;
1258  SCLogDebug("sb %p dst %p src_start %p src_end %p data_offset %" PRIu64
1259  "/data_len %u/data_re %" PRIu64,
1260  sb, dst, src_start, src_end, data_offset, data_len, data_re);
1261 #endif
1262 
1263  // 1. determine size and offset for dst.
1264  const uint64_t dst_offset = MIN(src_start->stream_offset, data_offset);
1265  DEBUG_VALIDATE_BUG_ON(dst_offset < sb->region.stream_offset);
1266  const uint32_t dst_size = dst_buf_size;
1267  SCLogDebug("dst_size %u", dst_size);
1268 
1269  // 2. resize dst
1270  const uint32_t old_size = dst->buf_size;
1271  const uint32_t dst_copy_offset = dst->stream_offset - dst_offset;
1272 #ifdef DEBUG
1273  const uint32_t old_offset = dst->buf_offset;
1274  SCLogDebug("old_size %u, old_offset %u, dst_copy_offset %u", old_size, old_offset,
1275  dst_copy_offset);
1276 #endif
1277  if ((retval = GrowRegionToSize(sb, cfg, dst, dst_size)) != SC_OK) {
1278  sc_errno = retval;
1279  return NULL;
1280  }
1281  SCLogDebug("resized to %u -> %u", dst_size, dst->buf_size);
1282  /* validate that the size is exactly what we asked for */
1283  DEBUG_VALIDATE_BUG_ON(dst_size != dst->buf_size);
1284  if (dst_copy_offset != 0)
1285  memmove(dst->buf + dst_copy_offset, dst->buf, old_size);
1286  if (dst_offset != dst->stream_offset) {
1287  dst->stream_offset = dst_offset;
1288  // buf_offset no longer valid, reset.
1289  dst->buf_offset = 0;
1290  }
1291 
1292  uint32_t new_offset = src_start->buf_offset;
1293  if (data_offset == src_start->stream_offset + src_start->buf_offset) {
1294  new_offset += data_len;
1295  }
1296 
1297  bool start_is_main = false;
1298  if (src_start == &sb->region) {
1299  DEBUG_VALIDATE_BUG_ON(src_start->stream_offset != dst_offset);
1300 
1301  start_is_main = true;
1302  SCLogDebug("src_start is main region");
1303  if (src_start != dst)
1304  memcpy(dst->buf, src_start->buf, src_start->buf_offset);
1305  if (src_start == src_end) {
1306  SCLogDebug("src_start == src_end == main, we're done");
1307  DEBUG_VALIDATE_BUG_ON(src_start != dst);
1308  return src_start;
1309  }
1310  prev = src_start;
1311  src_start = src_start->next; // skip in the loop below
1312  }
1313 
1314  // 3. copy all regions from src_start to dst_start into the new region
1315  for (StreamingBufferRegion *r = src_start; r != NULL;) {
1316  SCLogDebug("r %p %" PRIu64 ", offset %u, len %u, %s, last %s", r, r->stream_offset,
1317  r->buf_offset, r->buf_size, r == &sb->region ? "main" : "aux",
1318  BOOL2STR(r == src_end));
1319  // skip dst
1320  if (r == dst) {
1321  SCLogDebug("skipping r %p as it is 'dst'", r);
1322  if (r == src_end)
1323  break;
1324  prev = r;
1325  r = r->next;
1326  continue;
1327  }
1328  const uint32_t target_offset = r->stream_offset - dst_offset;
1329  SCLogDebug("r %p: target_offset %u", r, target_offset);
1330  DEBUG_VALIDATE_BUG_ON(target_offset > dst->buf_size);
1331  DEBUG_VALIDATE_BUG_ON(target_offset + r->buf_size > dst->buf_size);
1332  memcpy(dst->buf + target_offset, r->buf, r->buf_size);
1333 
1334  StreamingBufferRegion *next = r->next;
1335  FREE(cfg, r->buf, r->buf_size);
1336  FREE(cfg, r, sizeof(*r));
1337  sb->regions--;
1338  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1339 
1340  DEBUG_VALIDATE_BUG_ON(prev == NULL && src_start != &sb->region);
1341  if (prev != NULL) {
1342  SCLogDebug("setting prev %p next to %p (was %p)", prev, next, prev->next);
1343  prev->next = next;
1344  } else {
1345  SCLogDebug("no prev yet");
1346  }
1347 
1348  if (r == src_end)
1349  break;
1350  r = next;
1351  }
1352 
1353  /* special handling of main region being the start, but not the
1354  * region we expand. In this case we'll have main and dst. We will
1355  * move the buffer from dst into main and free dst. */
1356  if (start_is_main && dst != &sb->region) {
1357  DEBUG_VALIDATE_BUG_ON(sb->region.next != dst);
1358  SCLogDebug("start_is_main && dst != main region");
1359  FREE(cfg, sb->region.buf, sb->region.buf_size);
1360  sb->region.buf = dst->buf;
1361  sb->region.buf_size = dst->buf_size;
1362  sb->region.buf_offset = new_offset;
1363  SCLogDebug("sb->region.buf_offset set to %u", sb->region.buf_offset);
1364  sb->region.next = dst->next;
1365  FREE(cfg, dst, sizeof(*dst));
1366  dst = &sb->region;
1367  sb->regions--;
1368  DEBUG_VALIDATE_BUG_ON(sb->regions == 0);
1369  } else {
1370  SCLogDebug("dst: %p next %p", dst, dst->next);
1371  }
1372 
1373  SCLogDebug("returning dst %p stream_offset %" PRIu64 " buf_offset %u buf_size %u", dst,
1374  dst->stream_offset, dst->buf_offset, dst->buf_size);
1375  return dst;
1376 }
1377 
1378 /** \note sets sc_errno */
1379 static StreamingBufferRegion *BufferInsertAtRegionDo(StreamingBuffer *sb,
1380  const StreamingBufferConfig *cfg, const uint64_t offset, const uint32_t len)
1381 {
1382  SCLogDebug("offset %" PRIu64 ", len %u", offset, len);
1383  StreamingBufferRegion *start_prev = NULL;
1384  StreamingBufferRegion *start =
1385  FindFirstRegionForOffset(cfg, &sb->region, offset, len, &start_prev);
1386  if (start) {
1387  const uint64_t insert_re = offset + len;
1388  const uint64_t insert_start_offset = MIN(start->stream_offset, offset);
1389  uint64_t insert_adjusted_re = insert_re;
1390 
1391  SCLogDebug("start region %p/%" PRIu64 "/%u", start, start->stream_offset, start->buf_size);
1392  StreamingBufferRegion *big = FindLargestRegionForOffset(cfg, start, offset, len);
1393  DEBUG_VALIDATE_BUG_ON(big == NULL);
1394  if (big == NULL) {
1395  sc_errno = SC_EINVAL;
1396  return NULL;
1397  }
1398  SCLogDebug("big region %p/%" PRIu64 "/%u", big, big->stream_offset, big->buf_size);
1399  StreamingBufferRegion *end = FindRightEdge(cfg, big, offset, len);
1400  DEBUG_VALIDATE_BUG_ON(end == NULL);
1401  if (end == NULL) {
1402  sc_errno = SC_EINVAL;
1403  return NULL;
1404  }
1405  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1406 
1407  uint32_t new_buf_size =
1408  ToNextMultipleOf(insert_adjusted_re - insert_start_offset, cfg->buf_size);
1409  SCLogDebug("new_buf_size %u", new_buf_size);
1410 
1411  /* see if our new buf size + cfg->buf_size margin leads to an overlap with the next region.
1412  * If so, include that in the consolidation. */
1413  if (end->next != NULL && new_buf_size + insert_start_offset > end->next->stream_offset) {
1414  SCLogDebug("adjusted end from %p to %p", end, end->next);
1415  end = end->next;
1416  insert_adjusted_re = MAX(insert_adjusted_re, (end->stream_offset + end->buf_size));
1417  new_buf_size =
1418  ToNextMultipleOf(insert_adjusted_re - insert_start_offset, cfg->buf_size);
1419  SCLogDebug("new_buf_size %u", new_buf_size);
1420  }
1421 
1422  SCLogDebug("end region %p/%" PRIu64 "/%u", end, end->stream_offset, end->buf_size);
1423  /* sets sc_errno */
1424  StreamingBufferRegion *ret = BufferInsertAtRegionConsolidate(
1425  sb, cfg, big, start, end, offset, len, start_prev, new_buf_size);
1426  return ret;
1427  } else {
1428  /* if there was no region we can use we add a new region and insert it */
1429  StreamingBufferRegion *append = &sb->region;
1430  for (StreamingBufferRegion *r = append; r != NULL; r = r->next) {
1431  if (r->stream_offset > offset) {
1432  break;
1433  } else {
1434  append = r;
1435  }
1436  }
1437 
1438  SCLogDebug("no matching region found, append to %p (%s)", append,
1439  append == &sb->region ? "main" : "aux");
1440  /* sets sc_errno */
1441  StreamingBufferRegion *add = InitBufferRegion(sb, cfg, len);
1442  if (add == NULL)
1443  return NULL;
1444  add->stream_offset = offset;
1445  add->next = append->next;
1446  append->next = add;
1447  SCLogDebug("new region %p offset %" PRIu64, add, add->stream_offset);
1448  return add;
1449  }
1450 }
1451 
1452 /** \internal
1453  * \brief return the region to put the new data in
1454  *
1455  * Will find an existing region, expand it if needed. If no existing region exists or is
1456  * a good fit, it will try to set up a new region. If the region then overlaps or gets
1457  * too close to the next, merge them.
1458  *
1459  * \note sets sc_errno
1460  */
1461 static StreamingBufferRegion *BufferInsertAtRegion(StreamingBuffer *sb,
1462  const StreamingBufferConfig *cfg, const uint32_t data_len, const uint64_t data_offset)
1463 {
1464  SCLogDebug("data_offset %" PRIu64 ", data_len %u, re %" PRIu64, data_offset, data_len,
1465  data_offset + data_len);
1466  ListRegions(sb);
1467 
1468  if (RegionsIntersect(cfg, &sb->region, data_offset, data_offset + data_len)) {
1469  SCLogDebug("data_offset %" PRIu64 ", data_len %u intersects with main region (next %p)",
1470  data_offset, data_len, sb->region.next);
1471  if (sb->region.next == NULL ||
1472  !RegionsIntersect(cfg, sb->region.next, data_offset, data_offset + data_len)) {
1473  SCLogDebug(
1474  "data_offset %" PRIu64
1475  ", data_len %u intersects with main region, no next or way before next region",
1476  data_offset, data_len);
1477  if (sb->region.buf == NULL) {
1478  int r;
1479  if ((r = InitBuffer(sb, cfg)) != SC_OK) { // TODO init with size
1480  sc_errno = r;
1481  return NULL;
1482  }
1483  }
1484  return &sb->region;
1485  }
1486  } else if (sb->region.next == NULL) {
1487  /* InitBufferRegion sets sc_errno */
1488  StreamingBufferRegion *aux_r = sb->region.next = InitBufferRegion(sb, cfg, data_len);
1489  if (aux_r == NULL)
1490  return NULL;
1491  aux_r->stream_offset = data_offset;
1492  DEBUG_VALIDATE_BUG_ON(data_len > aux_r->buf_size);
1493  SCLogDebug("created new region %p with offset %" PRIu64 ", size %u", aux_r,
1494  aux_r->stream_offset, aux_r->buf_size);
1495  return aux_r;
1496  }
1497  /* BufferInsertAtRegionDo sets sc_errno */
1498  StreamingBufferRegion *blob = BufferInsertAtRegionDo(sb, cfg, data_offset, data_len);
1499  SCLogDebug("blob %p (%s)", blob, blob == &sb->region ? "main" : "aux");
1500  return blob;
1501 }
1502 
1503 /**
1504  * \param offset offset relative to StreamingBuffer::stream_offset
1505  *
1506  * \return SC_OK in case of success
1507  * \return SC_E* errors otherwise
1508  */
1509 int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg,
1510  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
1511 {
1512  DEBUG_VALIDATE_BUG_ON(seg == NULL);
1513  DEBUG_VALIDATE_BUG_ON(data_len > BIT_U32(27)); // 128MiB is excessive already
1514  DEBUG_VALIDATE_BUG_ON(offset < sb->region.stream_offset);
1515  if (offset < sb->region.stream_offset) {
1516  return SC_EINVAL;
1517  }
1518 
1519  StreamingBufferRegion *region = BufferInsertAtRegion(sb, cfg, data_len, offset);
1520  if (region == NULL) {
1521  return sc_errno;
1522  }
1523 
1524  const bool region_is_main = region == &sb->region;
1525 
1526  SCLogDebug("inserting %" PRIu64 "/%u using %s region %p", offset, data_len,
1527  region == &sb->region ? "main" : "aux", region);
1528 
1529  uint32_t rel_offset = offset - region->stream_offset;
1530  int r = DataFitsAtOffset(region, data_len, rel_offset);
1531  if (r < 0) {
1532  DEBUG_VALIDATE_BUG_ON(1);
1533  return SC_ELIMIT;
1534  } else if (r == 0) {
1535  if ((r = GrowToSize(sb, cfg, (rel_offset + data_len))) != SC_OK)
1536  return r;
1537  }
1538  DEBUG_VALIDATE_BUG_ON(DataFitsAtOffset(region, data_len, rel_offset) != 1);
1539 
1540  SCLogDebug("offset %" PRIu64 " data_len %u, rel_offset %u into region offset %" PRIu64
1541  ", buf_offset %u, buf_size %u",
1542  offset, data_len, rel_offset, region->stream_offset, region->buf_offset,
1543  region->buf_size);
1544  memcpy(region->buf + rel_offset, data, data_len);
1545  seg->stream_offset = offset;
1546  seg->segment_len = data_len;
1547 
1548  SCLogDebug("rel_offset %u region->stream_offset %" PRIu64 ", buf_offset %u", rel_offset,
1549  region->stream_offset, region->buf_offset);
1550 
1551  if (RB_EMPTY(&sb->sbb_tree)) {
1552  SCLogDebug("empty sbb list");
1553 
1554  if (region_is_main) {
1555  if (sb->region.stream_offset == offset) {
1556  SCLogDebug("empty sbb list: block exactly what was expected, fall through");
1557  /* empty list, data is exactly what is expected (append),
1558  * so do nothing.
1559  * Update buf_offset if needed, but in case of overlaps it might be beyond us. */
1560  sb->region.buf_offset = MAX(sb->region.buf_offset, rel_offset + data_len);
1561  } else if ((rel_offset + data_len) <= sb->region.buf_offset) {
1562  SCLogDebug("empty sbb list: block is within existing main data region");
1563  } else {
1564  if (sb->region.buf_offset && rel_offset == sb->region.buf_offset) {
1565  SCLogDebug("exactly at expected offset");
1566  // nothing to do
1567  sb->region.buf_offset = rel_offset + data_len;
1568 
1569  } else if (rel_offset < sb->region.buf_offset) {
1570  // nothing to do
1571 
1572  SCLogDebug("before expected offset: %u < sb->region.buf_offset %u", rel_offset,
1573  sb->region.buf_offset);
1574  if (rel_offset + data_len > sb->region.buf_offset) {
1575  SCLogDebug("before expected offset, ends after: %u < sb->region.buf_offset "
1576  "%u, %u > %u",
1577  rel_offset, sb->region.buf_offset, rel_offset + data_len,
1578  sb->region.buf_offset);
1579  sb->region.buf_offset = rel_offset + data_len;
1580  }
1581 
1582  } else if (sb->region.buf_offset) {
1583  SCLogDebug("beyond expected offset: SBBInit");
1584  /* existing data, but there is a gap between us */
1585  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1586  return r;
1587  } else {
1588  /* gap before data in empty list */
1589  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1590  if ((r = SBBInitLeadingGap(sb, cfg, offset, data_len)) != SC_OK)
1591  return r;
1592  }
1593  }
1594  } else {
1595  if (sb->region.buf_offset) {
1596  /* existing data, but there is a gap between us */
1597  SCLogDebug("empty sbb list, no data in main: use SBBInit");
1598  if ((r = SBBInit(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1599  return r;
1600  } else {
1601  /* gap before data in empty list */
1602  SCLogDebug("empty sbb list: invoking SBBInitLeadingGap");
1603  if ((r = SBBInitLeadingGap(sb, cfg, offset, data_len)) != SC_OK)
1604  return r;
1605  }
1606  if (rel_offset == region->buf_offset) {
1607  SCLogDebug("pre region->buf_offset %u", region->buf_offset);
1608  region->buf_offset = rel_offset + data_len;
1609  SCLogDebug("post region->buf_offset %u", region->buf_offset);
1610  }
1611  }
1612  } else {
1613  SCLogDebug("updating sbb tree");
1614  /* already have blocks, so append new block based on new data */
1615  if ((r = SBBUpdate(sb, cfg, region, rel_offset, data_len)) != SC_OK)
1616  return r;
1617  }
1618  DEBUG_VALIDATE_BUG_ON(!region_is_main && sb->head == NULL);
1619 
1620  ListRegions(sb);
1621  if (RB_EMPTY(&sb->sbb_tree)) {
1622  DEBUG_VALIDATE_BUG_ON(offset + data_len > sb->region.stream_offset + sb->region.buf_offset);
1623  }
1624 
1625  return SC_OK;
1626 }
1627 
1628 int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb,
1629  const StreamingBufferSegment *seg)
1630 {
1631  if (seg->stream_offset < sb->region.stream_offset) {
1632  if (seg->stream_offset + seg->segment_len <= sb->region.stream_offset) {
1633  return 1;
1634  }
1635  }
1636  return 0;
1637 }
1638 
1639 static inline const StreamingBufferRegion *GetRegionForOffset(
1640  const StreamingBuffer *sb, const uint64_t offset)
1641 {
1642  if (sb == NULL)
1643  return NULL;
1644  if (sb->region.next == NULL) {
1645  return &sb->region;
1646  }
1647  if (offset >= sb->region.stream_offset &&
1648  offset < (sb->region.stream_offset + sb->region.buf_size)) {
1649  return &sb->region;
1650  }
1651  for (const StreamingBufferRegion *r = sb->region.next; r != NULL; r = r->next) {
1652  if (offset >= r->stream_offset && offset < (r->stream_offset + r->buf_size)) {
1653  return r;
1654  }
1655  }
1656  return NULL;
1657 }
1658 
1659 /** \brief get the data for one SBB */
1660 void StreamingBufferSBBGetData(const StreamingBuffer *sb,
1661  const StreamingBufferBlock *sbb,
1662  const uint8_t **data, uint32_t *data_len)
1663 {
1664  const StreamingBufferRegion *region = GetRegionForOffset(sb, sbb->offset);
1665  SCLogDebug("first find our region (offset %" PRIu64 ") -> %p", sbb->offset, region);
1666  if (region) {
1667  SCLogDebug("region %p found %" PRIu64 "/%u/%u", region, region->stream_offset,
1668  region->buf_size, region->buf_offset);
1669  DEBUG_VALIDATE_BUG_ON(
1670  region->stream_offset == sbb->offset && region->buf_offset > sbb->len);
1671  // buf_offset should match first sbb len if it has the same offset
1672 
1673  if (sbb->offset >= region->stream_offset) {
1674  SCLogDebug("1");
1675  uint64_t offset = sbb->offset - region->stream_offset;
1676  *data = region->buf + offset;
1677  DEBUG_VALIDATE_BUG_ON(offset + sbb->len > region->buf_size);
1678  *data_len = sbb->len;
1679  return;
1680  } else {
1681  SCLogDebug("2");
1682  uint64_t offset = region->stream_offset - sbb->offset;
1683  if (offset < sbb->len) {
1684  *data = region->buf;
1685  *data_len = sbb->len - offset;
1686  return;
1687  }
1688  SCLogDebug("3");
1689  }
1690  }
1691  *data = NULL;
1692  *data_len = 0;
1693 }
1694 
1695 /** \brief get the data for one SBB */
1696 void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb,
1697  const StreamingBufferBlock *sbb,
1698  const uint8_t **data, uint32_t *data_len,
1699  uint64_t offset)
1700 {
1701  /* validate that we are looking for a offset within the sbb */
1702  DEBUG_VALIDATE_BUG_ON(!(offset >= sbb->offset && offset < (sbb->offset + sbb->len)));
1703  if (!(offset >= sbb->offset && offset < (sbb->offset + sbb->len))) {
1704  *data = NULL;
1705  *data_len = 0;
1706  return;
1707  }
1708 
1709  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1710  if (region) {
1711  uint32_t sbblen = sbb->len - (offset - sbb->offset);
1712 
1713  if (offset >= region->stream_offset) {
1714  uint64_t data_offset = offset - region->stream_offset;
1715  *data = region->buf + data_offset;
1716  if (data_offset + sbblen > region->buf_size)
1717  *data_len = region->buf_size - data_offset;
1718  else
1719  *data_len = sbblen;
1720  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1721  return;
1722  } else {
1723  uint64_t data_offset = region->stream_offset - sbb->offset;
1724  if (data_offset < sbblen) {
1725  *data = region->buf;
1726  *data_len = sbblen - data_offset;
1727  DEBUG_VALIDATE_BUG_ON(*data_len > sbblen);
1728  return;
1729  }
1730  }
1731  }
1732 
1733  *data = NULL;
1734  *data_len = 0;
1735 }
1736 
1737 void StreamingBufferSegmentGetData(const StreamingBuffer *sb,
1738  const StreamingBufferSegment *seg,
1739  const uint8_t **data, uint32_t *data_len)
1740 {
1741  const StreamingBufferRegion *region = GetRegionForOffset(sb, seg->stream_offset);
1742  if (region) {
1743  if (seg->stream_offset >= region->stream_offset) {
1744  uint64_t offset = seg->stream_offset - region->stream_offset;
1745  *data = region->buf + offset;
1746  if (offset + seg->segment_len > region->buf_size)
1747  *data_len = region->buf_size - offset;
1748  else
1749  *data_len = seg->segment_len;
1750  SCLogDebug("*data_len %u", *data_len);
1751  return;
1752  } else {
1753  uint64_t offset = region->stream_offset - seg->stream_offset;
1754  if (offset < seg->segment_len) {
1755  *data = region->buf;
1756  *data_len = seg->segment_len - offset;
1757  SCLogDebug("*data_len %u", *data_len);
1758  return;
1759  }
1760  }
1761  }
1762  *data = NULL;
1763  *data_len = 0;
1764 }
1765 
1766 /**
1767  * \retval 1 data is the same
1768  * \retval 0 data is different
1769  */
1770 int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb,
1771  const StreamingBufferSegment *seg,
1772  const uint8_t *rawdata, uint32_t rawdata_len)
1773 {
1774  const uint8_t *segdata = NULL;
1775  uint32_t segdata_len = 0;
1776  StreamingBufferSegmentGetData(sb, seg, &segdata, &segdata_len);
1777  if (segdata && segdata_len &&
1778  segdata_len == rawdata_len &&
1779  memcmp(segdata, rawdata, segdata_len) == 0)
1780  {
1781  return 1;
1782  }
1783  return 0;
1784 }
1785 
1786 int StreamingBufferGetData(const StreamingBuffer *sb,
1787  const uint8_t **data, uint32_t *data_len,
1788  uint64_t *stream_offset)
1789 {
1790  if (sb != NULL && sb->region.buf != NULL) {
1791  *data = sb->region.buf;
1792  *data_len = sb->region.buf_offset;
1793  *stream_offset = sb->region.stream_offset;
1794  return 1;
1795  } else {
1796  *data = NULL;
1797  *data_len = 0;
1798  *stream_offset = 0;
1799  return 0;
1800  }
1801 }
1802 
1803 int StreamingBufferGetDataAtOffset (const StreamingBuffer *sb,
1804  const uint8_t **data, uint32_t *data_len,
1805  uint64_t offset)
1806 {
1807  const StreamingBufferRegion *region = GetRegionForOffset(sb, offset);
1808  if (region != NULL && region->buf != NULL && offset >= region->stream_offset &&
1809  offset < (region->stream_offset + region->buf_offset)) {
1810  uint32_t skip = offset - region->stream_offset;
1811  *data = region->buf + skip;
1812  *data_len = region->buf_offset - skip;
1813  return 1;
1814  } else {
1815  *data = NULL;
1816  *data_len = 0;
1817  return 0;
1818  }
1819 }
1820 
1821 /**
1822  * \retval 1 data is the same
1823  * \retval 0 data is different
1824  */
1825 int StreamingBufferCompareRawData(const StreamingBuffer *sb,
1826  const uint8_t *rawdata, uint32_t rawdata_len)
1827 {
1828  const uint8_t *sbdata = NULL;
1829  uint32_t sbdata_len = 0;
1830  uint64_t offset = 0;
1831  StreamingBufferGetData(sb, &sbdata, &sbdata_len, &offset);
1832  if (offset == 0 &&
1833  sbdata && sbdata_len &&
1834  sbdata_len == rawdata_len &&
1835  memcmp(sbdata, rawdata, sbdata_len) == 0)
1836  {
1837  return 1;
1838  }
1839  SCLogDebug("sbdata_len %u, offset %" PRIu64, sbdata_len, offset);
1840  printf("got:\n");
1841  PrintRawDataFp(stdout, sbdata,sbdata_len);
1842  printf("wanted:\n");
1843  PrintRawDataFp(stdout, rawdata,rawdata_len);
1844  return 0;
1845 }
1846 
1847 #ifdef UNITTESTS
1848 static void Dump(StreamingBuffer *sb)
1849 {
1850  PrintRawDataFp(stdout, sb->region.buf, sb->region.buf_offset);
1851 }
1852 
1853 static void DumpSegment(StreamingBuffer *sb, StreamingBufferSegment *seg)
1854 {
1855  const uint8_t *data = NULL;
1856  uint32_t data_len = 0;
1857  StreamingBufferSegmentGetData(sb, seg, &data, &data_len);
1858  if (data && data_len) {
1859  PrintRawDataFp(stdout, data, data_len);
1860  }
1861 }
1862 
1863 static int StreamingBufferTest02(void)
1864 {
1865  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1866  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1867  FAIL_IF(sb == NULL);
1868 
1869  StreamingBufferSegment seg1;
1870  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1871  StreamingBufferSegment seg2;
1872  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8) != 0);
1873  FAIL_IF(sb->region.stream_offset != 0);
1874  FAIL_IF(sb->region.buf_offset != 16);
1875  FAIL_IF(seg1.stream_offset != 0);
1876  FAIL_IF(seg2.stream_offset != 8);
1877  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1878  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1879  Dump(sb);
1880  DumpSegment(sb, &seg1);
1881  DumpSegment(sb, &seg2);
1882  FAIL_IF_NOT_NULL(sb->head);
1883  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1884 
1885  StreamingBufferSlideToOffset(sb, &cfg, 6);
1886  FAIL_IF_NOT_NULL(sb->head);
1887  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1888 
1889  StreamingBufferSegment seg3;
1890  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6) != 0);
1891  FAIL_IF(sb->region.stream_offset != 6);
1892  FAIL_IF(sb->region.buf_offset != 16);
1893  FAIL_IF(seg3.stream_offset != 16);
1894  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1895  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1896  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1897  Dump(sb);
1898  DumpSegment(sb, &seg1);
1899  DumpSegment(sb, &seg2);
1900  DumpSegment(sb, &seg3);
1901  FAIL_IF_NOT_NULL(sb->head);
1902  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1903 
1904  StreamingBufferSlideToOffset(sb, &cfg, 12);
1905  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1906  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1907  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1908  Dump(sb);
1909  DumpSegment(sb, &seg1);
1910  DumpSegment(sb, &seg2);
1911  DumpSegment(sb, &seg3);
1912  FAIL_IF_NOT_NULL(sb->head);
1913  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1914 
1915  StreamingBufferFree(sb, &cfg);
1916  PASS;
1917 }
1918 
1919 static int StreamingBufferTest03(void)
1920 {
1921  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1922  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1923  FAIL_IF(sb == NULL);
1924 
1925  StreamingBufferSegment seg1;
1926  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1927  StreamingBufferSegment seg2;
1928  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1929  FAIL_IF(sb->region.stream_offset != 0);
1930  FAIL_IF(sb->region.buf_offset != 8);
1931  FAIL_IF(seg1.stream_offset != 0);
1932  FAIL_IF(seg2.stream_offset != 14);
1933  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1934  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1935  Dump(sb);
1936  DumpSegment(sb, &seg1);
1937  DumpSegment(sb, &seg2);
1938  FAIL_IF_NULL(sb->head);
1939  FAIL_IF_NOT(sb->sbb_size == 16);
1940  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1941 
1942  StreamingBufferSegment seg3;
1943  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
1944  FAIL_IF(sb->region.stream_offset != 0);
1945  FAIL_IF(sb->region.buf_offset != 22);
1946  FAIL_IF(seg3.stream_offset != 8);
1947  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1948  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1949  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1950  Dump(sb);
1951  DumpSegment(sb, &seg1);
1952  DumpSegment(sb, &seg2);
1953  DumpSegment(sb, &seg3);
1954  FAIL_IF_NULL(sb->head);
1955  FAIL_IF_NOT(sb->sbb_size == 22);
1956  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1957 
1958  StreamingBufferSlideToOffset(sb, &cfg, 10);
1959  FAIL_IF(!StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1960  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1961  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
1962  Dump(sb);
1963  DumpSegment(sb, &seg1);
1964  DumpSegment(sb, &seg2);
1965  DumpSegment(sb, &seg3);
1966  FAIL_IF_NULL(sb->head);
1967  FAIL_IF_NOT(sb->sbb_size == 12);
1968  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
1969 
1970  StreamingBufferFree(sb, &cfg);
1971  PASS;
1972 }
1973 
1974 static int StreamingBufferTest04(void)
1975 {
1976  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
1977  StreamingBuffer *sb = StreamingBufferInit(&cfg);
1978  FAIL_IF(sb == NULL);
1979 
1980  StreamingBufferSegment seg1;
1981  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
1982  FAIL_IF(!RB_EMPTY(&sb->sbb_tree));
1983  StreamingBufferSegment seg2;
1984  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"01234567", 8, 14) != 0);
1985  FAIL_IF(sb->region.stream_offset != 0);
1986  FAIL_IF(sb->region.buf_offset != 8);
1987  FAIL_IF(seg1.stream_offset != 0);
1988  FAIL_IF(seg2.stream_offset != 14);
1989  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
1990  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
1991  FAIL_IF(RB_EMPTY(&sb->sbb_tree));
1992  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
1993  FAIL_IF(sbb1 != sb->head);
1994  FAIL_IF_NULL(sbb1);
1995  FAIL_IF(sbb1->offset != 0);
1996  FAIL_IF(sbb1->len != 8);
1997  StreamingBufferBlock *sbb2 = SBB_RB_NEXT(sbb1);
1998  FAIL_IF_NULL(sbb2);
1999  FAIL_IF(sbb2 == sb->head);
2000  FAIL_IF(sbb2->offset != 14);
2001  FAIL_IF(sbb2->len != 8);
2002  Dump(sb);
2003  DumpSegment(sb, &seg1);
2004  DumpSegment(sb, &seg2);
2005  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2006 
2007  StreamingBufferSegment seg3;
2008  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"QWERTY", 6, 8) != 0);
2009  FAIL_IF(sb->region.stream_offset != 0);
2010  FAIL_IF(sb->region.buf_offset != 22);
2011  FAIL_IF(seg3.stream_offset != 8);
2012  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2013  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2014  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
2015  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2016  FAIL_IF_NULL(sbb1);
2017  FAIL_IF(sbb1 != sb->head);
2018  FAIL_IF(sbb1->offset != 0);
2019  FAIL_IF(sbb1->len != 22);
2020  FAIL_IF(SBB_RB_NEXT(sbb1));
2021  Dump(sb);
2022  DumpSegment(sb, &seg1);
2023  DumpSegment(sb, &seg2);
2024  DumpSegment(sb, &seg3);
2025  FAIL_IF_NULL(sb->head);
2026  FAIL_IF_NOT(sb->sbb_size == 22);
2027  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2028 
2029  /* far ahead of curve: */
2030  StreamingBufferSegment seg4;
2031  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"XYZ", 3, 124) != 0);
2032  FAIL_IF(sb->region.stream_offset != 0);
2033  FAIL_IF(sb->region.buf_offset != 22);
2034  FAIL_IF(sb->region.buf_size != 128);
2035  FAIL_IF(seg4.stream_offset != 124);
2036  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg1));
2037  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg2));
2038  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg3));
2039  FAIL_IF(StreamingBufferSegmentIsBeforeWindow(sb,&seg4));
2040  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2041  FAIL_IF_NULL(sbb1);
2042  FAIL_IF(sbb1 != sb->head);
2043  FAIL_IF(sbb1->offset != 0);
2044  FAIL_IF(sbb1->len != 22);
2045  FAIL_IF(!SBB_RB_NEXT(sbb1));
2046  Dump(sb);
2047  DumpSegment(sb, &seg1);
2048  DumpSegment(sb, &seg2);
2049  DumpSegment(sb, &seg3);
2050  DumpSegment(sb, &seg4);
2051  FAIL_IF_NULL(sb->head);
2052  FAIL_IF_NOT(sb->sbb_size == 25);
2053  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2054 
2055  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg1,(const uint8_t *)"ABCDEFGH", 8));
2056  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg2,(const uint8_t *)"01234567", 8));
2057  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg3,(const uint8_t *)"QWERTY", 6));
2058  FAIL_IF(!StreamingBufferSegmentCompareRawData(sb,&seg4,(const uint8_t *)"XYZ", 3));
2059 
2060  StreamingBufferFree(sb, &cfg);
2061  PASS;
2062 }
2063 
2064 /** \test lots of gaps in block list */
2065 static int StreamingBufferTest06(void)
2066 {
2067  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2068  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2069  FAIL_IF(sb == NULL);
2070 
2071  StreamingBufferSegment seg1;
2072  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"A", 1) != 0);
2073  StreamingBufferSegment seg2;
2074  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"C", 1, 2) != 0);
2075  Dump(sb);
2076  FAIL_IF_NULL(sb->head);
2077  FAIL_IF_NOT(sb->sbb_size == 2);
2078  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2079 
2080  StreamingBufferSegment seg3;
2081  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2082  Dump(sb);
2083  FAIL_IF_NULL(sb->head);
2084  FAIL_IF_NOT(sb->sbb_size == 3);
2085  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2086 
2087  StreamingBufferSegment seg4;
2088  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2089  Dump(sb);
2090  FAIL_IF_NULL(sb->head);
2091  FAIL_IF_NOT(sb->sbb_size == 4);
2092  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2093 
2094  StreamingBufferSegment seg5;
2095  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2096  Dump(sb);
2097  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2098  FAIL_IF_NULL(sbb1);
2099  FAIL_IF(sbb1->offset != 0);
2100  FAIL_IF(sbb1->len != 10);
2101  FAIL_IF(SBB_RB_NEXT(sbb1));
2102  FAIL_IF_NULL(sb->head);
2103  FAIL_IF_NOT(sb->sbb_size == 10);
2104  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2105 
2106  StreamingBufferSegment seg6;
2107  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2108  Dump(sb);
2109  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2110  FAIL_IF_NULL(sbb1);
2111  FAIL_IF(sbb1->offset != 0);
2112  FAIL_IF(sbb1->len != 10);
2113  FAIL_IF(SBB_RB_NEXT(sbb1));
2114  FAIL_IF_NULL(sb->head);
2115  FAIL_IF_NOT(sb->sbb_size == 10);
2116  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2117 
2118  StreamingBufferFree(sb, &cfg);
2119  PASS;
2120 }
2121 
2122 /** \test lots of gaps in block list */
2123 static int StreamingBufferTest07(void)
2124 {
2125  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2126  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2127  FAIL_IF(sb == NULL);
2128 
2129  StreamingBufferSegment seg1;
2130  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2131  StreamingBufferSegment seg2;
2132  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2133  Dump(sb);
2134  FAIL_IF_NULL(sb->head);
2135  FAIL_IF_NOT(sb->sbb_size == 2);
2136  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2137 
2138  StreamingBufferSegment seg3;
2139  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2140  Dump(sb);
2141  FAIL_IF_NULL(sb->head);
2142  FAIL_IF_NOT(sb->sbb_size == 3);
2143  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2144 
2145  StreamingBufferSegment seg4;
2146  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2147  Dump(sb);
2148  FAIL_IF_NULL(sb->head);
2149  FAIL_IF_NOT(sb->sbb_size == 4);
2150  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2151 
2152  StreamingBufferSegment seg5;
2153  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2154  Dump(sb);
2155  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2156  FAIL_IF_NULL(sbb1);
2157  FAIL_IF(sbb1->offset != 0);
2158  FAIL_IF(sbb1->len != 10);
2159  FAIL_IF(SBB_RB_NEXT(sbb1));
2160  FAIL_IF_NULL(sb->head);
2161  FAIL_IF_NOT(sb->sbb_size == 10);
2162  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2163 
2164  StreamingBufferSegment seg6;
2165  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2166  Dump(sb);
2167  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2168  FAIL_IF_NULL(sbb1);
2169  FAIL_IF(sbb1->offset != 0);
2170  FAIL_IF(sbb1->len != 10);
2171  FAIL_IF(SBB_RB_NEXT(sbb1));
2172  FAIL_IF_NULL(sb->head);
2173  FAIL_IF_NOT(sb->sbb_size == 10);
2174  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2175 
2176  StreamingBufferFree(sb, &cfg);
2177  PASS;
2178 }
2179 
2180 /** \test lots of gaps in block list */
2181 static int StreamingBufferTest08(void)
2182 {
2183  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2184  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2185  FAIL_IF(sb == NULL);
2186 
2187  StreamingBufferSegment seg1;
2188  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2189  StreamingBufferSegment seg2;
2190  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2191  Dump(sb);
2192  FAIL_IF_NULL(sb->head);
2193  FAIL_IF_NOT(sb->sbb_size == 2);
2194  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2195 
2196  StreamingBufferSegment seg3;
2197  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"F", 1, 5) != 0);
2198  Dump(sb);
2199  FAIL_IF_NULL(sb->head);
2200  FAIL_IF_NOT(sb->sbb_size == 3);
2201  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2202 
2203  StreamingBufferSegment seg4;
2204  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"H", 1, 7) != 0);
2205  Dump(sb);
2206  FAIL_IF_NULL(sb->head);
2207  FAIL_IF_NOT(sb->sbb_size == 4);
2208  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2209 
2210  StreamingBufferSegment seg5;
2211  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2212  Dump(sb);
2213  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2214  FAIL_IF_NULL(sbb1);
2215  FAIL_IF(sbb1->offset != 0);
2216  FAIL_IF(sbb1->len != 10);
2217  FAIL_IF(SBB_RB_NEXT(sbb1));
2218  FAIL_IF_NULL(sb->head);
2219  FAIL_IF_NOT(sb->sbb_size == 10);
2220  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2221 
2222  StreamingBufferSegment seg6;
2223  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10) != 0);
2224  Dump(sb);
2225  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2226  FAIL_IF_NULL(sbb1);
2227  FAIL_IF(sbb1->offset != 0);
2228  FAIL_IF(sbb1->len != 20);
2229  FAIL_IF(SBB_RB_NEXT(sbb1));
2230  FAIL_IF_NULL(sb->head);
2231  FAIL_IF_NOT(sb->sbb_size == 20);
2232  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2233 
2234  StreamingBufferFree(sb, &cfg);
2235  PASS;
2236 }
2237 
2238 /** \test lots of gaps in block list */
2239 static int StreamingBufferTest09(void)
2240 {
2241  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2242  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2243  FAIL_IF(sb == NULL);
2244 
2245  StreamingBufferSegment seg1;
2246  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"B", 1, 1) != 0);
2247  StreamingBufferSegment seg2;
2248  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2249  Dump(sb);
2250  FAIL_IF_NULL(sb->head);
2251  FAIL_IF_NOT(sb->sbb_size == 2);
2252  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2253 
2254  StreamingBufferSegment seg3;
2255  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2256  Dump(sb);
2257  FAIL_IF_NULL(sb->head);
2258  FAIL_IF_NOT(sb->sbb_size == 3);
2259  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2260 
2261  StreamingBufferSegment seg4;
2262  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"F", 1, 5) != 0);
2263  Dump(sb);
2264  FAIL_IF_NULL(sb->head);
2265  FAIL_IF_NOT(sb->sbb_size == 4);
2266  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2267 
2268  StreamingBufferSegment seg5;
2269  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2270  Dump(sb);
2271  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2272  FAIL_IF_NULL(sbb1);
2273  FAIL_IF(sbb1->offset != 0);
2274  FAIL_IF(sbb1->len != 10);
2275  FAIL_IF(SBB_RB_NEXT(sbb1));
2276  FAIL_IF_NULL(sb->head);
2277  FAIL_IF_NOT(sb->sbb_size == 10);
2278  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2279 
2280  StreamingBufferSegment seg6;
2281  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2282  Dump(sb);
2283  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2284  FAIL_IF_NULL(sbb1);
2285  FAIL_IF(sbb1->offset != 0);
2286  FAIL_IF(sbb1->len != 10);
2287  FAIL_IF(SBB_RB_NEXT(sbb1));
2288  FAIL_IF_NULL(sb->head);
2289  FAIL_IF_NOT(sb->sbb_size == 10);
2290  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2291 
2292  StreamingBufferFree(sb, &cfg);
2293  PASS;
2294 }
2295 
2296 /** \test lots of gaps in block list */
2297 static int StreamingBufferTest10(void)
2298 {
2299  StreamingBufferConfig cfg = { 16, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2300  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2301  FAIL_IF(sb == NULL);
2302 
2303  StreamingBufferSegment seg1;
2304  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg1, (const uint8_t *)"A", 1, 0) != 0);
2305  Dump(sb);
2306  StreamingBufferSegment seg2;
2307  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg2, (const uint8_t *)"D", 1, 3) != 0);
2308  Dump(sb);
2309  StreamingBufferSegment seg3;
2310  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg3, (const uint8_t *)"H", 1, 7) != 0);
2311  Dump(sb);
2312  FAIL_IF_NULL(sb->head);
2313  FAIL_IF_NOT(sb->sbb_size == 3);
2314 
2315  StreamingBufferSegment seg4;
2316  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg4, (const uint8_t *)"B", 1, 1) != 0);
2317  Dump(sb);
2318  StreamingBufferSegment seg5;
2319  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg5, (const uint8_t *)"C", 1, 2) != 0);
2320  Dump(sb);
2321  StreamingBufferSegment seg6;
2322  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg6, (const uint8_t *)"G", 1, 6) != 0);
2323  Dump(sb);
2324  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2325  FAIL_IF_NULL(sb->head);
2326  FAIL_IF_NOT(sb->sbb_size == 6);
2327 
2328  StreamingBufferSegment seg7;
2329  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg7, (const uint8_t *)"ABCDEFGHIJ", 10, 0) != 0);
2330  Dump(sb);
2331  StreamingBufferBlock *sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2332  FAIL_IF_NULL(sbb1);
2333  FAIL_IF(sbb1->offset != 0);
2334  FAIL_IF(sbb1->len != 10);
2335  FAIL_IF(SBB_RB_NEXT(sbb1));
2336  FAIL_IF_NULL(sb->head);
2337  FAIL_IF_NOT(sb->sbb_size == 10);
2338 
2339  StreamingBufferSegment seg8;
2340  FAIL_IF(StreamingBufferInsertAt(sb, &cfg, &seg8, (const uint8_t *)"abcdefghij", 10, 0) != 0);
2341  Dump(sb);
2342  sbb1 = RB_MIN(SBB, &sb->sbb_tree);
2343  FAIL_IF_NOT(sb->head == RB_MIN(SBB, &sb->sbb_tree));
2344  FAIL_IF_NULL(sbb1);
2345  FAIL_IF(sbb1->offset != 0);
2346  FAIL_IF(sbb1->len != 10);
2347  FAIL_IF(SBB_RB_NEXT(sbb1));
2348  FAIL_IF_NULL(sb->head);
2349  FAIL_IF_NOT(sb->sbb_size == 10);
2350 
2351  StreamingBufferFree(sb, &cfg);
2352  PASS;
2353 }
2354 
2355 static int StreamingBufferTest11(void)
2356 {
2357  StreamingBufferConfig cfg = { 24, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL };
2358  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2359  FAIL_IF(sb == NULL);
2360 
2361  StreamingBufferSegment seg1;
2362  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGH", 8) != 0);
2363  StreamingBufferSegment seg2;
2364  unsigned int data_len = 0xffffffff;
2365  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2, (const uint8_t *)"unused", data_len) != -1);
2366  FAIL_IF(StreamingBufferInsertAt(
2367  sb, &cfg, &seg2, (const uint8_t *)"abcdefghij", data_len, 100000) != SC_ELIMIT);
2368  StreamingBufferFree(sb, &cfg);
2369  PASS;
2370 }
2371 
2372 static const char *dummy_conf_string = "%YAML 1.1\n"
2373  "---\n"
2374  "\n"
2375  "app-layer:\n"
2376  " protocols:\n"
2377  " http:\n"
2378  " enabled: yes\n"
2379  " memcap: 88\n"
2380  "\n";
2381 
2382 static int StreamingBufferTest12(void)
2383 {
2384  ConfCreateContextBackup();
2385  ConfInit();
2386  HtpConfigCreateBackup();
2387  ConfYamlLoadString((const char *)dummy_conf_string, strlen(dummy_conf_string));
2388  HTPConfigure();
2389 
2390  StreamingBufferConfig cfg = { 8, 1, STREAMING_BUFFER_REGION_GAP_DEFAULT, HTPCalloc, HTPRealloc,
2391  HTPFree };
2392  StreamingBuffer *sb = StreamingBufferInit(&cfg);
2393  FAIL_IF(sb == NULL);
2394 
2395  StreamingBufferSegment seg1;
2396  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg1, (const uint8_t *)"ABCDEFGHIJKLMNOP", 16) != 0);
2397 
2398  StreamingBufferSegment seg2;
2399  FAIL_IF(StreamingBufferAppend(sb, &cfg, &seg2,
2400  (const uint8_t *)"ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ",
2401  52) != -1);
2402  FAIL_IF(sc_errno != SC_ELIMIT);
2403 
2404  StreamingBufferFree(sb, &cfg);
2405  HtpConfigRestoreBackup();
2406  ConfRestoreContextBackup();
2407 
2408  PASS;
2409 }
2410 #endif
2411 
2412 void StreamingBufferRegisterTests(void)
2413 {
2414 #ifdef UNITTESTS
2415  UtRegisterTest("StreamingBufferTest02", StreamingBufferTest02);
2416  UtRegisterTest("StreamingBufferTest03", StreamingBufferTest03);
2417  UtRegisterTest("StreamingBufferTest04", StreamingBufferTest04);
2418  UtRegisterTest("StreamingBufferTest06", StreamingBufferTest06);
2419  UtRegisterTest("StreamingBufferTest07", StreamingBufferTest07);
2420  UtRegisterTest("StreamingBufferTest08", StreamingBufferTest08);
2421  UtRegisterTest("StreamingBufferTest09", StreamingBufferTest09);
2422  UtRegisterTest("StreamingBufferTest10", StreamingBufferTest10);
2423  UtRegisterTest("StreamingBufferTest11 Bug 6903", StreamingBufferTest11);
2424  UtRegisterTest("StreamingBufferTest12 Bug 6782", StreamingBufferTest12);
2425 #endif
2426 }
FREE
#define FREE(cfg, ptr, s)
Definition: util-streaming-buffer.c:66
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:1004
len
uint8_t len
Definition: app-layer-dnp3.h:2
FAIL_IF_NULL
#define FAIL_IF_NULL(expr)
Fail a test if expression evaluates to NULL.
Definition: util-unittest.h:89
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:66
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:111
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
StreamingBufferRegion_::next
struct StreamingBufferRegion_ * next
Definition: util-streaming-buffer.h:89
UtRegisterTest
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
Definition: util-unittest.c:103
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:297
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
RB_LEFT
#define RB_LEFT(elm, field)
Definition: tree.h:322
SC_EINVAL
@ SC_EINVAL
Definition: util-error.h:30
StreamingBuffer_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:114
StreamingBuffer_::regions
uint16_t regions
Definition: util-streaming-buffer.h:113
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1786
RB_MIN
#define RB_MIN(name, x)
Definition: tree.h:778
MIN
#define MIN(x, y)
Definition: suricata-common.h:391
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len)
Definition: util-streaming-buffer.c:1081
HTPRealloc
void * HTPRealloc(void *ptr, size_t orig_size, size_t size)
Definition: app-layer-htp-mem.c:175
segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:0
m
SCMutex m
Definition: flow-hash.h:6
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1803
MAX
#define MAX(x, y)
Definition: suricata-common.h:395
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1770
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:67
SC_ELIMIT
@ SC_ELIMIT
Definition: util-error.h:31
util-unittest.h
HTPConfigure
void HTPConfigure(void)
Definition: app-layer-htp.c:2556
FAIL_IF_NOT
#define FAIL_IF_NOT(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:82
SC_ENOMEM
@ SC_ENOMEM
Definition: util-error.h:29
app-layer-htp.h
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
FAIL_IF_NOT_NULL
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
util-debug.h
StreamingBufferRegion_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:86
PASS
#define PASS
Pass the test.
Definition: util-unittest.h:105
util-error.h
RB_ROOT
#define RB_ROOT(head)
Definition: tree.h:326
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:251
HtpConfigCreateBackup
void HtpConfigCreateBackup(void)
Definition: app-layer-htp.c:2870
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:1628
BIT_U32
#define BIT_U32(n)
Definition: suricata-common.h:400
REALLOC
#define REALLOC(cfg, ptr, orig_s, s)
Definition: util-streaming-buffer.c:64
CALLOC
#define CALLOC(cfg, n, s)
Definition: util-streaming-buffer.c:63
BOOL2STR
#define BOOL2STR(b)
Definition: util-debug.h:527
util-print.h
RB_FOREACH_SAFE
#define RB_FOREACH_SAFE(x, name, head, y)
Definition: tree.h:791
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:274
HTPCalloc
void * HTPCalloc(size_t n, size_t size)
Definition: app-layer-htp-mem.c:154
ConfYamlLoadString
int ConfYamlLoadString(const char *string, size_t len)
Load configuration from a YAML string.
Definition: conf-yaml-loader.c:523
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
PrintRawDataFp
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:111
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
RB_FOREACH
#define RB_FOREACH(x, name, head)
Definition: tree.h:781
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:112
STREAMING_BUFFER_REGION_GAP_DEFAULT
#define STREAMING_BUFFER_REGION_GAP_DEFAULT
Definition: util-streaming-buffer.h:63
conf-yaml-loader.h
StreamingBufferRegion_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:85
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1825
SC_OK
@ SC_OK
Definition: util-error.h:27
RB_GENERATE
RB_GENERATE(SBB, StreamingBufferBlock, rb, SBBCompare)
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:87
ConfCreateContextBackup
void ConfCreateContextBackup(void)
Creates a backup of the conf_hash hash_table used by the conf API.
Definition: conf.c:669
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:104
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:110
StreamingBuffer_
Definition: util-streaming-buffer.h:108
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1696
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
FAIL_IF
#define FAIL_IF(expr)
Fail a test if expression evaluates to true.
Definition: util-unittest.h:71
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
Definition: util-streaming-buffer.c:1125
suricata-common.h
util-streaming-buffer.h
ConfRestoreContextBackup
void ConfRestoreContextBackup(void)
Restores the backup of the hash_table present in backup_conf_hash back to conf_hash.
Definition: conf.c:679
util-validate.h
HtpConfigRestoreBackup
void HtpConfigRestoreBackup(void)
Definition: app-layer-htp.c:2875
StreamingBufferConfig_
Definition: util-streaming-buffer.h:65
ConfInit
void ConfInit(void)
Initialize the configuration system.
Definition: conf.c:120
HtpBodyChunk_::next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:178
RB_RIGHT
#define RB_RIGHT(elm, field)
Definition: tree.h:323
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1509
head
Flow * head
Definition: flow-hash.h:1
sc_errno
thread_local SCError sc_errno
Definition: util-error.c:31
StreamingBufferRegion_
Definition: util-streaming-buffer.h:84
app-layer-htp-mem.h
RB_FOREACH_REVERSE_FROM
#define RB_FOREACH_REVERSE_FROM(x, name, y)
Definition: tree.h:801
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:73
HTPFree
void HTPFree(void *ptr, size_t size)
Definition: app-layer-htp-mem.c:199
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
RB_FOREACH_FROM
#define RB_FOREACH_FROM(x, name, y)
Definition: tree.h:786
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:109
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:1737
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:68
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:98
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:96
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1660
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:95
WARN_UNUSED
#define WARN_UNUSED
Definition: suricata-common.h:403
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:88
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:2412