suricata
util-streaming-buffer.h
Go to the documentation of this file.
1 /* Copyright (C) 2015-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /*
19  * This API is meant to be used with streaming data. A single memory
20  * block is used to store the data. StreamingBufferSegment points to
21  * chunk of data in the single StreamingBuffer. It points by offset
22  * and length, so no pointers. The buffer is resized on demand and
23  * slides forward, either automatically or manually.
24  *
25  * When a segment needs it's data it uses StreamingBufferSegmentGetData
26  * which takes care of checking if the segment still has a valid offset
27  * and length.
28  *
29  * The StreamingBuffer::stream_offset is an absolute offset since the
30  * start of the data streaming.
31  *
32  * Similarly, StreamingBufferSegment::stream_offset is also an absolute
33  * offset.
34  *
35  * Using the segments is optional.
36  *
37  *
38  * stream_offset buf_offset stream_offset + buf_size
39  * ^ ^ ^
40  * | | |
41  * | | |
42  * +--------------------------------------------+
43  * | data | empty |
44  * | xxxxxxxxxx | |
45  * +------^--------^--------+-------------------+
46  * | |
47  * | |
48  * | |
49  * | |
50  * | |
51  * +------+--------+-------+
52  * | StreamingBufferSegment|
53  * +-----------+-----------+
54  * | offset | len |
55  * +-----------+-----------+
56  */
57 
58 
59 #ifndef __UTIL_STREAMING_BUFFER_H__
60 #define __UTIL_STREAMING_BUFFER_H__
61 
62 #include "tree.h"
63 
64 #define STREAMING_BUFFER_REGION_GAP_DEFAULT 262144
65 
66 typedef struct StreamingBufferConfig_ {
67  uint32_t buf_size;
68  uint16_t max_regions; /**< max concurrent memory regions. 0 means no limit. */
69  uint32_t region_gap; /**< max gap size before a new region will be created. */
70  void *(*Calloc)(size_t n, size_t size);
71  void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
72  void (*Free)(void *ptr, size_t size);
74 
75 #define STREAMING_BUFFER_CONFIG_INITIALIZER \
76  { \
77  2048, 8, STREAMING_BUFFER_REGION_GAP_DEFAULT, NULL, NULL, NULL, \
78  }
79 
80 #define STREAMING_BUFFER_REGION_INIT \
81  { \
82  NULL, 0, 0, 0ULL, NULL, \
83  }
84 
85 typedef struct StreamingBufferRegion_ {
86  uint8_t *buf; /**< memory block for reassembly */
87  uint32_t buf_size; /**< size of memory block */
88  uint32_t buf_offset; /**< how far we are in buf_size */
89  uint64_t stream_offset; /**< stream offset of this region */
92 
93 /**
94  * \brief block of continues data
95  */
96 typedef struct StreamingBufferBlock {
97  uint64_t offset;
99  uint32_t len;
100 } __attribute__((__packed__)) StreamingBufferBlock;
101 
102 int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b);
103 
104 /* red-black tree prototype for SACK records */
108 
109 typedef struct StreamingBuffer_ {
111  struct SBB sbb_tree; /**< red black tree of Stream Buffer Blocks */
112  StreamingBufferBlock *head; /**< head, should always be the same as RB_MIN */
113  uint32_t sbb_size; /**< data size covered by sbbs */
114  uint16_t regions;
115  uint16_t max_regions;
116 #ifdef DEBUG
117  uint32_t buf_size_max;
118 #endif
120 
121 static inline bool StreamingBufferHasData(const StreamingBuffer *sb)
122 {
123  return (sb->region.stream_offset || sb->region.buf_offset || sb->region.next != NULL ||
124  !RB_EMPTY(&sb->sbb_tree));
125 }
126 
127 static inline uint64_t StreamingBufferGetConsecutiveDataRightEdge(const StreamingBuffer *sb)
128 {
129  return sb->region.stream_offset + sb->region.buf_offset;
130 }
131 
132 static inline uint64_t StreamingBufferGetOffset(const StreamingBuffer *sb)
133 {
134  return sb->region.stream_offset;
135 }
136 
137 #ifndef DEBUG
138 #define STREAMING_BUFFER_INITIALIZER \
139  { \
140  STREAMING_BUFFER_REGION_INIT, \
141  { NULL }, \
142  NULL, \
143  0, \
144  1, \
145  1, \
146  };
147 #else
148 #define STREAMING_BUFFER_INITIALIZER { STREAMING_BUFFER_REGION_INIT, { NULL }, NULL, 0, 1, 1, 0 };
149 #endif
150 
151 typedef struct StreamingBufferSegment_ {
152  uint32_t segment_len;
153  uint64_t stream_offset;
154 } __attribute__((__packed__)) StreamingBufferSegment;
155 
159 
161  StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset);
162 
164  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len) WARN_UNUSED;
166  const uint8_t *data, uint32_t data_len) WARN_UNUSED;
168  StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len,
169  uint64_t offset) WARN_UNUSED;
170 
172  const StreamingBufferSegment *seg,
173  const uint8_t **data, uint32_t *data_len);
174 
176  const StreamingBufferBlock *sbb,
177  const uint8_t **data, uint32_t *data_len);
178 
180  const StreamingBufferBlock *sbb,
181  const uint8_t **data, uint32_t *data_len,
182  uint64_t offset);
183 
185  const StreamingBufferSegment *seg,
186  const uint8_t *rawdata, uint32_t rawdata_len);
188  const uint8_t *rawdata, uint32_t rawdata_len);
189 
191  const uint8_t **data, uint32_t *data_len,
192  uint64_t *stream_offset);
193 
195  const uint8_t **data, uint32_t *data_len,
196  uint64_t offset);
197 
199  const StreamingBufferSegment *seg);
200 
202 
203 #endif /* __UTIL_STREAMING_BUFFER_H__ */
StreamingBufferSegmentIsBeforeWindow
int StreamingBufferSegmentIsBeforeWindow(const StreamingBuffer *sb, const StreamingBufferSegment *seg)
Definition: util-streaming-buffer.c:1590
RB_PROTOTYPE
RB_PROTOTYPE(SBB, StreamingBufferBlock, rb, SBBCompare)
SBB_RB_FIND_INCLUSIVE
StreamingBufferBlock * SBB_RB_FIND_INCLUSIVE(struct SBB *head, StreamingBufferBlock *elm)
Definition: util-streaming-buffer.c:100
SBBCompare
int SBBCompare(struct StreamingBufferBlock *a, struct StreamingBufferBlock *b)
Definition: util-streaming-buffer.c:69
StreamingBufferAppendNoTrack
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const StreamingBufferConfig *cfg, const uint8_t *data, uint32_t data_len) WARN_UNUSED
add data w/o tracking a segment
Definition: util-streaming-buffer.c:1103
StreamingBufferConfig_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:67
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
StreamingBuffer_::head
StreamingBufferBlock * head
Definition: util-streaming-buffer.h:112
StreamingBufferRegion_::next
struct StreamingBufferRegion_ * next
Definition: util-streaming-buffer.h:90
StreamingBufferSegment_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:153
StreamingBuffer_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:115
StreamingBuffer_::regions
uint16_t regions
Definition: util-streaming-buffer.h:114
StreamingBufferSegment_
Definition: util-streaming-buffer.h:151
stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:1
StreamingBufferConfig_::max_regions
uint16_t max_regions
Definition: util-streaming-buffer.h:68
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:1768
__attribute__
struct StreamingBufferBlock __attribute__((__packed__)) StreamingBufferBlock
block of continues data
Definition: decode-vlan.c:103
StreamingBufferFree
void StreamingBufferFree(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:295
StreamingBufferInit
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:249
StreamingBufferSBBGetData
void StreamingBufferSBBGetData(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len)
get the data for one SBB
Definition: util-streaming-buffer.c:1622
RB_EMPTY
#define RB_EMPTY(head)
Definition: tree.h:327
StreamingBufferRegion_::buf_size
uint32_t buf_size
Definition: util-streaming-buffer.h:87
StreamingBufferSBBGetDataAtOffset
void StreamingBufferSBBGetDataAtOffset(const StreamingBuffer *sb, const StreamingBufferBlock *sbb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
get the data for one SBB
Definition: util-streaming-buffer.c:1659
StreamingBufferRegisterTests
void StreamingBufferRegisterTests(void)
Definition: util-streaming-buffer.c:2322
StreamingBuffer_::sbb_size
uint32_t sbb_size
Definition: util-streaming-buffer.h:113
StreamingBufferSlideToOffset
void StreamingBufferSlideToOffset(StreamingBuffer *sb, const StreamingBufferConfig *cfg, uint64_t offset)
slide to absolute offset
Definition: util-streaming-buffer.c:997
StreamingBufferRegion_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:86
StreamingBufferRegion_::buf_offset
uint32_t buf_offset
Definition: util-streaming-buffer.h:88
StreamingBufferSegmentCompareRawData
int StreamingBufferSegmentCompareRawData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1735
tree.h
StreamingBuffer_::sbb_tree
struct SBB sbb_tree
Definition: util-streaming-buffer.h:111
StreamingBuffer_
Definition: util-streaming-buffer.h:109
StreamingBufferBlock::RB_ENTRY
RB_ENTRY(StreamingBufferBlock) rb
StreamingBufferInsertAt
int StreamingBufferInsertAt(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len, uint64_t offset) WARN_UNUSED
Definition: util-streaming-buffer.c:1474
StreamingBuffer
struct StreamingBuffer_ StreamingBuffer
StreamingBufferRegion
struct StreamingBufferRegion_ StreamingBufferRegion
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:1751
StreamingBufferAppend
int StreamingBufferAppend(StreamingBuffer *sb, const StreamingBufferConfig *cfg, StreamingBufferSegment *seg, const uint8_t *data, uint32_t data_len) WARN_UNUSED
Definition: util-streaming-buffer.c:1066
StreamingBufferConfig
struct StreamingBufferConfig_ StreamingBufferConfig
RB_HEAD
RB_HEAD(SBB, StreamingBufferBlock)
StreamingBufferConfig_
Definition: util-streaming-buffer.h:66
StreamingBufferCompareRawData
int StreamingBufferCompareRawData(const StreamingBuffer *sb, const uint8_t *rawdata, uint32_t rawdata_len)
Definition: util-streaming-buffer.c:1790
head
Flow * head
Definition: flow-hash.h:1
StreamingBufferConfig_::Free
void(* Free)(void *ptr, size_t size)
Definition: util-streaming-buffer.h:72
StreamingBufferRegion_
Definition: util-streaming-buffer.h:85
StreamingBufferSegmentGetData
void StreamingBufferSegmentGetData(const StreamingBuffer *sb, const StreamingBufferSegment *seg, const uint8_t **data, uint32_t *data_len)
Definition: util-streaming-buffer.c:1701
StreamingBufferClear
void StreamingBufferClear(StreamingBuffer *sb, const StreamingBufferConfig *cfg)
Definition: util-streaming-buffer.c:272
StreamingBufferSegment_::segment_len
uint32_t segment_len
Definition: util-streaming-buffer.h:152
StreamingBuffer_::region
StreamingBufferRegion region
Definition: util-streaming-buffer.h:110
StreamingBufferConfig_::region_gap
uint32_t region_gap
Definition: util-streaming-buffer.h:69
StreamingBufferBlock::len
uint32_t len
Definition: util-streaming-buffer.h:99
StreamingBufferBlock::offset
uint64_t offset
Definition: util-streaming-buffer.h:97
StreamingBufferBlock
block of continues data
Definition: util-streaming-buffer.h:96
WARN_UNUSED
#define WARN_UNUSED
Definition: suricata-common.h:403
StreamingBufferRegion_::stream_offset
uint64_t stream_offset
Definition: util-streaming-buffer.h:89