suricata
app-layer-frames.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "util-print.h"
27 
28 #include "flow.h"
29 #include "stream-tcp.h"
30 #include "app-layer-frames.h"
31 #include "app-layer-parser.h"
32 
33 struct FrameConfig {
34  SC_ATOMIC_DECLARE(uint64_t, types);
35 };
36 static struct FrameConfig frame_config[ALPROTO_MAX];
37 
38 void FrameConfigInit(void)
39 {
40  for (AppProto p = 0; p < ALPROTO_MAX; p++) {
41  SC_ATOMIC_INIT(frame_config[p].types);
42  }
43 }
44 
46 {
47  const uint64_t bits = UINT64_MAX;
48  for (AppProto p = 0; p < ALPROTO_MAX; p++) {
49  struct FrameConfig *fc = &frame_config[p];
50  SC_ATOMIC_OR(fc->types, bits);
51  }
52 }
53 
54 void FrameConfigEnable(const AppProto p, const uint8_t type)
55 {
56  const uint64_t bits = BIT_U64(type);
57  struct FrameConfig *fc = &frame_config[p];
58  SC_ATOMIC_OR(fc->types, bits);
59 }
60 
61 static inline bool FrameConfigTypeIsEnabled(const AppProto p, const uint8_t type)
62 {
63  struct FrameConfig *fc = &frame_config[p];
64  const uint64_t bits = BIT_U64(type);
65  const bool enabled = (SC_ATOMIC_GET(fc->types) & bits) != 0;
66  return enabled;
67 }
68 
69 static void FrameDebug(const char *prefix, const Frames *frames, const Frame *frame)
70 {
71 #ifdef DEBUG
72  const char *type_name = "unknown";
73  if (frame->type == FRAME_STREAM_TYPE) {
74  type_name = "stream";
75  } else if (frames != NULL) {
76  type_name = AppLayerParserGetFrameNameById(frames->ipproto, frames->alproto, frame->type);
77  }
78  SCLogDebug("[%s] %p: frame:%p type:%u/%s id:%" PRIi64 " flags:%02x offset:%" PRIu64
79  ", len:%" PRIi64 ", inspect_progress:%" PRIu64 ", events:%u %u/%u/%u/%u",
80  prefix, frames, frame, frame->type, type_name, frame->id, frame->flags, frame->offset,
81  frame->len, frame->inspect_progress, frame->event_cnt, frame->events[0],
82  frame->events[1], frame->events[2], frame->events[3]);
83 #endif
84 }
85 
86 Frame *FrameGetById(Frames *frames, const int64_t id)
87 {
88  SCLogDebug("frames %p cnt %u, looking for %" PRIi64, frames, frames->cnt, id);
89  for (uint16_t i = 0; i < frames->cnt; i++) {
90  if (i < FRAMES_STATIC_CNT) {
91  Frame *frame = &frames->sframes[i];
92  FrameDebug("get_by_id(static)", frames, frame);
93  if (frame->id == id)
94  return frame;
95  } else {
96  const uint16_t o = i - FRAMES_STATIC_CNT;
97  Frame *frame = &frames->dframes[o];
98  FrameDebug("get_by_id(dynamic)", frames, frame);
99  if (frame->id == id)
100  return frame;
101  }
102  }
103  return NULL;
104 }
105 
106 Frame *FrameGetByIndex(Frames *frames, const uint32_t idx)
107 {
108  if (idx >= frames->cnt)
109  return NULL;
110 
111  if (idx < FRAMES_STATIC_CNT) {
112  Frame *frame = &frames->sframes[idx];
113  FrameDebug("get_by_idx(s)", frames, frame);
114  return frame;
115  } else {
116  const uint32_t o = idx - FRAMES_STATIC_CNT;
117  Frame *frame = &frames->dframes[o];
118  FrameDebug("get_by_idx(d)", frames, frame);
119  return frame;
120  }
121 }
122 
123 static Frame *FrameNew(Frames *frames, uint64_t offset, int64_t len)
124 {
125  BUG_ON(frames == NULL);
126 
127  if (frames->cnt < FRAMES_STATIC_CNT) {
128  Frame *frame = &frames->sframes[frames->cnt];
129  frames->sframes[frames->cnt].offset = offset;
130  frames->sframes[frames->cnt].len = len;
131  frames->sframes[frames->cnt].id = ++frames->base_id;
132  frames->cnt++;
133  return frame;
134  } else if (frames->dframes == NULL) {
135  BUG_ON(frames->dyn_size != 0);
136  BUG_ON(frames->cnt != FRAMES_STATIC_CNT);
137 
138  frames->dframes = SCCalloc(8, sizeof(Frame));
139  if (frames->dframes == NULL) {
140  return NULL;
141  }
142  frames->cnt++;
143  BUG_ON(frames->cnt != FRAMES_STATIC_CNT + 1);
144 
145  frames->dyn_size = 8;
146  frames->dframes[0].offset = offset;
147  frames->dframes[0].len = len;
148  frames->dframes[0].id = ++frames->base_id;
149  return &frames->dframes[0];
150  } else {
151  BUG_ON(frames->cnt < FRAMES_STATIC_CNT);
152 
153  /* need to handle dynamic storage of frames now */
154  const uint16_t dyn_cnt = frames->cnt - FRAMES_STATIC_CNT;
155  if (dyn_cnt < frames->dyn_size) {
156  BUG_ON(frames->dframes == NULL);
157 
158  // fall through
159  } else {
160  if (frames->dyn_size == 256) {
161  SCLogDebug("limit reached! 256 dynamic frames already");
162  // limit reached
163  // TODO figure out if this should lead to an event of sorts
164  return NULL;
165  }
166 
167  /* realloc time */
168  uint16_t new_dyn_size = frames->dyn_size * 2;
169  uint32_t new_alloc_size = new_dyn_size * sizeof(Frame);
170 
171  void *ptr = SCRealloc(frames->dframes, new_alloc_size);
172  if (ptr == NULL) {
173  return NULL;
174  }
175 
176  memset((uint8_t *)ptr + (frames->dyn_size * sizeof(Frame)), 0x00,
177  (frames->dyn_size * sizeof(Frame)));
178  frames->dframes = ptr;
179  frames->dyn_size = new_dyn_size;
180  }
181 
182  frames->cnt++;
183  frames->dframes[dyn_cnt].offset = offset;
184  frames->dframes[dyn_cnt].len = len;
185  frames->dframes[dyn_cnt].id = ++frames->base_id;
186  return &frames->dframes[dyn_cnt];
187  }
188 }
189 
190 static void FrameClean(Frame *frame)
191 {
192  memset(frame, 0, sizeof(*frame));
193 }
194 
195 static void FrameCopy(Frame *dst, Frame *src)
196 {
197  memcpy(dst, src, sizeof(*dst));
198 }
199 
200 static void AppLayerFrameDumpForFrames(const char *prefix, const Frames *frames)
201 {
202  SCLogDebug("prefix: %s", prefix);
203  for (uint16_t i = 0; i < frames->cnt; i++) {
204  if (i < FRAMES_STATIC_CNT) {
205  const Frame *frame = &frames->sframes[i];
206  FrameDebug(prefix, frames, frame);
207  } else {
208  const uint16_t o = i - FRAMES_STATIC_CNT;
209  const Frame *frame = &frames->dframes[o];
210  FrameDebug(prefix, frames, frame);
211  }
212  }
213  SCLogDebug("prefix: %s", prefix);
214 }
215 
216 static inline uint64_t FrameLeftEdge(const TcpStream *stream, const Frame *frame)
217 {
218  const int64_t app_progress = STREAM_APP_PROGRESS(stream);
219 
220  const int64_t frame_offset = frame->offset;
221  const int64_t frame_data = app_progress - frame_offset;
222 
223  SCLogDebug("frame_offset %" PRIi64 ", frame_data %" PRIi64 ", frame->len %" PRIi64,
224  frame_offset, frame_data, frame->len);
225  BUG_ON(frame_offset > app_progress);
226 
227  /* length unknown, make sure to have at least 2500 */
228  if (frame->len < 0) {
229  if (frame_data <= 2500) {
230  SCLogDebug("got <= 2500 bytes (%" PRIu64 "), returning offset %" PRIu64, frame_data,
231  frame_offset);
232  return frame_offset;
233  } else {
234  SCLogDebug("got > 2500 bytes (%" PRIu64 "), returning offset %" PRIu64, frame_data,
235  (frame_offset + (frame_data - 2500)));
236  return frame_offset + (frame_data - 2500);
237  }
238 
239  /* length specified */
240  } else {
241  /* have all data for the frame, we can skip it */
242  if (frame->len <= frame_data) {
243  uint64_t x = frame_offset + frame_data;
244  SCLogDebug("x %" PRIu64, x);
245  return x;
246  /*
247 
248  [ stream <frame_data> ]
249  [ frame .......]
250 
251  */
252  } else if (frame_data < 2500) {
253  uint64_t x = frame_offset;
254  SCLogDebug("x %" PRIu64, x);
255  return x;
256  } else {
257  uint64_t x = frame_offset + (frame_data - 2500);
258  SCLogDebug("x %" PRIu64, x);
259  return x;
260  }
261  }
262 }
263 
264 /** Stream buffer slides forward, we need to update and age out
265  * frame offsets/frames. Aging out means we move existing frames
266  * into the slots we'd free up.
267  *
268  * Start:
269  *
270  * [ stream ]
271  * [ frame ...........]
272  * offset: 2
273  * len: 19
274  *
275  * Slide:
276  * [ stream ]
277  * [ frame .... .]
278  * offset: 2
279  * len: 19
280  *
281  * Slide:
282  * [ stream ]
283  * [ frame ........... ]
284  * offset: 2
285  * len: 19
286  */
287 static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, const uint32_t slide)
288 {
289  SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
290  ", next %" PRIu64,
291  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
292  STREAM_BASE_OFFSET(stream), STREAM_BASE_OFFSET(stream) + slide);
293  BUG_ON(frames == NULL);
294  SCLogDebug("%s frames %p: sliding %u bytes", ds, frames, slide);
295  uint64_t le = STREAM_APP_PROGRESS(stream);
296  const uint64_t next_base = STREAM_BASE_OFFSET(stream) + slide;
297  const uint16_t start = frames->cnt;
298  uint16_t removed = 0;
299  uint16_t x = 0;
300  for (uint16_t i = 0; i < frames->cnt; i++) {
301  if (i < FRAMES_STATIC_CNT) {
302  Frame *frame = &frames->sframes[i];
303  FrameDebug("slide(s)", frames, frame);
304  if (frame->len >= 0 && frame->offset + frame->len <= next_base) {
305  // remove by not incrementing 'x'
306  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
307  FrameClean(frame);
308  removed++;
309  } else {
310  Frame *nframe = &frames->sframes[x];
311  FrameCopy(nframe, frame);
312  if (frame != nframe) {
313  FrameClean(frame);
314  }
315  le = MIN(le, FrameLeftEdge(stream, nframe));
316  x++;
317  }
318  } else {
319  const uint16_t o = i - FRAMES_STATIC_CNT;
320  Frame *frame = &frames->dframes[o];
321  FrameDebug("slide(d)", frames, frame);
322  if (frame->len >= 0 && frame->offset + frame->len <= next_base) {
323  // remove by not incrementing 'x'
324  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
325  FrameClean(frame);
326  removed++;
327  } else {
328  Frame *nframe;
329  if (x >= FRAMES_STATIC_CNT) {
330  nframe = &frames->dframes[x - FRAMES_STATIC_CNT];
331  } else {
332  nframe = &frames->sframes[x];
333  }
334  FrameCopy(nframe, frame);
335  if (frame != nframe) {
336  FrameClean(frame);
337  }
338  le = MIN(le, FrameLeftEdge(stream, nframe));
339  x++;
340  }
341  }
342  }
343  frames->cnt = x;
344  uint64_t o = STREAM_BASE_OFFSET(stream) + slide;
345  frames->left_edge_rel = le - (STREAM_BASE_OFFSET(stream) + slide);
346 
347 #ifdef DEBUG
348  SCLogDebug("end: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
349  " (+slide), cnt %u, removed %u, start %u",
350  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream) + slide,
351  frames->left_edge_rel, STREAM_BASE_OFFSET(stream) + slide, frames->cnt, removed, start);
352  char pf[32] = "";
353  snprintf(pf, sizeof(pf), "%s:post_slide", ds);
354  AppLayerFrameDumpForFrames(pf, frames);
355 #endif
356  BUG_ON(o > le);
357  BUG_ON(x != start - removed);
358  return 0;
359 }
360 
361 void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction)
362 {
363  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
364  if (frames_container == NULL)
365  return;
366  Frames *frames;
367  TcpSession *ssn = f->protoctx;
368  TcpStream *stream;
369  if (direction == STREAM_TOSERVER) {
370  stream = &ssn->client;
371  frames = &frames_container->toserver;
372  FrameSlide("toserver", frames, stream, slide);
373  } else {
374  stream = &ssn->server;
375  frames = &frames_container->toclient;
376  FrameSlide("toclient", frames, stream, slide);
377  }
378 }
379 
380 static void FrameFreeSingleFrame(Frames *frames, Frame *r)
381 {
382  FrameDebug("free", frames, r);
383  FrameClean(r);
384 }
385 
386 static void FramesClear(Frames *frames)
387 {
388  BUG_ON(frames == NULL);
389 
390  SCLogDebug("frames %u", frames->cnt);
391  for (uint16_t i = 0; i < frames->cnt; i++) {
392  if (i < FRAMES_STATIC_CNT) {
393  Frame *r = &frames->sframes[i];
394  SCLogDebug("removing frame %p", r);
395  FrameFreeSingleFrame(frames, r);
396  } else {
397  const uint16_t o = i - FRAMES_STATIC_CNT;
398  Frame *r = &frames->dframes[o];
399  SCLogDebug("removing frame %p", r);
400  FrameFreeSingleFrame(frames, r);
401  }
402  }
403  frames->cnt = 0;
404 }
405 
406 void FramesFree(Frames *frames)
407 {
408  BUG_ON(frames == NULL);
409  FramesClear(frames);
410  SCFree(frames->dframes);
411  frames->dframes = NULL;
412 }
413 
414 /** \brief create new frame using a pointer to start of the frame
415  */
416 Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
417  const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type)
418 {
419  SCLogDebug("frame_start:%p stream_slice->input:%p stream_slice->offset:%" PRIu64, frame_start,
420  stream_slice->input, stream_slice->offset);
421 
422  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
423  return NULL;
424 
425  /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
426 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
427  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
428  return NULL;
429  if (frame_start < stream_slice->input ||
430  frame_start >= stream_slice->input + stream_slice->input_len)
431  return NULL;
432 #endif
433  BUG_ON(frame_start < stream_slice->input);
434  BUG_ON(stream_slice->input == NULL);
435  BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
436 
437  ptrdiff_t ptr_offset = frame_start - stream_slice->input;
438 #ifdef DEBUG
439  uint64_t offset = ptr_offset + stream_slice->offset;
440  SCLogDebug("flow %p direction %s frame %p starting at %" PRIu64 " len %" PRIi64
441  " (offset %" PRIu64 ")",
442  f, dir == 0 ? "toserver" : "toclient", frame_start, offset, len, stream_slice->offset);
443 #endif
444  BUG_ON(f->alparser == NULL);
445 
446  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
447  if (frames_container == NULL)
448  return NULL;
449 
450  Frames *frames;
451  if (dir == 0) {
452  frames = &frames_container->toserver;
453  } else {
454  frames = &frames_container->toclient;
455  }
456 
457  uint64_t abs_frame_offset = stream_slice->offset + ptr_offset;
458 
459  Frame *r = FrameNew(frames, abs_frame_offset, len);
460  if (r != NULL) {
461  r->type = frame_type;
462  FrameDebug("new_by_ptr", frames, r);
463  }
464  return r;
465 }
466 
467 static Frame *AppLayerFrameUdp(Flow *f, const StreamSlice *stream_slice,
468  const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
469 {
470  BUG_ON(f->proto != IPPROTO_UDP);
471 
472  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
473  return NULL;
474 
475  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
476  if (frames_container == NULL)
477  return NULL;
478 
479  Frames *frames;
480  if (dir == 0) {
481  frames = &frames_container->toserver;
482  } else {
483  frames = &frames_container->toclient;
484  }
485 
486  Frame *r = FrameNew(frames, frame_start_rel, len);
487  if (r != NULL) {
488  r->type = frame_type;
489  }
490  return r;
491 }
492 
493 /** \brief create new frame using a relative offset from the start of the stream slice
494  */
495 Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice,
496  const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
497 {
498  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
499  return NULL;
500 
501  /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
502 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
503  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
504  return NULL;
505  if (stream_slice->input == NULL)
506  return NULL;
507 #else
508  BUG_ON(stream_slice->input == NULL);
509 #endif
510  BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
511  BUG_ON(f->alparser == NULL);
512 
513  if (f->proto == IPPROTO_UDP) {
514  return AppLayerFrameUdp(f, stream_slice, frame_start_rel, len, dir, frame_type);
515  }
516 
517  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
518  if (frames_container == NULL)
519  return NULL;
520 
521  Frames *frames;
522  if (dir == 0) {
523  frames = &frames_container->toserver;
524  } else {
525  frames = &frames_container->toclient;
526  }
527 
528  const uint64_t frame_abs_offset = (uint64_t)frame_start_rel + stream_slice->offset;
529 #ifdef DEBUG_VALIDATION
530  const TcpSession *ssn = f->protoctx;
531  const TcpStream *stream = dir == 0 ? &ssn->client : &ssn->server;
532  BUG_ON(stream_slice->offset != STREAM_APP_PROGRESS(stream));
533  BUG_ON(frame_abs_offset > STREAM_APP_PROGRESS(stream) + stream_slice->input_len);
534 #endif
535  Frame *r = FrameNew(frames, frame_abs_offset, len);
536  if (r != NULL) {
537  r->type = frame_type;
538  }
539  return r;
540 }
541 
543 {
544  if (f->proto == IPPROTO_TCP && f->protoctx && f->alparser) {
545  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
546  if (frames_container != NULL) {
547  AppLayerFrameDumpForFrames("toserver::dump", &frames_container->toserver);
548  AppLayerFrameDumpForFrames("toclient::dump", &frames_container->toclient);
549  }
550  }
551 }
552 
553 /** \brief create new frame using the absolute offset from the start of the stream
554  */
555 Frame *AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice,
556  const uint64_t frame_start, const int64_t len, int dir, uint8_t frame_type)
557 {
558  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
559  return NULL;
560 
561  /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
562 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
563  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
564  return NULL;
565  if (stream_slice->input == NULL)
566  return NULL;
567 #else
568  BUG_ON(stream_slice->input == NULL);
569 #endif
570  BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
571  BUG_ON(f->alparser == NULL);
572  BUG_ON(frame_start < stream_slice->offset);
573  BUG_ON(frame_start - stream_slice->offset >= (uint64_t)INT_MAX);
574 
575  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
576  if (frames_container == NULL)
577  return NULL;
578 
579  Frames *frames;
580  if (dir == 0) {
581  frames = &frames_container->toserver;
582  } else {
583  frames = &frames_container->toclient;
584  }
585 
586  SCLogDebug("flow %p direction %s frame type %u offset %" PRIu64 " len %" PRIi64
587  " (slice offset %" PRIu64 ")",
588  f, dir == 0 ? "toserver" : "toclient", frame_type, frame_start, len,
589  stream_slice->offset);
590  Frame *r = FrameNew(frames, frame_start, len);
591  if (r != NULL) {
592  r->type = frame_type;
593  }
594  return r;
595 }
596 
597 void AppLayerFrameAddEvent(Frame *r, uint8_t e)
598 {
599  if (r != NULL) {
600  if (r->event_cnt < 4) { // TODO
601  r->events[r->event_cnt++] = e;
602  }
603  FrameDebug("add_event", NULL, r);
604  }
605 }
606 
607 void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e)
608 {
609  Frame *frame = AppLayerFrameGetById(f, dir, id);
610  AppLayerFrameAddEvent(frame, e);
611 }
612 
614 {
615  if (r != NULL) {
616  return r->id;
617  } else {
618  return -1;
619  }
620 }
621 
622 void AppLayerFrameSetLength(Frame *frame, int64_t len)
623 {
624  if (frame != NULL) {
625  frame->len = len;
626  FrameDebug("set_length", NULL, frame);
627  }
628 }
629 
630 void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_t len)
631 {
632  Frame *frame = AppLayerFrameGetById(f, dir, id);
633  AppLayerFrameSetLength(frame, len);
634 }
635 
636 void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id)
637 {
638  if (r != NULL) {
640  r->tx_id = tx_id;
641  FrameDebug("set_txid", NULL, r);
642  }
643 }
644 
645 void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id)
646 {
647  Frame *frame = AppLayerFrameGetById(f, dir, id);
648  AppLayerFrameSetTxId(frame, tx_id);
649 }
650 
651 Frame *AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
652 {
653  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
654  SCLogDebug("get frame_id %" PRIi64 " direction %u/%s frames_container %p", frame_id, dir,
655  dir == 0 ? "toserver" : "toclient", frames_container);
656  if (frames_container == NULL)
657  return NULL;
658 
659  Frames *frames;
660  if (dir == 0) {
661  frames = &frames_container->toserver;
662  } else {
663  frames = &frames_container->toclient;
664  }
665  SCLogDebug("frames %p", frames);
666  return FrameGetById(frames, frame_id);
667 }
668 
669 static inline bool FrameIsDone(
670  const Frame *frame, const uint64_t abs_offset, const uint64_t abs_right_edge)
671 {
672  /* frame with negative length means we don't know the size yet. */
673  if (frame->len < 0)
674  return false;
675 
676  const int64_t frame_abs_offset = frame->offset;
677  const int64_t frame_right_edge = frame_abs_offset + frame->len;
678  if ((uint64_t)frame_right_edge <= abs_right_edge) {
679  SCLogDebug("frame %p id %" PRIi64 " is done", frame, frame->id);
680  return true;
681  }
682  return false;
683 }
684 
685 static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof)
686 {
687  const uint64_t frames_le_start = (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream);
688  SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64,
689  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
690  STREAM_BASE_OFFSET(stream));
691  const uint64_t abs_offset = STREAM_BASE_OFFSET(stream);
692  const uint64_t acked = StreamTcpGetUsable(stream, eof);
693  uint64_t le = STREAM_APP_PROGRESS(stream);
694 
695  const uint16_t start = frames->cnt;
696  uint16_t removed = 0;
697  uint16_t x = 0;
698  for (uint16_t i = 0; i < frames->cnt; i++) {
699  if (i < FRAMES_STATIC_CNT) {
700  Frame *frame = &frames->sframes[i];
701  FrameDebug("prune(s)", frames, frame);
702  if (eof || FrameIsDone(frame, abs_offset, acked)) {
703  // remove by not incrementing 'x'
704  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
705  FrameDebug("remove(s)", frames, frame);
706  FrameClean(frame);
707  removed++;
708  } else {
709  const uint64_t fle = FrameLeftEdge(stream, frame);
710  le = MIN(le, fle);
711  SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
712  Frame *nframe = &frames->sframes[x];
713  FrameCopy(nframe, frame);
714  if (frame != nframe) {
715  FrameClean(frame);
716  }
717  x++;
718  }
719  } else {
720  const uint16_t o = i - FRAMES_STATIC_CNT;
721  Frame *frame = &frames->dframes[o];
722  FrameDebug("prune(d)", frames, frame);
723  if (eof || FrameIsDone(frame, abs_offset, acked)) {
724  // remove by not incrementing 'x'
725  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
726  FrameDebug("remove(d)", frames, frame);
727  FrameClean(frame);
728  removed++;
729  } else {
730  const uint64_t fle = FrameLeftEdge(stream, frame);
731  le = MIN(le, fle);
732  SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
733  Frame *nframe;
734  if (x >= FRAMES_STATIC_CNT) {
735  nframe = &frames->dframes[x - FRAMES_STATIC_CNT];
736  } else {
737  nframe = &frames->sframes[x];
738  }
739  FrameCopy(nframe, frame);
740  if (frame != nframe) {
741  FrameClean(frame);
742  }
743  x++;
744  }
745  }
746  }
747  frames->cnt = x;
748  frames->left_edge_rel = le - STREAM_BASE_OFFSET(stream);
749 #ifdef DEBUG
750  SCLogDebug("end: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
751  ", cnt %u, removed %u, start %u",
752  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
753  STREAM_BASE_OFFSET(stream), frames->cnt, removed, start);
754  AppLayerFrameDumpForFrames("post_slide", frames);
755 #endif
756  BUG_ON(le < STREAM_BASE_OFFSET(stream));
757  if (frames->cnt > 0) { // if we removed all this can fail
758  BUG_ON(frames_le_start > le);
759  }
760  BUG_ON(x != start - removed);
761 }
762 
763 void FramesPrune(Flow *f, Packet *p)
764 {
765  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
766  return;
767  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
768  if (frames_container == NULL)
769  return;
770 
771  Frames *frames;
772 
773  if (p->proto == IPPROTO_UDP) {
774  SCLogDebug("clearing all UDP frames");
775  if (PKT_IS_TOSERVER(p)) {
776  frames = &frames_container->toserver;
777  } else {
778  frames = &frames_container->toclient;
779  }
780  FramesClear(frames);
781  return;
782  }
783 
784  TcpSession *ssn = f->protoctx;
785 
788  return;
789  }
790 
791  TcpStream *stream;
792  if (PKT_IS_TOSERVER(p)) {
793  stream = &ssn->client;
794  frames = &frames_container->toserver;
795  } else {
796  stream = &ssn->server;
797  frames = &frames_container->toclient;
798  }
799 
800  const bool eof = ssn->state == TCP_CLOSED || PKT_IS_PSEUDOPKT(p);
801  SCLogDebug("eof %s", eof ? "TRUE" : "false");
802  FramePrune(frames, stream, eof);
803 }
AppLayerFrameSetLengthById
void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_t len)
Definition: app-layer-frames.c:630
Packet_::proto
uint8_t proto
Definition: decode.h:458
TcpStream_
Definition: stream-tcp-private.h:106
Frame::inspect_progress
uint64_t inspect_progress
Definition: app-layer-frames.h:55
len
uint8_t len
Definition: app-layer-dnp3.h:2
Frame::tx_id
uint64_t tx_id
Definition: app-layer-frames.h:54
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
FrameConfig::SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, types)
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
FramesFree
void FramesFree(Frames *frames)
Definition: app-layer-frames.c:406
AppLayerFrameAddEvent
void AppLayerFrameAddEvent(Frame *r, uint8_t e)
Definition: app-layer-frames.c:597
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1071
stream-tcp.h
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
Frame::events
uint8_t events[4]
Definition: app-layer-frames.h:50
Flow_::proto
uint8_t proto
Definition: flow.h:372
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:80
FramesContainer::toserver
Frames toserver
Definition: app-layer-frames.h:74
AppLayerFramesSetupContainer
FramesContainer * AppLayerFramesSetupContainer(Flow *f)
Definition: app-layer-parser.c:189
Frame::offset
uint64_t offset
Definition: app-layer-frames.h:51
Frame
Definition: app-layer-frames.h:45
Flow_
Flow data structure.
Definition: flow.h:350
AppLayerFrameSetTxIdById
void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id)
Definition: app-layer-frames.c:645
FrameConfig
Definition: app-layer-frames.c:33
Frames::cnt
uint16_t cnt
Definition: app-layer-frames.h:61
AppLayerFrameSetTxId
void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id)
Definition: app-layer-frames.c:636
Frame::id
int64_t id
Definition: app-layer-frames.h:53
FrameGetByIndex
Frame * FrameGetByIndex(Frames *frames, const uint32_t idx)
Definition: app-layer-frames.c:106
MIN
#define MIN(x, y)
Definition: suricata-common.h:391
Frames
Definition: app-layer-frames.h:60
FramesContainer
Definition: app-layer-frames.h:73
AppLayerFrameGetId
FrameId AppLayerFrameGetId(Frame *r)
Definition: app-layer-frames.c:613
Frames::left_edge_rel
uint32_t left_edge_rel
Definition: app-layer-frames.h:63
AppLayerFrameNewByAbsoluteOffset
Frame * AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice, const uint64_t frame_start, const int64_t len, int dir, uint8_t frame_type)
create new frame using the absolute offset from the start of the stream
Definition: app-layer-frames.c:555
ALPROTO_MAX
@ ALPROTO_MAX
Definition: app-layer-protos.h:75
Flow_::protoctx
void * protoctx
Definition: flow.h:440
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:430
Frames::dframes
Frame * dframes
Definition: app-layer-frames.h:66
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:292
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:474
FramesContainer::toclient
Frames toclient
Definition: app-layer-frames.h:75
type
uint8_t type
Definition: decode-icmpv4.h:0
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:251
AppLayerParserGetFrameNameById
const char * AppLayerParserGetFrameNameById(uint8_t ipproto, AppProto alproto, const uint8_t id)
Definition: app-layer-parser.c:1623
AppLayerFramesFreeContainer
void AppLayerFramesFreeContainer(Flow *f)
Definition: app-layer-parser.c:174
FrameGetById
Frame * FrameGetById(Frames *frames, const int64_t id)
Definition: app-layer-frames.c:86
FRAME_FLAG_TX_ID_SET
#define FRAME_FLAG_TX_ID_SET
Definition: app-layer-frames.h:38
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
util-print.h
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:285
FrameId
int64_t FrameId
Definition: app-layer-frames.h:34
AppLayerFramesSlide
void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction)
Definition: app-layer-frames.c:361
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
AppLayerFrameAddEventById
void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e)
Definition: app-layer-frames.c:607
FRAMES_STATIC_CNT
#define FRAMES_STATIC_CNT
Definition: app-layer-frames.h:58
Packet_
Definition: decode.h:436
AppLayerFramesGetContainer
FramesContainer * AppLayerFramesGetContainer(Flow *f)
Definition: app-layer-parser.c:182
Frame::len
int64_t len
Definition: app-layer-frames.h:52
Frame
struct Frame Frame
BIT_U64
#define BIT_U64(n)
Definition: suricata-common.h:401
Frame::flags
uint8_t flags
Definition: app-layer-frames.h:47
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
app-layer-frames.h
Frame::type
uint8_t type
Definition: app-layer-frames.h:46
suricata-common.h
Frame::event_cnt
uint8_t event_cnt
Definition: app-layer-frames.h:48
Frames::sframes
Frame sframes[FRAMES_STATIC_CNT]
Definition: app-layer-frames.h:65
AppLayerFrameNewByPointer
Frame * AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice, const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type)
create new frame using a pointer to start of the frame
Definition: app-layer-frames.c:416
FrameConfigEnableAll
void FrameConfigEnableAll(void)
Definition: app-layer-frames.c:45
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:295
FrameConfigInit
void FrameConfigInit(void)
Definition: app-layer-frames.c:38
AppLayerFrameGetById
Frame * AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
Definition: app-layer-frames.c:651
FRAME_STREAM_TYPE
#define FRAME_STREAM_TYPE
Definition: app-layer-frames.h:30
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:294
FramesPrune
void FramesPrune(Flow *f, Packet *p)
Definition: app-layer-frames.c:763
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Frames::base_id
uint64_t base_id
Definition: app-layer-frames.h:64
src
uint16_t src
Definition: app-layer-dnp3.h:5
FrameConfigEnable
void FrameConfigEnable(const AppProto p, const uint8_t type)
Definition: app-layer-frames.c:54
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:542
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
AppLayerFrameNewByRelativeOffset
Frame * AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice, const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
create new frame using a relative offset from the start of the stream slice
Definition: app-layer-frames.c:495
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TcpSession_
Definition: stream-tcp-private.h:283
flow.h
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:449
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
Frames::dyn_size
uint16_t dyn_size
Definition: app-layer-frames.h:62
AppLayerFrameSetLength
void AppLayerFrameSetLength(Frame *frame, int64_t len)
Definition: app-layer-frames.c:622
SC_ATOMIC_OR
#define SC_ATOMIC_OR(name, val)
Bitwise OR a value to our atomic variable.
Definition: util-atomic.h:350