suricata
app-layer-frames.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2024 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "util-print.h"
27 
28 #include "flow.h"
29 #include "stream-tcp.h"
30 #include "app-layer-frames.h"
31 #include "app-layer-parser.h"
32 
33 struct FrameConfig {
34  SC_ATOMIC_DECLARE(uint64_t, types);
35 };
36 /* This array should be allocated to contain g_alproto_max protocols. */
37 static struct FrameConfig *frame_config;
38 
39 void FrameConfigInit(void)
40 {
41  frame_config = SCCalloc(g_alproto_max, sizeof(struct FrameConfig));
42  if (unlikely(frame_config == NULL)) {
43  FatalError("Unable to alloc frame_config.");
44  }
45  for (AppProto p = 0; p < g_alproto_max; p++) {
46  SC_ATOMIC_INIT(frame_config[p].types);
47  }
48 }
49 
51 {
52  SCFree(frame_config);
53 }
54 
56 {
57  const uint64_t bits = UINT64_MAX;
58  for (AppProto p = 0; p < g_alproto_max; p++) {
59  struct FrameConfig *fc = &frame_config[p];
60  SC_ATOMIC_OR(fc->types, bits);
61  }
62 }
63 
64 void FrameConfigEnable(const AppProto p, const uint8_t type)
65 {
66  const uint64_t bits = BIT_U64(type);
67  struct FrameConfig *fc = &frame_config[p];
68  SC_ATOMIC_OR(fc->types, bits);
69 }
70 
71 static inline bool FrameConfigTypeIsEnabled(const AppProto p, const uint8_t type)
72 {
73  struct FrameConfig *fc = &frame_config[p];
74  const uint64_t bits = BIT_U64(type);
75  const bool enabled = (SC_ATOMIC_GET(fc->types) & bits) != 0;
76  return enabled;
77 }
78 
79 #ifdef DEBUG
80 static void FrameDebug(const char *prefix, const Frames *frames, const Frame *frame)
81 {
82  const char *type_name = "unknown";
83  if (frame->type == FRAME_STREAM_TYPE) {
84  type_name = "stream";
85  } else if (frames != NULL) {
86  type_name = AppLayerParserGetFrameNameById(frames->ipproto, frames->alproto, frame->type);
87  }
88  SCLogDebug("[%s] %p: frame:%p type:%u/%s id:%" PRIi64 " flags:%02x offset:%" PRIu64
89  ", len:%" PRIi64 ", inspect_progress:%" PRIu64 ", events:%u %u/%u/%u/%u",
90  prefix, frames, frame, frame->type, type_name, frame->id, frame->flags, frame->offset,
91  frame->len, frame->inspect_progress, frame->event_cnt, frame->events[0],
92  frame->events[1], frame->events[2], frame->events[3]);
93 }
94 #else
95 #define FrameDebug(prefix, frames, frame)
96 #endif
97 
98 /**
99  * \note "open" means a frame that has no length set (len == -1)
100  * \todo perhaps we can search backwards */
101 Frame *FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type)
102 {
103  Frame *candidate = NULL;
104 
105  SCLogDebug(
106  "frames %p cnt %u, looking for last of type %" PRIu8, frames, frames->cnt, frame_type);
107  for (uint16_t i = 0; i < frames->cnt; i++) {
108  if (i < FRAMES_STATIC_CNT) {
109  Frame *frame = &frames->sframes[i];
110  FrameDebug("get_by_id(static)", frames, frame);
111  if (frame->type == frame_type && frame->len == -1)
112  candidate = frame;
113  } else {
114  const uint16_t o = i - FRAMES_STATIC_CNT;
115  Frame *frame = &frames->dframes[o];
116  FrameDebug("get_by_id(dynamic)", frames, frame);
117  if (frame->type == frame_type && frame->len == -1)
118  candidate = frame;
119  }
120  }
121  return candidate;
122 }
123 
124 Frame *FrameGetById(Frames *frames, const int64_t id)
125 {
126  SCLogDebug("frames %p cnt %u, looking for %" PRIi64, frames, frames->cnt, id);
127  for (uint16_t i = 0; i < frames->cnt; i++) {
128  if (i < FRAMES_STATIC_CNT) {
129  Frame *frame = &frames->sframes[i];
130  FrameDebug("get_by_id(static)", frames, frame);
131  if (frame->id == id)
132  return frame;
133  } else {
134  const uint16_t o = i - FRAMES_STATIC_CNT;
135  Frame *frame = &frames->dframes[o];
136  FrameDebug("get_by_id(dynamic)", frames, frame);
137  if (frame->id == id)
138  return frame;
139  }
140  }
141  return NULL;
142 }
143 
144 Frame *FrameGetByIndex(Frames *frames, const uint32_t idx)
145 {
146  if (idx >= frames->cnt)
147  return NULL;
148 
149  if (idx < FRAMES_STATIC_CNT) {
150  Frame *frame = &frames->sframes[idx];
151  FrameDebug("get_by_idx(s)", frames, frame);
152  return frame;
153  } else {
154  const uint32_t o = idx - FRAMES_STATIC_CNT;
155  Frame *frame = &frames->dframes[o];
156  FrameDebug("get_by_idx(d)", frames, frame);
157  return frame;
158  }
159 }
160 
161 static Frame *FrameNew(Frames *frames, uint64_t offset, int64_t len)
162 {
163  DEBUG_VALIDATE_BUG_ON(frames == NULL);
164 
165  if (frames->cnt < FRAMES_STATIC_CNT) {
166  Frame *frame = &frames->sframes[frames->cnt];
167  frames->sframes[frames->cnt].offset = offset;
168  frames->sframes[frames->cnt].len = len;
169  frames->sframes[frames->cnt].id = ++frames->base_id;
170  frames->cnt++;
171  return frame;
172  } else if (frames->dframes == NULL) {
173  DEBUG_VALIDATE_BUG_ON(frames->dyn_size != 0);
175 
176  frames->dframes = SCCalloc(8, sizeof(Frame));
177  if (frames->dframes == NULL) {
178  return NULL;
179  }
180  frames->cnt++;
182 
183  frames->dyn_size = 8;
184  frames->dframes[0].offset = offset;
185  frames->dframes[0].len = len;
186  frames->dframes[0].id = ++frames->base_id;
187  return &frames->dframes[0];
188  } else {
190 
191  /* need to handle dynamic storage of frames now */
192  const uint16_t dyn_cnt = frames->cnt - FRAMES_STATIC_CNT;
193  if (dyn_cnt < frames->dyn_size) {
194  DEBUG_VALIDATE_BUG_ON(frames->dframes == NULL);
195 
196  // fall through
197  } else {
198  if (frames->dyn_size == 256) {
199  SCLogDebug("limit reached! 256 dynamic frames already");
200  // limit reached
201  // TODO figure out if this should lead to an event of sorts
202  return NULL;
203  }
204 
205  /* realloc time */
206  uint16_t new_dyn_size = frames->dyn_size * 2;
207  uint32_t new_alloc_size = new_dyn_size * sizeof(Frame);
208 
209  void *ptr = SCRealloc(frames->dframes, new_alloc_size);
210  if (ptr == NULL) {
211  return NULL;
212  }
213 
214  memset((uint8_t *)ptr + (frames->dyn_size * sizeof(Frame)), 0x00,
215  (frames->dyn_size * sizeof(Frame)));
216  frames->dframes = ptr;
217  frames->dyn_size = new_dyn_size;
218  }
219 
220  frames->cnt++;
221  frames->dframes[dyn_cnt].offset = offset;
222  frames->dframes[dyn_cnt].len = len;
223  frames->dframes[dyn_cnt].id = ++frames->base_id;
224  return &frames->dframes[dyn_cnt];
225  }
226 }
227 
228 static void FrameClean(Frame *frame)
229 {
230  memset(frame, 0, sizeof(*frame));
231 }
232 
233 static void FrameCopy(Frame *dst, Frame *src)
234 {
235  memcpy(dst, src, sizeof(*dst));
236 }
237 
238 #ifdef DEBUG
239 static void AppLayerFrameDumpForFrames(const char *prefix, const Frames *frames)
240 {
241  SCLogDebug("prefix: %s", prefix);
242  for (uint16_t i = 0; i < frames->cnt; i++) {
243  if (i < FRAMES_STATIC_CNT) {
244  const Frame *frame = &frames->sframes[i];
245  FrameDebug(prefix, frames, frame);
246  } else {
247  const uint16_t o = i - FRAMES_STATIC_CNT;
248  const Frame *frame = &frames->dframes[o];
249  FrameDebug(prefix, frames, frame);
250  }
251  }
252  SCLogDebug("prefix: %s", prefix);
253 }
254 #endif
255 
256 static inline uint64_t FrameLeftEdge(const TcpStream *stream, const Frame *frame)
257 {
258  const int64_t app_progress = STREAM_APP_PROGRESS(stream);
259 
260  const int64_t frame_offset = frame->offset;
261  const int64_t frame_data = app_progress - frame_offset;
262 
263  SCLogDebug("frame_offset %" PRIi64 ", frame_data %" PRIi64 ", frame->len %" PRIi64,
264  frame_offset, frame_data, frame->len);
265  DEBUG_VALIDATE_BUG_ON(frame_offset > app_progress);
266 
267  /* length unknown, make sure to have at least 2500 */
268  if (frame->len < 0) {
269  if (frame_data <= 2500) {
270  SCLogDebug("got <= 2500 bytes (%" PRIu64 "), returning offset %" PRIu64, frame_data,
271  frame_offset);
272  return frame_offset;
273  } else {
274  SCLogDebug("got > 2500 bytes (%" PRIu64 "), returning offset %" PRIu64, frame_data,
275  (frame_offset + (frame_data - 2500)));
276  return frame_offset + (frame_data - 2500);
277  }
278 
279  /* length specified */
280  } else {
281  /* have all data for the frame, we can skip it */
282  if (frame->len <= frame_data) {
283  uint64_t x = frame_offset + frame_data;
284  SCLogDebug("x %" PRIu64, x);
285  return x;
286  /*
287 
288  [ stream <frame_data> ]
289  [ frame .......]
290 
291  */
292  } else if (frame_data < 2500) {
293  uint64_t x = frame_offset;
294  SCLogDebug("x %" PRIu64, x);
295  return x;
296  } else {
297  uint64_t x = frame_offset + (frame_data - 2500);
298  SCLogDebug("x %" PRIu64, x);
299  return x;
300  }
301  }
302 }
303 
304 /** Stream buffer slides forward, we need to update and age out
305  * frame offsets/frames. Aging out means we move existing frames
306  * into the slots we'd free up.
307  *
308  * Start:
309  *
310  * [ stream ]
311  * [ frame ...........]
312  * offset: 2
313  * len: 19
314  *
315  * Slide:
316  * [ stream ]
317  * [ frame .... .]
318  * offset: 2
319  * len: 19
320  *
321  * Slide:
322  * [ stream ]
323  * [ frame ........... ]
324  * offset: 2
325  * len: 19
326  */
327 static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, const uint32_t slide)
328 {
329  SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
330  ", next %" PRIu64,
331  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
332  STREAM_BASE_OFFSET(stream), STREAM_BASE_OFFSET(stream) + slide);
333  DEBUG_VALIDATE_BUG_ON(frames == NULL);
334  SCLogDebug("%s frames %p: sliding %u bytes", ds, frames, slide);
335  uint64_t le = STREAM_APP_PROGRESS(stream);
336  const uint64_t next_base = STREAM_BASE_OFFSET(stream) + slide;
337 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
338  const uint16_t start = frames->cnt;
339  uint16_t removed = 0;
340 #endif
341  uint16_t x = 0;
342  for (uint16_t i = 0; i < frames->cnt; i++) {
343  if (i < FRAMES_STATIC_CNT) {
344  Frame *frame = &frames->sframes[i];
345  FrameDebug("slide(s)", frames, frame);
346  if (frame->len >= 0 && frame->offset + frame->len <= next_base) {
347  // remove by not incrementing 'x'
348  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
349  FrameClean(frame);
350 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
351  removed++;
352 #endif
353  } else {
354  Frame *nframe = &frames->sframes[x];
355  FrameCopy(nframe, frame);
356  if (frame != nframe) {
357  FrameClean(frame);
358  }
359  le = MIN(le, FrameLeftEdge(stream, nframe));
360  x++;
361  }
362  } else {
363  const uint16_t o = i - FRAMES_STATIC_CNT;
364  Frame *frame = &frames->dframes[o];
365  FrameDebug("slide(d)", frames, frame);
366  if (frame->len >= 0 && frame->offset + frame->len <= next_base) {
367  // remove by not incrementing 'x'
368  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
369  FrameClean(frame);
370 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
371  removed++;
372 #endif
373  } else {
374  Frame *nframe;
375  if (x >= FRAMES_STATIC_CNT) {
376  nframe = &frames->dframes[x - FRAMES_STATIC_CNT];
377  } else {
378  nframe = &frames->sframes[x];
379  }
380  FrameCopy(nframe, frame);
381  if (frame != nframe) {
382  FrameClean(frame);
383  }
384  le = MIN(le, FrameLeftEdge(stream, nframe));
385  x++;
386  }
387  }
388  }
389  frames->cnt = x;
390  uint64_t o = STREAM_BASE_OFFSET(stream) + slide;
391  DEBUG_VALIDATE_BUG_ON(o > le);
392  DEBUG_VALIDATE_BUG_ON(le - o > UINT32_MAX);
393  frames->left_edge_rel = (uint32_t)(le - o);
394 
395 #ifdef DEBUG
396  SCLogDebug("end: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
397  " (+slide), cnt %u, removed %u, start %u",
398  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream) + slide,
399  frames->left_edge_rel, STREAM_BASE_OFFSET(stream) + slide, frames->cnt, removed, start);
400  char pf[32] = "";
401  snprintf(pf, sizeof(pf), "%s:post_slide", ds);
402  AppLayerFrameDumpForFrames(pf, frames);
403 #endif
404  DEBUG_VALIDATE_BUG_ON(x != start - removed);
405  return 0;
406 }
407 
408 void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction)
409 {
410  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
411  if (frames_container == NULL)
412  return;
413  Frames *frames;
414  TcpSession *ssn = f->protoctx;
415  TcpStream *stream;
416  if (direction == STREAM_TOSERVER) {
417  stream = &ssn->client;
418  frames = &frames_container->toserver;
419  FrameSlide("toserver", frames, stream, slide);
420  } else {
421  stream = &ssn->server;
422  frames = &frames_container->toclient;
423  FrameSlide("toclient", frames, stream, slide);
424  }
425 }
426 
427 static void FrameFreeSingleFrame(Frames *frames, Frame *r)
428 {
429  FrameDebug("free", frames, r);
430  FrameClean(r);
431 }
432 
433 static void FramesClear(Frames *frames)
434 {
435  DEBUG_VALIDATE_BUG_ON(frames == NULL);
436 
437  SCLogDebug("frames %u", frames->cnt);
438  for (uint16_t i = 0; i < frames->cnt; i++) {
439  if (i < FRAMES_STATIC_CNT) {
440  Frame *r = &frames->sframes[i];
441  SCLogDebug("removing frame %p", r);
442  FrameFreeSingleFrame(frames, r);
443  } else {
444  const uint16_t o = i - FRAMES_STATIC_CNT;
445  Frame *r = &frames->dframes[o];
446  SCLogDebug("removing frame %p", r);
447  FrameFreeSingleFrame(frames, r);
448  }
449  }
450  frames->cnt = 0;
451 }
452 
453 void FramesFree(Frames *frames)
454 {
455  DEBUG_VALIDATE_BUG_ON(frames == NULL);
456  FramesClear(frames);
457  SCFree(frames->dframes);
458  frames->dframes = NULL;
459 }
460 
461 /** \brief create new frame using a pointer to start of the frame
462  */
464  const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type)
465 {
466  SCLogDebug("frame_start:%p stream_slice->input:%p stream_slice->offset:%" PRIu64, frame_start,
467  stream_slice->input, stream_slice->offset);
468 
469  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
470  return NULL;
471 
472  /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
473 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
474  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
475  return NULL;
476  if (frame_start < stream_slice->input ||
477  frame_start > stream_slice->input + stream_slice->input_len)
478  return NULL;
479 #endif
480  DEBUG_VALIDATE_BUG_ON(frame_start < stream_slice->input);
481  DEBUG_VALIDATE_BUG_ON(stream_slice->input == NULL);
482  DEBUG_VALIDATE_BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
483 
484  ptrdiff_t ptr_offset = frame_start - stream_slice->input;
485 #ifdef DEBUG
486  uint64_t offset = ptr_offset + stream_slice->offset;
487  SCLogDebug("flow %p direction %s frame %p starting at %" PRIu64 " len %" PRIi64
488  " (offset %" PRIu64 ")",
489  f, dir == 0 ? "toserver" : "toclient", frame_start, offset, len, stream_slice->offset);
490 #endif
491  DEBUG_VALIDATE_BUG_ON(f->alparser == NULL);
492 
493  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
494  if (frames_container == NULL)
495  return NULL;
496 
497  Frames *frames;
498  if (dir == 0) {
499  frames = &frames_container->toserver;
500  } else {
501  frames = &frames_container->toclient;
502  }
503 
504  uint64_t abs_frame_offset = stream_slice->offset + ptr_offset;
505 
506  Frame *r = FrameNew(frames, abs_frame_offset, len);
507  if (r != NULL) {
508  r->type = frame_type;
509  FrameDebug("new_by_ptr", frames, r);
510  }
511  return r;
512 }
513 
514 static Frame *AppLayerFrameUdp(
515  Flow *f, const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
516 {
517  DEBUG_VALIDATE_BUG_ON(f->proto != IPPROTO_UDP);
518 
519  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
520  return NULL;
521 
522  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
523  if (frames_container == NULL)
524  return NULL;
525 
526  Frames *frames;
527  if (dir == 0) {
528  frames = &frames_container->toserver;
529  } else {
530  frames = &frames_container->toclient;
531  }
532 
533  Frame *r = FrameNew(frames, frame_start_rel, len);
534  if (r != NULL) {
535  r->type = frame_type;
536  }
537  return r;
538 }
539 
540 /** \brief create new frame using a relative offset from the start of the stream slice
541  */
543  const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
544 {
545  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
546  return NULL;
547 
548  /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
549 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
550  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
551  return NULL;
552  if (stream_slice->input == NULL)
553  return NULL;
554 #else
555  DEBUG_VALIDATE_BUG_ON(stream_slice->input == NULL);
556 #endif
557  DEBUG_VALIDATE_BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
558  DEBUG_VALIDATE_BUG_ON(f->alparser == NULL);
559 
560  if (f->proto == IPPROTO_UDP) {
561  return AppLayerFrameUdp(f, frame_start_rel, len, dir, frame_type);
562  }
563 
564  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
565  if (frames_container == NULL)
566  return NULL;
567 
568  Frames *frames;
569  if (dir == 0) {
570  frames = &frames_container->toserver;
571  } else {
572  frames = &frames_container->toclient;
573  }
574 
575  const uint64_t frame_abs_offset = (uint64_t)frame_start_rel + stream_slice->offset;
576 #ifdef DEBUG_VALIDATION
577  const TcpSession *ssn = f->protoctx;
578  const TcpStream *stream = dir == 0 ? &ssn->client : &ssn->server;
579  BUG_ON(stream_slice->offset != STREAM_APP_PROGRESS(stream));
580  BUG_ON(frame_abs_offset > STREAM_APP_PROGRESS(stream) + stream_slice->input_len);
581 #endif
582  Frame *r = FrameNew(frames, frame_abs_offset, len);
583  if (r != NULL) {
584  r->type = frame_type;
585  }
586  return r;
587 }
588 
590 {
591 #ifdef DEBUG
592  if (f->proto == IPPROTO_TCP && f->protoctx && f->alparser) {
593  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
594  if (frames_container != NULL) {
595  AppLayerFrameDumpForFrames("toserver::dump", &frames_container->toserver);
596  AppLayerFrameDumpForFrames("toclient::dump", &frames_container->toclient);
597  }
598  }
599 #endif
600 }
601 
602 /** \brief create new frame using the absolute offset from the start of the stream
603  */
605  const uint64_t frame_start, const int64_t len, int dir, uint8_t frame_type)
606 {
607  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
608  return NULL;
609 
610  /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
611 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
612  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
613  return NULL;
614  if (stream_slice->input == NULL)
615  return NULL;
616 #else
617  DEBUG_VALIDATE_BUG_ON(stream_slice->input == NULL);
618 #endif
619  DEBUG_VALIDATE_BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
620  DEBUG_VALIDATE_BUG_ON(f->alparser == NULL);
621  DEBUG_VALIDATE_BUG_ON(frame_start < stream_slice->offset);
622  DEBUG_VALIDATE_BUG_ON(frame_start - stream_slice->offset >= (uint64_t)INT_MAX);
623 
624  FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
625  if (frames_container == NULL)
626  return NULL;
627 
628  Frames *frames;
629  if (dir == 0) {
630  frames = &frames_container->toserver;
631  } else {
632  frames = &frames_container->toclient;
633  }
634 
635  SCLogDebug("flow %p direction %s frame type %u offset %" PRIu64 " len %" PRIi64
636  " (slice offset %" PRIu64 ")",
637  f, dir == 0 ? "toserver" : "toclient", frame_type, frame_start, len,
638  stream_slice->offset);
639  Frame *r = FrameNew(frames, frame_start, len);
640  if (r != NULL) {
641  r->type = frame_type;
642  }
643  return r;
644 }
645 
646 void AppLayerFrameAddEvent(Frame *r, uint8_t e)
647 {
648  if (r != NULL) {
649  if (r->event_cnt < 4) { // TODO
650  r->events[r->event_cnt++] = e;
651  }
652  FrameDebug("add_event", NULL, r);
653  }
654 }
655 
656 void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e)
657 {
658  Frame *frame = AppLayerFrameGetById(f, dir, id);
659  AppLayerFrameAddEvent(frame, e);
660 }
661 
663 {
664  if (r != NULL) {
665  return r->id;
666  } else {
667  return -1;
668  }
669 }
670 
671 void AppLayerFrameSetLength(Frame *frame, int64_t len)
672 {
673  if (frame != NULL) {
674  frame->len = len;
675  FrameDebug("set_length", NULL, frame);
676  }
677 }
678 
679 void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_t len)
680 {
681  Frame *frame = AppLayerFrameGetById(f, dir, id);
682  AppLayerFrameSetLength(frame, len);
683 }
684 
685 void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id)
686 {
687  if (r != NULL) {
689  r->tx_id = tx_id;
690  FrameDebug("set_txid", NULL, r);
691  }
692 }
693 
694 void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id)
695 {
696  Frame *frame = AppLayerFrameGetById(f, dir, id);
697  AppLayerFrameSetTxId(frame, tx_id);
698 }
699 
700 Frame *AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
701 {
702  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
703  SCLogDebug("get frame_id %" PRIi64 " direction %u/%s frames_container %p", frame_id, dir,
704  dir == 0 ? "toserver" : "toclient", frames_container);
705  if (frames_container == NULL)
706  return NULL;
707 
708  Frames *frames;
709  if (dir == 0) {
710  frames = &frames_container->toserver;
711  } else {
712  frames = &frames_container->toclient;
713  }
714  SCLogDebug("frames %p", frames);
715  return FrameGetById(frames, frame_id);
716 }
717 
718 Frame *AppLayerFrameGetLastOpenByType(Flow *f, const int dir, const uint8_t frame_type)
719 {
720  if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type)))
721  return NULL;
722 
723  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
724  SCLogDebug("get frame_type %" PRIu8 " direction %u/%s frames_container %p", frame_type, dir,
725  dir == 0 ? "toserver" : "toclient", frames_container);
726  if (frames_container == NULL)
727  return NULL;
728 
729  Frames *frames;
730  if (dir == 0) {
731  frames = &frames_container->toserver;
732  } else {
733  frames = &frames_container->toclient;
734  }
735  SCLogDebug("frames %p", frames);
736  return FrameGetLastOpenByType(frames, frame_type);
737 }
738 
739 static inline bool FrameIsDone(const Frame *frame, const uint64_t abs_right_edge)
740 {
741  /* frame with negative length means we don't know the size yet. */
742  if (frame->len < 0)
743  return false;
744 
745  const int64_t frame_abs_offset = frame->offset;
746  const int64_t frame_right_edge = frame_abs_offset + frame->len;
747  if ((uint64_t)frame_right_edge <= abs_right_edge) {
748  SCLogDebug("frame %p id %" PRIi64 " is done", frame, frame->id);
749  return true;
750  }
751  return false;
752 }
753 
754 static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof)
755 {
756 #ifdef DEBUG_VALIDATION
757  const uint64_t frames_le_start = (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream);
758 #endif
759  SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64,
760  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
761  STREAM_BASE_OFFSET(stream));
762  const uint64_t acked = StreamTcpGetUsable(stream, eof);
763  uint64_t le = STREAM_APP_PROGRESS(stream);
764 
765 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
766  const uint16_t start = frames->cnt;
767  uint16_t removed = 0;
768 #endif
769  uint16_t x = 0;
770  for (uint16_t i = 0; i < frames->cnt; i++) {
771  if (i < FRAMES_STATIC_CNT) {
772  Frame *frame = &frames->sframes[i];
773  FrameDebug("prune(s)", frames, frame);
774  if (eof || FrameIsDone(frame, acked)) {
775  // remove by not incrementing 'x'
776  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
777  FrameDebug("remove(s)", frames, frame);
778  FrameClean(frame);
779 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
780  removed++;
781 #endif
782  } else {
783  const uint64_t fle = FrameLeftEdge(stream, frame);
784  le = MIN(le, fle);
785  SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
786  Frame *nframe = &frames->sframes[x];
787  FrameCopy(nframe, frame);
788  if (frame != nframe) {
789  FrameClean(frame);
790  }
791  x++;
792  }
793  } else {
794  const uint16_t o = i - FRAMES_STATIC_CNT;
795  Frame *frame = &frames->dframes[o];
796  FrameDebug("prune(d)", frames, frame);
797  if (eof || FrameIsDone(frame, acked)) {
798  // remove by not incrementing 'x'
799  SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
800  FrameDebug("remove(d)", frames, frame);
801  FrameClean(frame);
802 #if defined(DEBUG) || defined(DEBUG_VALIDATION)
803  removed++;
804 #endif
805  } else {
806  const uint64_t fle = FrameLeftEdge(stream, frame);
807  le = MIN(le, fle);
808  SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
809  Frame *nframe;
810  if (x >= FRAMES_STATIC_CNT) {
811  nframe = &frames->dframes[x - FRAMES_STATIC_CNT];
812  } else {
813  nframe = &frames->sframes[x];
814  }
815  FrameCopy(nframe, frame);
816  if (frame != nframe) {
817  FrameClean(frame);
818  }
819  x++;
820  }
821  }
822  }
823  frames->cnt = x;
825  DEBUG_VALIDATE_BUG_ON(le - STREAM_BASE_OFFSET(stream) > UINT32_MAX);
826  frames->left_edge_rel = (uint32_t)(le - STREAM_BASE_OFFSET(stream));
827 #ifdef DEBUG
828  SCLogDebug("end: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
829  ", cnt %u, removed %u, start %u",
830  (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
831  STREAM_BASE_OFFSET(stream), frames->cnt, removed, start);
832  AppLayerFrameDumpForFrames("post_slide", frames);
833 #endif
834  if (frames->cnt > 0) { // if we removed all this can fail
835  DEBUG_VALIDATE_BUG_ON(frames_le_start > le);
836  }
837  DEBUG_VALIDATE_BUG_ON(x != start - removed);
838 }
839 
840 void FramesPrune(Flow *f, Packet *p)
841 {
842  if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
843  return;
844  FramesContainer *frames_container = AppLayerFramesGetContainer(f);
845  if (frames_container == NULL)
846  return;
847 
848  Frames *frames;
849 
850  if (p->proto == IPPROTO_UDP) {
851  SCLogDebug("clearing all UDP frames");
852  if (PKT_IS_TOSERVER(p)) {
853  frames = &frames_container->toserver;
854  } else {
855  frames = &frames_container->toclient;
856  }
857  FramesClear(frames);
858  return;
859  }
860 
861  TcpSession *ssn = f->protoctx;
862 
865  return;
866  }
867 
868  TcpStream *stream;
869  if (PKT_IS_TOSERVER(p)) {
870  stream = &ssn->client;
871  frames = &frames_container->toserver;
872  } else {
873  stream = &ssn->server;
874  frames = &frames_container->toclient;
875  }
876 
877  const bool eof = ssn->state == TCP_CLOSED || PKT_IS_PSEUDOPKT(p);
878  SCLogDebug("eof %s", eof ? "TRUE" : "false");
879  FramePrune(frames, stream, eof);
880 }
AppLayerFrameSetLengthById
void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_t len)
Definition: app-layer-frames.c:679
Packet_::proto
uint8_t proto
Definition: decode.h:523
TcpStream_
Definition: stream-tcp-private.h:106
Frame::inspect_progress
uint64_t inspect_progress
Definition: app-layer-frames.h:55
len
uint8_t len
Definition: app-layer-dnp3.h:2
Frame::tx_id
uint64_t tx_id
Definition: app-layer-frames.h:54
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
FrameConfig::SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, types)
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:314
FramesFree
void FramesFree(Frames *frames)
Definition: app-layer-frames.c:453
AppLayerFrameAddEvent
void AppLayerFrameAddEvent(Frame *r, uint8_t e)
Definition: app-layer-frames.c:646
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1321
stream-tcp.h
StreamSlice
struct StreamSlice StreamSlice
Definition: app-layer-parser.h:38
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:275
Frame::events
uint8_t events[4]
Definition: app-layer-frames.h:50
Flow_::proto
uint8_t proto
Definition: flow.h:378
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:86
FramesContainer::toserver
Frames toserver
Definition: app-layer-frames.h:74
AppLayerFramesSetupContainer
FramesContainer * AppLayerFramesSetupContainer(Flow *f)
Definition: app-layer-parser.c:191
Frame::offset
uint64_t offset
Definition: app-layer-frames.h:51
Frame
Definition: app-layer-frames.h:45
Flow_
Flow data structure.
Definition: flow.h:356
AppLayerFrameSetTxIdById
void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id)
Definition: app-layer-frames.c:694
FrameConfig
Definition: app-layer-frames.c:33
Frames::cnt
uint16_t cnt
Definition: app-layer-frames.h:61
AppLayerFrameSetTxId
void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id)
Definition: app-layer-frames.c:685
Frame::id
int64_t id
Definition: app-layer-frames.h:53
FrameGetByIndex
Frame * FrameGetByIndex(Frames *frames, const uint32_t idx)
Definition: app-layer-frames.c:144
MIN
#define MIN(x, y)
Definition: suricata-common.h:408
Frames
Definition: app-layer-frames.h:60
FramesContainer
Definition: app-layer-frames.h:73
AppLayerFrameGetId
FrameId AppLayerFrameGetId(Frame *r)
Definition: app-layer-frames.c:662
Frames::left_edge_rel
uint32_t left_edge_rel
Definition: app-layer-frames.h:63
AppLayerFrameNewByAbsoluteOffset
Frame * AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice, const uint64_t frame_start, const int64_t len, int dir, uint8_t frame_type)
create new frame using the absolute offset from the start of the stream
Definition: app-layer-frames.c:604
AppLayerFrameGetLastOpenByType
Frame * AppLayerFrameGetLastOpenByType(Flow *f, const int dir, const uint8_t frame_type)
Definition: app-layer-frames.c:718
Flow_::protoctx
void * protoctx
Definition: flow.h:441
StreamTcpGetUsable
uint64_t StreamTcpGetUsable(const TcpStream *stream, const bool eof)
Definition: stream-tcp-reassemble.c:426
Frames::dframes
Frame * dframes
Definition: app-layer-frames.h:66
TcpSession_::flags
uint32_t flags
Definition: stream-tcp-private.h:294
Flow_::alparser
AppLayerParserState * alparser
Definition: flow.h:478
FramesContainer::toclient
Frames toclient
Definition: app-layer-frames.h:75
PKT_IS_TOSERVER
#define PKT_IS_TOSERVER(p)
Definition: decode.h:238
AppLayerParserGetFrameNameById
const char * AppLayerParserGetFrameNameById(uint8_t ipproto, AppProto alproto, const uint8_t id)
Definition: app-layer-parser.c:1636
g_alproto_max
AppProto g_alproto_max
Definition: app-layer-protos.c:29
AppLayerFramesFreeContainer
void AppLayerFramesFreeContainer(Flow *f)
Definition: app-layer-parser.c:176
FrameGetById
Frame * FrameGetById(Frames *frames, const int64_t id)
Definition: app-layer-frames.c:124
FRAME_FLAG_TX_ID_SET
#define FRAME_FLAG_TX_ID_SET
Definition: app-layer-frames.h:38
STREAM_BASE_OFFSET
#define STREAM_BASE_OFFSET(stream)
Definition: stream-tcp-private.h:144
util-print.h
TcpSession_::state
uint8_t state
Definition: stream-tcp-private.h:285
FrameId
int64_t FrameId
Definition: app-layer-frames.h:34
AppLayerFramesSlide
void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction)
Definition: app-layer-frames.c:408
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:317
AppLayerFrameAddEventById
void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e)
Definition: app-layer-frames.c:656
FRAMES_STATIC_CNT
#define FRAMES_STATIC_CNT
Definition: app-layer-frames.h:58
Packet_
Definition: decode.h:501
type
uint16_t type
Definition: decode-vlan.c:106
AppLayerFramesGetContainer
FramesContainer * AppLayerFramesGetContainer(Flow *f)
Definition: app-layer-parser.c:184
Frame::len
int64_t len
Definition: app-layer-frames.h:52
Frame
struct Frame Frame
BIT_U64
#define BIT_U64(n)
Definition: suricata-common.h:418
Frame::flags
uint8_t flags
Definition: app-layer-frames.h:47
SCRealloc
#define SCRealloc(ptr, sz)
Definition: util-mem.h:50
TCP_CLOSED
@ TCP_CLOSED
Definition: stream-tcp-private.h:162
app-layer-frames.h
Frame::type
uint8_t type
Definition: app-layer-frames.h:46
suricata-common.h
Frame::event_cnt
uint8_t event_cnt
Definition: app-layer-frames.h:48
FatalError
#define FatalError(...)
Definition: util-debug.h:510
Frames::sframes
Frame sframes[FRAMES_STATIC_CNT]
Definition: app-layer-frames.h:65
AppLayerFrameNewByPointer
Frame * AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice, const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type)
create new frame using a pointer to start of the frame
Definition: app-layer-frames.c:463
FrameConfigEnableAll
void FrameConfigEnableAll(void)
Definition: app-layer-frames.c:55
TcpSession_::client
TcpStream client
Definition: stream-tcp-private.h:297
FrameConfigInit
void FrameConfigInit(void)
Definition: app-layer-frames.c:39
AppLayerFrameGetById
Frame * AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
Definition: app-layer-frames.c:700
FRAME_STREAM_TYPE
#define FRAME_STREAM_TYPE
Definition: app-layer-frames.h:32
TcpSession_::server
TcpStream server
Definition: stream-tcp-private.h:296
FramesPrune
void FramesPrune(Flow *f, Packet *p)
Definition: app-layer-frames.c:840
SCFree
#define SCFree(p)
Definition: util-mem.h:61
FrameGetLastOpenByType
Frame * FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type)
Definition: app-layer-frames.c:101
Frames::base_id
uint64_t base_id
Definition: app-layer-frames.h:64
src
uint16_t src
Definition: app-layer-dnp3.h:5
FrameConfigEnable
void FrameConfigEnable(const AppProto p, const uint8_t type)
Definition: app-layer-frames.c:64
AppLayerFrameDump
void AppLayerFrameDump(Flow *f)
Definition: app-layer-frames.c:589
STREAMTCP_FLAG_APP_LAYER_DISABLED
#define STREAMTCP_FLAG_APP_LAYER_DISABLED
Definition: stream-tcp-private.h:201
AppLayerFrameNewByRelativeOffset
Frame * AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice, const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
create new frame using a relative offset from the start of the stream slice
Definition: app-layer-frames.c:542
STREAM_APP_PROGRESS
#define STREAM_APP_PROGRESS(stream)
Definition: stream-tcp-private.h:145
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:375
TcpSession_
Definition: stream-tcp-private.h:283
FrameConfigDeInit
void FrameConfigDeInit(void)
Definition: app-layer-frames.c:50
flow.h
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:450
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
Frames::dyn_size
uint16_t dyn_size
Definition: app-layer-frames.h:62
AppLayerFrameSetLength
void AppLayerFrameSetLength(Frame *frame, int64_t len)
Definition: app-layer-frames.c:671
SC_ATOMIC_OR
#define SC_ATOMIC_OR(name, val)
Bitwise OR a value to our atomic variable.
Definition: util-atomic.h:350
FrameDebug
#define FrameDebug(prefix, frames, frame)
Definition: app-layer-frames.c:95