suricata
detect-file-data.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "threads.h"
27 #include "debug.h"
28 #include "decode.h"
29 
30 #include "detect.h"
31 #include "detect-parse.h"
32 
33 #include "detect-engine.h"
34 #include "detect-engine-mpm.h"
35 #include "detect-engine-state.h"
38 #include "detect-file-data.h"
39 
40 #include "app-layer-parser.h"
41 #include "app-layer-htp.h"
42 #include "app-layer-smtp.h"
43 
44 #include "flow.h"
45 #include "flow-var.h"
46 #include "flow-util.h"
47 
48 #include "util-debug.h"
49 #include "util-spm-bm.h"
50 #include "util-unittest.h"
51 #include "util-unittest-helper.h"
53 
54 static int DetectFiledataSetup (DetectEngineCtx *, Signature *, const char *);
55 #ifdef UNITTESTS
56 static void DetectFiledataRegisterTests(void);
57 #endif
58 static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
59  Signature *s);
60 static int g_file_data_buffer_id = 0;
61 
62 static inline HtpBody *GetResponseBody(htp_tx_t *tx);
63 
64 /* HTTP */
65 static int PrefilterMpmHTTPFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
66  MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id);
67 
68 /* file API */
69 static int DetectEngineInspectFiledata(
71  const DetectEngineAppInspectionEngine *engine,
72  const Signature *s,
73  Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
75  SigGroupHead *sgh, MpmCtx *mpm_ctx,
76  const DetectBufferMpmRegistery *mpm_reg, int list_id);
77 
78 static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
80  const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
81 
82 /**
83  * \brief Registration function for keyword: file_data
84  */
86 {
87  sigmatch_table[DETECT_FILE_DATA].name = "file.data";
88  sigmatch_table[DETECT_FILE_DATA].alias = "file_data";
89  sigmatch_table[DETECT_FILE_DATA].desc = "make content keywords match on file data";
90  sigmatch_table[DETECT_FILE_DATA].url = "/rules/http-keywords.html#file-data";
91  sigmatch_table[DETECT_FILE_DATA].Setup = DetectFiledataSetup;
92 #ifdef UNITTESTS
94 #endif
96 
99  ALPROTO_SMTP, 0);
100  DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2, PrefilterMpmHTTPFiledataRegister,
101  NULL, ALPROTO_HTTP1, HTP_RESPONSE_BODY);
104  ALPROTO_SMB, 0);
107  ALPROTO_SMB, 0);
110  ALPROTO_HTTP2, HTTP2StateDataClient);
113  ALPROTO_HTTP2, HTTP2StateDataServer);
115  "file_data", SIG_FLAG_TOSERVER, 2, PrefilterMpmFiledataRegister, NULL, ALPROTO_NFS, 0);
117  "file_data", SIG_FLAG_TOCLIENT, 2, PrefilterMpmFiledataRegister, NULL, ALPROTO_NFS, 0);
119  NULL, ALPROTO_FTPDATA, 0);
121  NULL, ALPROTO_FTPDATA, 0);
123  "file_data", SIG_FLAG_TOSERVER, 2, PrefilterMpmFiledataRegister, NULL, ALPROTO_FTP, 0);
125  "file_data", SIG_FLAG_TOCLIENT, 2, PrefilterMpmFiledataRegister, NULL, ALPROTO_FTP, 0);
126 
128  HTP_RESPONSE_BODY, DetectEngineInspectBufferHttpBody, NULL);
131  DetectEngineInspectFiledata, NULL);
133  DetectFiledataSetupCallback);
136  DetectEngineInspectFiledata, NULL);
139  DetectEngineInspectFiledata, NULL);
141  ALPROTO_HTTP2, SIG_FLAG_TOSERVER, HTTP2StateDataClient,
142  DetectEngineInspectFiledata, NULL);
144  ALPROTO_HTTP2, SIG_FLAG_TOCLIENT, HTTP2StateDataServer,
145  DetectEngineInspectFiledata, NULL);
147  "file_data", ALPROTO_NFS, SIG_FLAG_TOSERVER, 0, DetectEngineInspectFiledata, NULL);
149  "file_data", ALPROTO_NFS, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectFiledata, NULL);
151  "file_data", ALPROTO_FTPDATA, SIG_FLAG_TOSERVER, 0, DetectEngineInspectFiledata, NULL);
153  "file_data", ALPROTO_FTPDATA, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectFiledata, NULL);
155  "file_data", ALPROTO_FTP, SIG_FLAG_TOSERVER, 0, DetectEngineInspectFiledata, NULL);
157  "file_data", ALPROTO_FTP, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectFiledata, NULL);
158 
160  "http response body, smb files or smtp attachments data");
161 
162  g_file_data_buffer_id = DetectBufferTypeGetByName("file_data");
163 }
164 
165 #define FILEDATA_CONTENT_LIMIT 100000
166 #define FILEDATA_CONTENT_INSPECT_MIN_SIZE 32768
167 #define FILEDATA_CONTENT_INSPECT_WINDOW 4096
168 
169 static void SetupDetectEngineConfig(DetectEngineCtx *de_ctx) {
171  return;
172 
173  /* initialize default */
174  for (int i = 0; i < (int)ALPROTO_MAX; i++) {
178  }
179 
180  /* add protocol specific settings here */
181 
182  /* SMTP */
186 
188 }
189 
190 /**
191  * \brief this function is used to parse filedata options
192  * \brief into the current signature
193  *
194  * \param de_ctx pointer to the Detection Engine Context
195  * \param s pointer to the Current Signature
196  * \param str pointer to the user provided "filestore" option
197  *
198  * \retval 0 on Success
199  * \retval -1 on Failure
200  */
201 static int DetectFiledataSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str)
202 {
203  SCEnter();
204 
205  if (!DetectProtoContainsProto(&s->proto, IPPROTO_TCP) ||
206  (s->alproto != ALPROTO_UNKNOWN && s->alproto != ALPROTO_HTTP1 &&
207  s->alproto != ALPROTO_SMTP && s->alproto != ALPROTO_SMB &&
208  s->alproto != ALPROTO_HTTP2 && s->alproto != ALPROTO_FTP &&
209  s->alproto != ALPROTO_FTPDATA && s->alproto != ALPROTO_HTTP &&
210  s->alproto != ALPROTO_NFS)) {
211  SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
212  return -1;
213  }
214 
215  if ((s->alproto == ALPROTO_HTTP1 || s->alproto == ALPROTO_HTTP) &&
217  !(s->flags & SIG_FLAG_TOCLIENT)) {
218  SCLogError(SC_ERR_INVALID_SIGNATURE, "Can't use file_data with "
219  "flow:to_server or flow:from_client with http.");
220  return -1;
221  }
222 
224  !(s->flags & SIG_FLAG_TOSERVER) && (s->flags & SIG_FLAG_TOCLIENT)) {
225  SCLogError(SC_ERR_INVALID_SIGNATURE, "Can't use file_data with "
226  "flow:to_client or flow:from_server with smtp.");
227  return -1;
228  }
229 
230  if (DetectBufferSetActiveList(s, DetectBufferTypeGetByName("file_data")) < 0)
231  return -1;
232 
234  SetupDetectEngineConfig(de_ctx);
235  return 0;
236 }
237 
238 static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
239  Signature *s)
240 {
241  if (s->alproto == ALPROTO_HTTP1 || s->alproto == ALPROTO_UNKNOWN ||
242  s->alproto == ALPROTO_HTTP) {
244  }
245 
246  /* server body needs to be inspected in sync with stream if possible */
248 
249  SCLogDebug("callback invoked by %u", s->id);
250 }
251 
252 /* common */
253 
254 typedef struct PrefilterMpmFiledata {
255  int list_id;
257  const MpmCtx *mpm_ctx;
260 
261 static void PrefilterMpmFiledataFree(void *ptr)
262 {
263  SCFree(ptr);
264 }
265 
266 /* HTTP based detection */
267 
268 static inline HtpBody *GetResponseBody(htp_tx_t *tx)
269 {
270  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
271  if (htud == NULL) {
272  SCLogDebug("no htud");
273  return NULL;
274  }
275 
276  return &htud->response_body;
277 }
278 
279 static inline InspectionBuffer *HttpServerBodyXformsGetDataCallback(DetectEngineThreadCtx *det_ctx,
280  const DetectEngineTransforms *transforms, const int list_id, InspectionBuffer *base_buffer)
281 {
282  InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
283  if (buffer->inspect != NULL)
284  return buffer;
285 
286  InspectionBufferSetup(det_ctx, list_id, buffer, base_buffer->inspect, base_buffer->inspect_len);
287  buffer->inspect_offset = base_buffer->inspect_offset;
288  InspectionBufferApplyTransforms(buffer, transforms);
289  SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len);
290  SCReturnPtr(buffer, "InspectionBuffer");
291 }
292 
293 static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
294  const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
295  const int list_id, const int base_id)
296 {
297  SCEnter();
298 
299  InspectionBuffer *buffer = InspectionBufferGet(det_ctx, base_id);
300  if (base_id != list_id && buffer->inspect != NULL)
301  return HttpServerBodyXformsGetDataCallback(det_ctx, transforms, list_id, buffer);
302  else if (buffer->inspect != NULL)
303  return buffer;
304 
305  htp_tx_t *tx = txv;
306  HtpState *htp_state = f->alstate;
307  const uint8_t flags = flow_flags;
308 
309  HtpBody *body = GetResponseBody(tx);
310  if (body == NULL) {
311  return NULL;
312  }
313 
314  /* no new data */
315  if (body->body_inspected == body->content_len_so_far) {
316  SCLogDebug("no new data");
317  return NULL;
318  }
319 
320  HtpBodyChunk *cur = body->first;
321  if (cur == NULL) {
322  SCLogDebug("No http chunks to inspect for this transaction");
323  return NULL;
324  }
325 
326  SCLogDebug("response.body_limit %u response_body.content_len_so_far %" PRIu64
327  ", response.inspect_min_size %" PRIu32 ", EOF %s, progress > body? %s",
328  htp_state->cfg->response.body_limit, body->content_len_so_far,
329  htp_state->cfg->response.inspect_min_size, flags & STREAM_EOF ? "true" : "false",
331  HTP_RESPONSE_BODY)
332  ? "true"
333  : "false");
334 
335  if (!htp_state->cfg->http_body_inline) {
336  /* inspect the body if the transfer is complete or we have hit
337  * our body size limit */
338  if ((htp_state->cfg->response.body_limit == 0 ||
339  body->content_len_so_far < htp_state->cfg->response.body_limit) &&
340  body->content_len_so_far < htp_state->cfg->response.inspect_min_size &&
341  !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, flags) >
342  HTP_RESPONSE_BODY) &&
343  !(flags & STREAM_EOF)) {
344  SCLogDebug("we still haven't seen the entire response body. "
345  "Let's defer body inspection till we see the "
346  "entire body.");
347  return NULL;
348  }
349  }
350 
351  /* get the inspect buffer
352  *
353  * make sure that we have at least the configured inspect_win size.
354  * If we have more, take at least 1/4 of the inspect win size before
355  * the new data.
356  */
357  uint64_t offset = 0;
358  if (body->body_inspected > htp_state->cfg->response.inspect_min_size) {
360  uint64_t inspect_win = body->content_len_so_far - body->body_inspected;
361  SCLogDebug("inspect_win %"PRIu64, inspect_win);
362  if (inspect_win < htp_state->cfg->response.inspect_window) {
363  uint64_t inspect_short = htp_state->cfg->response.inspect_window - inspect_win;
364  if (body->body_inspected < inspect_short)
365  offset = 0;
366  else
367  offset = body->body_inspected - inspect_short;
368  } else {
369  offset = body->body_inspected - (htp_state->cfg->response.inspect_window / 4);
370  }
371  }
372 
373  const uint8_t *data;
374  uint32_t data_len;
375 
377  &data, &data_len, offset);
378  InspectionBufferSetup(det_ctx, base_id, buffer, data, data_len);
379  buffer->inspect_offset = offset;
380  body->body_inspected = body->content_len_so_far;
381  SCLogDebug("body->body_inspected now: %" PRIu64, body->body_inspected);
382 
383  /* built-in 'transformation' */
384  if (htp_state->cfg->swf_decompression_enabled) {
385  int swf_file_type = FileIsSwfFile(data, data_len);
386  if (swf_file_type == FILE_SWF_ZLIB_COMPRESSION ||
387  swf_file_type == FILE_SWF_LZMA_COMPRESSION)
388  {
389  (void)FileSwfDecompression(data, data_len,
390  det_ctx,
391  buffer,
392  htp_state->cfg->swf_compression_type,
393  htp_state->cfg->swf_decompress_depth,
394  htp_state->cfg->swf_compress_depth);
395  }
396  }
397 
398  if (base_id != list_id) {
399  buffer = HttpServerBodyXformsGetDataCallback(det_ctx, transforms, list_id, buffer);
400  }
401  SCReturnPtr(buffer, "InspectionBuffer");
402 }
403 
404 static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
406  const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
407 {
408  bool eof =
409  (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
410  const InspectionBuffer *buffer = HttpServerBodyGetDataCallback(
411  det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base);
412  if (buffer == NULL || buffer->inspect == NULL) {
414  }
415 
416  const uint32_t data_len = buffer->inspect_len;
417  const uint8_t *data = buffer->inspect;
418  const uint64_t offset = buffer->inspect_offset;
419 
420  uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0;
421  ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0);
422  ci_flags |= buffer->flags;
423 
424  det_ctx->discontinue_matching = 0;
425  det_ctx->buffer_offset = 0;
426  det_ctx->inspection_recursion_counter = 0;
427 
428  /* Inspect all the uricontents fetched on each
429  * transaction at the app layer */
430  int r = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, (uint8_t *)data,
432  if (r == 1) {
434  }
435 
436  if (flags & STREAM_TOSERVER) {
437  if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
438  HTP_REQUEST_BODY)
440  } else {
441  if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
442  HTP_RESPONSE_BODY)
444  }
446 }
447 
448 /** \brief Filedata Filedata Mpm prefilter callback
449  *
450  * \param det_ctx detection engine thread ctx
451  * \param pectx inspection context
452  * \param p packet to inspect
453  * \param f flow to inspect
454  * \param txv tx to inspect
455  * \param idx transaction id
456  * \param flags STREAM_* flags including direction
457  */
458 static void PrefilterTxHTTPFiledata(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p,
459  Flow *f, void *txv, const uint64_t idx, const uint8_t flags)
460 {
461  SCEnter();
462 
463  const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
464  const MpmCtx *mpm_ctx = ctx->mpm_ctx;
465  const int list_id = ctx->list_id;
466 
467  InspectionBuffer *buffer = HttpServerBodyGetDataCallback(
468  det_ctx, ctx->transforms, f, flags, txv, list_id, ctx->base_list_id);
469  if (buffer == NULL)
470  return;
471 
472  if (buffer->inspect_len >= mpm_ctx->minlen) {
473  (void)mpm_table[mpm_ctx->mpm_type].Search(
474  mpm_ctx, &det_ctx->mtcu, &det_ctx->pmq, buffer->inspect, buffer->inspect_len);
475  }
476 }
477 
478 static int PrefilterMpmHTTPFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
479  MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id)
480 {
481  PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
482  if (pectx == NULL)
483  return -1;
484  pectx->list_id = list_id;
485  pectx->base_list_id = mpm_reg->sm_list_base;
486  SCLogDebug("list_id %d base_list_id %d", list_id, pectx->base_list_id);
487  pectx->mpm_ctx = mpm_ctx;
488  pectx->transforms = &mpm_reg->transforms;
489 
490  return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHTTPFiledata, mpm_reg->app_v2.alproto,
491  mpm_reg->app_v2.tx_min_progress, pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
492 }
493 
494 /* file API based inspection */
495 
496 static inline InspectionBuffer *FiledataWithXformsGetDataCallback(DetectEngineThreadCtx *det_ctx,
497  const DetectEngineTransforms *transforms, const int list_id, int local_file_id,
498  InspectionBuffer *base_buffer, const bool first)
499 {
500  InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, local_file_id);
501  if (buffer == NULL) {
502  SCLogDebug("list_id: %d: no buffer", list_id);
503  return NULL;
504  }
505  if (!first && buffer->inspect != NULL) {
506  SCLogDebug("list_id: %d: returning %p", list_id, buffer);
507  return buffer;
508  }
509 
510  InspectionBufferSetupMulti(buffer, transforms, base_buffer->inspect, base_buffer->inspect_len);
511  buffer->inspect_offset = base_buffer->inspect_offset;
512  SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len);
513  SCReturnPtr(buffer, "InspectionBuffer");
514 }
515 
516 static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
517  const DetectEngineTransforms *transforms, Flow *f, uint8_t flow_flags, File *cur_file,
518  const int list_id, const int base_id, int local_file_id, bool first)
519 {
520  SCEnter();
521  SCLogDebug(
522  "starting: list_id %d base_id %d first %s", list_id, base_id, first ? "true" : "false");
523 
524  InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, base_id, local_file_id);
525  SCLogDebug("base: buffer %p", buffer);
526  if (buffer == NULL)
527  return NULL;
528  if (base_id != list_id && buffer->inspect != NULL) {
529  SCLogDebug("handle xform %s", (list_id != base_id) ? "true" : "false");
530  return FiledataWithXformsGetDataCallback(
531  det_ctx, transforms, list_id, local_file_id, buffer, first);
532  }
533  if (!first && buffer->inspect != NULL) {
534  SCLogDebug("base_id: %d, not first: use %p", base_id, buffer);
535  return buffer;
536  }
537 
538  const uint64_t file_size = FileDataSize(cur_file);
539  const DetectEngineCtx *de_ctx = det_ctx->de_ctx;
540  const uint32_t content_limit = de_ctx->filedata_config[f->alproto].content_limit;
541  const uint32_t content_inspect_min_size = de_ctx->filedata_config[f->alproto].content_inspect_min_size;
542  // TODO this is unused, is that right?
543  //const uint32_t content_inspect_window = de_ctx->filedata_config[f->alproto].content_inspect_window;
544 
545  SCLogDebug("[list %d] first: %d, content_limit %u, content_inspect_min_size %u", list_id,
546  first ? 1 : 0, content_limit, content_inspect_min_size);
547 
548  SCLogDebug("[list %d] file %p size %" PRIu64 ", state %d", list_id, cur_file, file_size,
549  cur_file->state);
550 
551  /* no new data */
552  if (cur_file->content_inspected == file_size) {
553  SCLogDebug("no new data");
554  return NULL;
555  }
556 
557  if (file_size == 0) {
558  SCLogDebug("no data to inspect for this transaction");
559  return NULL;
560  }
561 
562  if ((content_limit == 0 || file_size < content_limit) &&
563  file_size < content_inspect_min_size &&
564  !(flow_flags & STREAM_EOF) && !(cur_file->state > FILE_STATE_OPENED)) {
565  SCLogDebug("we still haven't seen the entire content. "
566  "Let's defer content inspection till we see the "
567  "entire content.");
568  return NULL;
569  }
570 
571  const uint8_t *data;
572  uint32_t data_len;
573 
575  &data, &data_len,
576  cur_file->content_inspected);
577  InspectionBufferSetupMulti(buffer, NULL, data, data_len);
578  SCLogDebug("[list %d] [before] buffer offset %" PRIu64 "; buffer len %" PRIu32
579  "; data_len %" PRIu32 "; file_size %" PRIu64,
580  list_id, buffer->inspect_offset, buffer->inspect_len, data_len, file_size);
581  buffer->inspect_offset = cur_file->content_inspected;
582 
583  /* update inspected tracker */
584  cur_file->content_inspected = buffer->inspect_len + buffer->inspect_offset;
585  SCLogDebug("content inspected: %" PRIu64, cur_file->content_inspected);
586 
587  /* get buffer for the list id if it is different from the base id */
588  if (list_id != base_id) {
589  SCLogDebug("regular %d has been set up: now handle xforms id %d", base_id, list_id);
590  InspectionBuffer *tbuffer = FiledataWithXformsGetDataCallback(
591  det_ctx, transforms, list_id, local_file_id, buffer, first);
592  SCReturnPtr(tbuffer, "InspectionBuffer");
593  } else {
594  SCLogDebug("regular buffer %p size %u", buffer, buffer->inspect_len);
595  SCReturnPtr(buffer, "InspectionBuffer");
596  }
597 }
598 
599 static int DetectEngineInspectFiledata(
601  const DetectEngineAppInspectionEngine *engine,
602  const Signature *s,
603  Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
604 {
605  int r = 0;
606  int match = 0;
607 
608  const DetectEngineTransforms *transforms = NULL;
609  if (!engine->mpm) {
610  transforms = engine->v2.transforms;
611  }
612 
614  if (ffc == NULL) {
616  }
617 
618  int local_file_id = 0;
619  File *file = ffc->head;
620  for (; file != NULL; file = file->next) {
621  if (file->txid != tx_id)
622  continue;
623 
624  InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, transforms, f, flags, file,
625  engine->sm_list, engine->sm_list_base, local_file_id, false);
626  if (buffer == NULL)
627  continue;
628 
629  bool eof = (file->state == FILE_STATE_CLOSED);
630  uint8_t ciflags = eof ? DETECT_CI_FLAGS_END : 0;
631  if (buffer->inspect_offset == 0)
632  ciflags |= DETECT_CI_FLAGS_START;
633 
634  det_ctx->buffer_offset = 0;
635  det_ctx->discontinue_matching = 0;
636  det_ctx->inspection_recursion_counter = 0;
637  match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd,
638  NULL, f,
639  (uint8_t *)buffer->inspect,
640  buffer->inspect_len,
641  buffer->inspect_offset, ciflags,
643  if (match == 1) {
644  r = 1;
645  break;
646  }
647  local_file_id++;
648  }
649 
650  if (r == 1)
652  else
654 }
655 
656 /** \brief Filedata Filedata Mpm prefilter callback
657  *
658  * \param det_ctx detection engine thread ctx
659  * \param pectx inspection context
660  * \param p packet to inspect
661  * \param f flow to inspect
662  * \param txv tx to inspect
663  * \param idx transaction id
664  * \param flags STREAM_* flags including direction
665  */
666 static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx,
667  const void *pectx,
668  Packet *p, Flow *f, void *txv,
669  const uint64_t idx, const uint8_t flags)
670 {
671  SCEnter();
672 
673  const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
674  const MpmCtx *mpm_ctx = ctx->mpm_ctx;
675  const int list_id = ctx->list_id;
676 
678  int local_file_id = 0;
679  if (ffc != NULL) {
680  File *file = ffc->head;
681  for (; file != NULL; file = file->next) {
682  if (file->txid != idx)
683  continue;
684 
685  InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, ctx->transforms, f, flags,
686  file, list_id, ctx->base_list_id, local_file_id, true);
687  if (buffer == NULL)
688  continue;
689 
690  if (buffer->inspect_len >= mpm_ctx->minlen) {
691  (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
692  &det_ctx->mtcu, &det_ctx->pmq,
693  buffer->inspect, buffer->inspect_len);
694  }
695  local_file_id++;
696  }
697  }
698 }
699 
701  SigGroupHead *sgh, MpmCtx *mpm_ctx,
702  const DetectBufferMpmRegistery *mpm_reg, int list_id)
703 {
704  PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
705  if (pectx == NULL)
706  return -1;
707  pectx->list_id = list_id;
708  pectx->base_list_id = mpm_reg->sm_list_base;
709  pectx->mpm_ctx = mpm_ctx;
710  pectx->transforms = &mpm_reg->transforms;
711 
712  return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFiledata,
713  mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
714  pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
715 }
716 
717 #ifdef UNITTESTS
718 #include "tests/detect-file-data.c"
719 #endif
HtpState_::cfg
const struct HTPCfgRec_ * cfg
Definition: app-layer-htp.h:255
SMTPConfig::content_limit
uint32_t content_limit
Definition: app-layer-smtp.h:95
FILE_SWF_LZMA_COMPRESSION
@ FILE_SWF_LZMA_COMPRESSION
Definition: util-file-decompression.h:34
DetectEngineAppInspectionEngine_
Definition: detect.h:397
SigTableElmt_::url
const char * url
Definition: detect.h:1204
DetectEngineAppInspectionEngine_::mpm
bool mpm
Definition: detect.h:401
FileContainer_
Definition: util-file.h:107
MpmCtx_::mpm_type
uint8_t mpm_type
Definition: util-mpm.h:90
DetectEngineThreadCtx_::buffer_offset
uint32_t buffer_offset
Definition: detect.h:1031
detect-engine.h
FILE_SWF_ZLIB_COMPRESSION
@ FILE_SWF_ZLIB_COMPRESSION
Definition: util-file-decompression.h:33
DetectEngineCtx_::filedata_config
struct DetectEngineCtx_::@89 filedata_config[ALPROTO_MAX]
SigTableElmt_::desc
const char * desc
Definition: detect.h:1203
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
DETECT_CI_FLAGS_START
#define DETECT_CI_FLAGS_START
Definition: detect-engine-content-inspection.h:38
flow-util.h
SCFPSupportSMList_::list_id
int list_id
Definition: detect-fast-pattern.h:29
SigTableElmt_::name
const char * name
Definition: detect.h:1201
SigGroupHead_
Container for matching data for a signature group.
Definition: detect.h:1346
HtpBody_::sb
StreamingBuffer * sb
Definition: app-layer-htp.h:194
SIG_FLAG_INIT_FLOW
#define SIG_FLAG_INIT_FLOW
Definition: detect.h:258
DetectEngineTransforms
Definition: detect.h:378
HTPCfgRec_::response
HTPCfgDir response
Definition: app-layer-htp.h:178
SIG_FLAG_INIT_NEED_FLUSH
#define SIG_FLAG_INIT_NEED_FLUSH
Definition: detect.h:263
FILEDATA_CONTENT_LIMIT
#define FILEDATA_CONTENT_LIMIT
Definition: detect-file-data.c:165
Signature_::alproto
AppProto alproto
Definition: detect.h:521
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
SIG_FLAG_INIT_FILEDATA
#define SIG_FLAG_INIT_FILEDATA
Definition: detect.h:265
HTPCfgDir_::body_limit
uint32_t body_limit
Definition: app-layer-htp.h:154
Flow_::proto
uint8_t proto
Definition: flow.h:375
FILEDATA_CONTENT_INSPECT_WINDOW
#define FILEDATA_CONTENT_INSPECT_WINDOW
Definition: detect-file-data.c:167
DetectBufferMpmRegistery_::transforms
DetectEngineTransforms transforms
Definition: detect.h:615
AppLayerParserGetStateProgress
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t flags)
get the progress value for a tx/protocol
Definition: app-layer-parser.c:1041
FileSwfDecompression
int FileSwfDecompression(const uint8_t *buffer, uint32_t buffer_len, DetectEngineThreadCtx *det_ctx, InspectionBuffer *out_buffer, int swf_type, uint32_t decompress_depth, uint32_t compress_depth)
This function decompresses a buffer with zlib/lzma algorithm.
Definition: util-file-decompression.c:71
InspectionBuffer
Definition: detect.h:344
threads.h
FILE_STATE_OPENED
@ FILE_STATE_OPENED
Definition: util-file.h:63
Flow_
Flow data structure.
Definition: flow.h:353
File_::state
FileState state
Definition: util-file.h:75
DetectBufferTypeRegisterSetupCallback
void DetectBufferTypeRegisterSetupCallback(const char *name, void(*SetupCallback)(const DetectEngineCtx *, Signature *))
Definition: detect-engine.c:902
DetectEngineThreadCtx_::pmq
PrefilterRuleStore pmq
Definition: detect.h:1101
SigTableElmt_::flags
uint16_t flags
Definition: detect.h:1195
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:758
SC_ERR_INVALID_SIGNATURE
@ SC_ERR_INVALID_SIGNATURE
Definition: util-error.h:69
AppLayerParserGetFiles
FileContainer * AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
Definition: app-layer-parser.c:833
detect-file-data.h
DetectEngineAppInspectionEngine_::sm_list_base
uint16_t sm_list_base
Definition: detect.h:404
ALPROTO_FTP
@ ALPROTO_FTP
Definition: app-layer-protos.h:31
DetectBufferMpmRegistery_
one time registration of keywords at start up
Definition: detect.h:601
InspectionBuffer::flags
uint8_t flags
Definition: detect.h:348
SignatureInitData_::init_flags
uint32_t init_flags
Definition: detect.h:482
DetectEngineContentInspection
int DetectEngineContentInspection(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const uint8_t *buffer, uint32_t buffer_len, uint32_t stream_start_offset, uint8_t flags, uint8_t inspection_mode)
Run the actual payload match functions.
Definition: detect-engine-content-inspection.c:102
HTPCfgRec_::swf_compress_depth
uint32_t swf_compress_depth
Definition: app-layer-htp.h:175
StreamingBufferGetDataAtOffset
int StreamingBufferGetDataAtOffset(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t offset)
Definition: util-streaming-buffer.c:884
SIG_FLAG_TOCLIENT
#define SIG_FLAG_TOCLIENT
Definition: detect.h:237
ALPROTO_MAX
@ ALPROTO_MAX
Definition: app-layer-protos.h:72
SigTableElmt_::Setup
int(* Setup)(DetectEngineCtx *, Signature *, const char *)
Definition: detect.h:1186
PrefilterMpmFiledata
struct PrefilterMpmFiledata PrefilterMpmFiledata
DetectEngineCtx_::content_inspect_window
uint32_t content_inspect_window
Definition: detect.h:876
detect-engine-prefilter.h
HTPCfgDir_::inspect_window
uint32_t inspect_window
Definition: app-layer-htp.h:156
DetectEngineThreadCtx_::mtcu
MpmThreadCtx mtcu
Definition: detect.h:1099
util-unittest.h
InspectionBufferGet
InspectionBuffer * InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id)
Definition: detect-engine.c:1010
HtpBody_::first
HtpBodyChunk * first
Definition: app-layer-htp.h:191
HtpState_
Definition: app-layer-htp.h:245
SMTPConfig::content_inspect_min_size
uint32_t content_inspect_min_size
Definition: app-layer-smtp.h:96
util-unittest-helper.h
DetectBufferTypeGetByName
int DetectBufferTypeGetByName(const char *name)
Definition: detect-engine.c:829
File_::sb
StreamingBuffer * sb
Definition: util-file.h:76
HtpBody_::content_len_so_far
uint64_t content_len_so_far
Definition: app-layer-htp.h:197
DetectEngineAppInspectionEngine_::sm_list
uint16_t sm_list
Definition: detect.h:403
PrefilterMpmFiledata::transforms
const DetectEngineTransforms * transforms
Definition: detect-file-data.c:258
SIG_FLAG_TOSERVER
#define SIG_FLAG_TOSERVER
Definition: detect.h:236
app-layer-htp.h
decode.h
util-debug.h
HTPCfgRec_::http_body_inline
int http_body_inline
Definition: app-layer-htp.h:170
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
DetectEngineThreadCtx_
Definition: detect.h:1003
HTPCfgDir_::inspect_min_size
uint32_t inspect_min_size
Definition: app-layer-htp.h:155
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
STREAM_TOSERVER
#define STREAM_TOSERVER
Definition: stream.h:31
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
detect-engine-mpm.h
FileContainer_::head
File * head
Definition: util-file.h:108
detect.h
PrefilterMpmFiledataRegister
int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id)
Definition: detect-file-data.c:700
HTPCfgRec_::swf_decompress_depth
uint32_t swf_decompress_depth
Definition: app-layer-htp.h:174
DETECT_ENGINE_INSPECT_SIG_MATCH
#define DETECT_ENGINE_INSPECT_SIG_MATCH
Definition: detect-engine-state.h:39
PrefilterMpmFiledata::mpm_ctx
const MpmCtx * mpm_ctx
Definition: detect-file-data.c:257
DetectFiledataRegister
void DetectFiledataRegister(void)
Registration function for keyword: file_data.
Definition: detect-file-data.c:85
HTPCfgRec_::swf_decompression_enabled
int swf_decompression_enabled
Definition: app-layer-htp.h:172
InspectionBuffer::inspect_offset
uint64_t inspect_offset
Definition: detect.h:346
app-layer-parser.h
MpmCtx_::minlen
uint16_t minlen
Definition: util-mpm.h:99
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
smtp_config
SMTPConfig smtp_config
Definition: app-layer-smtp.c:240
PrefilterMpmFiledata::base_list_id
int base_list_id
Definition: detect-file-data.c:256
Signature_::flags
uint32_t flags
Definition: detect.h:518
Packet_
Definition: decode.h:414
DETECT_CI_FLAGS_END
#define DETECT_CI_FLAGS_END
Definition: detect-engine-content-inspection.h:39
DetectAppLayerInspectEngineRegister2
void DetectAppLayerInspectEngineRegister2(const char *name, AppProto alproto, uint32_t dir, int progress, InspectEngineFuncPtr2 Callback2, InspectionBufferGetDataPtr GetData)
register inspect engine at start up time
Definition: detect-engine.c:174
Signature_::init_data
SignatureInitData * init_data
Definition: detect.h:587
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
FileIsSwfFile
int FileIsSwfFile(const uint8_t *buffer, uint32_t buffer_len)
Definition: util-file-decompression.c:41
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
AppLayerHtpEnableResponseBodyCallback
void AppLayerHtpEnableResponseBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
Definition: app-layer-htp.c:438
HTPCfgRec_::swf_compression_type
HtpSwfCompressType swf_compression_type
Definition: app-layer-htp.h:173
ALPROTO_HTTP2
@ ALPROTO_HTTP2
Definition: app-layer-protos.h:59
MpmTableElmt_::Search
uint32_t(* Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t)
Definition: util-mpm.h:165
HtpTxUserData_::response_body
HtpBody response_body
Definition: app-layer-htp.h:222
DETECT_FILE_DATA
@ DETECT_FILE_DATA
Definition: detect-engine-register.h:172
DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH
Definition: detect-engine-state.h:40
DetectBufferMpmRegistery_::app_v2
struct DetectBufferMpmRegistery_::@85::@87 app_v2
FileDataSize
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:294
detect-engine-content-inspection.h
DetectEngineThreadCtx_::discontinue_matching
uint16_t discontinue_matching
Definition: detect.h:1066
DetectEngineAppInspectionEngine_::smd
SigMatchData * smd
Definition: detect.h:414
File_::content_inspected
uint64_t content_inspected
Definition: util-file.h:93
FILE_STATE_CLOSED
@ FILE_STATE_CLOSED
Definition: util-file.h:64
File_
Definition: util-file.h:72
DetectAppLayerMpmRegister2
void DetectAppLayerMpmRegister2(const char *name, int direction, int priority, int(*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id), InspectionBufferGetDataPtr GetData, AppProto alproto, int tx_min_progress)
register a MPM engine
Definition: detect-engine-mpm.c:89
HtpBody_
Definition: app-layer-htp.h:190
DetectBufferMpmRegistery_::pname
char pname[32]
Definition: detect.h:603
flags
uint8_t flags
Definition: decode-gre.h:0
Signature_::proto
DetectProto proto
Definition: detect.h:535
SigTableElmt_::alias
const char * alias
Definition: detect.h:1202
suricata-common.h
FILEDATA_CONTENT_INSPECT_MIN_SIZE
#define FILEDATA_CONTENT_INSPECT_MIN_SIZE
Definition: detect-file-data.c:166
InspectionBufferApplyTransforms
void InspectionBufferApplyTransforms(InspectionBuffer *buffer, const DetectEngineTransforms *transforms)
Definition: detect-engine.c:1193
HtpBodyChunk_
Definition: app-layer-htp.h:182
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:30
sigmatch_table
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect-parse.c:73
DetectEngineThreadCtx_::inspection_recursion_counter
int inspection_recursion_counter
Definition: detect.h:1078
File_::next
struct File_ * next
Definition: util-file.h:86
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
ALPROTO_FTPDATA
@ ALPROTO_FTPDATA
Definition: app-layer-protos.h:47
util-spm-bm.h
InspectionBufferSetupMulti
void InspectionBufferSetupMulti(InspectionBuffer *buffer, const DetectEngineTransforms *transforms, const uint8_t *data, const uint32_t data_len)
setup the buffer with our initial data
Definition: detect-engine.c:1077
HtpTxUserData_
Definition: app-layer-htp.h:213
DETECT_ENGINE_INSPECT_SIG_NO_MATCH
#define DETECT_ENGINE_INSPECT_SIG_NO_MATCH
Definition: detect-engine-state.h:38
DetectEngineAppInspectionEngine_::progress
int16_t progress
Definition: detect.h:405
InspectionBuffer::inspect_len
uint32_t inspect_len
Definition: detect.h:347
DetectEngineAppInspectionEngine_::v2
struct DetectEngineAppInspectionEngine_::@83 v2
DetectFiledataRegisterTests
void DetectFiledataRegisterTests(void)
Definition: detect-file-data.c:338
InspectionBuffer::inspect
const uint8_t * inspect
Definition: detect.h:345
str
#define str(s)
Definition: suricata-common.h:272
InspectionBufferSetup
void InspectionBufferSetup(DetectEngineThreadCtx *det_ctx, const int list_id, InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
setup the buffer with our initial data
Definition: detect-engine.c:1091
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alstate
void * alstate
Definition: flow.h:486
Signature_::id
uint32_t id
Definition: detect.h:551
DetectEngineCtx_::content_limit
uint32_t content_limit
Definition: detect.h:874
detect-parse.h
Signature_
Signature container.
Definition: detect.h:517
PrefilterAppendTxEngine
int PrefilterAppendTxEngine(DetectEngineCtx *de_ctx, SigGroupHead *sgh, void(*PrefilterTxFunc)(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f, void *tx, const uint64_t idx, const uint8_t flags), AppProto alproto, int tx_min_progress, void *pectx, void(*FreeFunc)(void *pectx), const char *name)
Definition: detect-engine-prefilter.c:259
DetectEngineAppInspectionEngine_::transforms
const DetectEngineTransforms * transforms
Definition: detect.h:411
util-file-decompression.h
ALPROTO_HTTP
@ ALPROTO_HTTP
Definition: app-layer-protos.h:63
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
STREAM_EOF
#define STREAM_EOF
Definition: stream.h:30
mpm_table
MpmTableElmt mpm_table[MPM_TABLE_SIZE]
Definition: util-mpm.c:48
DetectEngineCtx_::filedata_config_initialized
bool filedata_config_initialized
Definition: detect.h:878
DetectEngineCtx_::content_inspect_min_size
uint32_t content_inspect_min_size
Definition: detect.h:875
DetectEngineThreadCtx_::de_ctx
DetectEngineCtx * de_ctx
Definition: detect.h:1126
InspectionBufferMultipleForListGet
InspectionBuffer * InspectionBufferMultipleForListGet(DetectEngineThreadCtx *det_ctx, const int list_id, const uint32_t local_id)
for a InspectionBufferMultipleForList get a InspectionBuffer
Definition: detect-engine.c:1030
File_::txid
uint64_t txid
Definition: util-file.h:77
ALPROTO_SMB
@ ALPROTO_SMB
Definition: app-layer-protos.h:37
DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE
@ DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE
Definition: detect-engine-content-inspection.h:35
SIGMATCH_NOOPT
#define SIGMATCH_NOOPT
Definition: detect.h:1373
DetectBufferSetActiveList
int DetectBufferSetActiveList(Signature *s, const int list)
Definition: detect-engine.c:941
app-layer-smtp.h
DetectBufferTypeSetDescriptionByName
void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc)
Definition: detect-engine.c:857
PrefilterMpmFiledata
Definition: detect-file-data.c:254
MpmCtx_
Definition: util-mpm.h:88
flow.h
detect-file-data.c
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:460
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
PrefilterMpmFiledata::list_id
int list_id
Definition: detect-file-data.c:255
flow-var.h
SMTPConfig::content_inspect_window
uint32_t content_inspect_window
Definition: app-layer-smtp.h:97
ALPROTO_NFS
@ ALPROTO_NFS
Definition: app-layer-protos.h:45
debug.h
SigTableElmt_::RegisterTests
void(* RegisterTests)(void)
Definition: detect.h:1193
HtpBody_::body_inspected
uint64_t body_inspected
Definition: app-layer-htp.h:201
DetectBufferMpmRegistery_::sm_list_base
int16_t sm_list_base
Definition: detect.h:606
DetectProtoContainsProto
int DetectProtoContainsProto(const DetectProto *dp, int proto)
see if a DetectProto contains a certain proto
Definition: detect-engine-proto.c:135
SC_ERR_CONFLICTING_RULE_KEYWORDS
@ SC_ERR_CONFLICTING_RULE_KEYWORDS
Definition: util-error.h:171