suricata
detect-filestore.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements the filestore keyword
24  */
25 
26 #include "suricata-common.h"
27 #include "threads.h"
28 #include "debug.h"
29 #include "decode.h"
30 
31 #include "detect.h"
32 #include "detect-parse.h"
33 
34 #include "detect-engine.h"
35 #include "detect-engine-mpm.h"
36 #include "detect-engine-state.h"
37 
38 #include "flow.h"
39 #include "flow-var.h"
40 #include "flow-util.h"
41 
42 #include "util-debug.h"
43 #include "util-spm-bm.h"
44 #include "util-unittest.h"
45 #include "util-unittest-helper.h"
46 
47 #include "app-layer.h"
48 #include "app-layer-parser.h"
49 #include "app-layer-htp.h"
50 
51 #include "stream-tcp.h"
52 
53 #include "detect-filestore.h"
54 
55 /**
56  * \brief Regex for parsing our flow options
57  */
58 #define PARSE_REGEX "^\\s*([A-z_]+)\\s*(?:,\\s*([A-z_]+))?\\s*(?:,\\s*([A-z_]+))?\\s*$"
59 
60 static pcre *parse_regex;
61 static pcre_extra *parse_regex_study;
62 
63 static int DetectFilestoreMatch (ThreadVars *, DetectEngineThreadCtx *,
64  Flow *, uint8_t, File *, const Signature *, const SigMatchCtx *);
65 static int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx,
66  Packet *p, const Signature *s, const SigMatchCtx *ctx);
67 static int DetectFilestoreSetup (DetectEngineCtx *, Signature *, const char *);
68 static void DetectFilestoreFree(void *);
69 static void DetectFilestoreRegisterTests(void);
70 static int g_file_match_list_id = 0;
71 
72 /**
73  * \brief Registration function for keyword: filestore
74  */
76 {
77  sigmatch_table[DETECT_FILESTORE].name = "filestore";
78  sigmatch_table[DETECT_FILESTORE].desc = "stores files to disk if the rule matched";
79  sigmatch_table[DETECT_FILESTORE].url = DOC_URL DOC_VERSION "/rules/file-keywords.html#filestore";
80  sigmatch_table[DETECT_FILESTORE].FileMatch = DetectFilestoreMatch;
81  sigmatch_table[DETECT_FILESTORE].Setup = DetectFilestoreSetup;
82  sigmatch_table[DETECT_FILESTORE].Free = DetectFilestoreFree;
83  sigmatch_table[DETECT_FILESTORE].RegisterTests = DetectFilestoreRegisterTests;
85 
86  sigmatch_table[DETECT_FILESTORE_POSTMATCH].name = "__filestore__postmatch__";
87  sigmatch_table[DETECT_FILESTORE_POSTMATCH].Match = DetectFilestorePostMatch;
88  sigmatch_table[DETECT_FILESTORE_POSTMATCH].Free = DetectFilestoreFree;
89 
90  DetectSetupParseRegexes(PARSE_REGEX, &parse_regex, &parse_regex_study);
91 
92  g_file_match_list_id = DetectBufferTypeRegister("files");
93 }
94 
95 /**
96  * \brief apply the post match filestore with options
97  */
98 static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilestoreData *filestore,
99  FileContainer *fc, uint32_t file_id, uint64_t tx_id)
100 {
101  if (filestore == NULL) {
102  SCReturnInt(0);
103  }
104 
105  int this_file = 0;
106  int this_tx = 0;
107  int this_flow = 0;
108  int rule_dir = 0;
109  int toserver_dir = 0;
110  int toclient_dir = 0;
111 
112  switch (filestore->direction) {
114  rule_dir = 1;
115  break;
116  case FILESTORE_DIR_BOTH:
117  toserver_dir = 1;
118  toclient_dir = 1;
119  break;
121  toserver_dir = 1;
122  break;
124  toclient_dir = 1;
125  break;
126  }
127 
128  switch (filestore->scope) {
130  if (rule_dir) {
131  this_file = 1;
132  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && toclient_dir) {
133  this_file = 1;
134  } else if ((p->flowflags & FLOW_PKT_TOSERVER) && toserver_dir) {
135  this_file = 1;
136  }
137  break;
138  case FILESTORE_SCOPE_TX:
139  this_tx = 1;
140  break;
141  case FILESTORE_SCOPE_SSN:
142  this_flow = 1;
143  break;
144  }
145 
146  if (this_file) {
147  FileStoreFileById(fc, file_id);
148  } else if (this_tx) {
149  /* flag tx all files will be stored */
150  if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
151  HtpState *htp_state = f->alstate;
152  if (toserver_dir) {
153  htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS;
154  FileStoreAllFilesForTx(htp_state->files_ts, tx_id);
155  }
156  if (toclient_dir) {
157  htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC;
158  FileStoreAllFilesForTx(htp_state->files_tc, tx_id);
159  }
160  htp_state->store_tx_id = tx_id;
161  }
162  } else if (this_flow) {
163  /* flag flow all files will be stored */
164  if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
165  HtpState *htp_state = f->alstate;
166  if (toserver_dir) {
167  htp_state->flags |= HTP_FLAG_STORE_FILES_TS;
168  FileStoreAllFiles(htp_state->files_ts);
169  }
170  if (toclient_dir) {
171  htp_state->flags |= HTP_FLAG_STORE_FILES_TC;
172  FileStoreAllFiles(htp_state->files_tc);
173  }
174  }
175  } else {
176  FileStoreFileById(fc, file_id);
177  }
178 
179  SCReturnInt(0);
180 }
181 
182 /**
183  * \brief post-match function for filestore
184  *
185  * \param t thread local vars
186  * \param det_ctx pattern matcher thread local data
187  * \param p packet
188  *
189  * The match function for filestore records store candidates in the det_ctx.
190  * When we are sure all parts of the signature matched, we run this function
191  * to finalize the filestore.
192  */
193 static int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx,
194  Packet *p, const Signature *s, const SigMatchCtx *ctx)
195 {
196  uint8_t flags = 0;
197 
198  SCEnter();
199 
200  if (det_ctx->filestore_cnt == 0) {
201  SCReturnInt(0);
202  }
203 
204  if ((s->filestore_ctx == NULL && !(s->flags & SIG_FLAG_FILESTORE)) || p->flow == NULL) {
205 #ifndef DEBUG
206  SCReturnInt(0);
207 #else
208  BUG_ON(1);
209 #endif
210  }
211 
212  /* set filestore depth for stream reassembling */
213  TcpSession *ssn = (TcpSession *)p->flow->protoctx;
215 
216  if (p->flowflags & FLOW_PKT_TOCLIENT)
217  flags |= STREAM_TOCLIENT;
218  else
219  flags |= STREAM_TOSERVER;
220 
222  p->flow->alstate, flags);
223 
224  /* filestore for single files only */
225  if (s->filestore_ctx == NULL) {
226  for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
227  FileStoreFileById(ffc, det_ctx->filestore[u].file_id);
228  }
229  } else {
230  for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
231  FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc,
232  det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
233  }
234  }
235 
236  SCReturnInt(0);
237 }
238 
239 /**
240  * \brief match the specified filestore
241  *
242  * \param t thread local vars
243  * \param det_ctx pattern matcher thread local data
244  * \param f *LOCKED* flow
245  * \param flags direction flags
246  * \param file file being inspected
247  * \param s signature being inspected
248  * \param m sigmatch that we will cast into DetectFilestoreData
249  *
250  * \retval 0 no match
251  * \retval 1 match
252  *
253  * \todo when we start supporting more protocols, the logic in this function
254  * needs to be put behind a api.
255  */
256 static int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f,
257  uint8_t flags, File *file, const Signature *s, const SigMatchCtx *m)
258 {
259  uint32_t file_id = 0;
260 
261  SCEnter();
262 
263  if (det_ctx->filestore_cnt >= DETECT_FILESTORE_MAX) {
264  SCReturnInt(1);
265  }
266 
267  /* file can be NULL when a rule with filestore scope > file
268  * matches. */
269  if (file != NULL) {
270  file_id = file->file_store_id;
271  if (file->sid != NULL && s->id > 0) {
272  if (file->sid_cnt >= file->sid_max) {
273  void *p = SCRealloc(file->sid, sizeof(uint32_t) * (file->sid_max + 8));
274  if (p == NULL) {
275  SCFree(file->sid);
276  file->sid = NULL;
277  file->sid_cnt = 0;
278  file->sid_max = 0;
279  goto continue_after_realloc_fail;
280  } else {
281  file->sid = p;
282  file->sid_max += 8;
283  }
284  }
285  file->sid[file->sid_cnt] = s->id;
286  file->sid_cnt++;
287  }
288  }
289 
290 continue_after_realloc_fail:
291 
292  det_ctx->filestore[det_ctx->filestore_cnt].file_id = file_id;
293  det_ctx->filestore[det_ctx->filestore_cnt].tx_id = det_ctx->tx_id;
294 
295  SCLogDebug("%u, file %u, tx %"PRIu64, det_ctx->filestore_cnt,
296  det_ctx->filestore[det_ctx->filestore_cnt].file_id,
297  det_ctx->filestore[det_ctx->filestore_cnt].tx_id);
298 
299  det_ctx->filestore_cnt++;
300  SCReturnInt(1);
301 }
302 
303 /**
304  * \brief this function is used to parse filestore options
305  * \brief into the current signature
306  *
307  * \param de_ctx pointer to the Detection Engine Context
308  * \param s pointer to the Current Signature
309  * \param str pointer to the user provided "filestore" option
310  *
311  * \retval 0 on Success
312  * \retval -1 on Failure
313  */
314 static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str)
315 {
316  SCEnter();
317 
318  DetectFilestoreData *fd = NULL;
319  SigMatch *sm = NULL;
320  char *args[3] = {NULL,NULL,NULL};
321 #define MAX_SUBSTRINGS 30
322  int ret = 0, res = 0;
323  int ov[MAX_SUBSTRINGS];
324 
325  /* filestore and bypass keywords can't work together */
326  if (s->flags & SIG_FLAG_BYPASS) {
328  "filestore can't work with bypass keyword");
329  return -1;
330  }
331 
332  sm = SigMatchAlloc();
333  if (sm == NULL)
334  goto error;
335 
336  sm->type = DETECT_FILESTORE;
337 
338  if (str != NULL && strlen(str) > 0) {
339  SCLogDebug("str %s", str);
340 
341  ret = pcre_exec(parse_regex, parse_regex_study, str, strlen(str), 0, 0, ov, MAX_SUBSTRINGS);
342  if (ret < 1 || ret > 4) {
343  SCLogError(SC_ERR_PCRE_MATCH, "parse error, ret %" PRId32 ", string %s", ret, str);
344  goto error;
345  }
346 
347  if (ret > 1) {
348  const char *str_ptr;
349  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr);
350  if (res < 0) {
351  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
352  goto error;
353  }
354  args[0] = (char *)str_ptr;
355 
356  if (ret > 2) {
357  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 2, &str_ptr);
358  if (res < 0) {
359  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
360  goto error;
361  }
362  args[1] = (char *)str_ptr;
363  }
364  if (ret > 3) {
365  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 3, &str_ptr);
366  if (res < 0) {
367  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
368  goto error;
369  }
370  args[2] = (char *)str_ptr;
371  }
372  }
373 
374  fd = SCMalloc(sizeof(DetectFilestoreData));
375  if (unlikely(fd == NULL))
376  goto error;
377  memset(fd, 0x00, sizeof(DetectFilestoreData));
378 
379  if (args[0] != NULL) {
380  SCLogDebug("first arg %s", args[0]);
381 
382  if (strcasecmp(args[0], "request") == 0 ||
383  strcasecmp(args[0], "to_server") == 0)
384  {
387  }
388  else if (strcasecmp(args[0], "response") == 0 ||
389  strcasecmp(args[0], "to_client") == 0)
390  {
393  }
394  else if (strcasecmp(args[0], "both") == 0)
395  {
398  }
399  } else {
401  }
402 
403  if (args[1] != NULL) {
404  SCLogDebug("second arg %s", args[1]);
405 
406  if (strcasecmp(args[1], "file") == 0)
407  {
409  } else if (strcasecmp(args[1], "tx") == 0)
410  {
412  } else if (strcasecmp(args[1], "ssn") == 0 ||
413  strcasecmp(args[1], "flow") == 0)
414  {
416  }
417  } else {
418  if (fd->scope == 0)
420  }
421 
422  sm->ctx = (SigMatchCtx*)fd;
423  } else {
424  sm->ctx = (SigMatchCtx*)NULL;
425  }
426 
427  if (s->alproto == ALPROTO_HTTP) {
429  }
430 
431  SigMatchAppendSMToList(s, sm, g_file_match_list_id);
432  s->filestore_ctx = (const DetectFilestoreData *)sm->ctx;
433 
434  sm = SigMatchAlloc();
435  if (unlikely(sm == NULL))
436  goto error;
438  sm->ctx = NULL;
440 
441 
443  return 0;
444 
445 error:
446  if (sm != NULL)
447  SCFree(sm);
448  return -1;
449 }
450 
451 static void DetectFilestoreFree(void *ptr)
452 {
453  if (ptr != NULL) {
454  SCFree(ptr);
455  }
456 }
457 
458 #ifdef UNITTESTS
459 /*
460  * The purpose of this test is to confirm that
461  * filestore and bypass keywords can't
462  * can't work together
463  */
464 static int DetectFilestoreTest01(void)
465 {
466  DetectEngineCtx *de_ctx = NULL;
467  int result = 1;
468 
469  de_ctx = DetectEngineCtxInit();
470  FAIL_IF(de_ctx == NULL);
471 
472  de_ctx->flags |= DE_QUIET;
473 
474  de_ctx->sig_list = SigInit(de_ctx,"alert http any any -> any any "
475  "(bypass; filestore; "
476  "content:\"message\"; http_host; "
477  "sid:1;)");
478  FAIL_IF_NOT_NULL(de_ctx->sig_list);
479 
480  DetectEngineCtxFree(de_ctx);
481 
482  return result;
483 }
484 #endif /* UNITTESTS */
485 
486 void DetectFilestoreRegisterTests(void)
487 {
488 #ifdef UNITTESTS
489  UtRegisterTest("DetectFilestoreTest01", DetectFilestoreTest01);
490 #endif /* UNITTESTS */
491 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:239
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1406
#define MAX_SUBSTRINGS
uint16_t flags
int(* Setup)(DetectEngineCtx *, Signature *, const char *)
Definition: detect.h:1149
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
#define FILESTORE_DIR_BOTH
uint16_t flags
#define BUG_ON(x)
uint32_t flags
Definition: detect.h:496
uint8_t proto
Definition: flow.h:343
uint32_t id
Definition: detect.h:528
#define FILESTORE_DIR_TOCLIENT
#define unlikely(expr)
Definition: util-optimize.h:35
Signature * SigInit(DetectEngineCtx *, const char *)
Parses a signature and adds it to the Detection Engine Context.
Signature * sig_list
Definition: detect.h:729
uint32_t sid_cnt
Definition: util-file.h:96
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define HTP_FLAG_STORE_FILES_TX_TC
Definition: app-layer-htp.h:70
#define HTP_FLAG_STORE_FILES_TX_TS
Definition: app-layer-htp.h:69
#define DETECT_FILESTORE_MAX
Definition: detect.h:939
const char * name
Definition: detect.h:1163
Signature container.
Definition: detect.h:495
Used to start a pointer to SigMatch context Should never be dereferenced without casting to something...
Definition: detect.h:317
void * protoctx
Definition: flow.h:395
main detection engine ctx
Definition: detect.h:723
#define PARSE_REGEX
Regex for parsing our flow options.
void AppLayerHtpNeedFileInspection(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request file...
uint32_t FileReassemblyDepth(void)
Definition: util-file.c:122
void * alstate
Definition: flow.h:433
#define DE_QUIET
Definition: detect.h:296
#define str(s)
int(* FileMatch)(ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t flags, File *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1140
uint8_t flags
Definition: detect.h:724
void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id)
Definition: util-file.c:1288
uint16_t filestore_cnt
Definition: detect.h:999
uint32_t * sid
Definition: util-file.h:95
void FileStoreFileById(FileContainer *fc, uint32_t file_id)
flag a file with id "file_id" to be stored.
Definition: util-file.c:1273
Data structures and function prototypes for keeping state for the detection engine.
void(* Free)(void *)
Definition: detect.h:1154
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
uint32_t sid_max
Definition: util-file.h:97
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
void DetectSetupParseRegexes(const char *parse_str, pcre **parse_regex, pcre_extra **parse_regex_study)
#define SCEnter(...)
Definition: util-debug.h:337
const struct DetectFilestoreData_ * filestore_ctx
Definition: detect.h:550
uint8_t flowflags
Definition: decode.h:437
#define FILESTORE_DIR_TOSERVER
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:200
AppProto alproto
Definition: detect.h:499
#define HTP_FLAG_STORE_FILES_TC
Definition: app-layer-htp.h:68
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
uint8_t type
Definition: detect.h:323
#define SCReturnInt(x)
Definition: util-debug.h:341
void FileStoreAllFiles(FileContainer *fc)
Definition: util-file.c:1303
const char * desc
Definition: detect.h:1165
#define SCRealloc(x, a)
Definition: util-mem.h:182
void SigMatchAppendSMToList(Signature *s, SigMatch *new, int list)
Append a SigMatch to the list type.
Definition: detect-parse.c:288
int DetectBufferTypeRegister(const char *name)
#define FILESTORE_SCOPE_TX
struct DetectEngineThreadCtx_::@101 filestore[DETECT_FILESTORE_MAX]
SigMatchCtx * ctx
Definition: detect.h:325
#define SCMalloc(a)
Definition: util-mem.h:166
FileContainer * files_ts
#define SCFree(a)
Definition: util-mem.h:228
PoolThreadReserved res
FileContainer * files_tc
uint16_t tx_id
int(* Match)(ThreadVars *, DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1132
const char * url
Definition: detect.h:1166
#define SIGMATCH_OPTIONAL_OPT
Definition: detect.h:1340
#define STREAM_TOSERVER
Definition: stream.h:31
#define FILESTORE_SCOPE_DEFAULT
SCMutex m
Definition: flow-hash.h:105
#define SIG_FLAG_BYPASS
Definition: detect.h:246
uint32_t file_store_id
Definition: util-file.h:72
uint64_t store_tx_id
void TcpSessionSetReassemblyDepth(TcpSession *ssn, uint32_t size)
Definition: stream-tcp.c:6288
#define DOC_URL
Definition: suricata.h:86
#define FILESTORE_DIR_DEFAULT
SigMatch * SigMatchAlloc(void)
Definition: detect-parse.c:232
Per thread variable structure.
Definition: threadvars.h:57
#define FLOW_PKT_TOCLIENT
Definition: flow.h:201
AppProto alproto
application level protocol
Definition: flow.h:404
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
#define HTP_FLAG_STORE_FILES_TS
Definition: app-layer-htp.h:67
#define DOC_VERSION
Definition: suricata.h:91
uint16_t flags
Definition: detect.h:1157
#define FILESTORE_SCOPE_SSN
void DetectEngineCtxFree(DetectEngineCtx *)
Free a DetectEngineCtx::
Flow data structure.
Definition: flow.h:324
void(* RegisterTests)(void)
Definition: detect.h:1155
a single match condition for a signature
Definition: detect.h:322
void DetectFilestoreRegister(void)
Registration function for keyword: filestore.
DetectEngineCtx * DetectEngineCtxInit(void)