suricata
detect-filestore.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements the filestore keyword
24  */
25 
26 #include "suricata-common.h"
27 #include "threads.h"
28 #include "debug.h"
29 #include "decode.h"
30 
31 #include "detect.h"
32 #include "detect-parse.h"
33 
34 #include "detect-engine.h"
35 #include "detect-engine-mpm.h"
36 #include "detect-engine-state.h"
37 
38 #include "flow.h"
39 #include "flow-var.h"
40 #include "flow-util.h"
41 
42 #include "util-debug.h"
43 #include "util-spm-bm.h"
44 #include "util-unittest.h"
45 #include "util-unittest-helper.h"
46 
47 #include "app-layer.h"
48 #include "app-layer-parser.h"
49 #include "app-layer-htp.h"
50 
51 #include "stream-tcp.h"
52 
53 #include "detect-filestore.h"
54 
55 /**
56  * \brief Regex for parsing our flow options
57  */
58 #define PARSE_REGEX "^\\s*([A-z_]+)\\s*(?:,\\s*([A-z_]+))?\\s*(?:,\\s*([A-z_]+))?\\s*$"
59 
60 static pcre *parse_regex;
61 static pcre_extra *parse_regex_study;
62 
63 static int DetectFilestoreMatch (ThreadVars *, DetectEngineThreadCtx *,
64  Flow *, uint8_t, File *, const Signature *, const SigMatchCtx *);
65 static int DetectFilestoreSetup (DetectEngineCtx *, Signature *, const char *);
66 static void DetectFilestoreFree(void *);
67 static void DetectFilestoreRegisterTests(void);
68 static int g_file_match_list_id = 0;
69 
70 /**
71  * \brief Registration function for keyword: filestore
72  */
74 {
75  sigmatch_table[DETECT_FILESTORE].name = "filestore";
76  sigmatch_table[DETECT_FILESTORE].desc = "stores files to disk if the rule matched";
77  sigmatch_table[DETECT_FILESTORE].url = DOC_URL DOC_VERSION "/rules/file-keywords.html#filestore";
78  sigmatch_table[DETECT_FILESTORE].FileMatch = DetectFilestoreMatch;
79  sigmatch_table[DETECT_FILESTORE].Setup = DetectFilestoreSetup;
80  sigmatch_table[DETECT_FILESTORE].Free = DetectFilestoreFree;
81  sigmatch_table[DETECT_FILESTORE].RegisterTests = DetectFilestoreRegisterTests;
83 
84  DetectSetupParseRegexes(PARSE_REGEX, &parse_regex, &parse_regex_study);
85 
86  g_file_match_list_id = DetectBufferTypeRegister("files");
87 }
88 
89 /**
90  * \brief apply the post match filestore with options
91  */
92 static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilestoreData *filestore,
93  FileContainer *fc, uint32_t file_id, uint64_t tx_id)
94 {
95  if (filestore == NULL) {
96  SCReturnInt(0);
97  }
98 
99  int this_file = 0;
100  int this_tx = 0;
101  int this_flow = 0;
102  int rule_dir = 0;
103  int toserver_dir = 0;
104  int toclient_dir = 0;
105 
106  switch (filestore->direction) {
108  rule_dir = 1;
109  break;
110  case FILESTORE_DIR_BOTH:
111  toserver_dir = 1;
112  toclient_dir = 1;
113  break;
115  toserver_dir = 1;
116  break;
118  toclient_dir = 1;
119  break;
120  }
121 
122  switch (filestore->scope) {
124  if (rule_dir) {
125  this_file = 1;
126  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && toclient_dir) {
127  this_file = 1;
128  } else if ((p->flowflags & FLOW_PKT_TOSERVER) && toserver_dir) {
129  this_file = 1;
130  }
131  break;
132  case FILESTORE_SCOPE_TX:
133  this_tx = 1;
134  break;
135  case FILESTORE_SCOPE_SSN:
136  this_flow = 1;
137  break;
138  }
139 
140  if (this_file) {
141  FileStoreFileById(fc, file_id);
142  } else if (this_tx) {
143  /* flag tx all files will be stored */
144  if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
145  HtpState *htp_state = f->alstate;
146  if (toserver_dir) {
147  htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS;
148  FileStoreAllFilesForTx(htp_state->files_ts, tx_id);
149  }
150  if (toclient_dir) {
151  htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC;
152  FileStoreAllFilesForTx(htp_state->files_tc, tx_id);
153  }
154  htp_state->store_tx_id = tx_id;
155  }
156  } else if (this_flow) {
157  /* flag flow all files will be stored */
158  if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
159  HtpState *htp_state = f->alstate;
160  if (toserver_dir) {
161  htp_state->flags |= HTP_FLAG_STORE_FILES_TS;
162  FileStoreAllFiles(htp_state->files_ts);
163  }
164  if (toclient_dir) {
165  htp_state->flags |= HTP_FLAG_STORE_FILES_TC;
166  FileStoreAllFiles(htp_state->files_tc);
167  }
168  }
169  } else {
170  FileStoreFileById(fc, file_id);
171  }
172 
173  SCReturnInt(0);
174 }
175 
176 /**
177  * \brief post-match function for filestore
178  *
179  * \param t thread local vars
180  * \param det_ctx pattern matcher thread local data
181  * \param p packet
182  *
183  * The match function for filestore records store candidates in the det_ctx.
184  * When we are sure all parts of the signature matched, we run this function
185  * to finalize the filestore.
186  */
188 {
189  uint8_t flags = 0;
190 
191  SCEnter();
192 
193  if (det_ctx->filestore_cnt == 0) {
194  SCReturnInt(0);
195  }
196 
197  if ((s->filestore_ctx == NULL && !(s->flags & SIG_FLAG_FILESTORE)) || p->flow == NULL) {
198 #ifndef DEBUG
199  SCReturnInt(0);
200 #else
201  BUG_ON(1);
202 #endif
203  }
204 
205  /* set filestore depth for stream reassembling */
206  TcpSession *ssn = (TcpSession *)p->flow->protoctx;
208 
209  if (p->flowflags & FLOW_PKT_TOCLIENT)
210  flags |= STREAM_TOCLIENT;
211  else
212  flags |= STREAM_TOSERVER;
213 
215  p->flow->alstate, flags);
216 
217  /* filestore for single files only */
218  if (s->filestore_ctx == NULL) {
219  uint16_t u;
220  for (u = 0; u < det_ctx->filestore_cnt; u++) {
221  FileStoreFileById(ffc, det_ctx->filestore[u].file_id);
222  }
223  } else {
224  uint16_t u;
225 
226  for (u = 0; u < det_ctx->filestore_cnt; u++) {
227  FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc,
228  det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
229  }
230  }
231 
232  SCReturnInt(0);
233 }
234 
235 /**
236  * \brief match the specified filestore
237  *
238  * \param t thread local vars
239  * \param det_ctx pattern matcher thread local data
240  * \param f *LOCKED* flow
241  * \param flags direction flags
242  * \param file file being inspected
243  * \param s signature being inspected
244  * \param m sigmatch that we will cast into DetectFilestoreData
245  *
246  * \retval 0 no match
247  * \retval 1 match
248  *
249  * \todo when we start supporting more protocols, the logic in this function
250  * needs to be put behind a api.
251  */
252 static int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f,
253  uint8_t flags, File *file, const Signature *s, const SigMatchCtx *m)
254 {
255  uint32_t file_id = 0;
256 
257  SCEnter();
258 
259  if (det_ctx->filestore_cnt >= DETECT_FILESTORE_MAX) {
260  SCReturnInt(1);
261  }
262 
263  /* file can be NULL when a rule with filestore scope > file
264  * matches. */
265  if (file != NULL) {
266  file_id = file->file_store_id;
267  if (file->sid != NULL && s->id > 0) {
268  if (file->sid_cnt >= file->sid_max) {
269  void *p = SCRealloc(file->sid, sizeof(uint32_t) * (file->sid_max + 8));
270  if (p == NULL) {
271  SCFree(file->sid);
272  file->sid = NULL;
273  file->sid_cnt = 0;
274  file->sid_max = 0;
275  goto continue_after_realloc_fail;
276  } else {
277  file->sid = p;
278  file->sid_max += 8;
279  }
280  }
281  file->sid[file->sid_cnt] = s->id;
282  file->sid_cnt++;
283  }
284  }
285 
286 continue_after_realloc_fail:
287 
288  det_ctx->filestore[det_ctx->filestore_cnt].file_id = file_id;
289  det_ctx->filestore[det_ctx->filestore_cnt].tx_id = det_ctx->tx_id;
290 
291  SCLogDebug("%u, file %u, tx %"PRIu64, det_ctx->filestore_cnt,
292  det_ctx->filestore[det_ctx->filestore_cnt].file_id,
293  det_ctx->filestore[det_ctx->filestore_cnt].tx_id);
294 
295  det_ctx->filestore_cnt++;
296  SCReturnInt(1);
297 }
298 
299 /**
300  * \brief this function is used to parse filestore options
301  * \brief into the current signature
302  *
303  * \param de_ctx pointer to the Detection Engine Context
304  * \param s pointer to the Current Signature
305  * \param str pointer to the user provided "filestore" option
306  *
307  * \retval 0 on Success
308  * \retval -1 on Failure
309  */
310 static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str)
311 {
312  SCEnter();
313 
314  DetectFilestoreData *fd = NULL;
315  SigMatch *sm = NULL;
316  char *args[3] = {NULL,NULL,NULL};
317 #define MAX_SUBSTRINGS 30
318  int ret = 0, res = 0;
319  int ov[MAX_SUBSTRINGS];
320 
321  /* filestore and bypass keywords can't work together */
322  if (s->flags & SIG_FLAG_BYPASS) {
324  "filestore can't work with bypass keyword");
325  return -1;
326  }
327 
328  sm = SigMatchAlloc();
329  if (sm == NULL)
330  goto error;
331 
332  sm->type = DETECT_FILESTORE;
333 
334  if (str != NULL && strlen(str) > 0) {
335  SCLogDebug("str %s", str);
336 
337  ret = pcre_exec(parse_regex, parse_regex_study, str, strlen(str), 0, 0, ov, MAX_SUBSTRINGS);
338  if (ret < 1 || ret > 4) {
339  SCLogError(SC_ERR_PCRE_MATCH, "parse error, ret %" PRId32 ", string %s", ret, str);
340  goto error;
341  }
342 
343  if (ret > 1) {
344  const char *str_ptr;
345  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr);
346  if (res < 0) {
347  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
348  goto error;
349  }
350  args[0] = (char *)str_ptr;
351 
352  if (ret > 2) {
353  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 2, &str_ptr);
354  if (res < 0) {
355  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
356  goto error;
357  }
358  args[1] = (char *)str_ptr;
359  }
360  if (ret > 3) {
361  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 3, &str_ptr);
362  if (res < 0) {
363  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
364  goto error;
365  }
366  args[2] = (char *)str_ptr;
367  }
368  }
369 
370  fd = SCMalloc(sizeof(DetectFilestoreData));
371  if (unlikely(fd == NULL))
372  goto error;
373  memset(fd, 0x00, sizeof(DetectFilestoreData));
374 
375  if (args[0] != NULL) {
376  SCLogDebug("first arg %s", args[0]);
377 
378  if (strcasecmp(args[0], "request") == 0 ||
379  strcasecmp(args[0], "to_server") == 0)
380  {
383  }
384  else if (strcasecmp(args[0], "response") == 0 ||
385  strcasecmp(args[0], "to_client") == 0)
386  {
389  }
390  else if (strcasecmp(args[0], "both") == 0)
391  {
394  }
395  } else {
397  }
398 
399  if (args[1] != NULL) {
400  SCLogDebug("second arg %s", args[1]);
401 
402  if (strcasecmp(args[1], "file") == 0)
403  {
405  } else if (strcasecmp(args[1], "tx") == 0)
406  {
408  } else if (strcasecmp(args[1], "ssn") == 0 ||
409  strcasecmp(args[1], "flow") == 0)
410  {
412  }
413  } else {
414  if (fd->scope == 0)
416  }
417 
418  sm->ctx = (SigMatchCtx*)fd;
419  } else {
420  sm->ctx = (SigMatchCtx*)NULL;
421  }
422 
423  if (s->alproto == ALPROTO_HTTP) {
425  }
426 
427  SigMatchAppendSMToList(s, sm, g_file_match_list_id);
428  s->filestore_ctx = (const DetectFilestoreData *)sm->ctx;
429 
431  return 0;
432 
433 error:
434  if (sm != NULL)
435  SCFree(sm);
436  return -1;
437 }
438 
439 static void DetectFilestoreFree(void *ptr)
440 {
441  if (ptr != NULL) {
442  SCFree(ptr);
443  }
444 }
445 
446 #ifdef UNITTESTS
447 /*
448  * The purpose of this test is to confirm that
449  * filestore and bypass keywords can't
450  * can't work together
451  */
452 static int DetectFilestoreTest01(void)
453 {
454  DetectEngineCtx *de_ctx = NULL;
455  int result = 1;
456 
457  de_ctx = DetectEngineCtxInit();
458  FAIL_IF(de_ctx == NULL);
459 
460  de_ctx->flags |= DE_QUIET;
461 
462  de_ctx->sig_list = SigInit(de_ctx,"alert http any any -> any any "
463  "(bypass; filestore; "
464  "content:\"message\"; http_host; "
465  "sid:1;)");
466  FAIL_IF_NOT_NULL(de_ctx->sig_list);
467 
468  DetectEngineCtxFree(de_ctx);
469 
470  return result;
471 }
472 #endif /* UNITTESTS */
473 
474 void DetectFilestoreRegisterTests(void)
475 {
476 #ifdef UNITTESTS
477  UtRegisterTest("DetectFilestoreTest01", DetectFilestoreTest01);
478 #endif /* UNITTESTS */
479 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:241
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1403
#define MAX_SUBSTRINGS
uint16_t flags
int(* Setup)(DetectEngineCtx *, Signature *, const char *)
Definition: detect.h:1146
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:444
#define FILESTORE_DIR_BOTH
uint16_t flags
#define BUG_ON(x)
uint32_t flags
Definition: detect.h:493
uint8_t proto
Definition: flow.h:346
uint32_t id
Definition: detect.h:525
#define FILESTORE_DIR_TOCLIENT
#define unlikely(expr)
Definition: util-optimize.h:35
Signature * SigInit(DetectEngineCtx *, const char *)
Parses a signature and adds it to the Detection Engine Context.
Signature * sig_list
Definition: detect.h:726
uint32_t sid_cnt
Definition: util-file.h:94
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define HTP_FLAG_STORE_FILES_TX_TC
Definition: app-layer-htp.h:70
#define HTP_FLAG_STORE_FILES_TX_TS
Definition: app-layer-htp.h:69
#define DETECT_FILESTORE_MAX
Definition: detect.h:936
const char * name
Definition: detect.h:1160
Signature container.
Definition: detect.h:492
Used to start a pointer to SigMatch context Should never be dereferenced without casting to something...
Definition: detect.h:319
void * protoctx
Definition: flow.h:398
main detection engine ctx
Definition: detect.h:720
struct DetectEngineThreadCtx_::@109 filestore[DETECT_FILESTORE_MAX]
#define PARSE_REGEX
Regex for parsing our flow options.
void AppLayerHtpNeedFileInspection(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request file...
uint32_t FileReassemblyDepth(void)
Definition: util-file.c:122
void * alstate
Definition: flow.h:436
#define DE_QUIET
Definition: detect.h:298
#define str(s)
int(* FileMatch)(ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t flags, File *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1137
uint8_t flags
Definition: detect.h:721
void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id)
Definition: util-file.c:1266
uint16_t filestore_cnt
Definition: detect.h:996
uint32_t * sid
Definition: util-file.h:93
void FileStoreFileById(FileContainer *fc, uint32_t file_id)
flag a file with id "file_id" to be stored.
Definition: util-file.c:1251
Data structures and function prototypes for keeping state for the detection engine.
void(* Free)(void *)
Definition: detect.h:1151
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
uint32_t sid_max
Definition: util-file.h:95
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
void DetectSetupParseRegexes(const char *parse_str, pcre **parse_regex, pcre_extra **parse_regex_study)
#define SCEnter(...)
Definition: util-debug.h:337
const struct DetectFilestoreData_ * filestore_ctx
Definition: detect.h:547
uint8_t flowflags
Definition: decode.h:438
#define FILESTORE_DIR_TOSERVER
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:193
AppProto alproto
Definition: detect.h:496
#define HTP_FLAG_STORE_FILES_TC
Definition: app-layer-htp.h:68
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
uint8_t type
Definition: detect.h:325
#define SCReturnInt(x)
Definition: util-debug.h:341
void FileStoreAllFiles(FileContainer *fc)
Definition: util-file.c:1281
const char * desc
Definition: detect.h:1162
#define SCRealloc(x, a)
Definition: util-mem.h:190
void SigMatchAppendSMToList(Signature *s, SigMatch *new, int list)
Append a SigMatch to the list type.
Definition: detect-parse.c:282
int DetectBufferTypeRegister(const char *name)
#define FILESTORE_SCOPE_TX
SigMatchCtx * ctx
Definition: detect.h:327
#define SCMalloc(a)
Definition: util-mem.h:174
FileContainer * files_ts
#define SCFree(a)
Definition: util-mem.h:236
PoolThreadReserved res
FileContainer * files_tc
uint16_t tx_id
const char * url
Definition: detect.h:1163
#define SIGMATCH_OPTIONAL_OPT
Definition: detect.h:1337
#define STREAM_TOSERVER
Definition: stream.h:31
#define FILESTORE_SCOPE_DEFAULT
SCMutex m
Definition: flow-hash.h:105
#define SIG_FLAG_BYPASS
Definition: detect.h:248
uint32_t file_store_id
Definition: util-file.h:72
uint64_t store_tx_id
void TcpSessionSetReassemblyDepth(TcpSession *ssn, uint32_t size)
Definition: stream-tcp.c:6289
#define DOC_URL
Definition: suricata.h:86
#define FILESTORE_DIR_DEFAULT
SigMatch * SigMatchAlloc(void)
Definition: detect-parse.c:226
Per thread variable structure.
Definition: threadvars.h:57
#define FLOW_PKT_TOCLIENT
Definition: flow.h:194
AppProto alproto
application level protocol
Definition: flow.h:407
int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s)
post-match function for filestore
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
#define HTP_FLAG_STORE_FILES_TS
Definition: app-layer-htp.h:67
#define DOC_VERSION
Definition: suricata.h:91
uint16_t flags
Definition: detect.h:1154
#define FILESTORE_SCOPE_SSN
void DetectEngineCtxFree(DetectEngineCtx *)
Free a DetectEngineCtx::
Flow data structure.
Definition: flow.h:327
void(* RegisterTests)(void)
Definition: detect.h:1152
a single match condition for a signature
Definition: detect.h:324
void DetectFilestoreRegister(void)
Registration function for keyword: filestore.
DetectEngineCtx * DetectEngineCtxInit(void)