suricata
detect-filestore.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements the filestore keyword
24  */
25 
26 #include "suricata-common.h"
27 #include "threads.h"
28 #include "debug.h"
29 #include "decode.h"
30 
31 #include "detect.h"
32 #include "detect-parse.h"
33 
34 #include "detect-engine.h"
35 #include "detect-engine-mpm.h"
36 #include "detect-engine-state.h"
37 
38 #include "flow.h"
39 #include "flow-var.h"
40 #include "flow-util.h"
41 
42 #include "util-debug.h"
43 #include "util-spm-bm.h"
44 #include "util-unittest.h"
45 #include "util-unittest-helper.h"
46 
47 #include "app-layer.h"
48 #include "app-layer-parser.h"
49 #include "app-layer-htp.h"
50 
51 #include "stream-tcp.h"
52 
53 #include "detect-filestore.h"
54 
55 /**
56  * \brief Regex for parsing our flow options
57  */
58 #define PARSE_REGEX "^\\s*([A-z_]+)\\s*(?:,\\s*([A-z_]+))?\\s*(?:,\\s*([A-z_]+))?\\s*$"
59 
60 static pcre *parse_regex;
61 static pcre_extra *parse_regex_study;
62 
63 static int DetectFilestoreMatch (DetectEngineThreadCtx *,
64  Flow *, uint8_t, File *, const Signature *, const SigMatchCtx *);
65 static int DetectFilestorePostMatch(DetectEngineThreadCtx *det_ctx,
66  Packet *p, const Signature *s, const SigMatchCtx *ctx);
67 static int DetectFilestoreSetup (DetectEngineCtx *, Signature *, const char *);
68 static void DetectFilestoreFree(void *);
69 static void DetectFilestoreRegisterTests(void);
70 static int g_file_match_list_id = 0;
71 
72 /**
73  * \brief Registration function for keyword: filestore
74  */
76 {
77  sigmatch_table[DETECT_FILESTORE].name = "filestore";
78  sigmatch_table[DETECT_FILESTORE].desc = "stores files to disk if the rule matched";
79  sigmatch_table[DETECT_FILESTORE].url = DOC_URL DOC_VERSION "/rules/file-keywords.html#filestore";
80  sigmatch_table[DETECT_FILESTORE].FileMatch = DetectFilestoreMatch;
81  sigmatch_table[DETECT_FILESTORE].Setup = DetectFilestoreSetup;
82  sigmatch_table[DETECT_FILESTORE].Free = DetectFilestoreFree;
83  sigmatch_table[DETECT_FILESTORE].RegisterTests = DetectFilestoreRegisterTests;
85 
86  sigmatch_table[DETECT_FILESTORE_POSTMATCH].name = "__filestore__postmatch__";
87  sigmatch_table[DETECT_FILESTORE_POSTMATCH].Match = DetectFilestorePostMatch;
88  sigmatch_table[DETECT_FILESTORE_POSTMATCH].Free = DetectFilestoreFree;
89 
90  DetectSetupParseRegexes(PARSE_REGEX, &parse_regex, &parse_regex_study);
91 
92  g_file_match_list_id = DetectBufferTypeRegister("files");
93 }
94 
95 /**
96  * \brief apply the post match filestore with options
97  */
98 static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilestoreData *filestore,
99  FileContainer *fc, uint32_t file_id, uint64_t tx_id)
100 {
101  if (filestore == NULL) {
102  SCReturnInt(0);
103  }
104 
105  int this_file = 0;
106  int this_tx = 0;
107  int this_flow = 0;
108  int rule_dir = 0;
109  int toserver_dir = 0;
110  int toclient_dir = 0;
111 
112  switch (filestore->direction) {
114  rule_dir = 1;
115  break;
116  case FILESTORE_DIR_BOTH:
117  toserver_dir = 1;
118  toclient_dir = 1;
119  break;
121  toserver_dir = 1;
122  break;
124  toclient_dir = 1;
125  break;
126  }
127 
128  switch (filestore->scope) {
130  if (rule_dir) {
131  this_file = 1;
132  } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && toclient_dir) {
133  this_file = 1;
134  } else if ((p->flowflags & FLOW_PKT_TOSERVER) && toserver_dir) {
135  this_file = 1;
136  }
137  break;
138  case FILESTORE_SCOPE_TX:
139  this_tx = 1;
140  break;
141  case FILESTORE_SCOPE_SSN:
142  this_flow = 1;
143  break;
144  }
145 
146  if (this_file) {
147  FileStoreFileById(fc, file_id);
148  } else if (this_tx) {
149  /* flag tx all files will be stored */
150  if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
151  HtpState *htp_state = f->alstate;
152  if (toserver_dir) {
153  htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS;
154  FileStoreAllFilesForTx(htp_state->files_ts, tx_id);
155  }
156  if (toclient_dir) {
157  htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC;
158  FileStoreAllFilesForTx(htp_state->files_tc, tx_id);
159  }
160  htp_state->store_tx_id = tx_id;
161  }
162  } else if (this_flow) {
163  /* flag flow all files will be stored */
164  if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
165  HtpState *htp_state = f->alstate;
166  if (toserver_dir) {
167  htp_state->flags |= HTP_FLAG_STORE_FILES_TS;
168  FileStoreAllFiles(htp_state->files_ts);
169  }
170  if (toclient_dir) {
171  htp_state->flags |= HTP_FLAG_STORE_FILES_TC;
172  FileStoreAllFiles(htp_state->files_tc);
173  }
174  }
175  } else {
176  FileStoreFileById(fc, file_id);
177  }
178 
179  SCReturnInt(0);
180 }
181 
182 /**
183  * \brief post-match function for filestore
184  *
185  * \param t thread local vars
186  * \param det_ctx pattern matcher thread local data
187  * \param p packet
188  *
189  * The match function for filestore records store candidates in the det_ctx.
190  * When we are sure all parts of the signature matched, we run this function
191  * to finalize the filestore.
192  */
193 static int DetectFilestorePostMatch(DetectEngineThreadCtx *det_ctx,
194  Packet *p, const Signature *s, const SigMatchCtx *ctx)
195 {
196  uint8_t flags = 0;
197 
198  SCEnter();
199 
200  if (det_ctx->filestore_cnt == 0) {
201  SCReturnInt(0);
202  }
203 
204  if ((s->filestore_ctx == NULL && !(s->flags & SIG_FLAG_FILESTORE)) || p->flow == NULL) {
205 #ifndef DEBUG
206  SCReturnInt(0);
207 #else
208  BUG_ON(1);
209 #endif
210  }
211 
212  if (p->proto == IPPROTO_TCP && p->flow->protoctx != NULL) {
213  /* set filestore depth for stream reassembling */
214  TcpSession *ssn = (TcpSession *)p->flow->protoctx;
216  }
217  if (p->flowflags & FLOW_PKT_TOCLIENT)
218  flags |= STREAM_TOCLIENT;
219  else
220  flags |= STREAM_TOSERVER;
221 
222  for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
224  FlowGetAppState(p->flow),
225  det_ctx->filestore[u].tx_id,
226  flags);
227  }
228 
230  p->flow->alstate, flags);
231 
232  /* filestore for single files only */
233  if (s->filestore_ctx == NULL) {
234  for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
235  FileStoreFileById(ffc, det_ctx->filestore[u].file_id);
236  }
237  } else {
238  for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
239  FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc,
240  det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
241  }
242  }
243 
244  SCReturnInt(0);
245 }
246 
247 /**
248  * \brief match the specified filestore
249  *
250  * \param t thread local vars
251  * \param det_ctx pattern matcher thread local data
252  * \param f *LOCKED* flow
253  * \param flags direction flags
254  * \param file file being inspected
255  * \param s signature being inspected
256  * \param m sigmatch that we will cast into DetectFilestoreData
257  *
258  * \retval 0 no match
259  * \retval 1 match
260  *
261  * \todo when we start supporting more protocols, the logic in this function
262  * needs to be put behind a api.
263  */
264 static int DetectFilestoreMatch (DetectEngineThreadCtx *det_ctx, Flow *f,
265  uint8_t flags, File *file, const Signature *s, const SigMatchCtx *m)
266 {
267  uint32_t file_id = 0;
268 
269  SCEnter();
270 
271  if (det_ctx->filestore_cnt >= DETECT_FILESTORE_MAX) {
272  SCReturnInt(1);
273  }
274 
275  /* file can be NULL when a rule with filestore scope > file
276  * matches. */
277  if (file != NULL) {
278  file_id = file->file_track_id;
279  if (file->sid != NULL && s->id > 0) {
280  if (file->sid_cnt >= file->sid_max) {
281  void *p = SCRealloc(file->sid, sizeof(uint32_t) * (file->sid_max + 8));
282  if (p == NULL) {
283  SCFree(file->sid);
284  file->sid = NULL;
285  file->sid_cnt = 0;
286  file->sid_max = 0;
287  goto continue_after_realloc_fail;
288  } else {
289  file->sid = p;
290  file->sid_max += 8;
291  }
292  }
293  file->sid[file->sid_cnt] = s->id;
294  file->sid_cnt++;
295  }
296  }
297 
298 continue_after_realloc_fail:
299 
300  det_ctx->filestore[det_ctx->filestore_cnt].file_id = file_id;
301  det_ctx->filestore[det_ctx->filestore_cnt].tx_id = det_ctx->tx_id;
302 
303  SCLogDebug("%u, file %u, tx %"PRIu64, det_ctx->filestore_cnt,
304  det_ctx->filestore[det_ctx->filestore_cnt].file_id,
305  det_ctx->filestore[det_ctx->filestore_cnt].tx_id);
306 
307  det_ctx->filestore_cnt++;
308  SCReturnInt(1);
309 }
310 
311 /**
312  * \brief this function is used to parse filestore options
313  * \brief into the current signature
314  *
315  * \param de_ctx pointer to the Detection Engine Context
316  * \param s pointer to the Current Signature
317  * \param str pointer to the user provided "filestore" option
318  *
319  * \retval 0 on Success
320  * \retval -1 on Failure
321  */
322 static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str)
323 {
324  SCEnter();
325 
326  DetectFilestoreData *fd = NULL;
327  SigMatch *sm = NULL;
328  char *args[3] = {NULL,NULL,NULL};
329 #define MAX_SUBSTRINGS 30
330  int ret = 0, res = 0;
331  int ov[MAX_SUBSTRINGS];
332 
333  /* filestore and bypass keywords can't work together */
334  if (s->flags & SIG_FLAG_BYPASS) {
336  "filestore can't work with bypass keyword");
337  return -1;
338  }
339 
340  sm = SigMatchAlloc();
341  if (sm == NULL)
342  goto error;
343 
344  sm->type = DETECT_FILESTORE;
345 
346  if (str != NULL && strlen(str) > 0) {
347  SCLogDebug("str %s", str);
348 
349  ret = pcre_exec(parse_regex, parse_regex_study, str, strlen(str), 0, 0, ov, MAX_SUBSTRINGS);
350  if (ret < 1 || ret > 4) {
351  SCLogError(SC_ERR_PCRE_MATCH, "parse error, ret %" PRId32 ", string %s", ret, str);
352  goto error;
353  }
354 
355  if (ret > 1) {
356  const char *str_ptr;
357  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr);
358  if (res < 0) {
359  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
360  goto error;
361  }
362  args[0] = (char *)str_ptr;
363 
364  if (ret > 2) {
365  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 2, &str_ptr);
366  if (res < 0) {
367  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
368  goto error;
369  }
370  args[1] = (char *)str_ptr;
371  }
372  if (ret > 3) {
373  res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 3, &str_ptr);
374  if (res < 0) {
375  SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
376  goto error;
377  }
378  args[2] = (char *)str_ptr;
379  }
380  }
381 
382  fd = SCMalloc(sizeof(DetectFilestoreData));
383  if (unlikely(fd == NULL))
384  goto error;
385  memset(fd, 0x00, sizeof(DetectFilestoreData));
386 
387  if (args[0] != NULL) {
388  SCLogDebug("first arg %s", args[0]);
389 
390  if (strcasecmp(args[0], "request") == 0 ||
391  strcasecmp(args[0], "to_server") == 0)
392  {
395  }
396  else if (strcasecmp(args[0], "response") == 0 ||
397  strcasecmp(args[0], "to_client") == 0)
398  {
401  }
402  else if (strcasecmp(args[0], "both") == 0)
403  {
406  }
407  } else {
409  }
410 
411  if (args[1] != NULL) {
412  SCLogDebug("second arg %s", args[1]);
413 
414  if (strcasecmp(args[1], "file") == 0)
415  {
417  } else if (strcasecmp(args[1], "tx") == 0)
418  {
420  } else if (strcasecmp(args[1], "ssn") == 0 ||
421  strcasecmp(args[1], "flow") == 0)
422  {
424  }
425  } else {
426  if (fd->scope == 0)
428  }
429 
430  sm->ctx = (SigMatchCtx*)fd;
431  } else {
432  sm->ctx = (SigMatchCtx*)NULL;
433  }
434 
435  if (s->alproto == ALPROTO_HTTP) {
437  }
438 
439  SigMatchAppendSMToList(s, sm, g_file_match_list_id);
440  s->filestore_ctx = (const DetectFilestoreData *)sm->ctx;
441 
442  sm = SigMatchAlloc();
443  if (unlikely(sm == NULL))
444  goto error;
446  sm->ctx = NULL;
448 
449 
451  return 0;
452 
453 error:
454  if (sm != NULL)
455  SCFree(sm);
456  return -1;
457 }
458 
459 static void DetectFilestoreFree(void *ptr)
460 {
461  if (ptr != NULL) {
462  SCFree(ptr);
463  }
464 }
465 
466 #ifdef UNITTESTS
467 /*
468  * The purpose of this test is to confirm that
469  * filestore and bypass keywords can't
470  * can't work together
471  */
472 static int DetectFilestoreTest01(void)
473 {
474  DetectEngineCtx *de_ctx = NULL;
475  int result = 1;
476 
477  de_ctx = DetectEngineCtxInit();
478  FAIL_IF(de_ctx == NULL);
479 
480  de_ctx->flags |= DE_QUIET;
481 
482  de_ctx->sig_list = SigInit(de_ctx,"alert http any any -> any any "
483  "(bypass; filestore; "
484  "content:\"message\"; http_host; "
485  "sid:1;)");
486  FAIL_IF_NOT_NULL(de_ctx->sig_list);
487 
488  DetectEngineCtxFree(de_ctx);
489 
490  return result;
491 }
492 #endif /* UNITTESTS */
493 
494 void DetectFilestoreRegisterTests(void)
495 {
496 #ifdef UNITTESTS
497  UtRegisterTest("DetectFilestoreTest01", DetectFilestoreTest01);
498 #endif /* UNITTESTS */
499 }
#define SIG_FLAG_FILESTORE
Definition: detect.h:234
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect.h:1448
#define MAX_SUBSTRINGS
uint16_t flags
int(* Setup)(DetectEngineCtx *, Signature *, const char *)
Definition: detect.h:1186
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:445
#define FILESTORE_DIR_BOTH
uint16_t flags
#define BUG_ON(x)
uint32_t flags
Definition: detect.h:523
uint8_t proto
Definition: flow.h:344
uint32_t id
Definition: detect.h:555
#define FILESTORE_DIR_TOCLIENT
#define unlikely(expr)
Definition: util-optimize.h:35
Signature * SigInit(DetectEngineCtx *, const char *)
Parses a signature and adds it to the Detection Engine Context.
Signature * sig_list
Definition: detect.h:767
uint32_t sid_cnt
Definition: util-file.h:96
#define FAIL_IF(expr)
Fail a test if expression evaluates to false.
Definition: util-unittest.h:71
#define HTP_FLAG_STORE_FILES_TX_TC
Definition: app-layer-htp.h:74
#define HTP_FLAG_STORE_FILES_TX_TS
Definition: app-layer-htp.h:73
#define DETECT_FILESTORE_MAX
Definition: detect.h:977
const char * name
Definition: detect.h:1200
Signature container.
Definition: detect.h:522
Used to start a pointer to SigMatch context Should never be dereferenced without casting to something...
Definition: detect.h:313
void * protoctx
Definition: flow.h:400
main detection engine ctx
Definition: detect.h:761
#define PARSE_REGEX
Regex for parsing our flow options.
void AppLayerHtpNeedFileInspection(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request file...
uint32_t FileReassemblyDepth(void)
Definition: util-file.c:122
void * alstate
Definition: flow.h:438
#define DE_QUIET
Definition: detect.h:292
#define str(s)
uint8_t proto
Definition: decode.h:430
uint8_t flags
Definition: detect.h:762
int(* FileMatch)(DetectEngineThreadCtx *, Flow *, uint8_t flags, File *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1178
void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id)
Definition: util-file.c:1289
uint16_t filestore_cnt
Definition: detect.h:1037
uint32_t * sid
Definition: util-file.h:95
void FileStoreFileById(FileContainer *fc, uint32_t file_id)
flag a file with id "file_id" to be stored.
Definition: util-file.c:1274
Data structures and function prototypes for keeping state for the detection engine.
void(* Free)(void *)
Definition: detect.h:1191
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
uint32_t sid_max
Definition: util-file.h:97
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
void DetectSetupParseRegexes(const char *parse_str, pcre **parse_regex, pcre_extra **parse_regex_study)
#define SCEnter(...)
Definition: util-debug.h:337
const struct DetectFilestoreData_ * filestore_ctx
Definition: detect.h:578
uint8_t flowflags
Definition: decode.h:439
#define FILESTORE_DIR_TOSERVER
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
AppProto alproto
Definition: detect.h:526
#define HTP_FLAG_STORE_FILES_TC
Definition: app-layer-htp.h:72
#define FAIL_IF_NOT_NULL(expr)
Fail a test if expression evaluates to non-NULL.
Definition: util-unittest.h:96
int(* Match)(DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Definition: detect.h:1170
uint8_t type
Definition: detect.h:319
#define SCReturnInt(x)
Definition: util-debug.h:341
void FileStoreAllFiles(FileContainer *fc)
Definition: util-file.c:1304
const char * desc
Definition: detect.h:1202
#define SCRealloc(x, a)
Definition: util-mem.h:238
void SigMatchAppendSMToList(Signature *s, SigMatch *new, int list)
Append a SigMatch to the list type.
Definition: detect-parse.c:346
int DetectBufferTypeRegister(const char *name)
#define FILESTORE_SCOPE_TX
SigMatchCtx * ctx
Definition: detect.h:321
#define SCMalloc(a)
Definition: util-mem.h:222
FileContainer * files_ts
#define SCFree(a)
Definition: util-mem.h:322
PoolThreadReserved res
FileContainer * files_tc
uint16_t tx_id
const char * url
Definition: detect.h:1203
#define SIGMATCH_OPTIONAL_OPT
Definition: detect.h:1378
#define STREAM_TOSERVER
Definition: stream.h:31
#define FILESTORE_SCOPE_DEFAULT
SCMutex m
Definition: flow-hash.h:105
#define SIG_FLAG_BYPASS
Definition: detect.h:241
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1068
uint32_t file_track_id
Definition: util-file.h:70
uint64_t store_tx_id
void TcpSessionSetReassemblyDepth(TcpSession *ssn, uint32_t size)
Definition: stream-tcp.c:6326
#define DOC_URL
Definition: suricata.h:86
#define FILESTORE_DIR_DEFAULT
SigMatch * SigMatchAlloc(void)
Definition: detect-parse.c:232
void AppLayerParserSetStreamDepthFlag(uint8_t ipproto, AppProto alproto, void *state, uint64_t tx_id, uint8_t flags)
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
struct DetectEngineThreadCtx_::@106 filestore[DETECT_FILESTORE_MAX]
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
#define HTP_FLAG_STORE_FILES_TS
Definition: app-layer-htp.h:71
#define DOC_VERSION
Definition: suricata.h:91
uint16_t flags
Definition: detect.h:1194
#define FILESTORE_SCOPE_SSN
void DetectEngineCtxFree(DetectEngineCtx *)
Free a DetectEngineCtx::
Flow data structure.
Definition: flow.h:325
void(* RegisterTests)(void)
Definition: detect.h:1192
a single match condition for a signature
Definition: detect.h:318
void DetectFilestoreRegister(void)
Registration function for keyword: filestore.
DetectEngineCtx * DetectEngineCtxInit(void)