suricata
detect-dataset.c
Go to the documentation of this file.
1 /* Copyright (C) 2018-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements the dataset keyword
24  */
25 
26 #include "suricata-common.h"
27 #include "decode.h"
28 #include "detect.h"
29 #include "threads.h"
30 #include "datasets.h"
31 #include "detect-dataset.h"
32 
33 #include "detect-parse.h"
34 #include "detect-engine.h"
35 #include "detect-engine-mpm.h"
36 #include "detect-engine-state.h"
37 
38 #include "util-debug.h"
39 #include "util-print.h"
40 #include "util-misc.h"
41 
42 #define PARSE_REGEX "([a-z]+)(?:,\\s*([\\-_A-z0-9\\s\\.]+)){1,4}"
43 static DetectParseRegex parse_regex;
44 
46  const Signature *, const SigMatchCtx *);
47 static int DetectDatasetSetup (DetectEngineCtx *, Signature *, const char *);
48 void DetectDatasetFree (DetectEngineCtx *, void *);
49 
51 {
52  sigmatch_table[DETECT_DATASET].name = "dataset";
53  sigmatch_table[DETECT_DATASET].desc = "match sticky buffer against datasets (experimental)";
54  sigmatch_table[DETECT_DATASET].url = "/rules/dataset-keywords.html#dataset";
55  sigmatch_table[DETECT_DATASET].Setup = DetectDatasetSetup;
57 
58  DetectSetupParseRegexes(PARSE_REGEX, &parse_regex);
59 }
60 
61 /*
62  1 match
63  0 no match
64  -1 can't match
65  */
67  const DetectDatasetData *sd,
68  const uint8_t *data, const uint32_t data_len)
69 {
70  if (data == NULL || data_len == 0)
71  return 0;
72 
73  switch (sd->cmd) {
75  //PrintRawDataFp(stdout, data, data_len);
76  int r = DatasetLookup(sd->set, data, data_len);
77  SCLogDebug("r %d", r);
78  if (r == 1)
79  return 1;
80  break;
81  }
83  //PrintRawDataFp(stdout, data, data_len);
84  int r = DatasetLookup(sd->set, data, data_len);
85  SCLogDebug("r %d", r);
86  if (r < 1)
87  return 1;
88  break;
89  }
91  //PrintRawDataFp(stdout, data, data_len);
92  int r = DatasetAdd(sd->set, data, data_len);
93  if (r == 1)
94  return 1;
95  break;
96  }
97  default:
98  abort();
99  }
100  return 0;
101 }
102 
103 static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *name, int name_len,
104  enum DatasetTypes *type, char *load, size_t load_size, char *save, size_t save_size,
105  uint64_t *memcap, uint32_t *hashsize)
106 {
107  bool cmd_set = false;
108  bool name_set = false;
109  bool load_set = false;
110  bool save_set = false;
111  bool state_set = false;
112 
113  char copy[strlen(str)+1];
114  strlcpy(copy, str, sizeof(copy));
115  char *xsaveptr = NULL;
116  char *key = strtok_r(copy, ",", &xsaveptr);
117  while (key != NULL) {
118  while (*key != '\0' && isblank(*key)) {
119  key++;
120  }
121  char *val = strchr(key, ' ');
122  if (val != NULL) {
123  *val++ = '\0';
124  while (*val != '\0' && isblank(*val)) {
125  val++;
126  SCLogDebug("cmd %s val %s", key, val);
127  }
128  } else {
129  SCLogDebug("cmd %s", key);
130  }
131 
132  if (strlen(key) == 0) {
133  goto next;
134  }
135 
136  if (!cmd_set) {
137  if (val) {
138  return -1;
139  }
140  strlcpy(cmd, key, cmd_len);
141  cmd_set = true;
142  } else if (!name_set) {
143  if (val) {
144  return -1;
145  }
146  strlcpy(name, key, name_len);
147  name_set = true;
148  } else {
149  if (val == NULL) {
150  return -1;
151  }
152 
153  if (strcmp(key, "type") == 0) {
154  SCLogDebug("type %s", val);
155 
156  if (strcmp(val, "md5") == 0) {
158  } else if (strcmp(val, "sha256") == 0) {
160  } else if (strcmp(val, "string") == 0) {
162  } else {
163  SCLogError(SC_ERR_INVALID_SIGNATURE, "bad type %s", val);
164  return -1;
165  }
166 
167  } else if (strcmp(key, "save") == 0) {
168  if (save_set) {
170  "'save' can only appear once");
171  return -1;
172  }
173  SCLogDebug("save %s", val);
174  strlcpy(save, val, save_size);
175  save_set = true;
176  } else if (strcmp(key, "load") == 0) {
177  if (load_set) {
179  "'load' can only appear once");
180  return -1;
181  }
182  SCLogDebug("load %s", val);
183  strlcpy(load, val, load_size);
184  load_set = true;
185  } else if (strcmp(key, "state") == 0) {
186  if (state_set) {
188  "'state' can only appear once");
189  return -1;
190  }
191  SCLogDebug("state %s", val);
192  strlcpy(load, val, load_size);
193  strlcpy(save, val, save_size);
194  state_set = true;
195  }
196  if (strcmp(key, "memcap") == 0) {
197  if (ParseSizeStringU64(val, memcap) < 0) {
199  "invalid value for memcap: %s,"
200  " resetting to default",
201  val);
202  *memcap = 0;
203  }
204  }
205  if (strcmp(key, "hashsize") == 0) {
206  if (ParseSizeStringU32(val, hashsize) < 0) {
208  "invalid value for hashsize: %s,"
209  " resetting to default",
210  val);
211  *hashsize = 0;
212  }
213  }
214  }
215 
216  SCLogDebug("key: %s, value: %s", key, val);
217 
218  next:
219  key = strtok_r(NULL, ",", &xsaveptr);
220  }
221 
222  if ((load_set || save_set) && state_set) {
224  "'state' can not be mixed with 'load' and 'save'");
225  return -1;
226  }
227 
228  /* Trim trailing whitespace. */
229  while (strlen(name) > 0 && isblank(name[strlen(name) - 1])) {
230  name[strlen(name) - 1] = '\0';
231  }
232 
233  /* Validate name, spaces are not allowed. */
234  for (size_t i = 0; i < strlen(name); i++) {
235  if (isblank(name[i])) {
237  "spaces not allowed in dataset names");
238  return 0;
239  }
240  }
241 
242  return 1;
243 }
244 
245 /** \brief wrapper around dirname that does leave input untouched */
246 static void GetDirName(const char *in, char *out, size_t outs)
247 {
248  if (strlen(in) == 0) {
249  return;
250  }
251 
252  size_t size = strlen(in) + 1;
253  char tmp[size];
254  strlcpy(tmp, in, size);
255 
256  char *dir = dirname(tmp);
257  BUG_ON(dir == NULL);
258  strlcpy(out, dir, outs);
259  return;
260 }
261 
262 static int SetupLoadPath(const DetectEngineCtx *de_ctx,
263  char *load, size_t load_size)
264 {
265  SCLogDebug("load %s", load);
266 
267  if (PathIsAbsolute(load)) {
268  return 0;
269  }
270 
271  bool done = false;
272 #ifdef HAVE_LIBGEN_H
273  BUG_ON(de_ctx->rule_file == NULL);
274 
275  char dir[PATH_MAX] = "";
276  GetDirName(de_ctx->rule_file, dir, sizeof(dir));
277 
278  SCLogDebug("rule_file %s dir %s", de_ctx->rule_file, dir);
279  char path[PATH_MAX];
280  if (snprintf(path, sizeof(path), "%s/%s", dir, load) >= (int)sizeof(path)) // TODO windows path
281  return -1;
282 
283  if (SCPathExists(path)) {
284  done = true;
285  strlcpy(load, path, load_size);
286  SCLogDebug("using path '%s' (HAVE_LIBGEN_H)", load);
287  }
288 #endif
289  if (!done) {
290  char *loadp = DetectLoadCompleteSigPath(de_ctx, load);
291  if (loadp == NULL) {
292  return -1;
293  }
294  SCLogDebug("loadp %s", loadp);
295 
296  if (SCPathExists(loadp)) {
297  strlcpy(load, loadp, load_size);
298  SCLogDebug("using path '%s' (non-HAVE_LIBGEN_H)", load);
299  }
300  SCFree(loadp);
301  }
302  return 0;
303 }
304 
305 static int SetupSavePath(const DetectEngineCtx *de_ctx,
306  char *save, size_t save_size)
307 {
308  SCLogDebug("save %s", save);
309 
310  if (PathIsAbsolute(save)) {
311  return 0;
312  }
313 
314  // data dir
315  const char *dir = ConfigGetDataDirectory();
316  BUG_ON(dir == NULL); // should not be able to fail
317  char path[PATH_MAX];
318  if (snprintf(path, sizeof(path), "%s/%s", dir, save) >= (int)sizeof(path)) // TODO windows path
319  return -1;
320 
321  /* TODO check if location exists and is writable */
322 
323  strlcpy(save, path, save_size);
324 
325  return 0;
326 }
327 
328 int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr)
329 {
330  DetectDatasetData *cd = NULL;
331  SigMatch *sm = NULL;
332  uint8_t cmd = 0;
333  uint64_t memcap = 0;
334  uint32_t hashsize = 0;
335  char cmd_str[16] = "", name[DATASET_NAME_MAX_LEN + 1] = "";
337  char load[PATH_MAX] = "";
338  char save[PATH_MAX] = "";
339 
340  if (DetectBufferGetActiveList(de_ctx, s) == -1) {
342  "datasets are only supported for sticky buffers");
343  SCReturnInt(-1);
344  }
345 
346  int list = s->init_data->list;
347  if (list == DETECT_SM_LIST_NOTSET) {
349  "datasets are only supported for sticky buffers");
350  SCReturnInt(-1);
351  }
352 
353  if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name, sizeof(name), &type, load,
354  sizeof(load), save, sizeof(save), &memcap, &hashsize)) {
355  return -1;
356  }
357 
358  if (strcmp(cmd_str,"isset") == 0) {
360  } else if (strcmp(cmd_str,"isnotset") == 0) {
362  } else if (strcmp(cmd_str,"set") == 0) {
364  } else if (strcmp(cmd_str,"unset") == 0) {
366  } else {
368  "dataset action \"%s\" is not supported.", cmd_str);
369  return -1;
370  }
371 
372  /* if just 'load' is set, we load data from the same dir as the
373  * rule file. If load+save is used, we use data dir */
374  if (strlen(save) == 0 && strlen(load) != 0) {
375  if (SetupLoadPath(de_ctx, load, sizeof(load)) != 0)
376  return -1;
377  /* if just 'save' is set, we use either full path or the
378  * data-dir */
379  } else if (strlen(save) != 0 && strlen(load) == 0) {
380  if (SetupSavePath(de_ctx, save, sizeof(save)) != 0)
381  return -1;
382  /* use 'save' logic for 'state', but put the resulting
383  * path into 'load' as well. */
384  } else if (strlen(save) != 0 && strlen(load) != 0 &&
385  strcmp(save, load) == 0) {
386  if (SetupSavePath(de_ctx, save, sizeof(save)) != 0)
387  return -1;
388  strlcpy(load, save, sizeof(load));
389  }
390 
391  SCLogDebug("name '%s' load '%s' save '%s'", name, load, save);
392  Dataset *set = DatasetGet(name, type, save, load, memcap, hashsize);
393  if (set == NULL) {
395  "failed to set up dataset '%s'.", name);
396  return -1;
397  }
398  if (set->hash && SC_ATOMIC_GET(set->hash->memcap_reached)) {
399  SCLogError(SC_ERR_THASH_INIT, "dataset too large for set memcap");
400  return -1;
401  }
402 
403  cd = SCCalloc(1, sizeof(DetectDatasetData));
404  if (unlikely(cd == NULL))
405  goto error;
406 
407  cd->set = set;
408  cd->cmd = cmd;
409 
410  SCLogDebug("cmd %s, name %s",
411  cmd_str, strlen(name) ? name : "(none)");
412 
413  /* Okay so far so good, lets get this into a SigMatch
414  * and put it in the Signature. */
415  sm = SigMatchAlloc();
416  if (sm == NULL)
417  goto error;
418 
419  sm->type = DETECT_DATASET;
420  sm->ctx = (SigMatchCtx *)cd;
421  SigMatchAppendSMToList(s, sm, list);
422  return 0;
423 
424 error:
425  if (cd != NULL)
426  SCFree(cd);
427  if (sm != NULL)
428  SCFree(sm);
429  return -1;
430 }
431 
433 {
435  if (fd == NULL)
436  return;
437 
438  SCFree(fd);
439 }
DetectDatasetData_
Definition: detect-dataset.h:36
SigTableElmt_::url
const char * url
Definition: detect.h:1204
detect-engine.h
DetectDatasetData_::cmd
uint8_t cmd
Definition: detect-dataset.h:38
SigTableElmt_::desc
const char * desc
Definition: detect.h:1203
SC_ERR_INVALID_VALUE
@ SC_ERR_INVALID_VALUE
Definition: util-error.h:160
SigTableElmt_::Free
void(* Free)(DetectEngineCtx *, void *)
Definition: detect.h:1191
PARSE_REGEX
#define PARSE_REGEX
Definition: detect-dataset.c:42
DetectParseRegex
Definition: detect-parse.h:42
SigTableElmt_::name
const char * name
Definition: detect.h:1201
DetectEngineCtx_::rule_file
char * rule_file
Definition: detect.h:863
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
ParseSizeStringU64
int ParseSizeStringU64(const char *size, uint64_t *res)
Definition: util-misc.c:200
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
Dataset::hash
THashTableContext * hash
Definition: datasets.h:44
threads.h
DETECT_DATASET_CMD_SET
#define DETECT_DATASET_CMD_SET
Definition: detect-dataset.h:29
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:758
SC_ERR_INVALID_SIGNATURE
@ SC_ERR_INVALID_SIGNATURE
Definition: util-error.h:69
DATASET_TYPE_SHA256
@ DATASET_TYPE_SHA256
Definition: datasets.h:34
DetectDatasetData_::set
Dataset * set
Definition: detect-dataset.h:37
DetectBufferGetActiveList
int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s)
Definition: detect-engine.c:954
SigTableElmt_::Setup
int(* Setup)(DetectEngineCtx *, Signature *, const char *)
Definition: detect.h:1186
hashsize
#define hashsize(n)
Definition: util-hash-lookup3.c:67
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
DatasetGet
Dataset * DatasetGet(const char *name, enum DatasetTypes type, const char *save, const char *load, uint64_t memcap, uint32_t hashsize)
Definition: datasets.c:423
datasets.h
decode.h
util-debug.h
type
uint8_t type
Definition: decode-icmpv4.h:0
DatasetAdd
int DatasetAdd(Dataset *set, const uint8_t *data, const uint32_t data_len)
Definition: datasets.c:1103
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:17
DetectEngineThreadCtx_
Definition: detect.h:1003
SC_ERR_UNKNOWN_VALUE
@ SC_ERR_UNKNOWN_VALUE
Definition: util-error.h:159
DetectDatasetFree
void DetectDatasetFree(DetectEngineCtx *, void *)
Definition: detect-dataset.c:432
DATASET_TYPE_NOTSET
#define DATASET_TYPE_NOTSET
Definition: datasets.h:31
DetectSetupParseRegexes
void DetectSetupParseRegexes(const char *parse_str, DetectParseRegex *detect_parse)
Definition: detect-parse.c:2558
SignatureInitData_::list
int list
Definition: detect.h:494
util-print.h
detect-engine-mpm.h
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
SigMatch_::ctx
SigMatchCtx * ctx
Definition: detect.h:323
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
DetectDatasetRegister
void DetectDatasetRegister(void)
Definition: detect-dataset.c:50
Packet_
Definition: decode.h:414
DETECT_DATASET_CMD_ISSET
#define DETECT_DATASET_CMD_ISSET
Definition: detect-dataset.h:32
Signature_::init_data
SignatureInitData * init_data
Definition: detect.h:587
DatasetLookup
int DatasetLookup(Dataset *set, const uint8_t *data, const uint32_t data_len)
see if data is part of the set
Definition: datasets.c:954
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
SigMatchAlloc
SigMatch * SigMatchAlloc(void)
Definition: detect-parse.c:235
SigMatch_::type
uint8_t type
Definition: detect.h:321
SigMatchCtx_
Used to start a pointer to SigMatch context Should never be dereferenced without casting to something...
Definition: detect.h:315
DETECT_SM_LIST_NOTSET
#define DETECT_SM_LIST_NOTSET
Definition: detect.h:115
SCPathExists
bool SCPathExists(const char *path)
Check if a path exists.
Definition: util-path.c:170
DetectDatasetBufferMatch
int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd, const uint8_t *data, const uint32_t data_len)
Definition: detect-dataset.c:66
DatasetTypes
DatasetTypes
Definition: datasets.h:30
DETECT_DATASET_CMD_UNSET
#define DETECT_DATASET_CMD_UNSET
Definition: detect-dataset.h:30
suricata-common.h
DATASET_NAME_MAX_LEN
#define DATASET_NAME_MAX_LEN
Definition: datasets.h:37
sigmatch_table
SigTableElmt sigmatch_table[DETECT_TBLSIZE]
Definition: detect-parse.c:73
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:45
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
detect-dataset.h
ParseSizeStringU32
int ParseSizeStringU32(const char *size, uint32_t *res)
Definition: util-misc.c:183
SC_ERR_THASH_INIT
@ SC_ERR_THASH_INIT
Definition: util-error.h:354
DETECT_DATASET_CMD_ISNOTSET
#define DETECT_DATASET_CMD_ISNOTSET
Definition: detect-dataset.h:31
str
#define str(s)
Definition: suricata-common.h:272
SCLogWarning
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:244
SCFree
#define SCFree(p)
Definition: util-mem.h:61
detect-parse.h
Signature_
Signature container.
Definition: detect.h:517
SigMatch_
a single match condition for a signature
Definition: detect.h:320
DATASET_TYPE_MD5
@ DATASET_TYPE_MD5
Definition: datasets.h:33
DATASET_TYPE_STRING
@ DATASET_TYPE_STRING
Definition: datasets.h:32
DetectLoadCompleteSigPath
char * DetectLoadCompleteSigPath(const DetectEngineCtx *de_ctx, const char *sig_file)
Create the path if default-rule-path was specified.
Definition: detect-engine-loader.c:60
DetectDatasetMatch
int DetectDatasetMatch(ThreadVars *, DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *)
Dataset
Definition: datasets.h:38
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
util-misc.h
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SigMatchAppendSMToList
void SigMatchAppendSMToList(Signature *s, SigMatch *new, int list)
Append a SigMatch to the list type.
Definition: detect-parse.c:349
ConfigGetDataDirectory
const char * ConfigGetDataDirectory()
Definition: util-conf.c:82
DETECT_DATASET
@ DETECT_DATASET
Definition: detect-engine-register.h:99