suricata
detect-dataset.c
Go to the documentation of this file.
1 /* Copyright (C) 2018-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements the dataset keyword
24  */
25 
26 #include "suricata-common.h"
27 #include "decode.h"
28 #include "detect.h"
29 #include "threads.h"
30 #include "datasets.h"
31 #include "detect-dataset.h"
32 
33 #include "detect-parse.h"
34 #include "detect-engine.h"
35 #include "detect-engine-buffer.h"
36 #include "detect-engine-mpm.h"
37 #include "detect-engine-state.h"
38 
39 #include "util-debug.h"
40 #include "util-print.h"
41 #include "util-misc.h"
42 #include "util-path.h"
43 #include "util-conf.h"
44 #include "util-validate.h"
45 
46 #define DETECT_DATASET_CMD_SET 0
47 #define DETECT_DATASET_CMD_UNSET 1
48 #define DETECT_DATASET_CMD_ISNOTSET 2
49 #define DETECT_DATASET_CMD_ISSET 3
50 
51 static int DetectDatasetSetup (DetectEngineCtx *, Signature *, const char *);
52 void DetectDatasetFree (DetectEngineCtx *, void *);
53 
55 {
56  sigmatch_table[DETECT_DATASET].name = "dataset";
57  sigmatch_table[DETECT_DATASET].desc = "match sticky buffer against datasets (experimental)";
58  sigmatch_table[DETECT_DATASET].url = "/rules/dataset-keywords.html#dataset";
59  sigmatch_table[DETECT_DATASET].Setup = DetectDatasetSetup;
61 }
62 
63 /*
64  1 match
65  0 no match
66  -1 can't match
67  */
69  const DetectDatasetData *sd,
70  const uint8_t *data, const uint32_t data_len)
71 {
72  if (data == NULL || data_len == 0)
73  return 0;
74 
75  switch (sd->cmd) {
77  //PrintRawDataFp(stdout, data, data_len);
78  int r = DatasetLookup(sd->set, data, data_len);
79  SCLogDebug("r %d", r);
80  if (r == 1)
81  return 1;
82  break;
83  }
85  //PrintRawDataFp(stdout, data, data_len);
86  int r = DatasetLookup(sd->set, data, data_len);
87  SCLogDebug("r %d", r);
88  if (r < 1)
89  return 1;
90  break;
91  }
93  //PrintRawDataFp(stdout, data, data_len);
94  int r = DatasetAdd(sd->set, data, data_len);
95  if (r == 1)
96  return 1;
97  break;
98  }
100  int r = DatasetRemove(sd->set, data, data_len);
101  if (r == 1)
102  return 1;
103  break;
104  }
105  default:
106  DEBUG_VALIDATE_BUG_ON("unknown dataset command");
107  }
108  return 0;
109 }
110 
111 static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *name, int name_len,
112  enum DatasetTypes *type, char *load, size_t load_size, char *save, size_t save_size,
113  uint64_t *memcap, uint32_t *hashsize)
114 {
115  bool cmd_set = false;
116  bool name_set = false;
117  bool load_set = false;
118  bool save_set = false;
119  bool state_set = false;
120 
121  char copy[strlen(str)+1];
122  strlcpy(copy, str, sizeof(copy));
123  char *xsaveptr = NULL;
124  char *key = strtok_r(copy, ",", &xsaveptr);
125  while (key != NULL) {
126  while (*key != '\0' && isblank(*key)) {
127  key++;
128  }
129  char *val = strchr(key, ' ');
130  if (val != NULL) {
131  *val++ = '\0';
132  while (*val != '\0' && isblank(*val)) {
133  val++;
134  SCLogDebug("cmd %s val %s", key, val);
135  }
136  } else {
137  SCLogDebug("cmd %s", key);
138  }
139 
140  if (strlen(key) == 0) {
141  goto next;
142  }
143 
144  if (!cmd_set) {
145  if (val && strlen(val) != 0) {
146  return -1;
147  }
148  strlcpy(cmd, key, cmd_len);
149  cmd_set = true;
150  } else if (!name_set) {
151  if (val && strlen(val) != 0) {
152  return -1;
153  }
154  strlcpy(name, key, name_len);
155  name_set = true;
156  } else {
157  if (val == NULL) {
158  return -1;
159  }
160 
161  if (strcmp(key, "type") == 0) {
162  SCLogDebug("type %s", val);
163 
164  if (strcmp(val, "md5") == 0) {
166  } else if (strcmp(val, "sha256") == 0) {
168  } else if (strcmp(val, "string") == 0) {
170  } else if (strcmp(val, "ipv4") == 0) {
172  } else if (strcmp(val, "ipv6") == 0) {
174  } else if (strcmp(val, "ip") == 0) {
176  } else {
177  SCLogError("bad type %s", val);
178  return -1;
179  }
180 
181  } else if (strcmp(key, "save") == 0) {
182  if (save_set) {
183  SCLogWarning("'save' can only appear once");
184  return -1;
185  }
186  SCLogDebug("save %s", val);
187  strlcpy(save, val, save_size);
188  save_set = true;
189  } else if (strcmp(key, "load") == 0) {
190  if (load_set) {
191  SCLogWarning("'load' can only appear once");
192  return -1;
193  }
194  SCLogDebug("load %s", val);
195  strlcpy(load, val, load_size);
196  load_set = true;
197  } else if (strcmp(key, "state") == 0) {
198  if (state_set) {
199  SCLogWarning("'state' can only appear once");
200  return -1;
201  }
202  SCLogDebug("state %s", val);
203  strlcpy(load, val, load_size);
204  strlcpy(save, val, save_size);
205  state_set = true;
206  }
207  if (strcmp(key, "memcap") == 0) {
208  if (ParseSizeStringU64(val, memcap) < 0) {
209  SCLogWarning("invalid value for memcap: %s,"
210  " resetting to default",
211  val);
212  *memcap = 0;
213  }
214  }
215  if (strcmp(key, "hashsize") == 0) {
216  if (ParseSizeStringU32(val, hashsize) < 0) {
217  SCLogWarning("invalid value for hashsize: %s,"
218  " resetting to default",
219  val);
220  *hashsize = 0;
221  }
222  }
223  }
224 
225  SCLogDebug("key: %s, value: %s", key, val);
226 
227  next:
228  key = strtok_r(NULL, ",", &xsaveptr);
229  }
230 
231  if ((load_set || save_set) && state_set) {
232  SCLogWarning("'state' can not be mixed with 'load' and 'save'");
233  return -1;
234  }
235 
236  /* Trim trailing whitespace. */
237  while (strlen(name) > 0 && isblank(name[strlen(name) - 1])) {
238  name[strlen(name) - 1] = '\0';
239  }
240 
241  /* Validate name, spaces are not allowed. */
242  for (size_t i = 0; i < strlen(name); i++) {
243  if (isblank(name[i])) {
244  SCLogError("spaces not allowed in dataset names");
245  return 0;
246  }
247  }
248 
249  return 1;
250 }
251 
252 /** \brief wrapper around dirname that does leave input untouched */
253 static void GetDirName(const char *in, char *out, size_t outs)
254 {
255  if (strlen(in) == 0) {
256  return;
257  }
258 
259  size_t size = strlen(in) + 1;
260  char tmp[size];
261  strlcpy(tmp, in, size);
262 
263  char *dir = dirname(tmp);
264  BUG_ON(dir == NULL);
265  strlcpy(out, dir, outs);
266 }
267 
268 static int SetupLoadPath(const DetectEngineCtx *de_ctx,
269  char *load, size_t load_size)
270 {
271  SCLogDebug("load %s", load);
272 
273  if (PathIsAbsolute(load)) {
274  return 0;
275  }
276 
277  bool done = false;
278 #ifdef HAVE_LIBGEN_H
279  BUG_ON(de_ctx->rule_file == NULL);
280 
281  char dir[PATH_MAX] = "";
282  GetDirName(de_ctx->rule_file, dir, sizeof(dir));
283 
284  SCLogDebug("rule_file %s dir %s", de_ctx->rule_file, dir);
285  char path[PATH_MAX];
286  if (snprintf(path, sizeof(path), "%s/%s", dir, load) >= (int)sizeof(path)) // TODO windows path
287  return -1;
288 
289  if (SCPathExists(path)) {
290  done = true;
291  strlcpy(load, path, load_size);
292  SCLogDebug("using path '%s' (HAVE_LIBGEN_H)", load);
293  }
294 #endif
295  if (!done) {
296  char *loadp = DetectLoadCompleteSigPath(de_ctx, load);
297  if (loadp == NULL) {
298  return -1;
299  }
300  SCLogDebug("loadp %s", loadp);
301 
302  if (SCPathExists(loadp)) {
303  strlcpy(load, loadp, load_size);
304  SCLogDebug("using path '%s' (non-HAVE_LIBGEN_H)", load);
305  }
306  SCFree(loadp);
307  }
308  return 0;
309 }
310 
311 static int SetupSavePath(const DetectEngineCtx *de_ctx,
312  char *save, size_t save_size)
313 {
314  SCLogDebug("save %s", save);
315 
316  int allow_save = 1;
317  if (SCConfGetBool("datasets.rules.allow-write", &allow_save)) {
318  if (!allow_save) {
319  SCLogError("Rules containing save/state datasets have been disabled");
320  return -1;
321  }
322  }
323 
324  int allow_absolute = 0;
325  (void)SCConfGetBool("datasets.rules.allow-absolute-filenames", &allow_absolute);
326  if (allow_absolute) {
327  SCLogNotice("Allowing absolute filename for dataset rule: %s", save);
328  } else {
329  if (PathIsAbsolute(save)) {
330  SCLogError("Absolute paths not allowed: %s", save);
331  return -1;
332  }
333 
334  if (SCPathContainsTraversal(save)) {
335  SCLogError("Directory traversals not allowed: %s", save);
336  return -1;
337  }
338  }
339 
340  // data dir
341  const char *dir = ConfigGetDataDirectory();
342  BUG_ON(dir == NULL); // should not be able to fail
343  char path[PATH_MAX];
344  if (snprintf(path, sizeof(path), "%s/%s", dir, save) >= (int)sizeof(path)) // TODO windows path
345  return -1;
346 
347  /* TODO check if location exists and is writable */
348 
349  strlcpy(save, path, save_size);
350 
351  return 0;
352 }
353 
354 int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr)
355 {
356  DetectDatasetData *cd = NULL;
357  uint8_t cmd = 0;
358  uint64_t memcap = 0;
359  uint32_t hashsize = 0;
360  char cmd_str[16] = "", name[DATASET_NAME_MAX_LEN + 1] = "";
362  char load[PATH_MAX] = "";
363  char save[PATH_MAX] = "";
364 
365  if (DetectBufferGetActiveList(de_ctx, s) == -1) {
366  SCLogError("datasets are only supported for sticky buffers");
367  SCReturnInt(-1);
368  }
369 
370  int list = s->init_data->list;
371  if (list == DETECT_SM_LIST_NOTSET) {
372  SCLogError("datasets are only supported for sticky buffers");
373  SCReturnInt(-1);
374  }
375 
376  if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name, sizeof(name), &type, load,
377  sizeof(load), save, sizeof(save), &memcap, &hashsize)) {
378  return -1;
379  }
380 
381  if (strcmp(cmd_str,"isset") == 0) {
383  } else if (strcmp(cmd_str,"isnotset") == 0) {
385  } else if (strcmp(cmd_str,"set") == 0) {
387  } else if (strcmp(cmd_str,"unset") == 0) {
389  } else {
390  SCLogError("dataset action \"%s\" is not supported.", cmd_str);
391  return -1;
392  }
393 
394  /* if just 'load' is set, we load data from the same dir as the
395  * rule file. If load+save is used, we use data dir */
396  if (strlen(save) == 0 && strlen(load) != 0) {
397  if (SetupLoadPath(de_ctx, load, sizeof(load)) != 0)
398  return -1;
399  /* if just 'save' is set, we use either full path or the
400  * data-dir */
401  } else if (strlen(save) != 0 && strlen(load) == 0) {
402  if (SetupSavePath(de_ctx, save, sizeof(save)) != 0)
403  return -1;
404  /* use 'save' logic for 'state', but put the resulting
405  * path into 'load' as well. */
406  } else if (strlen(save) != 0 && strlen(load) != 0 &&
407  strcmp(save, load) == 0) {
408  if (SetupSavePath(de_ctx, save, sizeof(save)) != 0)
409  return -1;
410  strlcpy(load, save, sizeof(load));
411  }
412 
413  SCLogDebug("name '%s' load '%s' save '%s'", name, load, save);
414  Dataset *set = DatasetGet(name, type, save, load, memcap, hashsize);
415  if (set == NULL) {
416  SCLogError("failed to set up dataset '%s'.", name);
417  return -1;
418  }
419 
420  cd = SCCalloc(1, sizeof(DetectDatasetData));
421  if (unlikely(cd == NULL))
422  goto error;
423 
424  cd->set = set;
425  cd->cmd = cmd;
426 
427  SCLogDebug("cmd %s, name %s",
428  cmd_str, strlen(name) ? name : "(none)");
429 
430  /* Okay so far so good, lets get this into a SigMatch
431  * and put it in the Signature. */
432 
433  if (SigMatchAppendSMToList(de_ctx, s, DETECT_DATASET, (SigMatchCtx *)cd, list) == NULL) {
434  goto error;
435  }
436  return 0;
437 
438 error:
439  if (cd != NULL)
440  SCFree(cd);
441  return -1;
442 }
443 
445 {
447  if (fd == NULL)
448  return;
449 
450  SCFree(fd);
451 }
DetectDatasetData_
Definition: detect-dataset.h:29
SigTableElmt_::url
const char * url
Definition: detect.h:1431
detect-engine.h
DetectDatasetData_::cmd
uint8_t cmd
Definition: detect-dataset.h:31
SigTableElmt_::desc
const char * desc
Definition: detect.h:1430
sigmatch_table
SigTableElmt * sigmatch_table
Definition: detect-parse.c:155
SigTableElmt_::Free
void(* Free)(DetectEngineCtx *, void *)
Definition: detect.h:1418
SigTableElmt_::name
const char * name
Definition: detect.h:1428
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
DETECT_DATASET_CMD_SET
#define DETECT_DATASET_CMD_SET
Definition: detect-dataset.c:46
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
ParseSizeStringU64
int ParseSizeStringU64(const char *size, uint64_t *res)
Definition: util-misc.c:190
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
threads.h
DetectEngineCtx_
main detection engine ctx
Definition: detect.h:931
SCConfGetBool
int SCConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:497
DATASET_TYPE_SHA256
@ DATASET_TYPE_SHA256
Definition: datasets.h:35
DetectDatasetData_::set
Dataset * set
Definition: detect-dataset.h:30
ConfigGetDataDirectory
const char * ConfigGetDataDirectory(void)
Definition: util-conf.c:80
SigTableElmt_::Setup
int(* Setup)(DetectEngineCtx *, Signature *, const char *)
Definition: detect.h:1413
DATASET_TYPE_IPV6
@ DATASET_TYPE_IPV6
Definition: datasets.h:37
DETECT_DATASET_CMD_ISSET
#define DETECT_DATASET_CMD_ISSET
Definition: detect-dataset.c:49
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
DatasetGet
Dataset * DatasetGet(const char *name, enum DatasetTypes type, const char *save, const char *load, uint64_t memcap, uint32_t hashsize)
Definition: datasets.c:357
SCPathContainsTraversal
bool SCPathContainsTraversal(const char *path)
Check for directory traversal.
Definition: util-path.c:271
DatasetRemove
int DatasetRemove(Dataset *set, const uint8_t *data, const uint32_t data_len)
Definition: datasets.c:1523
datasets.h
decode.h
util-debug.h
DatasetAdd
int DatasetAdd(Dataset *set, const uint8_t *data, const uint32_t data_len)
Definition: datasets.c:1320
de_ctx
DetectEngineCtx * de_ctx
Definition: fuzz_siginit.c:18
DetectEngineThreadCtx_
Definition: detect.h:1223
DetectDatasetFree
void DetectDatasetFree(DetectEngineCtx *, void *)
Definition: detect-dataset.c:444
DATASET_TYPE_NOTSET
#define DATASET_TYPE_NOTSET
Definition: datasets.h:32
SignatureInitData_::list
int list
Definition: detect.h:630
util-print.h
detect-engine-mpm.h
detect.h
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:317
DetectDatasetRegister
void DetectDatasetRegister(void)
Definition: detect-dataset.c:54
type
uint16_t type
Definition: decode-vlan.c:106
Signature_::init_data
SignatureInitData * init_data
Definition: detect.h:749
name
const char * name
Definition: tm-threads.c:2123
DatasetLookup
int DatasetLookup(Dataset *set, const uint8_t *data, const uint32_t data_len)
see if data is part of the set
Definition: datasets.c:1085
DATASET_TYPE_IPV4
@ DATASET_TYPE_IPV4
Definition: datasets.h:36
detect-engine-state.h
Data structures and function prototypes for keeping state for the detection engine.
SigMatchCtx_
Used to start a pointer to SigMatch context Should never be dereferenced without casting to something...
Definition: detect.h:351
DETECT_SM_LIST_NOTSET
#define DETECT_SM_LIST_NOTSET
Definition: detect.h:143
SCPathExists
bool SCPathExists(const char *path)
Check if a path exists.
Definition: util-path.c:183
DetectDatasetBufferMatch
int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd, const uint8_t *data, const uint32_t data_len)
Definition: detect-dataset.c:68
DatasetTypes
DatasetTypes
Definition: datasets.h:31
util-conf.h
DetectEngineCtx_::rule_file
const char * rule_file
Definition: detect.h:1023
suricata-common.h
util-path.h
DATASET_NAME_MAX_LEN
#define DATASET_NAME_MAX_LEN
Definition: datasets.h:40
DETECT_DATASET_CMD_UNSET
#define DETECT_DATASET_CMD_UNSET
Definition: detect-dataset.c:47
PathIsAbsolute
int PathIsAbsolute(const char *path)
Check if a path is absolute.
Definition: util-path.c:44
detect-engine-buffer.h
detect-dataset.h
hashsize
#define hashsize(n)
Definition: util-hash-lookup3.h:40
ParseSizeStringU32
int ParseSizeStringU32(const char *size, uint32_t *res)
Definition: util-misc.c:173
util-validate.h
str
#define str(s)
Definition: suricata-common.h:308
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
detect-parse.h
Signature_
Signature container.
Definition: detect.h:670
DETECT_DATASET_CMD_ISNOTSET
#define DETECT_DATASET_CMD_ISNOTSET
Definition: detect-dataset.c:48
DATASET_TYPE_MD5
@ DATASET_TYPE_MD5
Definition: datasets.h:34
DATASET_TYPE_STRING
@ DATASET_TYPE_STRING
Definition: datasets.h:33
DetectLoadCompleteSigPath
char * DetectLoadCompleteSigPath(const DetectEngineCtx *de_ctx, const char *sig_file)
Create the path if default-rule-path was specified.
Definition: detect-engine-loader.c:106
SigMatchAppendSMToList
SigMatch * SigMatchAppendSMToList(DetectEngineCtx *de_ctx, Signature *s, uint16_t type, SigMatchCtx *ctx, const int list)
Append a SigMatch to the list type.
Definition: detect-parse.c:464
Dataset
Definition: datasets.h:41
util-misc.h
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:237
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
DetectBufferGetActiveList
int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s)
Definition: detect-engine-buffer.c:109
DEBUG_VALIDATE_BUG_ON
#define DEBUG_VALIDATE_BUG_ON(exp)
Definition: util-validate.h:102
DETECT_DATASET
@ DETECT_DATASET
Definition: detect-engine-register.h:86