45 static uint32_t set_ids = 0;
47 static int DatasetAddwRep(
Dataset *set,
const uint8_t *data,
const uint32_t data_len,
50 static inline void DatasetUnlockData(
THashData *d)
55 static bool DatasetIsStatic(
const char *save,
const char *load);
56 static void GetDefaultMemcap(uint64_t *memcap, uint32_t *
hashsize);
60 if (strcasecmp(
"md5", s) == 0)
62 if (strcasecmp(
"sha256", s) == 0)
64 if (strcasecmp(
"string", s) == 0)
66 if (strcasecmp(
"ipv4", s) == 0)
68 if (strcasecmp(
"ip", s) == 0)
73 static Dataset *DatasetAlloc(
const char *name)
82 static Dataset *DatasetSearchByName(
const char *name)
86 if (strcasecmp(name, set->
name) == 0 && set->
hidden ==
false) {
94 static int HexToRaw(
const uint8_t *in,
size_t ins, uint8_t *out,
size_t outs)
104 memset(hash, 0, outs);
106 for (x = 0, i = 0; i < ins; i+=2, x++) {
107 char buf[3] = { 0, 0, 0 };
111 long value = strtol(buf, NULL, 16);
112 if (value >= 0 && value <= 255)
113 hash[x] = (uint8_t)value;
115 SCLogError(
"hash byte out of range %ld", value);
120 memcpy(out, hash, outs);
124 static int ParseRepLine(
const char *in,
size_t ins,
DataRepType *rep_out)
128 memcpy(raw, in, ins);
132 char *ptrs[1] = {NULL};
136 while (i < ins + 1) {
137 if (line[i] ==
',' || line[i] ==
'\n' || line[i] ==
'\0') {
158 if (r != (
int)strlen(ptrs[0])) {
159 SCLogError(
"'%s' is not a valid reputation value (0-65535)", ptrs[0]);
168 static int DatasetLoadIPv4(
Dataset *set)
170 if (strlen(set->
load) == 0)
174 const char *fopen_mode =
"r";
175 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
179 FILE *fp = fopen(set->
load, fopen_mode);
187 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
188 char *r = strchr(line,
',');
190 line[strlen(line) - 1] =
'\0';
194 if (inet_pton(AF_INET, line, &in) != 1) {
199 if (
DatasetAdd(set, (
const uint8_t *)&in.s_addr, 4) < 0) {
207 line[strlen(line) - 1] =
'\0';
213 if (inet_pton(AF_INET, line, &in) != 1) {
221 if (ParseRepLine(r, strlen(r), &rep) < 0) {
227 if (DatasetAddwRep(set, (
const uint8_t *)&in.s_addr, 4, &rep) < 0) {
242 static int ParseIpv6String(
Dataset *set,
char *line,
struct in6_addr *in6)
245 char *got_colon = strchr(line,
':');
248 if (inet_pton(AF_INET6, line, in6) != 1) {
252 memcpy(&ip6addr, in6->s6_addr,
sizeof(ip6addr));
254 if (ip6addr[0] == 0 && ip6addr[1] == 0 && ip6addr[2] == 0xFFFF0000) {
255 ip6addr[0] = ip6addr[3];
258 memcpy(in6, ip6addr,
sizeof(
struct in6_addr));
263 if (inet_pton(AF_INET, line, &in) != 1) {
267 memset(in6, 0,
sizeof(
struct in6_addr));
268 memcpy(in6, &in,
sizeof(
struct in_addr));
273 static int DatasetLoadIPv6(
Dataset *set)
275 if (strlen(set->
load) == 0)
279 const char *fopen_mode =
"r";
280 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
284 FILE *fp = fopen(set->
load, fopen_mode);
292 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
293 char *r = strchr(line,
',');
295 line[strlen(line) - 1] =
'\0';
299 int ret = ParseIpv6String(set, line, &in6);
305 if (
DatasetAdd(set, (
const uint8_t *)&in6.s6_addr, 16) < 0) {
313 line[strlen(line) - 1] =
'\0';
319 int ret = ParseIpv6String(set, line, &in6);
328 if (ParseRepLine(r, strlen(r), &rep) < 0) {
334 if (DatasetAddwRep(set, (
const uint8_t *)&in6.s6_addr, 16, &rep) < 0) {
349 static int DatasetLoadMd5(
Dataset *set)
351 if (strlen(set->
load) == 0)
355 const char *fopen_mode =
"r";
356 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
360 FILE *fp = fopen(set->
load, fopen_mode);
368 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
370 if (strlen(line) == 33) {
371 line[strlen(line) - 1] =
'\0';
375 if (HexToRaw((
const uint8_t *)line, 32, hash,
sizeof(hash)) < 0) {
380 if (
DatasetAdd(set, (
const uint8_t *)hash, 16) < 0) {
387 }
else if (strlen(line) > 33 && line[32] ==
',') {
388 line[strlen(line) - 1] =
'\0';
392 if (HexToRaw((
const uint8_t *)line, 32, hash,
sizeof(hash)) < 0) {
398 if (ParseRepLine(line + 33, strlen(line) - 33, &rep) < 0) {
404 if (DatasetAddwRep(set, hash, 16, &rep) < 0) {
412 FatalErrorOnInit(
"MD5 bad line len %u: '%s'", (uint32_t)strlen(line), line);
423 static int DatasetLoadSha256(
Dataset *set)
425 if (strlen(set->
load) == 0)
429 const char *fopen_mode =
"r";
430 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
434 FILE *fp = fopen(set->
load, fopen_mode);
442 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
444 if (strlen(line) == 65) {
445 line[strlen(line) - 1] =
'\0';
449 if (HexToRaw((
const uint8_t *)line, 64, hash,
sizeof(hash)) < 0) {
454 if (
DatasetAdd(set, (
const uint8_t *)hash, (uint32_t)32) < 0) {
461 }
else if (strlen(line) > 65 && line[64] ==
',') {
462 line[strlen(line) - 1] =
'\0';
463 SCLogDebug(
"SHA-256 with REP line: '%s'", line);
466 if (HexToRaw((
const uint8_t *)line, 64, hash,
sizeof(hash)) < 0) {
472 if (ParseRepLine(line + 65, strlen(line) - 65, &rep) < 0) {
479 if (DatasetAddwRep(set, hash, 32, &rep) < 0) {
493 static int DatasetLoadString(
Dataset *set)
495 if (strlen(set->
load) == 0)
499 const char *fopen_mode =
"r";
500 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
504 FILE *fp = fopen(set->
load, fopen_mode);
512 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
513 if (strlen(line) <= 1)
516 char *r = strchr(line,
',');
518 line[strlen(line) - 1] =
'\0';
522 uint8_t decoded[strlen(line)];
523 uint32_t consumed = 0, num_decoded = 0;
531 if (
DatasetAdd(set, (
const uint8_t *)decoded, num_decoded) < 0) {
537 line[strlen(line) - 1] =
'\0';
543 uint8_t decoded[strlen(line)];
544 uint32_t consumed = 0, num_decoded = 0;
556 if (ParseRepLine(r, strlen(r), &rep) < 0) {
562 if (DatasetAddwRep(set, (
const uint8_t *)decoded, num_decoded, &rep) < 0) {
585 static void DatasetGetPath(
const char *in_path,
592 strlcpy(path, in_path,
sizeof(path));
593 strlcpy(out_path, path, out_size);
598 if (stat(data_dir, &st) != 0) {
599 SCLogDebug(
"data-dir '%s': %s", data_dir, strerror(errno));
603 snprintf(path,
sizeof(path),
"%s/%s", data_dir, in_path);
606 if (stat(path, &st) != 0) {
607 SCLogDebug(
"path %s: %s", path, strerror(errno));
609 snprintf(path,
sizeof(path),
"%s", in_path);
613 strlcpy(out_path, path, out_size);
614 SCLogDebug(
"in_path \'%s\' => \'%s\'", in_path, out_path);
621 Dataset *set = DatasetSearchByName(name);
635 uint64_t default_memcap = 0;
636 uint32_t default_hashsize = 0;
642 Dataset *set = DatasetSearchByName(name);
646 "exists and is of type %u",
652 if ((save == NULL || strlen(save) == 0) &&
653 (load == NULL || strlen(load) == 0)) {
657 if ((save == NULL && strlen(set->
save) > 0) ||
658 (save != NULL && strcmp(set->
save, save) != 0)) {
663 if ((load == NULL && strlen(set->
load) > 0) ||
664 (load != NULL && strcmp(set->
load, load) != 0)) {
680 set = DatasetAlloc(name);
687 if (save && strlen(save)) {
691 if (load && strlen(load)) {
697 snprintf(cnf_name,
sizeof(cnf_name),
"datasets.%s.hash", name);
699 GetDefaultMemcap(&default_memcap, &default_hashsize);
703 Md5StrCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
705 if (set->
hash == NULL)
707 if (DatasetLoadMd5(set) < 0)
712 StringCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
714 if (set->
hash == NULL)
716 if (DatasetLoadString(set) < 0)
722 memcap > 0 ? memcap : default_memcap,
724 if (set->
hash == NULL)
726 if (DatasetLoadSha256(set) < 0)
731 IPv4Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
733 if (set->
hash == NULL)
735 if (DatasetLoadIPv4(set) < 0)
740 IPv6Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
742 if (set->
hash == NULL)
744 if (DatasetLoadIPv6(set) < 0)
750 SCLogError(
"dataset too large for set memcap");
754 SCLogDebug(
"set %p/%s type %u save %s load %s",
773 static bool DatasetIsStatic(
const char *save,
const char *load)
778 if ((load != NULL && strlen(load) > 0) &&
779 (save == NULL || strlen(save) == 0)) {
809 SCLogDebug(
"Post Reload Cleanup starting.. Hidden sets will be removed");
815 if (cur->
hidden ==
false) {
833 static void GetDefaultMemcap(uint64_t *memcap, uint32_t *
hashsize)
835 const char *
str = NULL;
836 if (
ConfGet(
"datasets.defaults.memcap", &
str) == 1) {
839 " resetting to default",
844 if (
ConfGet(
"datasets.defaults.hashsize", &
str) == 1) {
847 " resetting to default",
858 uint64_t default_memcap = 0;
859 uint32_t default_hashsize = 0;
860 GetDefaultMemcap(&default_memcap, &default_hashsize);
861 if (datasets != NULL) {
865 if (iter->
name == NULL) {
870 char save[PATH_MAX] =
"";
871 char load[PATH_MAX] =
"";
875 const char *set_name = iter->
name;
884 if (set_type == NULL) {
892 DatasetGetPath(set_save->
val, save,
sizeof(save),
TYPE_STATE);
893 strlcpy(load, save,
sizeof(load));
898 DatasetGetPath(set_load->
val, load,
sizeof(load),
TYPE_LOAD);
906 " deduced: %s, resetting to default",
915 " deduced: %s, resetting to default",
921 snprintf(conf_str,
sizeof(conf_str),
"datasets.%d.%s", list_pos, set_name);
923 SCLogDebug(
"set %s type %s. Conf %s", set_name, set_type->
val, conf_str);
925 if (strcmp(set_type->
val,
"md5") == 0) {
927 memcap > 0 ? memcap : default_memcap,
933 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
936 }
else if (strcmp(set_type->
val,
"sha256") == 0) {
938 memcap > 0 ? memcap : default_memcap,
944 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
947 }
else if (strcmp(set_type->
val,
"string") == 0) {
949 memcap > 0 ? memcap : default_memcap,
955 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
958 }
else if (strcmp(set_type->
val,
"ipv4") == 0) {
960 memcap > 0 ? memcap : default_memcap,
966 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
969 }
else if (strcmp(set_type->
val,
"ip") == 0) {
971 memcap > 0 ? memcap : default_memcap,
977 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
1002 SCLogDebug(
"destroying datasets done: %p", sets);
1005 static int SaveCallback(
void *ctx,
const uint8_t *data,
const uint32_t data_len)
1010 return fwrite(data, data_len, 1, fp);
1015 static int Md5AsAscii(
const void *s,
char *out,
size_t out_size)
1025 static int Sha256AsAscii(
const void *s,
char *out,
size_t out_size)
1035 static int IPv4AsAscii(
const void *s,
char *out,
size_t out_size)
1045 static int IPv6AsAscii(
const void *s,
char *out,
size_t out_size)
1049 bool is_ipv4 =
true;
1050 for (
int i = 4; i <= 15; i++) {
1051 if (ip6->
ipv6[i] != 0) {
1072 if (strlen(set->
save) == 0)
1075 FILE *fp = fopen(set->
save,
"w");
1081 switch (set->
type) {
1107 static int DatasetLookupString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1115 DatasetUnlockData(rdata);
1122 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1129 StringType lookup = { .
ptr = (uint8_t *)data, .
len = data_len, .rep = *rep };
1135 DatasetUnlockData(rdata);
1141 static int DatasetLookupIPv4(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1150 memcpy(lookup.
ipv4, data, 4);
1153 DatasetUnlockData(rdata);
1171 memcpy(lookup.
ipv4, data, data_len);
1177 DatasetUnlockData(rdata);
1183 static int DatasetLookupIPv6(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1188 if (data_len != 16 && data_len != 4)
1192 memcpy(lookup.
ipv6, data, data_len);
1195 DatasetUnlockData(rdata);
1209 if (data_len != 16 && data_len != 4)
1213 memcpy(lookup.
ipv6, data, data_len);
1219 DatasetUnlockData(rdata);
1225 static int DatasetLookupMd5(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1234 memcpy(lookup.
md5, data, data_len);
1237 DatasetUnlockData(rdata);
1244 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1255 memcpy(lookup.
md5, data, data_len);
1261 DatasetUnlockData(rdata);
1267 static int DatasetLookupSha256(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1276 memcpy(lookup.
sha256, data, data_len);
1279 DatasetUnlockData(rdata);
1286 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1297 memcpy(lookup.
sha256, data, data_len);
1303 DatasetUnlockData(rdata);
1323 switch (set->
type) {
1325 return DatasetLookupString(set, data, data_len);
1327 return DatasetLookupMd5(set, data, data_len);
1329 return DatasetLookupSha256(set, data, data_len);
1331 return DatasetLookupIPv4(set, data, data_len);
1333 return DatasetLookupIPv6(set, data, data_len);
1345 switch (set->
type) {
1347 return DatasetLookupStringwRep(set, data, data_len, rep);
1349 return DatasetLookupMd5wRep(set, data, data_len, rep);
1351 return DatasetLookupSha256wRep(set, data, data_len, rep);
1353 return DatasetLookupIPv4wRep(set, data, data_len, rep);
1355 return DatasetLookupIPv6wRep(set, data, data_len, rep);
1365 static int DatasetAddString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1370 StringType lookup = { .ptr = (uint8_t *)data, .
len = data_len,
1374 DatasetUnlockData(res.
data);
1375 return res.
is_new ? 1 : 0;
1385 static int DatasetAddStringwRep(
1395 DatasetUnlockData(res.
data);
1396 return res.
is_new ? 1 : 0;
1401 static int DatasetAddIPv4(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1415 DatasetUnlockData(res.
data);
1416 return res.
is_new ? 1 : 0;
1421 static int DatasetAddIPv6(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1427 if (data_len != 16) {
1435 DatasetUnlockData(res.
data);
1436 return res.
is_new ? 1 : 0;
1441 static int DatasetAddIPv4wRep(
1454 DatasetUnlockData(res.
data);
1455 return res.
is_new ? 1 : 0;
1460 static int DatasetAddIPv6wRep(
1473 DatasetUnlockData(res.
data);
1474 return res.
is_new ? 1 : 0;
1479 static int DatasetAddMd5(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1491 DatasetUnlockData(res.
data);
1492 return res.
is_new ? 1 : 0;
1497 static int DatasetAddMd5wRep(
1510 DatasetUnlockData(res.
data);
1511 return res.
is_new ? 1 : 0;
1516 static int DatasetAddSha256wRep(
1529 DatasetUnlockData(res.
data);
1530 return res.
is_new ? 1 : 0;
1535 static int DatasetAddSha256(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1547 DatasetUnlockData(res.
data);
1548 return res.
is_new ? 1 : 0;
1558 switch (set->
type) {
1560 return DatasetAddString(set,
data, data_len);
1562 return DatasetAddMd5(set,
data, data_len);
1564 return DatasetAddSha256(set,
data, data_len);
1566 return DatasetAddIPv4(set,
data, data_len);
1568 return DatasetAddIPv6(set,
data, data_len);
1573 static int DatasetAddwRep(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len,
1579 switch (set->
type) {
1581 return DatasetAddStringwRep(set,
data, data_len, rep);
1583 return DatasetAddMd5wRep(set,
data, data_len, rep);
1585 return DatasetAddSha256wRep(set,
data, data_len, rep);
1587 return DatasetAddIPv4wRep(set,
data, data_len, rep);
1589 return DatasetAddIPv6wRep(set,
data, data_len, rep);
1596 static int DatasetOpSerialized(
Dataset *set,
const char *
string,
DatasetOpFunc DatasetOpString,
1603 switch (set->
type) {
1606 uint8_t decoded[strlen(
string)];
1607 uint32_t consumed = 0, num_decoded = 0;
1614 return DatasetOpString(set, decoded, num_decoded);
1617 if (strlen(
string) != 32)
1620 if (HexToRaw((
const uint8_t *)
string, 32, hash,
sizeof(hash)) < 0)
1622 return DatasetOpMd5(set, hash, 16);
1625 if (strlen(
string) != 64)
1628 if (HexToRaw((
const uint8_t *)
string, 64, hash,
sizeof(hash)) < 0)
1630 return DatasetOpSha256(set, hash, 32);
1634 if (inet_pton(AF_INET,
string, &in) != 1)
1636 return DatasetOpIPv4(set, (uint8_t *)&in.s_addr, 4);
1640 if (inet_pton(AF_INET6,
string, &in) != 1)
1642 return DatasetOpIPv6(set, (uint8_t *)&in.s6_addr, 16);
1656 return DatasetOpSerialized(set,
string, DatasetAddString, DatasetAddMd5, DatasetAddSha256,
1657 DatasetAddIPv4, DatasetAddIPv6);
1668 return DatasetOpSerialized(set,
string, DatasetLookupString, DatasetLookupMd5,
1669 DatasetLookupSha256, DatasetLookupIPv4, DatasetLookupIPv6);
1677 static int DatasetRemoveString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1682 StringType lookup = { .ptr = (uint8_t *)data, .
len = data_len,
1687 static int DatasetRemoveIPv4(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1696 memcpy(lookup.
ipv4, data, 4);
1700 static int DatasetRemoveIPv6(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1709 memcpy(lookup.
ipv6, data, 16);
1713 static int DatasetRemoveMd5(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1722 memcpy(lookup.
md5, data, 16);
1726 static int DatasetRemoveSha256(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1735 memcpy(lookup.
sha256, data, 32);
1746 return DatasetOpSerialized(set,
string, DatasetRemoveString, DatasetRemoveMd5,
1747 DatasetRemoveSha256, DatasetRemoveIPv4, DatasetRemoveIPv6);