45 static uint32_t set_ids = 0;
47 static int DatasetAddwRep(
Dataset *set,
const uint8_t *data,
const uint32_t data_len,
50 static inline void DatasetUnlockData(
THashData *d)
55 static bool DatasetIsStatic(
const char *save,
const char *load);
56 static void GetDefaultMemcap(uint64_t *memcap, uint32_t *
hashsize);
60 if (strcasecmp(
"md5", s) == 0)
62 if (strcasecmp(
"sha256", s) == 0)
64 if (strcasecmp(
"string", s) == 0)
66 if (strcasecmp(
"ipv4", s) == 0)
68 if (strcasecmp(
"ip", s) == 0)
73 static Dataset *DatasetAlloc(
const char *name)
82 static Dataset *DatasetSearchByName(
const char *name)
86 if (strcasecmp(name, set->
name) == 0 && set->
hidden ==
false) {
94 static int HexToRaw(
const uint8_t *in,
size_t ins, uint8_t *out,
size_t outs)
104 memset(hash, 0, outs);
106 for (x = 0, i = 0; i < ins; i+=2, x++) {
107 char buf[3] = { 0, 0, 0 };
111 long value = strtol(buf, NULL, 16);
112 if (value >= 0 && value <= 255)
113 hash[x] = (uint8_t)value;
115 SCLogError(
"hash byte out of range %ld", value);
120 memcpy(out, hash, outs);
124 static int ParseRepLine(
const char *in,
size_t ins,
DataRepType *rep_out)
128 memcpy(raw, in, ins);
132 char *ptrs[1] = {NULL};
136 while (i < ins + 1) {
137 if (line[i] ==
',' || line[i] ==
'\n' || line[i] ==
'\0') {
158 if (r != (
int)strlen(ptrs[0])) {
159 SCLogError(
"'%s' is not a valid reputation value (0-65535)", ptrs[0]);
168 static int DatasetLoadIPv4(
Dataset *set)
170 if (strlen(set->
load) == 0)
174 const char *fopen_mode =
"r";
175 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
179 FILE *fp = fopen(set->
load, fopen_mode);
187 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
188 char *r = strchr(line,
',');
190 line[strlen(line) - 1] =
'\0';
194 if (inet_pton(AF_INET, line, &in) != 1) {
199 if (
DatasetAdd(set, (
const uint8_t *)&in.s_addr, 4) < 0) {
207 line[strlen(line) - 1] =
'\0';
213 if (inet_pton(AF_INET, line, &in) != 1) {
221 if (ParseRepLine(r, strlen(r), &rep) < 0) {
227 if (DatasetAddwRep(set, (
const uint8_t *)&in.s_addr, 4, &rep) < 0) {
242 static int ParseIpv6String(
Dataset *set,
char *line,
struct in6_addr *in6)
245 char *got_colon = strchr(line,
':');
248 if (inet_pton(AF_INET6, line, in6) != 1) {
252 memcpy(&ip6addr, in6->s6_addr,
sizeof(ip6addr));
254 if (ip6addr[0] == 0 && ip6addr[1] == 0 && ip6addr[2] == 0xFFFF0000) {
255 ip6addr[0] = ip6addr[3];
258 memcpy(in6, ip6addr,
sizeof(
struct in6_addr));
263 if (inet_pton(AF_INET, line, &in) != 1) {
267 memset(in6, 0,
sizeof(
struct in6_addr));
268 memcpy(in6, &in,
sizeof(
struct in_addr));
273 static int DatasetLoadIPv6(
Dataset *set)
275 if (strlen(set->
load) == 0)
279 const char *fopen_mode =
"r";
280 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
284 FILE *fp = fopen(set->
load, fopen_mode);
292 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
293 char *r = strchr(line,
',');
295 line[strlen(line) - 1] =
'\0';
299 int ret = ParseIpv6String(set, line, &in6);
305 if (
DatasetAdd(set, (
const uint8_t *)&in6.s6_addr, 16) < 0) {
313 line[strlen(line) - 1] =
'\0';
319 int ret = ParseIpv6String(set, line, &in6);
328 if (ParseRepLine(r, strlen(r), &rep) < 0) {
334 if (DatasetAddwRep(set, (
const uint8_t *)&in6.s6_addr, 16, &rep) < 0) {
349 static int DatasetLoadMd5(
Dataset *set)
351 if (strlen(set->
load) == 0)
355 const char *fopen_mode =
"r";
356 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
360 FILE *fp = fopen(set->
load, fopen_mode);
368 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
370 if (strlen(line) == 33) {
371 line[strlen(line) - 1] =
'\0';
375 if (HexToRaw((
const uint8_t *)line, 32, hash,
sizeof(hash)) < 0) {
380 if (
DatasetAdd(set, (
const uint8_t *)hash, 16) < 0) {
387 }
else if (strlen(line) > 33 && line[32] ==
',') {
388 line[strlen(line) - 1] =
'\0';
392 if (HexToRaw((
const uint8_t *)line, 32, hash,
sizeof(hash)) < 0) {
398 if (ParseRepLine(line + 33, strlen(line) - 33, &rep) < 0) {
404 if (DatasetAddwRep(set, hash, 16, &rep) < 0) {
412 FatalErrorOnInit(
"MD5 bad line len %u: '%s'", (uint32_t)strlen(line), line);
423 static int DatasetLoadSha256(
Dataset *set)
425 if (strlen(set->
load) == 0)
429 const char *fopen_mode =
"r";
430 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
434 FILE *fp = fopen(set->
load, fopen_mode);
442 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
444 if (strlen(line) == 65) {
445 line[strlen(line) - 1] =
'\0';
449 if (HexToRaw((
const uint8_t *)line, 64, hash,
sizeof(hash)) < 0) {
454 if (
DatasetAdd(set, (
const uint8_t *)hash, (uint32_t)32) < 0) {
461 }
else if (strlen(line) > 65 && line[64] ==
',') {
462 line[strlen(line) - 1] =
'\0';
463 SCLogDebug(
"SHA-256 with REP line: '%s'", line);
466 if (HexToRaw((
const uint8_t *)line, 64, hash,
sizeof(hash)) < 0) {
472 if (ParseRepLine(line + 65, strlen(line) - 65, &rep) < 0) {
479 if (DatasetAddwRep(set, hash, 32, &rep) < 0) {
493 static int DatasetLoadString(
Dataset *set)
495 if (strlen(set->
load) == 0)
499 const char *fopen_mode =
"r";
500 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
504 FILE *fp = fopen(set->
load, fopen_mode);
512 while (fgets(line, (
int)
sizeof(line), fp) != NULL) {
513 if (strlen(line) <= 1)
516 char *r = strchr(line,
',');
518 line[strlen(line) - 1] =
'\0';
522 uint8_t decoded[strlen(line)];
523 uint32_t consumed = 0, num_decoded = 0;
531 if (
DatasetAdd(set, (
const uint8_t *)decoded, num_decoded) < 0) {
537 line[strlen(line) - 1] =
'\0';
543 uint8_t decoded[strlen(line)];
544 uint32_t consumed = 0, num_decoded = 0;
556 if (ParseRepLine(r, strlen(r), &rep) < 0) {
562 if (DatasetAddwRep(set, (
const uint8_t *)decoded, num_decoded, &rep) < 0) {
585 static void DatasetGetPath(
const char *in_path,
592 strlcpy(path, in_path,
sizeof(path));
593 strlcpy(out_path, path, out_size);
598 if (stat(data_dir, &st) != 0) {
599 SCLogDebug(
"data-dir '%s': %s", data_dir, strerror(errno));
603 snprintf(path,
sizeof(path),
"%s/%s", data_dir, in_path);
606 if (stat(path, &st) != 0) {
607 SCLogDebug(
"path %s: %s", path, strerror(errno));
609 snprintf(path,
sizeof(path),
"%s", in_path);
613 strlcpy(out_path, path, out_size);
614 SCLogDebug(
"in_path \'%s\' => \'%s\'", in_path, out_path);
621 Dataset *set = DatasetSearchByName(name);
635 uint64_t default_memcap = 0;
636 uint32_t default_hashsize = 0;
642 Dataset *set = DatasetSearchByName(name);
646 "exists and is of type %u",
652 if ((save == NULL || strlen(save) == 0) &&
653 (load == NULL || strlen(load) == 0)) {
657 if ((save == NULL && strlen(set->
save) > 0) ||
658 (save != NULL && strcmp(set->
save, save) != 0)) {
663 if ((load == NULL && strlen(set->
load) > 0) ||
664 (load != NULL && strcmp(set->
load, load) != 0)) {
680 set = DatasetAlloc(name);
687 if (save && strlen(save)) {
691 if (load && strlen(load)) {
697 snprintf(cnf_name,
sizeof(cnf_name),
"datasets.%s.hash", name);
699 GetDefaultMemcap(&default_memcap, &default_hashsize);
703 Md5StrCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
705 if (set->
hash == NULL)
707 if (DatasetLoadMd5(set) < 0)
712 StringCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
714 if (set->
hash == NULL)
716 if (DatasetLoadString(set) < 0)
722 memcap > 0 ? memcap : default_memcap,
724 if (set->
hash == NULL)
726 if (DatasetLoadSha256(set) < 0)
731 IPv4Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
733 if (set->
hash == NULL)
735 if (DatasetLoadIPv4(set) < 0)
740 IPv6Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
742 if (set->
hash == NULL)
744 if (DatasetLoadIPv6(set) < 0)
749 SCLogDebug(
"set %p/%s type %u save %s load %s",
768 static bool DatasetIsStatic(
const char *save,
const char *load)
773 if ((load != NULL && strlen(load) > 0) &&
774 (save == NULL || strlen(save) == 0)) {
804 SCLogDebug(
"Post Reload Cleanup starting.. Hidden sets will be removed");
810 if (cur->
hidden ==
false) {
828 static void GetDefaultMemcap(uint64_t *memcap, uint32_t *
hashsize)
830 const char *
str = NULL;
831 if (
ConfGet(
"datasets.defaults.memcap", &
str) == 1) {
834 " resetting to default",
839 if (
ConfGet(
"datasets.defaults.hashsize", &
str) == 1) {
842 " resetting to default",
853 uint64_t default_memcap = 0;
854 uint32_t default_hashsize = 0;
855 GetDefaultMemcap(&default_memcap, &default_hashsize);
856 if (datasets != NULL) {
860 if (iter->
name == NULL) {
865 char save[PATH_MAX] =
"";
866 char load[PATH_MAX] =
"";
870 const char *set_name = iter->
name;
879 if (set_type == NULL) {
887 DatasetGetPath(set_save->
val, save,
sizeof(save),
TYPE_STATE);
888 strlcpy(load, save,
sizeof(load));
893 DatasetGetPath(set_load->
val, load,
sizeof(load),
TYPE_LOAD);
901 " deduced: %s, resetting to default",
910 " deduced: %s, resetting to default",
916 snprintf(conf_str,
sizeof(conf_str),
"datasets.%d.%s", list_pos, set_name);
918 SCLogDebug(
"set %s type %s. Conf %s", set_name, set_type->
val, conf_str);
920 if (strcmp(set_type->
val,
"md5") == 0) {
922 memcap > 0 ? memcap : default_memcap,
928 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
931 }
else if (strcmp(set_type->
val,
"sha256") == 0) {
933 memcap > 0 ? memcap : default_memcap,
939 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
942 }
else if (strcmp(set_type->
val,
"string") == 0) {
944 memcap > 0 ? memcap : default_memcap,
950 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
953 }
else if (strcmp(set_type->
val,
"ipv4") == 0) {
955 memcap > 0 ? memcap : default_memcap,
961 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
964 }
else if (strcmp(set_type->
val,
"ip") == 0) {
966 memcap > 0 ? memcap : default_memcap,
972 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
997 SCLogDebug(
"destroying datasets done: %p", sets);
1000 static int SaveCallback(
void *ctx,
const uint8_t *data,
const uint32_t data_len)
1005 return fwrite(data, data_len, 1, fp);
1010 static int Md5AsAscii(
const void *s,
char *out,
size_t out_size)
1020 static int Sha256AsAscii(
const void *s,
char *out,
size_t out_size)
1030 static int IPv4AsAscii(
const void *s,
char *out,
size_t out_size)
1040 static int IPv6AsAscii(
const void *s,
char *out,
size_t out_size)
1044 bool is_ipv4 =
true;
1045 for (
int i = 4; i <= 15; i++) {
1046 if (ip6->
ipv6[i] != 0) {
1067 if (strlen(set->
save) == 0)
1070 FILE *fp = fopen(set->
save,
"w");
1076 switch (set->
type) {
1102 static int DatasetLookupString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1110 DatasetUnlockData(rdata);
1117 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1124 StringType lookup = { .
ptr = (uint8_t *)data, .
len = data_len, .rep = *rep };
1130 DatasetUnlockData(rdata);
1136 static int DatasetLookupIPv4(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1145 memcpy(lookup.
ipv4, data, 4);
1148 DatasetUnlockData(rdata);
1166 memcpy(lookup.
ipv4, data, data_len);
1172 DatasetUnlockData(rdata);
1178 static int DatasetLookupIPv6(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1183 if (data_len != 16 && data_len != 4)
1187 memcpy(lookup.
ipv6, data, data_len);
1190 DatasetUnlockData(rdata);
1204 if (data_len != 16 && data_len != 4)
1208 memcpy(lookup.
ipv6, data, data_len);
1214 DatasetUnlockData(rdata);
1220 static int DatasetLookupMd5(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1229 memcpy(lookup.
md5, data, data_len);
1232 DatasetUnlockData(rdata);
1239 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1250 memcpy(lookup.
md5, data, data_len);
1256 DatasetUnlockData(rdata);
1262 static int DatasetLookupSha256(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1271 memcpy(lookup.
sha256, data, data_len);
1274 DatasetUnlockData(rdata);
1281 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1292 memcpy(lookup.
sha256, data, data_len);
1298 DatasetUnlockData(rdata);
1318 switch (set->
type) {
1320 return DatasetLookupString(set, data, data_len);
1322 return DatasetLookupMd5(set, data, data_len);
1324 return DatasetLookupSha256(set, data, data_len);
1326 return DatasetLookupIPv4(set, data, data_len);
1328 return DatasetLookupIPv6(set, data, data_len);
1340 switch (set->
type) {
1342 return DatasetLookupStringwRep(set, data, data_len, rep);
1344 return DatasetLookupMd5wRep(set, data, data_len, rep);
1346 return DatasetLookupSha256wRep(set, data, data_len, rep);
1348 return DatasetLookupIPv4wRep(set, data, data_len, rep);
1350 return DatasetLookupIPv6wRep(set, data, data_len, rep);
1360 static int DatasetAddString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1365 StringType lookup = { .ptr = (uint8_t *)data, .
len = data_len,
1369 DatasetUnlockData(res.
data);
1370 return res.
is_new ? 1 : 0;
1380 static int DatasetAddStringwRep(
1390 DatasetUnlockData(res.
data);
1391 return res.
is_new ? 1 : 0;
1396 static int DatasetAddIPv4(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1410 DatasetUnlockData(res.
data);
1411 return res.
is_new ? 1 : 0;
1416 static int DatasetAddIPv6(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1422 if (data_len != 16) {
1430 DatasetUnlockData(res.
data);
1431 return res.
is_new ? 1 : 0;
1436 static int DatasetAddIPv4wRep(
1449 DatasetUnlockData(res.
data);
1450 return res.
is_new ? 1 : 0;
1455 static int DatasetAddIPv6wRep(
1468 DatasetUnlockData(res.
data);
1469 return res.
is_new ? 1 : 0;
1474 static int DatasetAddMd5(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1486 DatasetUnlockData(res.
data);
1487 return res.
is_new ? 1 : 0;
1492 static int DatasetAddMd5wRep(
1505 DatasetUnlockData(res.
data);
1506 return res.
is_new ? 1 : 0;
1511 static int DatasetAddSha256wRep(
1524 DatasetUnlockData(res.
data);
1525 return res.
is_new ? 1 : 0;
1530 static int DatasetAddSha256(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1542 DatasetUnlockData(res.
data);
1543 return res.
is_new ? 1 : 0;
1553 switch (set->
type) {
1555 return DatasetAddString(set,
data, data_len);
1557 return DatasetAddMd5(set,
data, data_len);
1559 return DatasetAddSha256(set,
data, data_len);
1561 return DatasetAddIPv4(set,
data, data_len);
1563 return DatasetAddIPv6(set,
data, data_len);
1568 static int DatasetAddwRep(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len,
1574 switch (set->
type) {
1576 return DatasetAddStringwRep(set,
data, data_len, rep);
1578 return DatasetAddMd5wRep(set,
data, data_len, rep);
1580 return DatasetAddSha256wRep(set,
data, data_len, rep);
1582 return DatasetAddIPv4wRep(set,
data, data_len, rep);
1584 return DatasetAddIPv6wRep(set,
data, data_len, rep);
1591 static int DatasetOpSerialized(
Dataset *set,
const char *
string,
DatasetOpFunc DatasetOpString,
1598 switch (set->
type) {
1601 uint8_t decoded[strlen(
string)];
1602 uint32_t consumed = 0, num_decoded = 0;
1609 return DatasetOpString(set, decoded, num_decoded);
1612 if (strlen(
string) != 32)
1615 if (HexToRaw((
const uint8_t *)
string, 32, hash,
sizeof(hash)) < 0)
1617 return DatasetOpMd5(set, hash, 16);
1620 if (strlen(
string) != 64)
1623 if (HexToRaw((
const uint8_t *)
string, 64, hash,
sizeof(hash)) < 0)
1625 return DatasetOpSha256(set, hash, 32);
1629 if (inet_pton(AF_INET,
string, &in) != 1)
1631 return DatasetOpIPv4(set, (uint8_t *)&in.s_addr, 4);
1635 if (inet_pton(AF_INET6,
string, &in) != 1)
1637 return DatasetOpIPv6(set, (uint8_t *)&in.s6_addr, 16);
1651 return DatasetOpSerialized(set,
string, DatasetAddString, DatasetAddMd5, DatasetAddSha256,
1652 DatasetAddIPv4, DatasetAddIPv6);
1663 return DatasetOpSerialized(set,
string, DatasetLookupString, DatasetLookupMd5,
1664 DatasetLookupSha256, DatasetLookupIPv4, DatasetLookupIPv6);
1672 static int DatasetRemoveString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1677 StringType lookup = { .ptr = (uint8_t *)data, .
len = data_len,
1682 static int DatasetRemoveIPv4(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1691 memcpy(lookup.
ipv4, data, 4);
1695 static int DatasetRemoveIPv6(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1704 memcpy(lookup.
ipv6, data, 16);
1708 static int DatasetRemoveMd5(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1717 memcpy(lookup.
md5, data, 16);
1721 static int DatasetRemoveSha256(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1730 memcpy(lookup.
sha256, data, 32);
1741 return DatasetOpSerialized(set,
string, DatasetRemoveString, DatasetRemoveMd5,
1742 DatasetRemoveSha256, DatasetRemoveIPv4, DatasetRemoveIPv6);