48 static uint32_t set_ids = 0;
55 static void DatasetUpdateHashsize(
const char *
name, uint32_t hash_size);
57 static inline void DatasetUnlockData(
THashData *d)
62 static bool DatasetIsStatic(
const char *save,
const char *load);
66 if (strcasecmp(
"md5", s) == 0)
68 if (strcasecmp(
"sha256", s) == 0)
70 if (strcasecmp(
"string", s) == 0)
72 if (strcasecmp(
"ipv4", s) == 0)
74 if (strcasecmp(
"ip", s) == 0)
82 if (set->
hash == NULL) {
87 SCLogError(
"dataset too large for set memcap");
92 "set %p/%s type %u save %s load %s", set, set->
name, set->
type, set->
save, set->
load);
133 static int DatasetLoadIPv4(
Dataset *set)
135 if (strlen(set->
load) == 0)
139 const char *fopen_mode =
"r";
140 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
144 int retval = ParseDatasets(set, set->
name, set->
load, fopen_mode, DSIpv4);
147 }
else if (retval == -1) {
159 char *got_colon = strchr(line,
':');
162 if (inet_pton(AF_INET6, line, in6) != 1) {
166 memcpy(&ip6addr, in6->s6_addr,
sizeof(ip6addr));
168 if (ip6addr[0] == 0 && ip6addr[1] == 0 && ip6addr[2] == 0xFFFF0000) {
169 ip6addr[0] = ip6addr[3];
172 memcpy(in6, ip6addr,
sizeof(
struct in6_addr));
177 if (inet_pton(AF_INET, line, &in) != 1) {
181 memset(in6, 0,
sizeof(
struct in6_addr));
182 memcpy(in6, &in,
sizeof(
struct in_addr));
187 static int DatasetLoadIPv6(
Dataset *set)
189 if (strlen(set->
load) == 0)
193 const char *fopen_mode =
"r";
194 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
198 int retval = ParseDatasets(set, set->
name, set->
load, fopen_mode, DSIpv6);
201 }
else if (retval == -1) {
210 static int DatasetLoadMd5(
Dataset *set)
212 if (strlen(set->
load) == 0)
216 const char *fopen_mode =
"r";
217 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
221 int retval = ParseDatasets(set, set->
name, set->
load, fopen_mode, DSMd5);
224 }
else if (retval == -1) {
233 static int DatasetLoadSha256(
Dataset *set)
235 if (strlen(set->
load) == 0)
239 const char *fopen_mode =
"r";
240 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
244 int retval = ParseDatasets(set, set->
name, set->
load, fopen_mode, DSSha256);
247 }
else if (retval == -1) {
256 static int DatasetLoadString(
Dataset *set)
258 if (strlen(set->
load) == 0)
263 const char *fopen_mode =
"r";
264 if (strlen(set->
save) > 0 && strcmp(set->
save, set->
load) == 0) {
268 int retval = ParseDatasets(set, set->
name, set->
load, fopen_mode, DSString);
271 }
else if (retval == -1) {
287 static void DatasetGetPath(
294 strlcpy(path, in_path,
sizeof(path));
295 strlcpy(out_path, path, out_size);
300 if (stat(data_dir, &st) != 0) {
301 SCLogDebug(
"data-dir '%s': %s", data_dir, strerror(errno));
305 snprintf(path,
sizeof(path),
"%s/%s", data_dir, in_path);
308 if (stat(path, &st) != 0) {
309 SCLogDebug(
"path %s: %s", path, strerror(errno));
311 snprintf(path,
sizeof(path),
"%s", in_path);
315 strlcpy(out_path, path, out_size);
316 SCLogDebug(
"in_path \'%s\' => \'%s\'", in_path, out_path);
334 static bool DatasetCheckHashsize(
const char *
name, uint32_t hash_size)
337 SCLogError(
"hashsize %u in dataset '%s' exceeds configured 'single-hashsize' limit (%u)",
345 SCLogError(
"hashsize %u in dataset '%s' exceeds configured 'total-hashsizes' limit (%u, in "
354 static void DatasetUpdateHashsize(
const char *
name, uint32_t hash_size)
372 uint64_t default_memcap = 0;
373 uint32_t default_hashsize = 0;
382 "exists and is of type %u",
387 if ((save == NULL || strlen(save) == 0) &&
388 (load == NULL || strlen(load) == 0)) {
392 if ((save == NULL && strlen(set->
save) > 0) ||
393 (save != NULL && strcmp(set->
save, save) != 0)) {
398 if ((load == NULL && strlen(set->
load) > 0) ||
399 (load != NULL && strcmp(set->
load, load) != 0)) {
419 *memcap = default_memcap;
433 if (save && strlen(save)) {
437 if (load && strlen(load)) {
470 snprintf(cnf_name,
sizeof(cnf_name),
"datasets.%s.hash",
name);
475 if (set->
hash == NULL)
477 if (DatasetLoadMd5(set) < 0)
483 if (set->
hash == NULL)
485 if (DatasetLoadString(set) < 0)
492 if (set->
hash == NULL)
494 if (DatasetLoadSha256(set) < 0)
500 if (set->
hash == NULL)
502 if (DatasetLoadIPv4(set) < 0)
508 if (set->
hash == NULL)
510 if (DatasetLoadIPv6(set) < 0)
531 static bool DatasetIsStatic(
const char *save,
const char *load)
536 if ((load != NULL && strlen(load) > 0) &&
537 (save == NULL || strlen(save) == 0)) {
571 SCLogDebug(
"Post Reload Cleanup starting.. Hidden sets will be removed");
597 #define DATASETS_HASHSIZE_DEFAULT 4096
601 const char *
str = NULL;
605 " resetting to default",
612 if (
SCConfGet(
"datasets.defaults.hashsize", &
str) == 1) {
616 " resetting to default: %u",
626 uint64_t default_memcap = 0;
627 uint32_t default_hashsize = 0;
629 if (datasets != NULL) {
630 const char *
str = NULL;
631 if (
SCConfGet(
"datasets.limits.total-hashsizes", &
str) == 1) {
633 FatalError(
"failed to parse datasets.limits.total-hashsizes value: %s",
str);
636 if (
SCConfGet(
"datasets.limits.single-hashsize", &
str) == 1) {
638 FatalError(
"failed to parse datasets.limits.single-hashsize value: %s",
str);
643 FatalError(
"total-hashsizes (%u) cannot be smaller than single-hashsize (%u)",
654 if (iter->
name == NULL) {
659 char save[PATH_MAX] =
"";
660 char load[PATH_MAX] =
"";
664 const char *set_name = iter->
name;
672 if (set_type == NULL) {
679 DatasetGetPath(set_save->
val, save,
sizeof(save),
TYPE_STATE);
680 strlcpy(load, save,
sizeof(load));
684 DatasetGetPath(set_load->
val, load,
sizeof(load),
TYPE_LOAD);
692 " deduced: %s, resetting to default",
701 " deduced: %s, resetting to default",
707 snprintf(conf_str,
sizeof(conf_str),
"datasets.%d.%s", list_pos, set_name);
709 SCLogDebug(
"set %s type %s. Conf %s", set_name, set_type->
val, conf_str);
711 if (strcmp(set_type->
val,
"md5") == 0) {
713 memcap > 0 ? memcap : default_memcap,
719 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
722 }
else if (strcmp(set_type->
val,
"sha256") == 0) {
724 memcap > 0 ? memcap : default_memcap,
730 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
733 }
else if (strcmp(set_type->
val,
"string") == 0) {
735 memcap > 0 ? memcap : default_memcap,
741 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
744 }
else if (strcmp(set_type->
val,
"ipv4") == 0) {
746 memcap > 0 ? memcap : default_memcap,
752 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
755 }
else if (strcmp(set_type->
val,
"ip") == 0) {
757 memcap > 0 ? memcap : default_memcap,
763 SCLogDebug(
"dataset %s: id %u type %s", set_name, dset->
id, set_type->
val);
788 SCLogDebug(
"destroying datasets done: %p", sets);
791 static int SaveCallback(
void *
ctx,
const uint8_t *data,
const uint32_t data_len)
796 return (
int)fwrite(data, data_len, 1, fp);
801 static int Md5AsAscii(
const void *s,
char *out,
size_t out_size)
808 return (
int)strlen(out);
811 static int Sha256AsAscii(
const void *s,
char *out,
size_t out_size)
818 return (
int)strlen(out);
821 static int IPv4AsAscii(
const void *s,
char *out,
size_t out_size)
828 return (
int)strlen(out);
831 static int IPv6AsAscii(
const void *s,
char *out,
size_t out_size)
836 for (
int i = 4; i <= 15; i++) {
837 if (ip6->
ipv6[i] != 0) {
849 return (
int)strlen(out);
858 if (strlen(set->
save) == 0)
861 FILE *fp = fopen(set->
save,
"w");
893 static int DatasetLookupString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
898 StringType lookup = { .
ptr = (uint8_t *)data, .
len = data_len, .rep.value = 0 };
901 DatasetUnlockData(rdata);
908 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
915 StringType lookup = { .
ptr = (uint8_t *)data, .
len = data_len, .rep = *rep };
921 DatasetUnlockData(rdata);
927 static int DatasetLookupIPv4(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
936 memcpy(lookup.
ipv4, data, 4);
939 DatasetUnlockData(rdata);
946 Dataset *set,
const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
957 memcpy(lookup.
ipv4, data, data_len);
963 DatasetUnlockData(rdata);
969 static int DatasetLookupIPv6(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
974 if (data_len != 16 && data_len != 4)
978 memcpy(lookup.
ipv6, data, data_len);
981 DatasetUnlockData(rdata);
988 Dataset *set,
const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
995 if (data_len != 16 && data_len != 4)
999 memcpy(lookup.
ipv6, data, data_len);
1005 DatasetUnlockData(rdata);
1011 static int DatasetLookupMd5(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1020 memcpy(lookup.
md5, data, data_len);
1023 DatasetUnlockData(rdata);
1030 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1041 memcpy(lookup.
md5, data, data_len);
1047 DatasetUnlockData(rdata);
1053 static int DatasetLookupSha256(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1062 memcpy(lookup.
sha256, data, data_len);
1065 DatasetUnlockData(rdata);
1072 const uint8_t *data,
const uint32_t data_len,
const DataRepType *rep)
1083 memcpy(lookup.
sha256, data, data_len);
1089 DatasetUnlockData(rdata);
1109 switch (set->
type) {
1111 return DatasetLookupString(set, data, data_len);
1113 return DatasetLookupMd5(set, data, data_len);
1115 return DatasetLookupSha256(set, data, data_len);
1117 return DatasetLookupIPv4(set, data, data_len);
1119 return DatasetLookupIPv6(set, data, data_len);
1125 const DataRepType *rep)
1131 switch (set->
type) {
1133 return DatasetLookupStringwRep(set, data, data_len, rep);
1135 return DatasetLookupMd5wRep(set, data, data_len, rep);
1137 return DatasetLookupSha256wRep(set, data, data_len, rep);
1139 return DatasetLookupIPv4wRep(set, data, data_len, rep);
1141 return DatasetLookupIPv6wRep(set, data, data_len, rep);
1151 static int DatasetAddString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1156 StringType lookup = { .ptr = (uint8_t *)data, .
len = data_len,
1160 DatasetUnlockData(res.
data);
1161 return res.
is_new ? 1 : 0;
1171 static int DatasetAddStringwRep(
1172 Dataset *set,
const uint8_t *
data,
const uint32_t data_len,
const DataRepType *rep)
1181 DatasetUnlockData(res.
data);
1182 return res.
is_new ? 1 : 0;
1187 static int DatasetAddIPv4(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1201 DatasetUnlockData(res.
data);
1202 return res.
is_new ? 1 : 0;
1207 static int DatasetAddIPv6(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1213 if (data_len != 16 && data_len != 4) {
1218 memcpy(lookup.
ipv6,
data, data_len);
1221 DatasetUnlockData(res.
data);
1222 return res.
is_new ? 1 : 0;
1227 static int DatasetAddIPv4wRep(
1228 Dataset *set,
const uint8_t *
data,
const uint32_t data_len,
const DataRepType *rep)
1240 DatasetUnlockData(res.
data);
1241 return res.
is_new ? 1 : 0;
1246 static int DatasetAddIPv6wRep(
1247 Dataset *set,
const uint8_t *
data,
const uint32_t data_len,
const DataRepType *rep)
1259 DatasetUnlockData(res.
data);
1260 return res.
is_new ? 1 : 0;
1265 static int DatasetAddMd5(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1277 DatasetUnlockData(res.
data);
1278 return res.
is_new ? 1 : 0;
1283 static int DatasetAddMd5wRep(
1284 Dataset *set,
const uint8_t *
data,
const uint32_t data_len,
const DataRepType *rep)
1296 DatasetUnlockData(res.
data);
1297 return res.
is_new ? 1 : 0;
1302 static int DatasetAddSha256wRep(
1303 Dataset *set,
const uint8_t *
data,
const uint32_t data_len,
const DataRepType *rep)
1315 DatasetUnlockData(res.
data);
1316 return res.
is_new ? 1 : 0;
1321 static int DatasetAddSha256(
Dataset *set,
const uint8_t *
data,
const uint32_t data_len)
1333 DatasetUnlockData(res.
data);
1334 return res.
is_new ? 1 : 0;
1344 switch (set->
type) {
1346 return DatasetAddString(set,
data, data_len);
1348 return DatasetAddMd5(set,
data, data_len);
1350 return DatasetAddSha256(set,
data, data_len);
1352 return DatasetAddIPv4(set,
data, data_len);
1354 return DatasetAddIPv6(set,
data, data_len);
1364 switch (set->
type) {
1366 return DatasetAddStringwRep(set,
data, data_len, rep);
1368 return DatasetAddMd5wRep(set,
data, data_len, rep);
1370 return DatasetAddSha256wRep(set,
data, data_len, rep);
1372 return DatasetAddIPv4wRep(set,
data, data_len, rep);
1374 return DatasetAddIPv6wRep(set,
data, data_len, rep);
1381 static int DatasetOpSerialized(
Dataset *set,
const char *
string,
DatasetOpFunc DatasetOpString,
1387 if (strlen(
string) == 0)
1390 switch (set->
type) {
1392 if (strlen(
string) > UINT16_MAX) {
1396 uint32_t decoded_size = SCBase64DecodeBufferSize((uint32_t)strlen(
string));
1397 uint8_t decoded[decoded_size];
1398 uint32_t num_decoded = SCBase64Decode(
1399 (
const uint8_t *)
string, strlen(
string), SCBase64ModeStrict, decoded);
1400 if (num_decoded == 0) {
1404 return DatasetOpString(set, decoded, num_decoded);
1407 if (strlen(
string) != 32)
1410 if (
HexToRaw((
const uint8_t *)
string, 32, hash,
sizeof(hash)) < 0)
1412 return DatasetOpMd5(set, hash, 16);
1415 if (strlen(
string) != 64)
1418 if (
HexToRaw((
const uint8_t *)
string, 64, hash,
sizeof(hash)) < 0)
1420 return DatasetOpSha256(set, hash, 32);
1424 if (inet_pton(AF_INET,
string, &in) != 1)
1426 return DatasetOpIPv4(set, (uint8_t *)&in.s_addr, 4);
1429 struct in6_addr in6;
1431 SCLogError(
"Dataset failed to import %s as IPv6",
string);
1434 return DatasetOpIPv6(set, (uint8_t *)&in6.s6_addr, 16);
1448 return DatasetOpSerialized(set,
string, DatasetAddString, DatasetAddMd5, DatasetAddSha256,
1449 DatasetAddIPv4, DatasetAddIPv6);
1460 return DatasetOpSerialized(set,
string, DatasetLookupString, DatasetLookupMd5,
1461 DatasetLookupSha256, DatasetLookupIPv4, DatasetLookupIPv6);
1469 static int DatasetRemoveString(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1474 StringType lookup = { .ptr = (uint8_t *)data, .
len = data_len,
1479 static int DatasetRemoveIPv4(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1488 memcpy(lookup.
ipv4, data, 4);
1492 static int DatasetRemoveIPv6(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1501 memcpy(lookup.
ipv6, data, 16);
1505 static int DatasetRemoveMd5(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1514 memcpy(lookup.
md5, data, 16);
1518 static int DatasetRemoveSha256(
Dataset *set,
const uint8_t *data,
const uint32_t data_len)
1527 memcpy(lookup.
sha256, data, 32);
1538 return DatasetOpSerialized(set,
string, DatasetRemoveString, DatasetRemoveMd5,
1539 DatasetRemoveSha256, DatasetRemoveIPv4, DatasetRemoveIPv6);
1547 switch (set->
type) {
1549 return DatasetRemoveString(set, data, data_len);
1551 return DatasetRemoveMd5(set, data, data_len);
1553 return DatasetRemoveSha256(set, data, data_len);
1555 return DatasetRemoveIPv4(set, data, data_len);
1557 return DatasetRemoveIPv6(set, data, data_len);