Go to the documentation of this file.
66 #define DEFAULT_LOG_FILENAME "eve.json"
67 #define MODULE_NAME "OutputJSON"
69 #define MAX_JSON_SIZE 2048
71 static void OutputJsonDeInitCtx(
OutputCtx *);
72 static void CreateEveCommunityFlowId(JsonBuilder *js,
const Flow *f,
const uint16_t seed);
73 static int CreateJSONEther(
76 static const char *TRAFFIC_ID_PREFIX =
"traffic/id/";
77 static const char *TRAFFIC_LABEL_PREFIX =
"traffic/label/";
78 static size_t traffic_id_prefix_len = 0;
79 static size_t traffic_label_prefix_len = 0;
87 traffic_id_prefix_len = strlen(TRAFFIC_ID_PREFIX);
88 traffic_label_prefix_len = strlen(TRAFFIC_LABEL_PREFIX);
101 json_t * retval = json_string(val);
103 if (retval == NULL) {
106 for (u = 0; u < strlen(val); u++) {
107 if (isprint(val[u])) {
116 retval = json_string(retbuf);
122 static int64_t sensor_id = -1;
126 jb_set_string_from_bytes(jb,
"filename", ff->
name, ff->
name_len);
129 jb_open_array(jb,
"sid");
130 for (uint32_t i = 0; ff->
sid != NULL && i < ff->
sid_cnt; i++) {
131 jb_append_uint(jb, ff->
sid[i]);
138 jb_set_string(jb,
"magic", (
char *)ff->magic);
145 jb_set_hex(jb,
"md5", (uint8_t *)ff->
md5, (uint32_t)
sizeof(ff->
md5));
148 jb_set_hex(jb,
"sha1", (uint8_t *)ff->
sha1, (uint32_t)
sizeof(ff->
sha1));
163 jb_set_hex(jb,
"sha256", (uint8_t *)ff->
sha256, (uint32_t)
sizeof(ff->
sha256));
178 jb_set_uint(jb,
"start", ff->
start);
179 jb_set_uint(jb,
"end", ff->
end);
181 jb_set_uint(jb,
"tx_id", tx_id);
184 static void EveAddPacketVars(
const Packet *p, JsonBuilder *js_vars)
186 if (p == NULL || p->
pktvar == NULL) {
192 if (pv->
key || pv->
id > 0) {
194 jb_open_array(js_vars,
"pktvars");
197 jb_start_object(js_vars);
199 if (pv->
key != NULL) {
201 uint8_t keybuf[pv->
key_len + 1];
206 uint8_t printable_buf[
len + 1];
209 sizeof(printable_buf),
211 jb_set_string(js_vars, (
char *)keybuf, (
char *)printable_buf);
215 uint8_t printable_buf[
len + 1];
218 sizeof(printable_buf),
220 jb_set_string(js_vars, varname, (
char *)printable_buf);
239 static bool SCStringHasPrefix(
const char *s,
const char *prefix)
241 if (strncmp(s, prefix, strlen(prefix)) == 0) {
247 static void EveAddFlowVars(
const Flow *f, JsonBuilder *js_root, JsonBuilder **js_traffic)
249 if (f == NULL || f->
flowvar == NULL) {
252 JsonBuilder *js_flowvars = NULL;
253 JsonBuilder *js_traffic_id = NULL;
254 JsonBuilder *js_traffic_label = NULL;
255 JsonBuilder *js_flowints = NULL;
256 JsonBuilder *js_flowbits = NULL;
265 if (js_flowvars == NULL) {
266 js_flowvars = jb_new_array();
267 if (js_flowvars == NULL)
272 uint8_t printable_buf[
len + 1];
275 sizeof(printable_buf),
278 jb_start_object(js_flowvars);
279 jb_set_string(js_flowvars, varname, (
char *)printable_buf);
280 jb_close(js_flowvars);
283 if (js_flowvars == NULL) {
284 js_flowvars = jb_new_array();
285 if (js_flowvars == NULL)
289 uint8_t keybuf[fv->
keylen + 1];
296 uint8_t printable_buf[
len + 1];
299 sizeof(printable_buf),
302 jb_start_object(js_flowvars);
303 jb_set_string(js_flowvars, (
const char *)keybuf, (
char *)printable_buf);
304 jb_close(js_flowvars);
309 if (js_flowints == NULL) {
310 js_flowints = jb_new_object();
311 if (js_flowints == NULL)
323 if (SCStringHasPrefix(varname, TRAFFIC_ID_PREFIX)) {
324 if (js_traffic_id == NULL) {
325 js_traffic_id = jb_new_array();
326 if (
unlikely(js_traffic_id == NULL)) {
330 jb_append_string(js_traffic_id, &varname[traffic_id_prefix_len]);
331 }
else if (SCStringHasPrefix(varname, TRAFFIC_LABEL_PREFIX)) {
332 if (js_traffic_label == NULL) {
333 js_traffic_label = jb_new_array();
334 if (
unlikely(js_traffic_label == NULL)) {
338 jb_append_string(js_traffic_label, &varname[traffic_label_prefix_len]);
340 if (js_flowbits == NULL) {
341 js_flowbits = jb_new_array();
345 jb_append_string(js_flowbits, varname);
352 jb_close(js_flowbits);
353 jb_set_object(js_root,
"flowbits", js_flowbits);
354 jb_free(js_flowbits);
357 jb_close(js_flowints);
358 jb_set_object(js_root,
"flowints", js_flowints);
359 jb_free(js_flowints);
362 jb_close(js_flowvars);
363 jb_set_object(js_root,
"flowvars", js_flowvars);
364 jb_free(js_flowvars);
367 if (js_traffic_id != NULL || js_traffic_label != NULL) {
368 *js_traffic = jb_new_object();
369 if (
likely(*js_traffic != NULL)) {
370 if (js_traffic_id != NULL) {
371 jb_close(js_traffic_id);
372 jb_set_object(*js_traffic,
"id", js_traffic_id);
373 jb_free(js_traffic_id);
375 if (js_traffic_label != NULL) {
376 jb_close(js_traffic_label);
377 jb_set_object(*js_traffic,
"label", js_traffic_label);
378 jb_free(js_traffic_label);
380 jb_close(*js_traffic);
388 JsonBuilder *js_vars = jb_new_object();
391 JsonBuilder *js_traffic = NULL;
392 EveAddFlowVars(f, js_vars, &js_traffic);
393 if (js_traffic != NULL) {
394 jb_set_object(js,
"traffic", js_traffic);
399 EveAddPacketVars(p, js_vars);
402 jb_set_object(js,
"metadata", js_vars);
415 CreateJSONEther(js, p, f, dir);
421 jb_set_uint(js,
"tenant_id", f->
tenant_id);
434 unsigned long max_len = max_length == 0 ?
GET_PKT_LEN(p) : max_length;
437 if (!jb_open_object(js,
"packet_info")) {
440 if (!jb_set_uint(js,
"linktype", p->
datalink)) {
471 char srcip[46] = {0}, dstip[46] = {0};
476 if (PacketIsIPv4(p)) {
478 srcip,
sizeof(srcip));
480 dstip,
sizeof(dstip));
481 }
else if (PacketIsIPv6(p)) {
483 srcip,
sizeof(srcip));
485 dstip,
sizeof(dstip));
496 if (PacketIsIPv4(p)) {
498 srcip,
sizeof(srcip));
500 dstip,
sizeof(dstip));
501 }
else if (PacketIsIPv6(p)) {
503 srcip,
sizeof(srcip));
505 dstip,
sizeof(dstip));
510 if (PacketIsIPv4(p)) {
512 srcip,
sizeof(srcip));
514 dstip,
sizeof(dstip));
515 }
else if (PacketIsIPv6(p)) {
517 srcip,
sizeof(srcip));
519 dstip,
sizeof(dstip));
527 if (PacketIsIPv4(p)) {
529 srcip,
sizeof(srcip));
531 dstip,
sizeof(dstip));
532 }
else if (PacketIsIPv6(p)) {
534 srcip,
sizeof(srcip));
536 dstip,
sizeof(dstip));
541 if (PacketIsIPv4(p)) {
543 srcip,
sizeof(srcip));
545 dstip,
sizeof(dstip));
546 }
else if (PacketIsIPv6(p)) {
548 srcip,
sizeof(srcip));
550 dstip,
sizeof(dstip));
580 snprintf(addr->
proto,
sizeof(addr->
proto),
"%" PRIu32, PacketGetIPProto(p));
584 #define COMMUNITY_ID_BUF_SIZE 64
586 static bool CalculateCommunityFlowIdv4(
const Flow *f,
587 const uint16_t seed,
unsigned char *base64buf)
599 uint32_t
src = f->
src.addr_data32[0];
600 uint32_t
dst = f->
dst.addr_data32[0];
602 if (f->
proto == IPPROTO_ICMP)
606 if (f->
proto == IPPROTO_ICMP)
610 ipv4.seed = htons(seed);
611 if (ntohl(
src) < ntohl(
dst) || (
src ==
dst && sp < dp)) {
622 ipv4.proto = f->
proto;
626 if (SCSha1HashBuffer((
const uint8_t *)&ipv4,
sizeof(ipv4), hash,
sizeof(hash)) == 1) {
629 if (Base64Encode(hash,
sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) {
636 static bool CalculateCommunityFlowIdv6(
const Flow *f,
637 const uint16_t seed,
unsigned char *base64buf)
650 if (f->
proto == IPPROTO_ICMPV6)
654 if (f->
proto == IPPROTO_ICMPV6)
658 ipv6.seed = htons(seed);
659 int cmp_r = memcmp(&f->
src, &f->
dst,
sizeof(f->
src));
660 if ((cmp_r < 0) || (cmp_r == 0 && sp < dp)) {
661 memcpy(&ipv6.src, &f->
src.addr_data32, 16);
662 memcpy(&ipv6.dst, &f->
dst.addr_data32, 16);
666 memcpy(&ipv6.src, &f->
dst.addr_data32, 16);
667 memcpy(&ipv6.dst, &f->
src.addr_data32, 16);
671 ipv6.proto = f->
proto;
675 if (SCSha1HashBuffer((
const uint8_t *)&ipv6,
sizeof(ipv6), hash,
sizeof(hash)) == 1) {
678 if (Base64Encode(hash,
sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) {
685 static void CreateEveCommunityFlowId(JsonBuilder *js,
const Flow *f,
const uint16_t seed)
689 if (CalculateCommunityFlowIdv4(f, seed, buf)) {
690 jb_set_string(js,
"community_id", (
const char *)buf);
693 if (CalculateCommunityFlowIdv6(f, seed, buf)) {
694 jb_set_string(js,
"community_id", (
const char *)buf);
704 int64_t flow_id = FlowGetId(f);
705 jb_set_uint(js,
"flow_id", flow_id);
707 jb_set_uint(js,
"parent_id", f->
parent_id);
714 (void) snprintf(eth_addr, 19,
"%02x:%02x:%02x:%02x:%02x:%02x",
715 val[0], val[1], val[2], val[3], val[4], val[5]);
717 jb_append_string(js, eth_addr);
719 jb_set_string(js, key, eth_addr);
728 static int MacSetIterateToJSON(uint8_t *val,
MacSetSide side,
void *data)
739 static int CreateJSONEther(
744 if (PacketIsEthernet(p)) {
745 const EthernetHdr *ethh = PacketGetEthernet(p);
746 jb_open_object(js,
"ether");
780 }
else if (f != NULL) {
785 jb_open_object(js,
"ether");
787 info.
dst = jb_new_array();
788 info.
src = jb_new_array();
801 jb_set_object(js,
"dest_macs", info.
src);
802 jb_set_object(js,
"src_macs", info.
dst);
805 jb_set_object(js,
"dest_macs", info.
dst);
806 jb_set_object(js,
"src_macs", info.
src);
822 JsonBuilder *js = jb_new_object();
829 jb_set_string(js,
"timestamp", timebuf);
834 if (sensor_id >= 0) {
835 jb_set_uint(js,
"sensor_id", sensor_id);
840 jb_set_string(js,
"in_iface", p->
livedev->
dev);
845 jb_set_uint(js,
"pcap_cnt", p->
pcap_cnt);
849 jb_set_string(js,
"event_type", event_type);
854 jb_open_array(js,
"vlan");
855 jb_append_uint(js, p->
vlan_id[0]);
857 jb_append_uint(js, p->
vlan_id[1]);
860 jb_append_uint(js, p->
vlan_id[2]);
871 if (addr->
src_ip[0] !=
'\0') {
872 jb_set_string(js,
"src_ip", addr->
src_ip);
875 jb_set_uint(js,
"src_port", addr->
sp);
877 if (addr->
dst_ip[0] !=
'\0') {
878 jb_set_string(js,
"dest_ip", addr->
dst_ip);
881 jb_set_uint(js,
"dest_port", addr->
dp);
883 if (addr->
proto[0] !=
'\0') {
884 jb_set_string(js,
"proto", addr->
proto);
890 if (PacketIsICMPv4(p)) {
891 jb_set_uint(js,
"icmp_type", p->
icmp_s.type);
892 jb_set_uint(js,
"icmp_code", p->
icmp_s.code);
896 if (PacketIsICMPv6(p)) {
897 jb_set_uint(js,
"icmp_type", PacketGetICMPv6(p)->
type);
898 jb_set_uint(js,
"icmp_code", PacketGetICMPv6(p)->code);
905 if (eve_ctx != NULL) {
920 jb_set_uint(js,
"tx_id", tx_id);
941 json_object_set_new(js,
"host",
981 MemBufferReset(*buffer);
987 size_t jslen = jb_len(js);
998 static inline enum LogFileType FileTypeFromConf(
const char *typestr)
1002 if (typestr == NULL) {
1004 }
else if (strcmp(typestr,
"file") == 0 || strcmp(typestr,
"regular") == 0) {
1006 }
else if (strcmp(typestr,
"unix_dgram") == 0) {
1008 }
else if (strcmp(typestr,
"unix_stream") == 0) {
1010 }
else if (strcmp(typestr,
"redis") == 0) {
1011 #ifdef HAVE_LIBHIREDIS
1014 FatalError(
"redis JSON output option is not compiled");
1017 SCLogDebug(
"type %s, file type value %d", typestr, log_filetype);
1018 return log_filetype;
1021 static int LogFileTypePrepare(
1032 #ifdef HAVE_LIBHIREDIS
1037 char hostname[1024];
1038 gethostname(hostname, 1023);
1045 if (SCConfLogOpenRedis(redis_node, json_ctx->
file_ctx) < 0) {
1083 SCLogDebug(
"could not create new OutputJsonCtx");
1090 if (sensor_name != NULL) {
1091 SCLogWarning(
"Found deprecated eve-log setting \"sensor-name\". "
1092 "Please set sensor-name globally.");
1095 (void)
ConfGet(
"sensor-name", &sensor_name);
1100 SCLogDebug(
"AlertJsonInitCtx: Could not create new LogFileCtx");
1114 if (
unlikely(output_ctx == NULL)) {
1118 output_ctx->
data = json_ctx;
1119 output_ctx->
DeInit = OutputJsonDeInitCtx;
1124 if (output_s == NULL) {
1128 enum LogFileType log_filetype = FileTypeFromConf(output_s);
1131 if (filetype != NULL) {
1135 FatalError(
"Invalid JSON output option: %s", output_s);
1141 SCLogInfo(
"Using prefix '%s' for JSON messages", prefix);
1145 FatalError(
"Failed to allocate memory for eve-log.prefix setting.");
1158 if (LogFileTypePrepare(json_ctx, log_filetype, conf) < 0) {
1163 if (sensor_id_s != NULL) {
1165 FatalError(
"Failed to initialize JSON output, "
1166 "invalid sensor-id: %s",
1183 SCLogConfig(
"Enabling Ethernet MAC address logging.");
1192 SCLogConfig(
"Enabling eve community_id logging.");
1198 if (cid_seed != NULL) {
1200 10, 0, cid_seed) < 0)
1202 FatalError(
"Failed to initialize JSON output, "
1203 "invalid community-id-seed: %s",
1225 SCLogDebug(
"returning output_ctx %p", output_ctx);
1227 result.
ctx = output_ctx;
1246 static void OutputJsonDeInitCtx(
OutputCtx *output_ctx)
1250 if (logfile_ctx->dropped) {
1251 SCLogWarning(
"%" PRIu64
" events were dropped due to slow or "
1252 "disconnected socket",
1253 logfile_ctx->dropped);
1255 if (json_ctx->
xff_cfg != NULL) {
#define PKT_IS_TOCLIENT(p)
void JSONFormatAndAddMACAddr(JsonBuilder *js, const char *key, const uint8_t *val, bool is_array)
union FlowVar_::@109 data
int SCRunmodeGet(void)
Get the current run mode.
void CreateIsoTimeString(const SCTime_t ts, char *str, size_t size)
OutputJsonCommonSettings cfg
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int MemBufferExpand(MemBuffer **buffer, uint32_t expand_by)
expand membuffer by size of 'expand_by'
#define JSON_OUTPUT_BUFFER_SIZE
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
const JsonAddrInfo json_addr_info_zero
#define COMMUNITY_ID_BUF_SIZE
bool include_community_id
void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const uint16_t flags)
struct Flow_::@116::@122 icmp_d
const char * known_proto[256]
struct Packet_::@29::@36 icmp_s
struct JSONMACAddrInfo JSONMACAddrInfo
uint8_t sha1[SC_SHA1_LEN]
struct Flow_::@114::@120 icmp_s
#define GET_IPV6_DST_ADDR(p)
json_t * SCJsonString(const char *val)
int ConfValIsTrue(const char *val)
Check if a value is true.
void PrintStringsToBuffer(uint8_t *dst_buf, uint32_t *dst_buf_offset_ptr, uint32_t dst_buf_size, const uint8_t *src_buf, const uint32_t src_buf_len)
void OutputJsonRegister(void)
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
size_t strlcpy(char *dst, const char *src, size_t siz)
enum @18 __attribute__
DNP3 application header.
void OutputRegisterFileRotationFlag(int *flag)
Register a flag for file rotation notification.
void EveAddMetadata(const Packet *p, const Flow *f, JsonBuilder *js)
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
#define JB_SET_STRING(jb, key, val)
struct GenericVar_ * next
#define GET_IPV4_DST_ADDR_PTR(p)
#define PKT_IS_TOSERVER(p)
#define DEFAULT_LOG_FILENAME
void OutputRegisterModule(const char *, const char *, OutputInitFunc)
JsonBuilder * CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, OutputJsonCtx *eve_ctx)
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js, enum OutputJsonLogDirection dir)
uint64_t FileTrackedSize(const File *file)
get the size of the file
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
const char * PktSrcToString(enum PktSrcEnum pkt_src)
#define JB_SET_TRUE(jb, key)
#define SCLogWarning(...)
Macro used to log WARNING messages.
void SyslogInitialize(void)
JsonBuilder * CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, uint64_t tx_id, OutputJsonCtx *eve_ctx)
void NullLogInitialize(void)
struct LiveDevice_ * livedev
@ LOGFILE_TYPE_UNIX_DGRAM
FlowStorageId MacSetGetFlowStorageID(void)
bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx)
@ LOGFILE_TYPE_UNIX_STREAM
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
void * FlowGetStorageById(const Flow *f, FlowStorageId id)
char proto[JSON_PROTO_LEN]
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
OutputInitResult OutputJsonInitCtx(ConfNode *conf)
Create a new LogFileCtx for "fast" output style.
int StringParseUint64(uint64_t *res, int base, size_t len, const char *str)
#define GET_IPV4_SRC_ADDR_PTR(p)
#define PrintBufferData(buf, buf_offset_ptr, buf_size,...)
void(* DeInit)(struct OutputCtx_ *)
const char * VarNameStoreLookupById(const uint32_t id, const enum VarTypes type)
find name for id+type at packet time.
int MacSetSize(const MacSet *ms)
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length)
Jsonify a packet.
uint8_t sha256[SC_SHA256_LEN]
#define JB_SET_FALSE(jb, key)
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
char src_ip[JSON_ADDR_LEN]
#define GET_IPV6_SRC_ADDR(p)
#define MEMBUFFER_SIZE(mem_buffer)
Get the MemBuffers current size.
int ConfValIsFalse(const char *val)
Check if a value is false.
int MacSetForEach(const MacSet *ms, MacSetIteratorFunc IterFunc, void *data)
char dst_ip[JSON_ADDR_LEN]
const char * PcapFileGetFilename(void)
int OutputJSONMemBufferCallback(const char *str, size_t size, void *data)
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
uint32_t MemBufferWriteRaw(MemBuffer *dst, const uint8_t *raw, const uint32_t raw_len)
Write a raw buffer to the MemBuffer dst.
SCEveFileType * SCEveFindFileType(const char *name)
void JsonAddrInfoInit(const Packet *p, enum OutputJsonLogDirection dir, JsonAddrInfo *addr)
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
uint16_t vlan_id[VLAN_MAX_LAYERS]
int(* Init)(const ConfNode *conf, const bool threaded, void **init_data)
Function to initialize this filetype.
int(* ThreadInit)(const void *init_data, const ThreadId thread_id, void **thread_data)
Initialize thread specific data.
uint16_t community_id_seed
#define MEMBUFFER_OFFSET(mem_buffer)
Get the MemBuffers current offset.
#define DEBUG_VALIDATE_BUG_ON(exp)
Structure used to define an EVE output file type plugin.
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.