Go to the documentation of this file.
75 #define DEFAULT_LOG_FILENAME "eve.json"
76 #define MODULE_NAME "OutputJSON"
78 #define MAX_JSON_SIZE 2048
80 static void OutputJsonDeInitCtx(
OutputCtx *);
81 static void CreateEveCommunityFlowId(JsonBuilder *js,
const Flow *f,
const uint16_t seed);
82 static int CreateJSONEther(JsonBuilder *parent,
const Packet *p,
const Flow *f);
84 static const char *TRAFFIC_ID_PREFIX =
"traffic/id/";
85 static const char *TRAFFIC_LABEL_PREFIX =
"traffic/label/";
86 static size_t traffic_id_prefix_len = 0;
87 static size_t traffic_label_prefix_len = 0;
95 traffic_id_prefix_len = strlen(TRAFFIC_ID_PREFIX);
96 traffic_label_prefix_len = strlen(TRAFFIC_LABEL_PREFIX);
109 json_t * retval = json_string(val);
111 if (retval == NULL) {
114 for (u = 0; u < strlen(val); u++) {
115 if (isprint(val[u])) {
124 retval = json_string(retbuf);
130 static int64_t sensor_id = -1;
134 jb_set_string_from_bytes(jb,
"filename", ff->
name, ff->
name_len);
137 jb_open_array(jb,
"sid");
138 for (uint32_t i = 0; ff->
sid != NULL && i < ff->
sid_cnt; i++) {
139 jb_append_uint(jb, ff->
sid[i]);
146 jb_set_string(jb,
"magic", (
char *)ff->magic);
153 jb_set_hex(jb,
"md5", (uint8_t *)ff->
md5, (uint32_t)
sizeof(ff->
md5));
156 jb_set_hex(jb,
"sha1", (uint8_t *)ff->
sha1, (uint32_t)
sizeof(ff->
sha1));
171 jb_set_hex(jb,
"sha256", (uint8_t *)ff->
sha256, (uint32_t)
sizeof(ff->
sha256));
186 jb_set_uint(jb,
"start", ff->
start);
187 jb_set_uint(jb,
"end", ff->
end);
189 jb_set_uint(jb,
"tx_id", tx_id);
192 static void EveAddPacketVars(
const Packet *p, JsonBuilder *js_vars)
194 if (p == NULL || p->
pktvar == NULL) {
200 if (pv->
key || pv->
id > 0) {
202 jb_open_array(js_vars,
"pktvars");
205 jb_start_object(js_vars);
207 if (pv->
key != NULL) {
209 uint8_t keybuf[pv->
key_len + 1];
214 uint8_t printable_buf[
len + 1];
217 sizeof(printable_buf),
219 jb_set_string(js_vars, (
char *)keybuf, (
char *)printable_buf);
223 uint8_t printable_buf[
len + 1];
226 sizeof(printable_buf),
228 jb_set_string(js_vars, varname, (
char *)printable_buf);
247 static bool SCStringHasPrefix(
const char *s,
const char *prefix)
249 if (strncmp(s, prefix, strlen(prefix)) == 0) {
255 static void EveAddFlowVars(
const Flow *f, JsonBuilder *js_root, JsonBuilder **js_traffic)
257 if (f == NULL || f->
flowvar == NULL) {
260 JsonBuilder *js_flowvars = NULL;
261 JsonBuilder *js_traffic_id = NULL;
262 JsonBuilder *js_traffic_label = NULL;
263 JsonBuilder *js_flowints = NULL;
264 JsonBuilder *js_flowbits = NULL;
273 if (js_flowvars == NULL) {
274 js_flowvars = jb_new_array();
275 if (js_flowvars == NULL)
280 uint8_t printable_buf[
len + 1];
283 sizeof(printable_buf),
286 jb_start_object(js_flowvars);
287 jb_set_string(js_flowvars, varname, (
char *)printable_buf);
288 jb_close(js_flowvars);
291 if (js_flowvars == NULL) {
292 js_flowvars = jb_new_array();
293 if (js_flowvars == NULL)
297 uint8_t keybuf[fv->
keylen + 1];
304 uint8_t printable_buf[
len + 1];
307 sizeof(printable_buf),
310 jb_start_object(js_flowvars);
311 jb_set_string(js_flowvars, (
const char *)keybuf, (
char *)printable_buf);
312 jb_close(js_flowvars);
317 if (js_flowints == NULL) {
318 js_flowints = jb_new_object();
319 if (js_flowints == NULL)
331 if (SCStringHasPrefix(varname, TRAFFIC_ID_PREFIX)) {
332 if (js_traffic_id == NULL) {
333 js_traffic_id = jb_new_array();
334 if (
unlikely(js_traffic_id == NULL)) {
338 jb_append_string(js_traffic_id, &varname[traffic_id_prefix_len]);
339 }
else if (SCStringHasPrefix(varname, TRAFFIC_LABEL_PREFIX)) {
340 if (js_traffic_label == NULL) {
341 js_traffic_label = jb_new_array();
342 if (
unlikely(js_traffic_label == NULL)) {
346 jb_append_string(js_traffic_label, &varname[traffic_label_prefix_len]);
348 if (js_flowbits == NULL) {
349 js_flowbits = jb_new_array();
353 jb_append_string(js_flowbits, varname);
360 jb_close(js_flowbits);
361 jb_set_object(js_root,
"flowbits", js_flowbits);
362 jb_free(js_flowbits);
365 jb_close(js_flowints);
366 jb_set_object(js_root,
"flowints", js_flowints);
367 jb_free(js_flowints);
370 jb_close(js_flowvars);
371 jb_set_object(js_root,
"flowvars", js_flowvars);
372 jb_free(js_flowvars);
375 if (js_traffic_id != NULL || js_traffic_label != NULL) {
376 *js_traffic = jb_new_object();
377 if (
likely(*js_traffic != NULL)) {
378 if (js_traffic_id != NULL) {
379 jb_close(js_traffic_id);
380 jb_set_object(*js_traffic,
"id", js_traffic_id);
381 jb_free(js_traffic_id);
383 if (js_traffic_label != NULL) {
384 jb_close(js_traffic_label);
385 jb_set_object(*js_traffic,
"label", js_traffic_label);
386 jb_free(js_traffic_label);
388 jb_close(*js_traffic);
396 JsonBuilder *js_vars = jb_new_object();
399 JsonBuilder *js_traffic = NULL;
400 EveAddFlowVars(f, js_vars, &js_traffic);
401 if (js_traffic != NULL) {
402 jb_set_object(js,
"traffic", js_traffic);
407 EveAddPacketVars(p, js_vars);
410 jb_set_object(js,
"metadata", js_vars);
417 const Packet *p,
const Flow *f, JsonBuilder *js)
423 CreateJSONEther(js, p, f);
429 jb_set_uint(js,
"tenant_id", f->
tenant_id);
442 unsigned long max_len = max_length == 0 ?
GET_PKT_LEN(p) : max_length;
445 if (!jb_open_object(js,
"packet_info")) {
448 if (!jb_set_uint(js,
"linktype", p->
datalink)) {
479 char srcip[46] = {0}, dstip[46] = {0};
486 srcip,
sizeof(srcip));
488 dstip,
sizeof(dstip));
491 srcip,
sizeof(srcip));
493 dstip,
sizeof(dstip));
506 srcip,
sizeof(srcip));
508 dstip,
sizeof(dstip));
511 srcip,
sizeof(srcip));
513 dstip,
sizeof(dstip));
520 srcip,
sizeof(srcip));
522 dstip,
sizeof(dstip));
525 srcip,
sizeof(srcip));
527 dstip,
sizeof(dstip));
537 srcip,
sizeof(srcip));
539 dstip,
sizeof(dstip));
542 srcip,
sizeof(srcip));
544 dstip,
sizeof(dstip));
551 srcip,
sizeof(srcip));
553 dstip,
sizeof(dstip));
556 srcip,
sizeof(srcip));
558 dstip,
sizeof(dstip));
590 #define COMMUNITY_ID_BUF_SIZE 64
592 static bool CalculateCommunityFlowIdv4(
const Flow *f,
593 const uint16_t seed,
unsigned char *base64buf)
605 uint32_t
src = f->
src.addr_data32[0];
606 uint32_t
dst = f->
dst.addr_data32[0];
608 if (f->
proto == IPPROTO_ICMP)
612 if (f->
proto == IPPROTO_ICMP)
616 ipv4.seed = htons(seed);
617 if (ntohl(
src) < ntohl(
dst) || (
src ==
dst && sp < dp)) {
628 ipv4.proto = f->
proto;
632 if (SCSha1HashBuffer((
const uint8_t *)&ipv4,
sizeof(ipv4), hash,
sizeof(hash)) == 1) {
635 if (Base64Encode(hash,
sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) {
642 static bool CalculateCommunityFlowIdv6(
const Flow *f,
643 const uint16_t seed,
unsigned char *base64buf)
656 if (f->
proto == IPPROTO_ICMPV6)
660 if (f->
proto == IPPROTO_ICMPV6)
664 ipv6.seed = htons(seed);
665 int cmp_r = memcmp(&f->
src, &f->
dst,
sizeof(f->
src));
666 if ((cmp_r < 0) || (cmp_r == 0 && sp < dp)) {
667 memcpy(&ipv6.src, &f->
src.addr_data32, 16);
668 memcpy(&ipv6.dst, &f->
dst.addr_data32, 16);
672 memcpy(&ipv6.src, &f->
dst.addr_data32, 16);
673 memcpy(&ipv6.dst, &f->
src.addr_data32, 16);
677 ipv6.proto = f->
proto;
681 if (SCSha1HashBuffer((
const uint8_t *)&ipv6,
sizeof(ipv6), hash,
sizeof(hash)) == 1) {
684 if (Base64Encode(hash,
sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) {
691 static void CreateEveCommunityFlowId(JsonBuilder *js,
const Flow *f,
const uint16_t seed)
695 if (CalculateCommunityFlowIdv4(f, seed, buf)) {
696 jb_set_string(js,
"community_id", (
const char *)buf);
699 if (CalculateCommunityFlowIdv6(f, seed, buf)) {
700 jb_set_string(js,
"community_id", (
const char *)buf);
710 int64_t flow_id = FlowGetId(f);
711 jb_set_uint(js,
"flow_id", flow_id);
713 jb_set_uint(js,
"parent_id", f->
parent_id);
717 static inline void JSONFormatAndAddMACAddr(JsonBuilder *js,
const char *key,
718 uint8_t *val,
bool is_array)
721 (void) snprintf(eth_addr, 19,
"%02x:%02x:%02x:%02x:%02x:%02x",
722 val[0], val[1], val[2], val[3], val[4], val[5]);
724 jb_append_string(js, eth_addr);
726 jb_set_string(js, key, eth_addr);
735 static int MacSetIterateToJSON(uint8_t *val,
MacSetSide side,
void *data)
739 JSONFormatAndAddMACAddr(info->
dst, NULL, val,
true);
741 JSONFormatAndAddMACAddr(info->
src, NULL, val,
true);
746 static int CreateJSONEther(JsonBuilder *js,
const Packet *p,
const Flow *f)
750 if (p->
ethh != NULL) {
751 jb_open_object(js,
"ether");
752 uint8_t *
src = p->
ethh->eth_src;
753 uint8_t *
dst = p->
ethh->eth_dst;
754 JSONFormatAndAddMACAddr(js,
"src_mac",
src,
false);
755 JSONFormatAndAddMACAddr(js,
"dest_mac",
dst,
false);
758 }
else if (f != NULL) {
763 jb_open_object(js,
"ether");
765 info.
dst = jb_new_array();
766 info.
src = jb_new_array();
777 jb_set_object(js,
"dest_macs", info.
dst);
778 jb_set_object(js,
"src_macs", info.
src);
793 JsonBuilder *js = jb_new_object();
800 jb_set_string(js,
"timestamp", timebuf);
805 if (sensor_id >= 0) {
806 jb_set_uint(js,
"sensor_id", sensor_id);
811 jb_set_string(js,
"in_iface", p->
livedev->
dev);
816 jb_set_uint(js,
"pcap_cnt", p->
pcap_cnt);
820 jb_set_string(js,
"event_type", event_type);
825 jb_open_array(js,
"vlan");
826 jb_append_uint(js, p->
vlan_id[0]);
828 jb_append_uint(js, p->
vlan_id[1]);
831 jb_append_uint(js, p->
vlan_id[2]);
842 jb_set_string(js,
"src_ip", addr->
src_ip);
843 jb_set_uint(js,
"src_port", addr->
sp);
844 jb_set_string(js,
"dest_ip", addr->
dst_ip);
845 jb_set_uint(js,
"dest_port", addr->
dp);
846 jb_set_string(js,
"proto", addr->
proto);
852 jb_set_uint(js,
"icmp_type", p->
icmpv4h->type);
853 jb_set_uint(js,
"icmp_code", p->
icmpv4h->code);
866 if (eve_ctx != NULL) {
881 jb_set_uint(js,
"tx_id", tx_id);
902 json_object_set_new(js,
"host",
942 MemBufferReset(*buffer);
948 size_t jslen = jb_len(js);
959 static inline enum LogFileType FileTypeFromConf(
const char *typestr)
963 if (typestr == NULL) {
965 }
else if (strcmp(typestr,
"file") == 0 || strcmp(typestr,
"regular") == 0) {
967 }
else if (strcmp(typestr,
"unix_dgram") == 0) {
969 }
else if (strcmp(typestr,
"unix_stream") == 0) {
971 }
else if (strcmp(typestr,
"redis") == 0) {
972 #ifdef HAVE_LIBHIREDIS
975 FatalError(
"redis JSON output option is not compiled");
978 SCLogDebug(
"type %s, file type value %d", typestr, log_filetype);
982 static int LogFileTypePrepare(
993 #ifdef HAVE_LIBHIREDIS
999 gethostname(hostname, 1023);
1006 if (SCConfLogOpenRedis(redis_node, json_ctx->
file_ctx) < 0) {
1044 SCLogDebug(
"could not create new OutputJsonCtx");
1051 if (sensor_name != NULL) {
1052 SCLogWarning(
"Found deprecated eve-log setting \"sensor-name\". "
1053 "Please set sensor-name globally.");
1056 (void)
ConfGet(
"sensor-name", &sensor_name);
1061 SCLogDebug(
"AlertJsonInitCtx: Could not create new LogFileCtx");
1075 if (
unlikely(output_ctx == NULL)) {
1079 output_ctx->
data = json_ctx;
1080 output_ctx->
DeInit = OutputJsonDeInitCtx;
1085 if (output_s == NULL) {
1089 enum LogFileType log_filetype = FileTypeFromConf(output_s);
1092 if (filetype != NULL) {
1096 FatalError(
"Invalid JSON output option: %s", output_s);
1102 SCLogInfo(
"Using prefix '%s' for JSON messages", prefix);
1106 FatalError(
"Failed to allocate memory for eve-log.prefix setting.");
1119 if (LogFileTypePrepare(json_ctx, log_filetype, conf) < 0) {
1124 if (sensor_id_s != NULL) {
1126 FatalError(
"Failed to initialize JSON output, "
1127 "invalid sensor-id: %s",
1144 SCLogConfig(
"Enabling Ethernet MAC address logging.");
1153 SCLogConfig(
"Enabling eve community_id logging.");
1159 if (cid_seed != NULL) {
1161 10, 0, cid_seed) < 0)
1163 FatalError(
"Failed to initialize JSON output, "
1164 "invalid community-id-seed: %s",
1187 SCLogDebug(
"returning output_ctx %p", output_ctx);
1189 result.
ctx = output_ctx;
1208 static void OutputJsonDeInitCtx(
OutputCtx *output_ctx)
1212 if (logfile_ctx->dropped) {
1213 SCLogWarning(
"%" PRIu64
" events were dropped due to slow or "
1214 "disconnected socket",
1215 logfile_ctx->dropped);
1217 if (json_ctx->
xff_cfg != NULL) {
#define PKT_IS_TOCLIENT(p)
void CreateIsoTimeString(const SCTime_t ts, char *str, size_t size)
struct Flow_::@119::@125 icmp_d
OutputJsonCommonSettings cfg
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int MemBufferExpand(MemBuffer **buffer, uint32_t expand_by)
expand membuffer by size of 'expand_by'
union FlowVar_::@112 data
#define JSON_OUTPUT_BUFFER_SIZE
int StringParseUint16(uint16_t *res, int base, size_t len, const char *str)
int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
bool SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
const JsonAddrInfo json_addr_info_zero
#define COMMUNITY_ID_BUF_SIZE
bool include_community_id
void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const uint16_t flags)
const char * known_proto[256]
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js)
struct JSONMACAddrInfo JSONMACAddrInfo
uint8_t sha1[SC_SHA1_LEN]
#define GET_IPV6_DST_ADDR(p)
json_t * SCJsonString(const char *val)
int ConfValIsTrue(const char *val)
Check if a value is true.
void PrintStringsToBuffer(uint8_t *dst_buf, uint32_t *dst_buf_offset_ptr, uint32_t dst_buf_size, const uint8_t *src_buf, const uint32_t src_buf_len)
void OutputJsonRegister(void)
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
size_t strlcpy(char *dst, const char *src, size_t siz)
enum @22 __attribute__
DNP3 application header.
void OutputRegisterFileRotationFlag(int *flag)
Register a flag for file rotation notification.
void EveAddMetadata(const Packet *p, const Flow *f, JsonBuilder *js)
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
#define JB_SET_STRING(jb, key, val)
struct GenericVar_ * next
#define GET_IPV4_DST_ADDR_PTR(p)
#define PKT_IS_TOSERVER(p)
#define DEFAULT_LOG_FILENAME
void OutputRegisterModule(const char *, const char *, OutputInitFunc)
JsonBuilder * CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, OutputJsonCtx *eve_ctx)
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
uint64_t FileTrackedSize(const File *file)
get the size of the file
void EveTcpFlags(const uint8_t flags, JsonBuilder *js)
jsonify tcp flags field Only add 'true' fields in an attempt to keep things reasonably compact.
const char * PktSrcToString(enum PktSrcEnum pkt_src)
#define JB_SET_TRUE(jb, key)
#define SCLogWarning(...)
Macro used to log WARNING messages.
void SyslogInitialize(void)
struct Flow_::@117::@123 icmp_s
JsonBuilder * CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, uint64_t tx_id, OutputJsonCtx *eve_ctx)
void NullLogInitialize(void)
struct LiveDevice_ * livedev
@ LOGFILE_TYPE_UNIX_DGRAM
FlowStorageId MacSetGetFlowStorageID(void)
bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx)
@ LOGFILE_TYPE_UNIX_STREAM
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
void * FlowGetStorageById(const Flow *f, FlowStorageId id)
char proto[JSON_PROTO_LEN]
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
OutputInitResult OutputJsonInitCtx(ConfNode *conf)
Create a new LogFileCtx for "fast" output style.
int StringParseUint64(uint64_t *res, int base, size_t len, const char *str)
#define GET_IPV4_SRC_ADDR_PTR(p)
#define PrintBufferData(buf, buf_offset_ptr, buf_size,...)
void(* DeInit)(struct OutputCtx_ *)
const char * VarNameStoreLookupById(const uint32_t id, const enum VarTypes type)
find name for id+type at packet time.
int MacSetSize(const MacSet *ms)
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length)
Jsonify a packet.
uint8_t sha256[SC_SHA256_LEN]
#define JB_SET_FALSE(jb, key)
int RunmodeGetCurrent(void)
void CreateEveFlowId(JsonBuilder *js, const Flow *f)
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
char src_ip[JSON_ADDR_LEN]
#define GET_IPV6_SRC_ADDR(p)
#define MEMBUFFER_SIZE(mem_buffer)
Get the MemBuffers current size.
#define IP_GET_IPPROTO(p)
int ConfValIsFalse(const char *val)
Check if a value is false.
int MacSetForEach(const MacSet *ms, MacSetIteratorFunc IterFunc, void *data)
char dst_ip[JSON_ADDR_LEN]
const char * PcapFileGetFilename(void)
int OutputJSONMemBufferCallback(const char *str, size_t size, void *data)
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
uint32_t MemBufferWriteRaw(MemBuffer *dst, const uint8_t *raw, const uint32_t raw_len)
Write a raw buffer to the MemBuffer dst.
SCEveFileType * SCEveFindFileType(const char *name)
void JsonAddrInfoInit(const Packet *p, enum OutputJsonLogDirection dir, JsonAddrInfo *addr)
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
uint16_t vlan_id[VLAN_MAX_LAYERS]
int(* Init)(const ConfNode *conf, const bool threaded, void **init_data)
Function to initialize this filetype.
int(* ThreadInit)(const void *init_data, const ThreadId thread_id, void **thread_data)
Initialize thread specific data.
uint16_t community_id_seed
#define MEMBUFFER_OFFSET(mem_buffer)
Get the MemBuffers current offset.
#define DEBUG_VALIDATE_BUG_ON(exp)
Structure used to define an EVE output file type plugin.
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.