suricata
output-json-metadata.c
Go to the documentation of this file.
1 /* Copyright (C) 2013-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Logs vars in JSON format.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "tm-threads.h"
35 #include "threadvars.h"
36 #include "util-debug.h"
37 
38 #include "util-misc.h"
39 #include "util-unittest.h"
40 #include "util-unittest-helper.h"
41 
42 #include "detect-parse.h"
43 #include "detect-engine.h"
44 #include "detect-engine-mpm.h"
45 #include "detect-reference.h"
46 #include "app-layer-parser.h"
47 #include "app-layer-dnp3.h"
48 #include "app-layer-htp.h"
49 #include "app-layer-htp-xff.h"
51 #include "util-syslog.h"
52 #include "util-logopenfile.h"
53 
54 #include "output.h"
55 #include "output-json.h"
56 #include "output-json-metadata.h"
57 
58 #include "util-byte.h"
59 #include "util-privs.h"
60 #include "util-print.h"
61 #include "util-proto-name.h"
62 #include "util-optimize.h"
63 #include "util-buffer.h"
64 #include "util-crypt.h"
65 
66 #define MODULE_NAME "JsonMetadataLog"
67 
68 #define JSON_STREAM_BUFFER_SIZE 4096
69 
70 typedef struct MetadataJsonOutputCtx_ {
74 
75 typedef struct JsonMetadataLogThread_ {
76  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
81 
82 static int MetadataJson(ThreadVars *tv, JsonMetadataLogThread *aft, const Packet *p)
83 {
84  json_t *js = CreateJSONHeader(p, LOG_DIR_PACKET, "metadata", NULL);
85  if (unlikely(js == NULL))
86  return TM_ECODE_OK;
87 
88  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
89  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
90  json_object_del(js, "metadata");
91  json_object_clear(js);
92  json_decref(js);
93 
94  return TM_ECODE_OK;
95 }
96 
97 static int JsonMetadataLogger(ThreadVars *tv, void *thread_data, const Packet *p)
98 {
99  JsonMetadataLogThread *aft = thread_data;
100 
101  return MetadataJson(tv, aft, p);
102 }
103 
104 static int JsonMetadataLogCondition(ThreadVars *tv, const Packet *p)
105 {
106  if (p->pktvar) {
107  return TRUE;
108  }
109  return FALSE;
110 }
111 
112 static TmEcode JsonMetadataLogThreadInit(ThreadVars *t, const void *initdata, void **data)
113 {
115  if (unlikely(aft == NULL))
116  return TM_ECODE_FAILED;
117  memset(aft, 0, sizeof(JsonMetadataLogThread));
118  if(initdata == NULL)
119  {
120  SCLogDebug("Error getting context for EveLogMetadata. \"initdata\" argument NULL");
121  SCFree(aft);
122  return TM_ECODE_FAILED;
123  }
124 
126  if (aft->json_buffer == NULL) {
127  SCFree(aft);
128  return TM_ECODE_FAILED;
129  }
130 
131  /** Use the Output Context (file pointer and mutex) */
132  MetadataJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
133  aft->file_ctx = json_output_ctx->file_ctx;
134  aft->json_output_ctx = json_output_ctx;
135 
136  *data = (void *)aft;
137  return TM_ECODE_OK;
138 }
139 
140 static TmEcode JsonMetadataLogThreadDeinit(ThreadVars *t, void *data)
141 {
143  if (aft == NULL) {
144  return TM_ECODE_OK;
145  }
146 
148 
149  /* clear memory */
150  memset(aft, 0, sizeof(JsonMetadataLogThread));
151 
152  SCFree(aft);
153  return TM_ECODE_OK;
154 }
155 
156 static void JsonMetadataLogDeInitCtx(OutputCtx *output_ctx)
157 {
158  MetadataJsonOutputCtx *json_output_ctx = (MetadataJsonOutputCtx *) output_ctx->data;
159  if (json_output_ctx != NULL) {
160  LogFileFreeCtx(json_output_ctx->file_ctx);
161  SCFree(json_output_ctx);
162  }
163  SCFree(output_ctx);
164 }
165 
166 static void JsonMetadataLogDeInitCtxSub(OutputCtx *output_ctx)
167 {
168  SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
169 
170  MetadataJsonOutputCtx *json_output_ctx = (MetadataJsonOutputCtx *) output_ctx->data;
171 
172  if (json_output_ctx != NULL) {
173  SCFree(json_output_ctx);
174  }
175  SCFree(output_ctx);
176 }
177 
178 #define DEFAULT_LOG_FILENAME "metadata.json"
179 
180 /**
181  * \brief Create a new LogFileCtx for "fast" output style.
182  * \param conf The configuration node for this output.
183  * \return A LogFileCtx pointer on success, NULL on failure.
184  */
185 static OutputInitResult JsonMetadataLogInitCtx(ConfNode *conf)
186 {
187  OutputInitResult result = { NULL, false };
188  MetadataJsonOutputCtx *json_output_ctx = NULL;
189  LogFileCtx *logfile_ctx = LogFileNewCtx();
190  if (logfile_ctx == NULL) {
191  SCLogDebug("MetadataFastLogInitCtx2: Could not create new LogFileCtx");
192  return result;
193  }
194 
195  if (SCConfLogOpenGeneric(conf, logfile_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
196  LogFileFreeCtx(logfile_ctx);
197  return result;
198  }
199 
200  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
201  if (unlikely(output_ctx == NULL)) {
202  LogFileFreeCtx(logfile_ctx);
203  return result;
204  }
205 
206  json_output_ctx = SCMalloc(sizeof(MetadataJsonOutputCtx));
207  if (unlikely(json_output_ctx == NULL)) {
208  LogFileFreeCtx(logfile_ctx);
209  SCFree(output_ctx);
210  return result;
211  }
212  memset(json_output_ctx, 0, sizeof(MetadataJsonOutputCtx));
213 
214  json_output_ctx->file_ctx = logfile_ctx;
215  json_output_ctx->cfg.include_metadata = true;
216 
217  output_ctx->data = json_output_ctx;
218  output_ctx->DeInit = JsonMetadataLogDeInitCtx;
219 
220  result.ctx = output_ctx;
221  result.ok = true;
222  return result;
223 }
224 
225 /**
226  * \brief Create a new LogFileCtx for "fast" output style.
227  * \param conf The configuration node for this output.
228  * \return A LogFileCtx pointer on success, NULL on failure.
229  */
230 static OutputInitResult JsonMetadataLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
231 {
232  OutputInitResult result = { NULL, false };
233  OutputJsonCtx *ajt = parent_ctx->data;
234  MetadataJsonOutputCtx *json_output_ctx = NULL;
235 
236  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
237  if (unlikely(output_ctx == NULL))
238  return result;
239 
240  json_output_ctx = SCMalloc(sizeof(MetadataJsonOutputCtx));
241  if (unlikely(json_output_ctx == NULL)) {
242  goto error;
243  }
244  memset(json_output_ctx, 0, sizeof(MetadataJsonOutputCtx));
245 
246  json_output_ctx->file_ctx = ajt->file_ctx;
247  json_output_ctx->cfg = ajt->cfg;
248  /* override config setting as this logger is about metadata */
249  json_output_ctx->cfg.include_metadata = true;
250 
251  output_ctx->data = json_output_ctx;
252  output_ctx->DeInit = JsonMetadataLogDeInitCtxSub;
253 
254  result.ctx = output_ctx;
255  result.ok = true;
256  return result;
257 
258 error:
259  if (json_output_ctx != NULL) {
260  SCFree(json_output_ctx);
261  }
262  if (output_ctx != NULL) {
263  SCFree(output_ctx);
264  }
265 
266  return result;
267 }
268 
270 {
272  "metadata-json-log", JsonMetadataLogInitCtx, JsonMetadataLogger,
273  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
274  JsonMetadataLogThreadDeinit, NULL);
276  "eve-log.metadata", JsonMetadataLogInitCtxSub, JsonMetadataLogger,
277  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
278  JsonMetadataLogThreadDeinit, NULL);
279 
280  /* Kept for compatibility. */
282  "vars-json-log", JsonMetadataLogInitCtx, JsonMetadataLogger,
283  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
284  JsonMetadataLogThreadDeinit, NULL);
286  "eve-log.vars", JsonMetadataLogInitCtxSub, JsonMetadataLogger,
287  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
288  JsonMetadataLogThreadDeinit, NULL);
289 }
util-byte.h
tm-threads.h
detect-engine.h
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:111
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
DEFAULT_LOG_FILENAME
#define DEFAULT_LOG_FILENAME
Definition: output-json-metadata.c:178
LogFileNewCtx
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
Definition: util-logopenfile.c:517
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
JsonMetadataLogThread_::json_output_ctx
MetadataJsonOutputCtx * json_output_ctx
Definition: output-json-metadata.c:79
JSON_OUTPUT_BUFFER_SIZE
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:62
threads.h
OutputJsonCtx_
Definition: output-json.h:108
OutputJsonCommonSettings_
Definition: output-json.h:99
util-syslog.h
LogFileCtx_
Definition: util-logopenfile.h:52
JsonMetadataLogThread_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-metadata.c:77
util-privs.h
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:79
MetadataJsonOutputCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json-metadata.c:72
util-unittest.h
util-unittest-helper.h
OutputCtx_::data
void * data
Definition: tm-modules.h:81
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:78
OutputCtx_
Definition: tm-modules.h:78
app-layer-htp-xff.h
SCConfLogOpenGeneric
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: util-logopenfile.c:305
detect-reference.h
app-layer-htp.h
util-debug.h
Packet_::pktvar
PktVar * pktvar
Definition: decode.h:493
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:42
JsonMetadataLogThread_
Definition: output-json-metadata.c:75
app-layer-dnp3.h
output-json.h
MetadataJsonOutputCtx_
Definition: output-json-metadata.c:70
util-print.h
detect-engine-mpm.h
util-crypt.h
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
OutputInitResult_::ok
bool ok
Definition: output.h:43
app-layer-parser.h
TRUE
#define TRUE
Definition: suricata-common.h:33
FALSE
#define FALSE
Definition: suricata-common.h:34
MODULE_NAME
#define MODULE_NAME
Definition: output-json-metadata.c:66
Packet_
Definition: decode.h:411
JsonMetadataLogThread
struct JsonMetadataLogThread_ JsonMetadataLogThread
conf.h
TmEcode
TmEcode
Definition: tm-threads-common.h:77
util-proto-name.h
MemBuffer_
Definition: util-buffer.h:27
OutputJsonCommonSettings_::include_metadata
bool include_metadata
Definition: output-json.h:100
OutputInitResult_
Definition: output.h:41
Packet_::flow
struct Flow_ * flow
Definition: decode.h:448
LOG_DIR_PACKET
@ LOG_DIR_PACKET
Definition: output-json.h:38
LOGGER_JSON_METADATA
@ LOGGER_JSON_METADATA
Definition: suricata-common.h:487
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
OutputRegisterPacketSubModule
void OutputRegisterPacketSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output sub-module.
Definition: output.c:212
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
LogFileFreeCtx
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
Definition: util-logopenfile.c:539
util-classification-config.h
JsonAddCommonOptions
void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, json_t *js)
Definition: output-json.c:662
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
JsonMetadataLogRegister
void JsonMetadataLogRegister(void)
Definition: output-json-metadata.c:269
util-optimize.h
OutputRegisterPacketModule
void OutputRegisterPacketModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output module.
Definition: output.c:171
threadvars.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
output-json-metadata.h
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
detect-parse.h
util-buffer.h
OutputJsonCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json.h:109
OutputJSONBuffer
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:1402
JsonMetadataLogThread_::json_buffer
MemBuffer * json_buffer
Definition: output-json-metadata.c:78
MetadataJsonOutputCtx
struct MetadataJsonOutputCtx_ MetadataJsonOutputCtx
util-misc.h
flow.h
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
debug.h
output.h
MetadataJsonOutputCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-metadata.c:71
CreateJSONHeader
json_t * CreateJSONHeader(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr)
Definition: output-json.c:1204