suricata
output-json-metadata.c
Go to the documentation of this file.
1 /* Copyright (C) 2013-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Logs vars in JSON format.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "tm-threads.h"
35 #include "threadvars.h"
36 #include "util-debug.h"
37 
38 #include "util-misc.h"
39 #include "util-unittest.h"
40 #include "util-unittest-helper.h"
41 
42 #include "detect-parse.h"
43 #include "detect-engine.h"
44 #include "detect-engine-mpm.h"
45 #include "detect-reference.h"
46 #include "app-layer-parser.h"
47 #include "app-layer-dnp3.h"
48 #include "app-layer-htp.h"
49 #include "app-layer-htp-xff.h"
51 #include "util-syslog.h"
52 #include "util-logopenfile.h"
53 
54 #include "output.h"
55 #include "output-json.h"
56 #include "output-json-metadata.h"
57 
58 #include "util-byte.h"
59 #include "util-privs.h"
60 #include "util-print.h"
61 #include "util-proto-name.h"
62 #include "util-optimize.h"
63 #include "util-buffer.h"
64 #include "util-crypt.h"
65 
66 #define MODULE_NAME "JsonMetadataLog"
67 
68 #define JSON_STREAM_BUFFER_SIZE 4096
69 
70 typedef struct MetadataJsonOutputCtx_ {
74 
75 typedef struct JsonMetadataLogThread_ {
76  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
81 
82 static int MetadataJson(ThreadVars *tv, JsonMetadataLogThread *aft, const Packet *p)
83 {
84  json_t *js = CreateJSONHeader(p, LOG_DIR_PACKET, "metadata");
85  if (unlikely(js == NULL))
86  return TM_ECODE_OK;
87 
88  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
89  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
90  json_object_del(js, "metadata");
91  json_object_clear(js);
92  json_decref(js);
93 
94  return TM_ECODE_OK;
95 }
96 
97 static int JsonMetadataLogger(ThreadVars *tv, void *thread_data, const Packet *p)
98 {
99  JsonMetadataLogThread *aft = thread_data;
100 
101  return MetadataJson(tv, aft, p);
102 }
103 
104 static int JsonMetadataLogCondition(ThreadVars *tv, const Packet *p)
105 {
106  if (p->pktvar) {
107  return TRUE;
108  }
109  return FALSE;
110 }
111 
112 static TmEcode JsonMetadataLogThreadInit(ThreadVars *t, const void *initdata, void **data)
113 {
115  if (unlikely(aft == NULL))
116  return TM_ECODE_FAILED;
117  memset(aft, 0, sizeof(JsonMetadataLogThread));
118  if(initdata == NULL)
119  {
120  SCLogDebug("Error getting context for EveLogMetadata. \"initdata\" argument NULL");
121  SCFree(aft);
122  return TM_ECODE_FAILED;
123  }
124 
126  if (aft->json_buffer == NULL) {
127  SCFree(aft);
128  return TM_ECODE_FAILED;
129  }
130 
131  /** Use the Output Context (file pointer and mutex) */
132  MetadataJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
133  aft->file_ctx = json_output_ctx->file_ctx;
134  aft->json_output_ctx = json_output_ctx;
135 
136  *data = (void *)aft;
137  return TM_ECODE_OK;
138 }
139 
140 static TmEcode JsonMetadataLogThreadDeinit(ThreadVars *t, void *data)
141 {
143  if (aft == NULL) {
144  return TM_ECODE_OK;
145  }
146 
148 
149  /* clear memory */
150  memset(aft, 0, sizeof(JsonMetadataLogThread));
151 
152  SCFree(aft);
153  return TM_ECODE_OK;
154 }
155 
156 static void JsonMetadataLogDeInitCtx(OutputCtx *output_ctx)
157 {
158  MetadataJsonOutputCtx *json_output_ctx = (MetadataJsonOutputCtx *) output_ctx->data;
159  if (json_output_ctx != NULL) {
160  LogFileFreeCtx(json_output_ctx->file_ctx);
161  SCFree(json_output_ctx);
162  }
163  SCFree(output_ctx);
164 }
165 
166 static void JsonMetadataLogDeInitCtxSub(OutputCtx *output_ctx)
167 {
168  SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
169 
170  MetadataJsonOutputCtx *json_output_ctx = (MetadataJsonOutputCtx *) output_ctx->data;
171 
172  if (json_output_ctx != NULL) {
173  SCFree(json_output_ctx);
174  }
175  SCFree(output_ctx);
176 }
177 
178 #define DEFAULT_LOG_FILENAME "metadata.json"
179 
180 /**
181  * \brief Create a new LogFileCtx for "fast" output style.
182  * \param conf The configuration node for this output.
183  * \return A LogFileCtx pointer on success, NULL on failure.
184  */
185 static OutputInitResult JsonMetadataLogInitCtx(ConfNode *conf)
186 {
187  OutputInitResult result = { NULL, false };
188  MetadataJsonOutputCtx *json_output_ctx = NULL;
189  LogFileCtx *logfile_ctx = LogFileNewCtx();
190  if (logfile_ctx == NULL) {
191  SCLogDebug("MetadataFastLogInitCtx2: Could not create new LogFileCtx");
192  return result;
193  }
194 
195  if (SCConfLogOpenGeneric(conf, logfile_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
196  LogFileFreeCtx(logfile_ctx);
197  return result;
198  }
199 
200  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
201  if (unlikely(output_ctx == NULL)) {
202  LogFileFreeCtx(logfile_ctx);
203  return result;
204  }
205 
206  json_output_ctx = SCMalloc(sizeof(MetadataJsonOutputCtx));
207  if (unlikely(json_output_ctx == NULL)) {
208  LogFileFreeCtx(logfile_ctx);
209  SCFree(output_ctx);
210  return result;
211  }
212  memset(json_output_ctx, 0, sizeof(MetadataJsonOutputCtx));
213 
214  json_output_ctx->file_ctx = logfile_ctx;
215  json_output_ctx->cfg.include_metadata = true;
216 
217  output_ctx->data = json_output_ctx;
218  output_ctx->DeInit = JsonMetadataLogDeInitCtx;
219 
220  result.ctx = output_ctx;
221  result.ok = true;
222  return result;
223 }
224 
225 /**
226  * \brief Create a new LogFileCtx for "fast" output style.
227  * \param conf The configuration node for this output.
228  * \return A LogFileCtx pointer on success, NULL on failure.
229  */
230 static OutputInitResult JsonMetadataLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
231 {
232  OutputInitResult result = { NULL, false };
233  OutputJsonCtx *ajt = parent_ctx->data;
234  MetadataJsonOutputCtx *json_output_ctx = NULL;
235 
236  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
237  if (unlikely(output_ctx == NULL))
238  return result;
239 
240  json_output_ctx = SCMalloc(sizeof(MetadataJsonOutputCtx));
241  if (unlikely(json_output_ctx == NULL)) {
242  goto error;
243  }
244  memset(json_output_ctx, 0, sizeof(MetadataJsonOutputCtx));
245 
246  json_output_ctx->file_ctx = ajt->file_ctx;
247  json_output_ctx->cfg = ajt->cfg;
248  /* override config setting as this logger is about metadata */
249  json_output_ctx->cfg.include_metadata = true;
250 
251  output_ctx->data = json_output_ctx;
252  output_ctx->DeInit = JsonMetadataLogDeInitCtxSub;
253 
254  result.ctx = output_ctx;
255  result.ok = true;
256  return result;
257 
258 error:
259  if (json_output_ctx != NULL) {
260  SCFree(json_output_ctx);
261  }
262  if (output_ctx != NULL) {
263  SCFree(output_ctx);
264  }
265 
266  return result;
267 }
268 
270 {
272  "metadata-json-log", JsonMetadataLogInitCtx, JsonMetadataLogger,
273  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
274  JsonMetadataLogThreadDeinit, NULL);
276  "eve-log.metadata", JsonMetadataLogInitCtxSub, JsonMetadataLogger,
277  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
278  JsonMetadataLogThreadDeinit, NULL);
279 
280  /* Kept for compatibility. */
282  "vars-json-log", JsonMetadataLogInitCtx, JsonMetadataLogger,
283  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
284  JsonMetadataLogThreadDeinit, NULL);
286  "eve-log.vars", JsonMetadataLogInitCtxSub, JsonMetadataLogger,
287  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
288  JsonMetadataLogThreadDeinit, NULL);
289 }
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:44
json_t * CreateJSONHeader(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type)
Definition: output-json.c:710
#define SCLogDebug(...)
Definition: util-debug.h:335
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:809
MetadataJsonOutputCtx * json_output_ctx
struct Flow_ * flow
Definition: decode.h:446
#define FALSE
#define unlikely(expr)
Definition: util-optimize.h:35
#define MODULE_NAME
struct JsonMetadataLogThread_ JsonMetadataLogThread
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, json_t *js)
Definition: output-json.c:390
#define TRUE
PktVar * pktvar
Definition: decode.h:491
#define SCCalloc(nm, a)
Definition: util-mem.h:253
#define DEFAULT_LOG_FILENAME
void OutputRegisterPacketModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output module.
Definition: output.c:163
void OutputRegisterPacketSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output sub-module.
Definition: output.c:204
OutputJsonCommonSettings cfg
Definition: output-json.h:81
LogFileCtx * file_ctx
Definition: output-json.h:79
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:222
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define SCFree(a)
Definition: util-mem.h:322
struct MetadataJsonOutputCtx_ MetadataJsonOutputCtx
void * data
Definition: tm-modules.h:81
OutputJsonCommonSettings cfg
void JsonMetadataLogRegister(void)
Per thread variable structure.
Definition: threadvars.h:57
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82