suricata
output-json-metadata.c
Go to the documentation of this file.
1 /* Copyright (C) 2013-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Logs vars in JSON format.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "tm-threads.h"
35 #include "threadvars.h"
36 #include "util-debug.h"
37 
38 #include "util-misc.h"
39 #include "util-unittest.h"
40 #include "util-unittest-helper.h"
41 
42 #include "detect-parse.h"
43 #include "detect-engine.h"
44 #include "detect-engine-mpm.h"
45 #include "detect-reference.h"
46 #include "app-layer-parser.h"
47 #include "app-layer-dnp3.h"
48 #include "app-layer-htp.h"
49 #include "app-layer-htp-xff.h"
51 #include "util-syslog.h"
52 #include "util-logopenfile.h"
53 
54 #include "output.h"
55 #include "output-json.h"
56 #include "output-json-metadata.h"
57 
58 #include "util-byte.h"
59 #include "util-privs.h"
60 #include "util-print.h"
61 #include "util-proto-name.h"
62 #include "util-optimize.h"
63 #include "util-buffer.h"
64 #include "util-crypt.h"
65 
66 #define MODULE_NAME "JsonMetadataLog"
67 
68 #ifdef HAVE_LIBJANSSON
69 
70 #define JSON_STREAM_BUFFER_SIZE 4096
71 
72 typedef struct MetadataJsonOutputCtx_ {
73  LogFileCtx* file_ctx;
74  OutputJsonCommonSettings cfg;
75 } MetadataJsonOutputCtx;
76 
77 typedef struct JsonMetadataLogThread_ {
78  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
79  LogFileCtx* file_ctx;
80  MemBuffer *json_buffer;
81  MetadataJsonOutputCtx* json_output_ctx;
82 } JsonMetadataLogThread;
83 
84 static int MetadataJson(ThreadVars *tv, JsonMetadataLogThread *aft, const Packet *p)
85 {
86  json_t *js = CreateJSONHeader(p, LOG_DIR_PACKET, "metadata");
87  if (unlikely(js == NULL))
88  return TM_ECODE_OK;
89 
90  JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
91  OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
92  json_object_del(js, "metadata");
93  json_object_clear(js);
94  json_decref(js);
95 
96  return TM_ECODE_OK;
97 }
98 
99 static int JsonMetadataLogger(ThreadVars *tv, void *thread_data, const Packet *p)
100 {
101  JsonMetadataLogThread *aft = thread_data;
102 
103  return MetadataJson(tv, aft, p);
104 }
105 
106 static int JsonMetadataLogCondition(ThreadVars *tv, const Packet *p)
107 {
108  if (p->pktvar) {
109  return TRUE;
110  }
111  return FALSE;
112 }
113 
114 #define OUTPUT_BUFFER_SIZE 65535
115 static TmEcode JsonMetadataLogThreadInit(ThreadVars *t, const void *initdata, void **data)
116 {
117  JsonMetadataLogThread *aft = SCMalloc(sizeof(JsonMetadataLogThread));
118  if (unlikely(aft == NULL))
119  return TM_ECODE_FAILED;
120  memset(aft, 0, sizeof(JsonMetadataLogThread));
121  if(initdata == NULL)
122  {
123  SCLogDebug("Error getting context for EveLogMetadata. \"initdata\" argument NULL");
124  SCFree(aft);
125  return TM_ECODE_FAILED;
126  }
127 
128  aft->json_buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
129  if (aft->json_buffer == NULL) {
130  SCFree(aft);
131  return TM_ECODE_FAILED;
132  }
133 
134  /** Use the Output Context (file pointer and mutex) */
135  MetadataJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
136  aft->file_ctx = json_output_ctx->file_ctx;
137  aft->json_output_ctx = json_output_ctx;
138 
139  *data = (void *)aft;
140  return TM_ECODE_OK;
141 }
142 
143 static TmEcode JsonMetadataLogThreadDeinit(ThreadVars *t, void *data)
144 {
145  JsonMetadataLogThread *aft = (JsonMetadataLogThread *)data;
146  if (aft == NULL) {
147  return TM_ECODE_OK;
148  }
149 
150  MemBufferFree(aft->json_buffer);
151 
152  /* clear memory */
153  memset(aft, 0, sizeof(JsonMetadataLogThread));
154 
155  SCFree(aft);
156  return TM_ECODE_OK;
157 }
158 
159 static void JsonMetadataLogDeInitCtx(OutputCtx *output_ctx)
160 {
161  MetadataJsonOutputCtx *json_output_ctx = (MetadataJsonOutputCtx *) output_ctx->data;
162  if (json_output_ctx != NULL) {
163  LogFileFreeCtx(json_output_ctx->file_ctx);
164  SCFree(json_output_ctx);
165  }
166  SCFree(output_ctx);
167 }
168 
169 static void JsonMetadataLogDeInitCtxSub(OutputCtx *output_ctx)
170 {
171  SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
172 
173  MetadataJsonOutputCtx *json_output_ctx = (MetadataJsonOutputCtx *) output_ctx->data;
174 
175  if (json_output_ctx != NULL) {
176  SCFree(json_output_ctx);
177  }
178  SCFree(output_ctx);
179 }
180 
181 #define DEFAULT_LOG_FILENAME "metadata.json"
182 
183 /**
184  * \brief Create a new LogFileCtx for "fast" output style.
185  * \param conf The configuration node for this output.
186  * \return A LogFileCtx pointer on success, NULL on failure.
187  */
188 static OutputInitResult JsonMetadataLogInitCtx(ConfNode *conf)
189 {
190  OutputInitResult result = { NULL, false };
191  MetadataJsonOutputCtx *json_output_ctx = NULL;
192  LogFileCtx *logfile_ctx = LogFileNewCtx();
193  if (logfile_ctx == NULL) {
194  SCLogDebug("MetadataFastLogInitCtx2: Could not create new LogFileCtx");
195  return result;
196  }
197 
198  if (SCConfLogOpenGeneric(conf, logfile_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
199  LogFileFreeCtx(logfile_ctx);
200  return result;
201  }
202 
203  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
204  if (unlikely(output_ctx == NULL)) {
205  LogFileFreeCtx(logfile_ctx);
206  return result;
207  }
208 
209  json_output_ctx = SCMalloc(sizeof(MetadataJsonOutputCtx));
210  if (unlikely(json_output_ctx == NULL)) {
211  LogFileFreeCtx(logfile_ctx);
212  SCFree(output_ctx);
213  return result;
214  }
215  memset(json_output_ctx, 0, sizeof(MetadataJsonOutputCtx));
216 
217  json_output_ctx->file_ctx = logfile_ctx;
218  json_output_ctx->cfg.include_metadata = true;
219 
220  output_ctx->data = json_output_ctx;
221  output_ctx->DeInit = JsonMetadataLogDeInitCtx;
222 
223  result.ctx = output_ctx;
224  result.ok = true;
225  return result;
226 }
227 
228 /**
229  * \brief Create a new LogFileCtx for "fast" output style.
230  * \param conf The configuration node for this output.
231  * \return A LogFileCtx pointer on success, NULL on failure.
232  */
233 static OutputInitResult JsonMetadataLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
234 {
235  OutputInitResult result = { NULL, false };
236  OutputJsonCtx *ajt = parent_ctx->data;
237  MetadataJsonOutputCtx *json_output_ctx = NULL;
238 
239  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
240  if (unlikely(output_ctx == NULL))
241  return result;
242 
243  json_output_ctx = SCMalloc(sizeof(MetadataJsonOutputCtx));
244  if (unlikely(json_output_ctx == NULL)) {
245  goto error;
246  }
247  memset(json_output_ctx, 0, sizeof(MetadataJsonOutputCtx));
248 
249  json_output_ctx->file_ctx = ajt->file_ctx;
250  json_output_ctx->cfg = ajt->cfg;
251  /* override config setting as this logger is about metadata */
252  json_output_ctx->cfg.include_metadata = true;
253 
254  output_ctx->data = json_output_ctx;
255  output_ctx->DeInit = JsonMetadataLogDeInitCtxSub;
256 
257  result.ctx = output_ctx;
258  result.ok = true;
259  return result;
260 
261 error:
262  if (json_output_ctx != NULL) {
263  SCFree(json_output_ctx);
264  }
265  if (output_ctx != NULL) {
266  SCFree(output_ctx);
267  }
268 
269  return result;
270 }
271 
272 void JsonMetadataLogRegister (void)
273 {
275  "metadata-json-log", JsonMetadataLogInitCtx, JsonMetadataLogger,
276  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
277  JsonMetadataLogThreadDeinit, NULL);
279  "eve-log.metadata", JsonMetadataLogInitCtxSub, JsonMetadataLogger,
280  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
281  JsonMetadataLogThreadDeinit, NULL);
282 
283  /* Kept for compatibility. */
285  "vars-json-log", JsonMetadataLogInitCtx, JsonMetadataLogger,
286  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
287  JsonMetadataLogThreadDeinit, NULL);
289  "eve-log.vars", JsonMetadataLogInitCtxSub, JsonMetadataLogger,
290  JsonMetadataLogCondition, JsonMetadataLogThreadInit,
291  JsonMetadataLogThreadDeinit, NULL);
292 }
293 
294 #else
295 
297 {
298 }
299 
300 #endif
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
#define FALSE
#define unlikely(expr)
Definition: util-optimize.h:35
#define MODULE_NAME
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
#define TRUE
PktVar * pktvar
Definition: decode.h:488
#define SCCalloc(nm, a)
Definition: util-mem.h:197
void OutputRegisterPacketModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output module.
Definition: output.c:160
void OutputRegisterPacketSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, PacketLogger PacketLogFunc, PacketLogCondition PacketConditionFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a packet output sub-module.
Definition: output.c:201
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define DEFAULT_LOG_FILENAME
#define SCMalloc(a)
Definition: util-mem.h:166
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define SCFree(a)
Definition: util-mem.h:228
void * data
Definition: tm-modules.h:81
void JsonMetadataLogRegister(void)
Per thread variable structure.
Definition: threadvars.h:57
#define OUTPUT_BUFFER_SIZE
Definition: log-httplog.c:58
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82