suricata
output-json-http.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Implements HTTP JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "app-layer-htp.h"
43 #include "app-layer-htp-xff.h"
44 #include "app-layer.h"
45 #include "app-layer-parser.h"
46 #include "util-privs.h"
47 #include "util-buffer.h"
48 #include "util-proto-name.h"
49 #include "util-logopenfile.h"
50 #include "util-time.h"
51 #include "util-crypt.h"
52 #include "output-json.h"
53 #include "output-json-alert.h"
54 #include "output-json-http.h"
55 
56 #ifdef HAVE_LIBJANSSON
57 
58 typedef struct LogHttpFileCtx_ {
60  uint32_t flags; /** Store mode */
61  uint64_t fields;/** Store fields */
62  HttpXFFCfg *xff_cfg;
63  HttpXFFCfg *parent_xff_cfg;
64  OutputJsonCommonSettings cfg;
66 
67 typedef struct JsonHttpLogThread_ {
68  LogHttpFileCtx *httplog_ctx;
69  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
70  uint32_t uri_cnt;
71 
72  MemBuffer *buffer;
73 } JsonHttpLogThread;
74 
75 #define MAX_SIZE_HEADER_NAME 256
76 #define MAX_SIZE_HEADER_VALUE 2048
77 
78 #define LOG_HTTP_DEFAULT 0
79 #define LOG_HTTP_EXTENDED 1
80 #define LOG_HTTP_REQUEST 2 /* request field */
81 #define LOG_HTTP_ARRAY 4 /* require array handling */
82 #define LOG_HTTP_REQ_HEADERS 8
83 #define LOG_HTTP_RES_HEADERS 16
84 
85 typedef enum {
86  HTTP_FIELD_ACCEPT = 0,
87  HTTP_FIELD_ACCEPT_CHARSET,
88  HTTP_FIELD_ACCEPT_ENCODING,
89  HTTP_FIELD_ACCEPT_LANGUAGE,
90  HTTP_FIELD_ACCEPT_DATETIME,
91  HTTP_FIELD_AUTHORIZATION,
92  HTTP_FIELD_CACHE_CONTROL,
93  HTTP_FIELD_CONNECTION,
94  HTTP_FIELD_FROM,
95  HTTP_FIELD_MAX_FORWARDS,
96  HTTP_FIELD_ORIGIN,
97  HTTP_FIELD_PRAGMA,
98  HTTP_FIELD_PROXY_AUTHORIZATION,
99  HTTP_FIELD_RANGE,
100  HTTP_FIELD_TE,
101  HTTP_FIELD_VIA,
102  HTTP_FIELD_X_REQUESTED_WITH,
103  HTTP_FIELD_DNT,
104  HTTP_FIELD_X_FORWARDED_PROTO,
105  HTTP_FIELD_X_AUTHENTICATED_USER,
106  HTTP_FIELD_X_FLASH_VERSION,
107  HTTP_FIELD_ACCEPT_RANGES,
108  HTTP_FIELD_AGE,
109  HTTP_FIELD_ALLOW,
110  HTTP_FIELD_CONTENT_ENCODING,
111  HTTP_FIELD_CONTENT_LANGUAGE,
112  HTTP_FIELD_CONTENT_LENGTH,
113  HTTP_FIELD_CONTENT_LOCATION,
114  HTTP_FIELD_CONTENT_MD5,
115  HTTP_FIELD_CONTENT_RANGE,
116  HTTP_FIELD_CONTENT_TYPE,
117  HTTP_FIELD_DATE,
118  HTTP_FIELD_ETAG,
119  HTTP_FIELD_EXPIRES,
120  HTTP_FIELD_LAST_MODIFIED,
121  HTTP_FIELD_LINK,
122  HTTP_FIELD_LOCATION,
123  HTTP_FIELD_PROXY_AUTHENTICATE,
124  HTTP_FIELD_REFERRER,
125  HTTP_FIELD_REFRESH,
126  HTTP_FIELD_RETRY_AFTER,
127  HTTP_FIELD_SERVER,
128  HTTP_FIELD_SET_COOKIE,
129  HTTP_FIELD_TRAILER,
130  HTTP_FIELD_TRANSFER_ENCODING,
131  HTTP_FIELD_UPGRADE,
132  HTTP_FIELD_VARY,
133  HTTP_FIELD_WARNING,
134  HTTP_FIELD_WWW_AUTHENTICATE,
135  HTTP_FIELD_SIZE
136 } HttpField;
137 
138 struct {
139  const char *config_field;
140  const char *htp_field;
141  uint32_t flags;
142 } http_fields[] = {
143  { "accept", "accept", LOG_HTTP_REQUEST },
144  { "accept_charset", "accept-charset", LOG_HTTP_REQUEST },
145  { "accept_encoding", "accept-encoding", LOG_HTTP_REQUEST },
146  { "accept_language", "accept-language", LOG_HTTP_REQUEST },
147  { "accept_datetime", "accept-datetime", LOG_HTTP_REQUEST },
148  { "authorization", "authorization", LOG_HTTP_REQUEST },
149  { "cache_control", "cache-control", LOG_HTTP_REQUEST },
150  { "cookie", "cookie", LOG_HTTP_REQUEST|LOG_HTTP_ARRAY },
151  { "from", "from", LOG_HTTP_REQUEST },
152  { "max_forwards", "max-forwards", LOG_HTTP_REQUEST },
153  { "origin", "origin", LOG_HTTP_REQUEST },
154  { "pragma", "pragma", LOG_HTTP_REQUEST },
155  { "proxy_authorization", "proxy-authorization", LOG_HTTP_REQUEST },
156  { "range", "range", LOG_HTTP_REQUEST },
157  { "te", "te", LOG_HTTP_REQUEST },
158  { "via", "via", LOG_HTTP_REQUEST },
159  { "x_requested_with", "x-requested-with", LOG_HTTP_REQUEST },
160  { "dnt", "dnt", LOG_HTTP_REQUEST },
161  { "x_forwarded_proto", "x-forwarded-proto", LOG_HTTP_REQUEST },
162  { "x_authenticated_user", "x-authenticated-user", LOG_HTTP_REQUEST },
163  { "x_flash_version", "x-flash-version", LOG_HTTP_REQUEST },
164  { "accept_range", "accept-range", 0 },
165  { "age", "age", 0 },
166  { "allow", "allow", 0 },
167  { "connection", "connection", 0 },
168  { "content_encoding", "content-encoding", 0 },
169  { "content_language", "content-language", 0 },
170  { "content_length", "content-length", 0 },
171  { "content_location", "content-location", 0 },
172  { "content_md5", "content-md5", 0 },
173  { "content_range", "content-range", 0 },
174  { "content_type", "content-type", 0 },
175  { "date", "date", 0 },
176  { "etag", "etags", 0 },
177  { "expires", "expires" , 0 },
178  { "last_modified", "last-modified", 0 },
179  { "link", "link", 0 },
180  { "location", "location", 0 },
181  { "proxy_authenticate", "proxy-authenticate", 0 },
182  { "referrer", "referrer", LOG_HTTP_EXTENDED },
183  { "refresh", "refresh", 0 },
184  { "retry_after", "retry-after", 0 },
185  { "server", "server", 0 },
186  { "set_cookie", "set-cookie", 0 },
187  { "trailer", "trailer", 0 },
188  { "transfer_encoding", "transfer-encoding", 0 },
189  { "upgrade", "upgrade", 0 },
190  { "vary", "vary", 0 },
191  { "warning", "warning", 0 },
192  { "www_authenticate", "www-authenticate", 0 },
193 };
194 
195 static void JsonHttpLogJSONBasic(json_t *js, htp_tx_t *tx)
196 {
197  char *c;
198 
199  /* hostname */
200  if (tx->request_hostname != NULL)
201  {
202  c = bstr_util_strdup_to_c(tx->request_hostname);
203  if (c != NULL) {
204  json_object_set_new(js, "hostname", SCJsonString(c));
205  SCFree(c);
206  }
207  }
208 
209  /* port */
210  /* NOTE: this field will be set ONLY if the port is present in the
211  * hostname. It may be present in the header "Host" or in the URL.
212  * There is no connection (from the suricata point of view) between this
213  * port and the TCP destination port of the flow.
214  */
215  if (tx->request_port_number >= 0)
216  {
217  json_object_set_new(js, "http_port",
218  json_integer(tx->request_port_number));
219  }
220 
221  /* uri */
222  if (tx->request_uri != NULL)
223  {
224  c = bstr_util_strdup_to_c(tx->request_uri);
225  if (c != NULL) {
226  json_object_set_new(js, "url", SCJsonString(c));
227  SCFree(c);
228  }
229  }
230 
231  /* user agent */
232  htp_header_t *h_user_agent = NULL;
233  if (tx->request_headers != NULL) {
234  h_user_agent = htp_table_get_c(tx->request_headers, "user-agent");
235  }
236  if (h_user_agent != NULL) {
237  c = bstr_util_strdup_to_c(h_user_agent->value);
238  if (c != NULL) {
239  json_object_set_new(js, "http_user_agent", SCJsonString(c));
240  SCFree(c);
241  }
242  }
243 
244  /* x-forwarded-for */
245  htp_header_t *h_x_forwarded_for = NULL;
246  if (tx->request_headers != NULL) {
247  h_x_forwarded_for = htp_table_get_c(tx->request_headers, "x-forwarded-for");
248  }
249  if (h_x_forwarded_for != NULL) {
250  c = bstr_util_strdup_to_c(h_x_forwarded_for->value);
251  if (c != NULL) {
252  json_object_set_new(js, "xff", json_string(c));
253  SCFree(c);
254  }
255  }
256 
257  /* content-type */
258  htp_header_t *h_content_type = NULL;
259  if (tx->response_headers != NULL) {
260  h_content_type = htp_table_get_c(tx->response_headers, "content-type");
261  }
262  if (h_content_type != NULL) {
263  char *p;
264  c = bstr_util_strdup_to_c(h_content_type->value);
265  if (c != NULL) {
266  p = strchr(c, ';');
267  if (p != NULL)
268  *p = '\0';
269  json_object_set_new(js, "http_content_type", SCJsonString(c));
270  SCFree(c);
271  }
272  }
273 }
274 
275 static void JsonHttpLogJSONCustom(LogHttpFileCtx *http_ctx, json_t *js, htp_tx_t *tx)
276 {
277  char *c;
278  HttpField f;
279 
280  for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++)
281  {
282  if ((http_ctx->fields & (1ULL<<f)) != 0)
283  {
284  /* prevent logging a field twice if extended logging is
285  enabled */
286  if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) ||
287  ((http_ctx->flags & LOG_HTTP_EXTENDED) !=
288  (http_fields[f].flags & LOG_HTTP_EXTENDED)))
289  {
290  htp_header_t *h_field = NULL;
291  if ((http_fields[f].flags & LOG_HTTP_REQUEST) != 0)
292  {
293  if (tx->request_headers != NULL) {
294  h_field = htp_table_get_c(tx->request_headers,
295  http_fields[f].htp_field);
296  }
297  } else {
298  if (tx->response_headers != NULL) {
299  h_field = htp_table_get_c(tx->response_headers,
300  http_fields[f].htp_field);
301  }
302  }
303  if (h_field != NULL) {
304  c = bstr_util_strdup_to_c(h_field->value);
305  if (c != NULL) {
306  json_object_set_new(js,
307  http_fields[f].config_field,
308  SCJsonString(c));
309  SCFree(c);
310  }
311  }
312  }
313  }
314  }
315 }
316 
317 static void JsonHttpLogJSONExtended(json_t *js, htp_tx_t *tx)
318 {
319  char *c;
320 
321  /* referer */
322  htp_header_t *h_referer = NULL;
323  if (tx->request_headers != NULL) {
324  h_referer = htp_table_get_c(tx->request_headers, "referer");
325  }
326  if (h_referer != NULL) {
327  c = bstr_util_strdup_to_c(h_referer->value);
328  if (c != NULL) {
329  json_object_set_new(js, "http_refer", SCJsonString(c));
330  SCFree(c);
331  }
332  }
333 
334  /* method */
335  if (tx->request_method != NULL) {
336  c = bstr_util_strdup_to_c(tx->request_method);
337  if (c != NULL) {
338  json_object_set_new(js, "http_method", SCJsonString(c));
339  SCFree(c);
340  }
341  }
342 
343  /* protocol */
344  if (tx->request_protocol != NULL) {
345  c = bstr_util_strdup_to_c(tx->request_protocol);
346  if (c != NULL) {
347  json_object_set_new(js, "protocol", SCJsonString(c));
348  SCFree(c);
349  }
350  }
351 
352  /* response status */
353  if (tx->response_status != NULL) {
354  c = bstr_util_strdup_to_c(tx->response_status);
355  if (c != NULL) {
356  unsigned int val = strtoul(c, NULL, 10);
357  json_object_set_new(js, "status", json_integer(val));
358  SCFree(c);
359  }
360 
361  htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
362  if (h_location != NULL) {
363  c = bstr_util_strdup_to_c(h_location->value);
364  if (c != NULL) {
365  json_object_set_new(js, "redirect", SCJsonString(c));
366  SCFree(c);
367  }
368  }
369  }
370 
371  /* length */
372  json_object_set_new(js, "length", json_integer(tx->response_message_len));
373 }
374 
375 static void JsonHttpLogJSONHeaders(json_t *js, uint32_t direction, htp_tx_t *tx)
376 {
377  htp_table_t * headers = direction & LOG_HTTP_REQ_HEADERS ?
378  tx->request_headers : tx->response_headers;
379  char name[MAX_SIZE_HEADER_NAME] = {0};
380  char value[MAX_SIZE_HEADER_VALUE] = {0};
381  size_t n = htp_table_size(headers);
382  json_t * arr = json_array();
383  if (arr == NULL) {
384  return;
385  }
386  for (size_t i = 0; i < n; i++) {
387  htp_header_t * h = htp_table_get_index(headers, i, NULL);
388  if (h == NULL) {
389  continue;
390  }
391  json_t * obj = json_object();
392  if (obj == NULL) {
393  continue;
394  }
395  size_t size_name = bstr_len(h->name) < MAX_SIZE_HEADER_NAME - 1 ?
396  bstr_len(h->name) : MAX_SIZE_HEADER_NAME - 1;
397  memcpy(name, bstr_ptr(h->name), size_name);
398  name[size_name] = '\0';
399  json_object_set_new(obj, "name", SCJsonString(name));
400  size_t size_value = bstr_len(h->value) < MAX_SIZE_HEADER_VALUE - 1 ?
401  bstr_len(h->value) : MAX_SIZE_HEADER_VALUE - 1;
402  memcpy(value, bstr_ptr(h->value), size_value);
403  value[size_value] = '\0';
404  json_object_set_new(obj, "value", SCJsonString(value));
405  json_array_append_new(arr, obj);
406  }
407  json_object_set_new(js, direction & LOG_HTTP_REQ_HEADERS ?
408  "request_headers" : "response_headers", arr);
409 }
410 
411 static void BodyPrintableBuffer(json_t *js, HtpBody *body, const char *key)
412 {
413  if (body->sb != NULL && body->sb->buf != NULL) {
414  uint32_t offset = 0;
415  const uint8_t *body_data;
416  uint32_t body_data_len;
417  uint64_t body_offset;
418 
419  if (StreamingBufferGetData(body->sb, &body_data,
420  &body_data_len, &body_offset) == 0) {
421  return;
422  }
423 
424  uint8_t printable_buf[body_data_len + 1];
425  PrintStringsToBuffer(printable_buf, &offset,
426  sizeof(printable_buf),
427  body_data, body_data_len);
428  if (offset > 0) {
429  json_object_set_new(js, key, json_string((char *)printable_buf));
430  }
431  }
432 }
433 
434 void JsonHttpLogJSONBodyPrintable(json_t *js, Flow *f, uint64_t tx_id)
435 {
436  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
437  if (htp_state) {
438  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
439  if (tx) {
440  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
441  if (htud != NULL) {
442  BodyPrintableBuffer(js, &htud->request_body, "http_request_body_printable");
443  BodyPrintableBuffer(js, &htud->response_body, "http_response_body_printable");
444  }
445  }
446  }
447 }
448 
449 static void BodyBase64Buffer(json_t *js, HtpBody *body, const char *key)
450 {
451  if (body->sb != NULL && body->sb->buf != NULL) {
452  const uint8_t *body_data;
453  uint32_t body_data_len;
454  uint64_t body_offset;
455 
456  if (StreamingBufferGetData(body->sb, &body_data,
457  &body_data_len, &body_offset) == 0) {
458  return;
459  }
460 
461  unsigned long len = body_data_len * 2 + 1;
462  uint8_t encoded[len];
463  if (Base64Encode(body_data, body_data_len, encoded, &len) == SC_BASE64_OK) {
464  json_object_set_new(js, key, json_string((char *)encoded));
465  }
466  }
467 }
468 
469 void JsonHttpLogJSONBodyBase64(json_t *js, Flow *f, uint64_t tx_id)
470 {
471  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
472  if (htp_state) {
473  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
474  if (tx) {
475  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
476  if (htud != NULL) {
477  BodyBase64Buffer(js, &htud->request_body, "http_request_body");
478  BodyBase64Buffer(js, &htud->response_body, "http_response_body");
479  }
480  }
481  }
482 }
483 
484 /* JSON format logging */
485 static void JsonHttpLogJSON(JsonHttpLogThread *aft, json_t *js, htp_tx_t *tx, uint64_t tx_id)
486 {
487  LogHttpFileCtx *http_ctx = aft->httplog_ctx;
488  json_t *hjs = json_object();
489  if (hjs == NULL) {
490  return;
491  }
492 
493  JsonHttpLogJSONBasic(hjs, tx);
494  /* log custom fields if configured */
495  if (http_ctx->fields != 0)
496  JsonHttpLogJSONCustom(http_ctx, hjs, tx);
497  if (http_ctx->flags & LOG_HTTP_EXTENDED)
498  JsonHttpLogJSONExtended(hjs, tx);
499  if (http_ctx->flags & LOG_HTTP_REQ_HEADERS)
500  JsonHttpLogJSONHeaders(hjs, LOG_HTTP_REQ_HEADERS, tx);
501  if (http_ctx->flags & LOG_HTTP_RES_HEADERS)
502  JsonHttpLogJSONHeaders(hjs, LOG_HTTP_RES_HEADERS, tx);
503 
504  json_object_set_new(js, "http", hjs);
505 }
506 
507 static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
508 {
509  SCEnter();
510 
511  htp_tx_t *tx = txptr;
512  JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data;
513 
514  json_t *js = CreateJSONHeaderWithTxId(p, LOG_DIR_FLOW, "http", tx_id);
515  if (unlikely(js == NULL))
516  return TM_ECODE_OK;
517 
518  JsonAddCommonOptions(&jhl->httplog_ctx->cfg, p, f, js);
519 
520  SCLogDebug("got a HTTP request and now logging !!");
521 
522  /* reset */
523  MemBufferReset(jhl->buffer);
524 
525  JsonHttpLogJSON(jhl, js, tx, tx_id);
526  HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg != NULL ?
527  jhl->httplog_ctx->xff_cfg : jhl->httplog_ctx->parent_xff_cfg;
528 
529  /* xff header */
530  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
531  int have_xff_ip = 0;
532  char buffer[XFF_MAXLEN];
533 
534  have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN);
535 
536  if (have_xff_ip) {
537  if (xff_cfg->flags & XFF_EXTRADATA) {
538  json_object_set_new(js, "xff", json_string(buffer));
539  }
540  else if (xff_cfg->flags & XFF_OVERWRITE) {
541  if (p->flowflags & FLOW_PKT_TOCLIENT) {
542  json_object_set(js, "dest_ip", json_string(buffer));
543  } else {
544  json_object_set(js, "src_ip", json_string(buffer));
545  }
546  }
547  }
548  }
549 
550  OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer);
551  json_object_del(js, "http");
552 
553  json_object_clear(js);
554  json_decref(js);
555 
557 }
558 
559 json_t *JsonHttpAddMetadata(const Flow *f, uint64_t tx_id)
560 {
561  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
562  if (htp_state) {
563  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
564 
565  if (tx) {
566  json_t *hjs = json_object();
567  if (unlikely(hjs == NULL))
568  return NULL;
569 
570  JsonHttpLogJSONBasic(hjs, tx);
571  JsonHttpLogJSONExtended(hjs, tx);
572  return hjs;
573  }
574  }
575 
576  return NULL;
577 }
578 
579 static void OutputHttpLogDeinit(OutputCtx *output_ctx)
580 {
581  LogHttpFileCtx *http_ctx = output_ctx->data;
582  LogFileCtx *logfile_ctx = http_ctx->file_ctx;
583  LogFileFreeCtx(logfile_ctx);
584  if (http_ctx->xff_cfg) {
585  SCFree(http_ctx->xff_cfg);
586  }
587  SCFree(http_ctx);
588  SCFree(output_ctx);
589 }
590 
591 #define DEFAULT_LOG_FILENAME "http.json"
592 static OutputInitResult OutputHttpLogInit(ConfNode *conf)
593 {
594  OutputInitResult result = { NULL, false };
596  if(file_ctx == NULL) {
597  SCLogError(SC_ERR_HTTP_LOG_GENERIC, "couldn't create new file_ctx");
598  return result;
599  }
600 
601  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
602  LogFileFreeCtx(file_ctx);
603  return result;
604  }
605 
606  LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx));
607  if (unlikely(http_ctx == NULL)) {
608  LogFileFreeCtx(file_ctx);
609  return result;
610  }
611 
612  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
613  if (unlikely(output_ctx == NULL)) {
614  LogFileFreeCtx(file_ctx);
615  SCFree(http_ctx);
616  return result;
617  }
618 
619  http_ctx->file_ctx = file_ctx;
620  http_ctx->flags = LOG_HTTP_DEFAULT;
621 
622  if (conf) {
623  const char *extended = ConfNodeLookupChildValue(conf, "extended");
624 
625  if (extended != NULL) {
626  if (ConfValIsTrue(extended)) {
627  http_ctx->flags = LOG_HTTP_EXTENDED;
628  }
629  }
630  const char *all_headers = ConfNodeLookupChildValue(
631  conf, "dump-all-headers");
632  if (all_headers != NULL) {
633  if (strcmp(all_headers, "both") == 0) {
634  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
635  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
636  } else if (strcmp(all_headers, "request") == 0) {
637  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
638  } else if (strcmp(all_headers, "response") == 0) {
639  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
640  }
641  }
642  }
643  http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
644  if (http_ctx->xff_cfg != NULL) {
645  HttpXFFGetCfg(conf, http_ctx->xff_cfg);
646  }
647 
648  output_ctx->data = http_ctx;
649  output_ctx->DeInit = OutputHttpLogDeinit;
650 
651  /* enable the logger for the app layer */
653 
654  result.ctx = output_ctx;
655  result.ok = true;
656  return result;
657 }
658 
659 static void OutputHttpLogDeinitSub(OutputCtx *output_ctx)
660 {
661  LogHttpFileCtx *http_ctx = output_ctx->data;
662  if (http_ctx->xff_cfg) {
663  SCFree(http_ctx->xff_cfg);
664  }
665  SCFree(http_ctx);
666  SCFree(output_ctx);
667 }
668 
669 static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
670 {
671  OutputInitResult result = { NULL, false };
672  OutputJsonCtx *ojc = parent_ctx->data;
673 
674  LogHttpFileCtx *http_ctx = SCCalloc(1, sizeof(LogHttpFileCtx));
675  if (unlikely(http_ctx == NULL))
676  return result;
677  memset(http_ctx, 0x00, sizeof(*http_ctx));
678 
679  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
680  if (unlikely(output_ctx == NULL)) {
681  SCFree(http_ctx);
682  return result;
683  }
684 
685  http_ctx->file_ctx = ojc->file_ctx;
686  http_ctx->flags = LOG_HTTP_DEFAULT;
687  http_ctx->cfg = ojc->cfg;
688 
689  if (conf) {
690  const char *extended = ConfNodeLookupChildValue(conf, "extended");
691 
692  if (extended != NULL) {
693  if (ConfValIsTrue(extended)) {
694  http_ctx->flags = LOG_HTTP_EXTENDED;
695  }
696  }
697 
698  ConfNode *custom;
699  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
700  ConfNode *field;
701  TAILQ_FOREACH(field, &custom->head, next)
702  {
703  if (field != NULL)
704  {
705  HttpField f;
706  for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++)
707  {
708  if ((strcmp(http_fields[f].config_field,
709  field->val) == 0) ||
710  (strcasecmp(http_fields[f].htp_field,
711  field->val) == 0))
712  {
713  http_ctx->fields |= (1ULL<<f);
714  break;
715  }
716  }
717  }
718  }
719  }
720  const char *all_headers = ConfNodeLookupChildValue(
721  conf, "dump-all-headers");
722  if (all_headers != NULL) {
723  if (strncmp(all_headers, "both", 4) == 0) {
724  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
725  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
726  } else if (strncmp(all_headers, "request", 7) == 0) {
727  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
728  } else if (strncmp(all_headers, "response", 8) == 0) {
729  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
730  }
731  }
732  }
733 
734  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
735  http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
736  if (http_ctx->xff_cfg != NULL) {
737  HttpXFFGetCfg(conf, http_ctx->xff_cfg);
738  }
739  } else if (ojc->xff_cfg) {
740  http_ctx->parent_xff_cfg = ojc->xff_cfg;
741  }
742 
743  output_ctx->data = http_ctx;
744  output_ctx->DeInit = OutputHttpLogDeinitSub;
745 
746  /* enable the logger for the app layer */
748 
749  result.ctx = output_ctx;
750  result.ok = true;
751  return result;
752 }
753 
754 #define OUTPUT_BUFFER_SIZE 65535
755 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data)
756 {
757  JsonHttpLogThread *aft = SCMalloc(sizeof(JsonHttpLogThread));
758  if (unlikely(aft == NULL))
759  return TM_ECODE_FAILED;
760  memset(aft, 0, sizeof(JsonHttpLogThread));
761 
762  if(initdata == NULL)
763  {
764  SCLogDebug("Error getting context for EveLogHTTP. \"initdata\" argument NULL");
765  SCFree(aft);
766  return TM_ECODE_FAILED;
767  }
768 
769  /* Use the Ouptut Context (file pointer and mutex) */
770  aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO
771 
772  aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
773  if (aft->buffer == NULL) {
774  SCFree(aft);
775  return TM_ECODE_FAILED;
776  }
777 
778  *data = (void *)aft;
779  return TM_ECODE_OK;
780 }
781 
782 static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data)
783 {
784  JsonHttpLogThread *aft = (JsonHttpLogThread *)data;
785  if (aft == NULL) {
786  return TM_ECODE_OK;
787  }
788 
789  MemBufferFree(aft->buffer);
790  /* clear memory */
791  memset(aft, 0, sizeof(JsonHttpLogThread));
792 
793  SCFree(aft);
794  return TM_ECODE_OK;
795 }
796 
797 void JsonHttpLogRegister (void)
798 {
799  /* register as separate module */
800  OutputRegisterTxModule(LOGGER_JSON_HTTP, "JsonHttpLog", "http-json-log",
801  OutputHttpLogInit, ALPROTO_HTTP, JsonHttpLogger, JsonHttpLogThreadInit,
802  JsonHttpLogThreadDeinit, NULL);
803 
804  /* also register as child of eve-log */
805  OutputRegisterTxSubModule(LOGGER_JSON_HTTP, "eve-log", "JsonHttpLog",
806  "eve-log.http", OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger,
807  JsonHttpLogThreadInit, JsonHttpLogThreadDeinit, NULL);
808 }
809 
810 #else
811 
813 {
814 }
815 
816 #endif
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define SCLogDebug(...)
Definition: util-debug.h:335
struct LogHttpFileCtx_ LogHttpFileCtx
struct Flow_ * flow
Definition: decode.h:444
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
struct HtpBodyChunk_ * next
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
uint64_t offset
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
HtpBody request_body
#define XFF_MAXLEN
#define SCCalloc(nm, a)
Definition: util-mem.h:205
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define LOG_HTTP_EXTENDED
Definition: log-httplog.c:93
HtpBody response_body
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow...
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
StreamingBuffer * sb
void AppLayerParserRegisterLogger(uint8_t ipproto, AppProto alproto)
#define SCEnter(...)
Definition: util-debug.h:337
uint8_t flowflags
Definition: decode.h:438
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
void PrintStringsToBuffer(uint8_t *dst_buf, uint32_t *dst_buf_offset_ptr, uint32_t dst_buf_size, const uint8_t *src_buf, const uint32_t src_buf_len)
Definition: util-print.c:224
#define SCReturnInt(x)
Definition: util-debug.h:341
void OutputRegisterTxSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output.c:399
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
void JsonHttpLogRegister(void)
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define DEFAULT_LOG_FILENAME
#define SCMalloc(a)
Definition: util-mem.h:174
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
LogFileCtx * file_ctx
Definition: log-httplog.c:87
#define XFF_OVERWRITE
#define SCFree(a)
Definition: util-mem.h:236
#define OUTPUT_BUFFER_SIZE
Definition: log-dnslog.c:58
#define LOG_HTTP_DEFAULT
Definition: log-httplog.c:92
uint16_t tx_id
void * data
Definition: tm-modules.h:81
void OutputRegisterTxModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a tx output module.
Definition: output.c:388
void * FlowGetAppState(const Flow *f)
Definition: flow.c:997
int Base64Encode(const unsigned char *in, unsigned long inlen, unsigned char *out, unsigned long *outlen)
Definition: util-crypt.c:272
#define XFF_DISABLED
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define FLOW_PKT_TOCLIENT
Definition: flow.h:194
uint32_t flags
Definition: log-httplog.c:88
Flow data structure.
Definition: flow.h:327
#define XFF_EXTRADATA
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82