suricata
output-json-http.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Implements HTTP JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "app-layer-htp.h"
43 #include "app-layer-htp-file.h"
44 #include "app-layer-htp-xff.h"
45 #include "app-layer.h"
46 #include "app-layer-parser.h"
47 #include "util-privs.h"
48 #include "util-buffer.h"
49 #include "util-proto-name.h"
50 #include "util-logopenfile.h"
51 #include "util-time.h"
52 #include "util-crypt.h"
53 #include "output-json.h"
54 #include "output-json-alert.h"
55 #include "output-json-http.h"
56 #include "util-byte.h"
57 
58 typedef struct LogHttpFileCtx_ {
60  uint32_t flags; /** Store mode */
61  uint64_t fields;/** Store fields */
66 
67 typedef struct JsonHttpLogThread_ {
69  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
70  uint32_t uri_cnt;
71 
74 
75 #define MAX_SIZE_HEADER_NAME 256
76 #define MAX_SIZE_HEADER_VALUE 2048
77 
78 #define LOG_HTTP_DEFAULT 0
79 #define LOG_HTTP_EXTENDED 1
80 #define LOG_HTTP_REQUEST 2 /* request field */
81 #define LOG_HTTP_ARRAY 4 /* require array handling */
82 #define LOG_HTTP_REQ_HEADERS 8
83 #define LOG_HTTP_RES_HEADERS 16
84 
85 typedef enum {
140 } HttpField;
141 
142 struct {
143  const char *config_field;
144  const char *htp_field;
145  uint32_t flags;
146 } http_fields[] = {
147  { "accept", "accept", LOG_HTTP_REQUEST },
148  { "accept_charset", "accept-charset", LOG_HTTP_REQUEST },
149  { "accept_encoding", "accept-encoding", LOG_HTTP_REQUEST },
150  { "accept_language", "accept-language", LOG_HTTP_REQUEST },
151  { "accept_datetime", "accept-datetime", LOG_HTTP_REQUEST },
152  { "authorization", "authorization", LOG_HTTP_REQUEST },
153  { "cache_control", "cache-control", LOG_HTTP_REQUEST },
154  { "cookie", "cookie", LOG_HTTP_REQUEST|LOG_HTTP_ARRAY },
155  { "from", "from", LOG_HTTP_REQUEST },
156  { "max_forwards", "max-forwards", LOG_HTTP_REQUEST },
157  { "origin", "origin", LOG_HTTP_REQUEST },
158  { "pragma", "pragma", LOG_HTTP_REQUEST },
159  { "proxy_authorization", "proxy-authorization", LOG_HTTP_REQUEST },
160  { "range", "range", LOG_HTTP_REQUEST },
161  { "te", "te", LOG_HTTP_REQUEST },
162  { "via", "via", LOG_HTTP_REQUEST },
163  { "x_requested_with", "x-requested-with", LOG_HTTP_REQUEST },
164  { "dnt", "dnt", LOG_HTTP_REQUEST },
165  { "x_forwarded_proto", "x-forwarded-proto", LOG_HTTP_REQUEST },
166  { "x_authenticated_user", "x-authenticated-user", LOG_HTTP_REQUEST },
167  { "x_flash_version", "x-flash-version", LOG_HTTP_REQUEST },
168  { "accept_range", "accept-range", 0 },
169  { "age", "age", 0 },
170  { "allow", "allow", 0 },
171  { "connection", "connection", 0 },
172  { "content_encoding", "content-encoding", 0 },
173  { "content_language", "content-language", 0 },
174  { "content_length", "content-length", 0 },
175  { "content_location", "content-location", 0 },
176  { "content_md5", "content-md5", 0 },
177  { "content_range", "content-range", 0 },
178  { "content_type", "content-type", 0 },
179  { "date", "date", 0 },
180  { "etag", "etags", 0 },
181  { "expires", "expires" , 0 },
182  { "last_modified", "last-modified", 0 },
183  { "link", "link", 0 },
184  { "location", "location", 0 },
185  { "proxy_authenticate", "proxy-authenticate", 0 },
186  { "referrer", "referrer", LOG_HTTP_EXTENDED },
187  { "refresh", "refresh", 0 },
188  { "retry_after", "retry-after", 0 },
189  { "server", "server", 0 },
190  { "set_cookie", "set-cookie", 0 },
191  { "trailer", "trailer", 0 },
192  { "transfer_encoding", "transfer-encoding", 0 },
193  { "upgrade", "upgrade", 0 },
194  { "vary", "vary", 0 },
195  { "warning", "warning", 0 },
196  { "www_authenticate", "www-authenticate", 0 },
197  { "true_client_ip", "true-client-ip", LOG_HTTP_REQUEST },
198  { "org_src_ip", "org-src-ip", LOG_HTTP_REQUEST },
199  { "x_bluecoat_via", "x-bluecoat-via", LOG_HTTP_REQUEST },
200 };
201 
202 static void JsonHttpLogJSONBasic(json_t *js, htp_tx_t *tx)
203 {
204  /* hostname */
205  if (tx->request_hostname != NULL) {
206  const size_t size = bstr_len(tx->request_hostname) * 2 + 1;
207  char string[size];
208  BytesToStringBuffer(bstr_ptr(tx->request_hostname), bstr_len(tx->request_hostname), string, size);
209  json_object_set_new(js, "hostname", SCJsonString(string));
210  }
211 
212  /* port */
213  /* NOTE: this field will be set ONLY if the port is present in the
214  * hostname. It may be present in the header "Host" or in the URL.
215  * There is no connection (from the suricata point of view) between this
216  * port and the TCP destination port of the flow.
217  */
218  if (tx->request_port_number >= 0) {
219  json_object_set_new(js, "http_port",
220  json_integer(tx->request_port_number));
221  }
222 
223  /* uri */
224  if (tx->request_uri != NULL) {
225  const size_t size = bstr_len(tx->request_uri) * 2 + 1;
226  char string[size];
227  BytesToStringBuffer(bstr_ptr(tx->request_uri), bstr_len(tx->request_uri), string, size);
228  json_object_set_new(js, "url", SCJsonString(string));
229  }
230 
231  if (tx->request_headers != NULL) {
232  /* user agent */
233  htp_header_t *h_user_agent = htp_table_get_c(tx->request_headers, "user-agent");
234  if (h_user_agent != NULL) {
235  const size_t size = bstr_len(h_user_agent->value) * 2 + 1;
236  char string[size];
237  BytesToStringBuffer(bstr_ptr(h_user_agent->value), bstr_len(h_user_agent->value), string, size);
238  json_object_set_new(js, "http_user_agent", SCJsonString(string));
239  }
240 
241  /* x-forwarded-for */
242  htp_header_t *h_x_forwarded_for = htp_table_get_c(tx->request_headers, "x-forwarded-for");
243  if (h_x_forwarded_for != NULL) {
244  const size_t size = bstr_len(h_x_forwarded_for->value) * 2 + 1;
245  char string[size];
246  BytesToStringBuffer(bstr_ptr(h_x_forwarded_for->value), bstr_len(h_x_forwarded_for->value), string, size);
247  json_object_set_new(js, "xff", json_string(string));
248  }
249  }
250 
251  /* content-type */
252  if (tx->response_headers != NULL) {
253  htp_header_t *h_content_type = htp_table_get_c(tx->response_headers, "content-type");
254  if (h_content_type != NULL) {
255  const size_t size = bstr_len(h_content_type->value) * 2 + 1;
256  char string[size];
257  BytesToStringBuffer(bstr_ptr(h_content_type->value), bstr_len(h_content_type->value), string, size);
258  char *p = strchr(string, ';');
259  if (p != NULL)
260  *p = '\0';
261  json_object_set_new(js, "http_content_type", SCJsonString(string));
262  }
263  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
264  if (h_content_range != NULL) {
265  const size_t size = bstr_len(h_content_range->value) * 2 + 1;
266  char string[size];
267  BytesToStringBuffer(bstr_ptr(h_content_range->value), bstr_len(h_content_range->value), string, size);
268  json_t *crjs = json_object();
269  if (crjs != NULL) {
270  json_object_set_new(crjs, "raw", SCJsonString(string));
271  HtpContentRange crparsed;
272  if (HTPParseContentRange(h_content_range->value, &crparsed) == 0) {
273  if (crparsed.start >= 0)
274  json_object_set_new(crjs, "start", json_integer(crparsed.start));
275  if (crparsed.end >= 0)
276  json_object_set_new(crjs, "end", json_integer(crparsed.end));
277  if (crparsed.size >= 0)
278  json_object_set_new(crjs, "size", json_integer(crparsed.size));
279  }
280  json_object_set_new(js, "content_range", crjs);
281  }
282  }
283  }
284 }
285 
286 static void JsonHttpLogJSONCustom(LogHttpFileCtx *http_ctx, json_t *js, htp_tx_t *tx)
287 {
288  char *c;
289  HttpField f;
290 
291  for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++)
292  {
293  if ((http_ctx->fields & (1ULL<<f)) != 0)
294  {
295  /* prevent logging a field twice if extended logging is
296  enabled */
297  if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) ||
298  ((http_ctx->flags & LOG_HTTP_EXTENDED) !=
299  (http_fields[f].flags & LOG_HTTP_EXTENDED)))
300  {
301  htp_header_t *h_field = NULL;
302  if ((http_fields[f].flags & LOG_HTTP_REQUEST) != 0)
303  {
304  if (tx->request_headers != NULL) {
305  h_field = htp_table_get_c(tx->request_headers,
306  http_fields[f].htp_field);
307  }
308  } else {
309  if (tx->response_headers != NULL) {
310  h_field = htp_table_get_c(tx->response_headers,
311  http_fields[f].htp_field);
312  }
313  }
314  if (h_field != NULL) {
315  c = bstr_util_strdup_to_c(h_field->value);
316  if (c != NULL) {
317  json_object_set_new(js,
319  SCJsonString(c));
320  SCFree(c);
321  }
322  }
323  }
324  }
325  }
326 }
327 
328 static void JsonHttpLogJSONExtended(json_t *js, htp_tx_t *tx)
329 {
330  /* referer */
331  htp_header_t *h_referer = NULL;
332  if (tx->request_headers != NULL) {
333  h_referer = htp_table_get_c(tx->request_headers, "referer");
334  }
335  if (h_referer != NULL) {
336  const size_t size = bstr_len(h_referer->value) * 2 + 1;
337  char string[size];
338  BytesToStringBuffer(bstr_ptr(h_referer->value), bstr_len(h_referer->value), string, size);
339 
340  json_object_set_new(js, "http_refer", SCJsonString(string));
341  }
342 
343  /* method */
344  if (tx->request_method != NULL) {
345  const size_t size = bstr_len(tx->request_method) * 2 + 1;
346  char string[size];
347  BytesToStringBuffer(bstr_ptr(tx->request_method), bstr_len(tx->request_method), string, size);
348  json_object_set_new(js, "http_method", SCJsonString(string));
349  }
350 
351  /* protocol */
352  if (tx->request_protocol != NULL) {
353  const size_t size = bstr_len(tx->request_protocol) * 2 + 1;
354  char string[size];
355  BytesToStringBuffer(bstr_ptr(tx->request_protocol), bstr_len(tx->request_protocol), string, size);
356  json_object_set_new(js, "protocol", SCJsonString(string));
357  }
358 
359  /* response status */
360  if (tx->response_status != NULL) {
361  const size_t status_size = bstr_len(tx->response_status) * 2 + 1;
362  char status_string[status_size];
363  BytesToStringBuffer(bstr_ptr(tx->response_status), bstr_len(tx->response_status),
364  status_string, status_size);
365  unsigned int val = strtoul(status_string, NULL, 10);
366  json_object_set_new(js, "status", json_integer(val));
367 
368  htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
369  if (h_location != NULL) {
370  const size_t size = bstr_len(h_location->value) * 2 + 1;
371  char string[size];
372  BytesToStringBuffer(bstr_ptr(h_location->value), bstr_len(h_location->value), string, size);
373  json_object_set_new(js, "redirect", SCJsonString(string));
374  }
375  }
376 
377  /* length */
378  json_object_set_new(js, "length", json_integer(tx->response_message_len));
379 }
380 
381 static void JsonHttpLogJSONHeaders(json_t *js, uint32_t direction, htp_tx_t *tx)
382 {
383  htp_table_t * headers = direction & LOG_HTTP_REQ_HEADERS ?
384  tx->request_headers : tx->response_headers;
385  char name[MAX_SIZE_HEADER_NAME] = {0};
386  char value[MAX_SIZE_HEADER_VALUE] = {0};
387  size_t n = htp_table_size(headers);
388  json_t * arr = json_array();
389  if (arr == NULL) {
390  return;
391  }
392  for (size_t i = 0; i < n; i++) {
393  htp_header_t * h = htp_table_get_index(headers, i, NULL);
394  if (h == NULL) {
395  continue;
396  }
397  json_t * obj = json_object();
398  if (obj == NULL) {
399  continue;
400  }
401  size_t size_name = bstr_len(h->name) < MAX_SIZE_HEADER_NAME - 1 ?
402  bstr_len(h->name) : MAX_SIZE_HEADER_NAME - 1;
403  memcpy(name, bstr_ptr(h->name), size_name);
404  name[size_name] = '\0';
405  json_object_set_new(obj, "name", SCJsonString(name));
406  size_t size_value = bstr_len(h->value) < MAX_SIZE_HEADER_VALUE - 1 ?
407  bstr_len(h->value) : MAX_SIZE_HEADER_VALUE - 1;
408  memcpy(value, bstr_ptr(h->value), size_value);
409  value[size_value] = '\0';
410  json_object_set_new(obj, "value", SCJsonString(value));
411  json_array_append_new(arr, obj);
412  }
413  json_object_set_new(js, direction & LOG_HTTP_REQ_HEADERS ?
414  "request_headers" : "response_headers", arr);
415 }
416 
417 static void BodyPrintableBuffer(json_t *js, HtpBody *body, const char *key)
418 {
419  if (body->sb != NULL && body->sb->buf != NULL) {
420  uint32_t offset = 0;
421  const uint8_t *body_data;
422  uint32_t body_data_len;
423  uint64_t body_offset;
424 
425  if (StreamingBufferGetData(body->sb, &body_data,
426  &body_data_len, &body_offset) == 0) {
427  return;
428  }
429 
430  uint8_t printable_buf[body_data_len + 1];
431  PrintStringsToBuffer(printable_buf, &offset,
432  sizeof(printable_buf),
433  body_data, body_data_len);
434  if (offset > 0) {
435  json_object_set_new(js, key, json_string((char *)printable_buf));
436  }
437  }
438 }
439 
440 void JsonHttpLogJSONBodyPrintable(json_t *js, Flow *f, uint64_t tx_id)
441 {
442  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
443  if (htp_state) {
444  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
445  if (tx) {
446  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
447  if (htud != NULL) {
448  BodyPrintableBuffer(js, &htud->request_body, "http_request_body_printable");
449  BodyPrintableBuffer(js, &htud->response_body, "http_response_body_printable");
450  }
451  }
452  }
453 }
454 
455 static void BodyBase64Buffer(json_t *js, HtpBody *body, const char *key)
456 {
457  if (body->sb != NULL && body->sb->buf != NULL) {
458  const uint8_t *body_data;
459  uint32_t body_data_len;
460  uint64_t body_offset;
461 
462  if (StreamingBufferGetData(body->sb, &body_data,
463  &body_data_len, &body_offset) == 0) {
464  return;
465  }
466 
467  unsigned long len = body_data_len * 2 + 1;
468  uint8_t encoded[len];
469  if (Base64Encode(body_data, body_data_len, encoded, &len) == SC_BASE64_OK) {
470  json_object_set_new(js, key, json_string((char *)encoded));
471  }
472  }
473 }
474 
475 void JsonHttpLogJSONBodyBase64(json_t *js, Flow *f, uint64_t tx_id)
476 {
477  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
478  if (htp_state) {
479  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
480  if (tx) {
481  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
482  if (htud != NULL) {
483  BodyBase64Buffer(js, &htud->request_body, "http_request_body");
484  BodyBase64Buffer(js, &htud->response_body, "http_response_body");
485  }
486  }
487  }
488 }
489 
490 /* JSON format logging */
491 static void JsonHttpLogJSON(JsonHttpLogThread *aft, json_t *js, htp_tx_t *tx, uint64_t tx_id)
492 {
493  LogHttpFileCtx *http_ctx = aft->httplog_ctx;
494  json_t *hjs = json_object();
495  if (hjs == NULL) {
496  return;
497  }
498 
499  JsonHttpLogJSONBasic(hjs, tx);
500  /* log custom fields if configured */
501  if (http_ctx->fields != 0)
502  JsonHttpLogJSONCustom(http_ctx, hjs, tx);
503  if (http_ctx->flags & LOG_HTTP_EXTENDED)
504  JsonHttpLogJSONExtended(hjs, tx);
505  if (http_ctx->flags & LOG_HTTP_REQ_HEADERS)
506  JsonHttpLogJSONHeaders(hjs, LOG_HTTP_REQ_HEADERS, tx);
507  if (http_ctx->flags & LOG_HTTP_RES_HEADERS)
508  JsonHttpLogJSONHeaders(hjs, LOG_HTTP_RES_HEADERS, tx);
509 
510  json_object_set_new(js, "http", hjs);
511 }
512 
513 static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
514 {
515  SCEnter();
516 
517  htp_tx_t *tx = txptr;
518  JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data;
519 
520  json_t *js = CreateJSONHeaderWithTxId(p, LOG_DIR_FLOW, "http", tx_id);
521  if (unlikely(js == NULL))
522  return TM_ECODE_OK;
523 
524  JsonAddCommonOptions(&jhl->httplog_ctx->cfg, p, f, js);
525 
526  SCLogDebug("got a HTTP request and now logging !!");
527 
528  /* reset */
529  MemBufferReset(jhl->buffer);
530 
531  JsonHttpLogJSON(jhl, js, tx, tx_id);
532  HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg != NULL ?
534 
535  /* xff header */
536  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
537  int have_xff_ip = 0;
538  char buffer[XFF_MAXLEN];
539 
540  have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN);
541 
542  if (have_xff_ip) {
543  if (xff_cfg->flags & XFF_EXTRADATA) {
544  json_object_set_new(js, "xff", json_string(buffer));
545  }
546  else if (xff_cfg->flags & XFF_OVERWRITE) {
547  if (p->flowflags & FLOW_PKT_TOCLIENT) {
548  json_object_set(js, "dest_ip", json_string(buffer));
549  } else {
550  json_object_set(js, "src_ip", json_string(buffer));
551  }
552  }
553  }
554  }
555 
556  OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer);
557  json_object_del(js, "http");
558 
559  json_object_clear(js);
560  json_decref(js);
561 
563 }
564 
565 json_t *JsonHttpAddMetadata(const Flow *f, uint64_t tx_id)
566 {
567  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
568  if (htp_state) {
569  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
570 
571  if (tx) {
572  json_t *hjs = json_object();
573  if (unlikely(hjs == NULL))
574  return NULL;
575 
576  JsonHttpLogJSONBasic(hjs, tx);
577  JsonHttpLogJSONExtended(hjs, tx);
578  return hjs;
579  }
580  }
581 
582  return NULL;
583 }
584 
585 static void OutputHttpLogDeinit(OutputCtx *output_ctx)
586 {
587  LogHttpFileCtx *http_ctx = output_ctx->data;
588  LogFileCtx *logfile_ctx = http_ctx->file_ctx;
589  LogFileFreeCtx(logfile_ctx);
590  if (http_ctx->xff_cfg) {
591  SCFree(http_ctx->xff_cfg);
592  }
593  SCFree(http_ctx);
594  SCFree(output_ctx);
595 }
596 
597 #define DEFAULT_LOG_FILENAME "http.json"
598 static OutputInitResult OutputHttpLogInit(ConfNode *conf)
599 {
600  OutputInitResult result = { NULL, false };
602  if(file_ctx == NULL) {
603  SCLogError(SC_ERR_HTTP_LOG_GENERIC, "couldn't create new file_ctx");
604  return result;
605  }
606 
607  if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
608  LogFileFreeCtx(file_ctx);
609  return result;
610  }
611 
612  LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx));
613  if (unlikely(http_ctx == NULL)) {
614  LogFileFreeCtx(file_ctx);
615  return result;
616  }
617 
618  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
619  if (unlikely(output_ctx == NULL)) {
620  LogFileFreeCtx(file_ctx);
621  SCFree(http_ctx);
622  return result;
623  }
624 
625  http_ctx->file_ctx = file_ctx;
626  http_ctx->flags = LOG_HTTP_DEFAULT;
627 
628  if (conf) {
629  const char *extended = ConfNodeLookupChildValue(conf, "extended");
630 
631  if (extended != NULL) {
632  if (ConfValIsTrue(extended)) {
633  http_ctx->flags = LOG_HTTP_EXTENDED;
634  }
635  }
636  const char *all_headers = ConfNodeLookupChildValue(
637  conf, "dump-all-headers");
638  if (all_headers != NULL) {
639  if (strcmp(all_headers, "both") == 0) {
640  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
641  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
642  } else if (strcmp(all_headers, "request") == 0) {
643  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
644  } else if (strcmp(all_headers, "response") == 0) {
645  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
646  }
647  }
648  }
649  http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
650  if (http_ctx->xff_cfg != NULL) {
651  HttpXFFGetCfg(conf, http_ctx->xff_cfg);
652  }
653 
654  output_ctx->data = http_ctx;
655  output_ctx->DeInit = OutputHttpLogDeinit;
656 
657  /* enable the logger for the app layer */
659 
660  result.ctx = output_ctx;
661  result.ok = true;
662  return result;
663 }
664 
665 static void OutputHttpLogDeinitSub(OutputCtx *output_ctx)
666 {
667  LogHttpFileCtx *http_ctx = output_ctx->data;
668  if (http_ctx->xff_cfg) {
669  SCFree(http_ctx->xff_cfg);
670  }
671  SCFree(http_ctx);
672  SCFree(output_ctx);
673 }
674 
675 static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
676 {
677  OutputInitResult result = { NULL, false };
678  OutputJsonCtx *ojc = parent_ctx->data;
679 
680  LogHttpFileCtx *http_ctx = SCCalloc(1, sizeof(LogHttpFileCtx));
681  if (unlikely(http_ctx == NULL))
682  return result;
683  memset(http_ctx, 0x00, sizeof(*http_ctx));
684 
685  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
686  if (unlikely(output_ctx == NULL)) {
687  SCFree(http_ctx);
688  return result;
689  }
690 
691  http_ctx->file_ctx = ojc->file_ctx;
692  http_ctx->flags = LOG_HTTP_DEFAULT;
693  http_ctx->cfg = ojc->cfg;
694 
695  if (conf) {
696  const char *extended = ConfNodeLookupChildValue(conf, "extended");
697 
698  if (extended != NULL) {
699  if (ConfValIsTrue(extended)) {
700  http_ctx->flags = LOG_HTTP_EXTENDED;
701  }
702  }
703 
704  ConfNode *custom;
705  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
706  ConfNode *field;
707  TAILQ_FOREACH(field, &custom->head, next)
708  {
709  if (field != NULL)
710  {
711  HttpField f;
712  for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++)
713  {
714  if ((strcmp(http_fields[f].config_field,
715  field->val) == 0) ||
716  (strcasecmp(http_fields[f].htp_field,
717  field->val) == 0))
718  {
719  http_ctx->fields |= (1ULL<<f);
720  break;
721  }
722  }
723  }
724  }
725  }
726  const char *all_headers = ConfNodeLookupChildValue(
727  conf, "dump-all-headers");
728  if (all_headers != NULL) {
729  if (strncmp(all_headers, "both", 4) == 0) {
730  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
731  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
732  } else if (strncmp(all_headers, "request", 7) == 0) {
733  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
734  } else if (strncmp(all_headers, "response", 8) == 0) {
735  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
736  }
737  }
738  }
739 
740  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
741  http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
742  if (http_ctx->xff_cfg != NULL) {
743  HttpXFFGetCfg(conf, http_ctx->xff_cfg);
744  }
745  } else if (ojc->xff_cfg) {
746  http_ctx->parent_xff_cfg = ojc->xff_cfg;
747  }
748 
749  output_ctx->data = http_ctx;
750  output_ctx->DeInit = OutputHttpLogDeinitSub;
751 
752  /* enable the logger for the app layer */
754 
755  result.ctx = output_ctx;
756  result.ok = true;
757  return result;
758 }
759 
760 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data)
761 {
763  if (unlikely(aft == NULL))
764  return TM_ECODE_FAILED;
765  memset(aft, 0, sizeof(JsonHttpLogThread));
766 
767  if(initdata == NULL)
768  {
769  SCLogDebug("Error getting context for EveLogHTTP. \"initdata\" argument NULL");
770  SCFree(aft);
771  return TM_ECODE_FAILED;
772  }
773 
774  /* Use the Ouptut Context (file pointer and mutex) */
775  aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO
776 
778  if (aft->buffer == NULL) {
779  SCFree(aft);
780  return TM_ECODE_FAILED;
781  }
782 
783  *data = (void *)aft;
784  return TM_ECODE_OK;
785 }
786 
787 static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data)
788 {
789  JsonHttpLogThread *aft = (JsonHttpLogThread *)data;
790  if (aft == NULL) {
791  return TM_ECODE_OK;
792  }
793 
794  MemBufferFree(aft->buffer);
795  /* clear memory */
796  memset(aft, 0, sizeof(JsonHttpLogThread));
797 
798  SCFree(aft);
799  return TM_ECODE_OK;
800 }
801 
803 {
804  /* register as separate module */
805  OutputRegisterTxModule(LOGGER_JSON_HTTP, "JsonHttpLog", "http-json-log",
806  OutputHttpLogInit, ALPROTO_HTTP, JsonHttpLogger, JsonHttpLogThreadInit,
807  JsonHttpLogThreadDeinit, NULL);
808 
809  /* also register as child of eve-log */
810  OutputRegisterTxSubModule(LOGGER_JSON_HTTP, "eve-log", "JsonHttpLog",
811  "eve-log.http", OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger,
812  JsonHttpLogThreadInit, JsonHttpLogThreadDeinit, NULL);
813 }
#define LOG_HTTP_RES_HEADERS
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:44
const char * htp_field
#define SCLogDebug(...)
Definition: util-debug.h:335
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:809
#define LOG_HTTP_EXTENDED
#define LOG_HTTP_REQUEST
HttpXFFCfg * parent_xff_cfg
struct Flow_ * flow
Definition: decode.h:446
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
struct HtpBodyChunk_ * next
struct JsonHttpLogThread_ JsonHttpLogThread
const char * config_field
HttpField
LogHttpFileCtx * httplog_ctx
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
struct @131 http_fields[]
#define unlikely(expr)
Definition: util-optimize.h:35
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
json_t * CreateJSONHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, uint64_t tx_id)
Definition: output-json.c:783
uint64_t offset
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, json_t *js)
Definition: output-json.c:390
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
HttpXFFCfg * xff_cfg
OutputJsonCommonSettings cfg
HtpBody request_body
json_t * JsonHttpAddMetadata(const Flow *f, uint64_t tx_id)
HttpXFFCfg * xff_cfg
Definition: output-json.h:82
#define XFF_MAXLEN
#define SCCalloc(nm, a)
Definition: util-mem.h:253
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define LOG_HTTP_REQ_HEADERS
HtpBody response_body
#define MAX_SIZE_HEADER_VALUE
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow...
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
StreamingBuffer * sb
OutputJsonCommonSettings cfg
Definition: output-json.h:81
void AppLayerParserRegisterLogger(uint8_t ipproto, AppProto alproto)
LogFileCtx * file_ctx
Definition: output-json.h:79
void JsonHttpLogJSONBodyBase64(json_t *js, Flow *f, uint64_t tx_id)
void BytesToStringBuffer(const uint8_t *bytes, size_t nbytes, char *outstr, size_t outlen)
Turn byte array into string.
Definition: util-byte.c:85
#define SCEnter(...)
Definition: util-debug.h:337
uint8_t flowflags
Definition: decode.h:440
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
json_t * SCJsonString(const char *val)
Definition: output-json.c:107
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
void PrintStringsToBuffer(uint8_t *dst_buf, uint32_t *dst_buf_offset_ptr, uint32_t dst_buf_size, const uint8_t *src_buf, const uint32_t src_buf_len)
Definition: util-print.c:224
#define SCReturnInt(x)
Definition: util-debug.h:341
void OutputRegisterTxSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output.c:402
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
void JsonHttpLogJSONBodyPrintable(json_t *js, Flow *f, uint64_t tx_id)
void JsonHttpLogRegister(void)
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:222
int HTPParseContentRange(bstr *rawvalue, HtpContentRange *range)
#define LOG_HTTP_ARRAY
#define DEFAULT_LOG_FILENAME
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define XFF_OVERWRITE
LogFileCtx * file_ctx
Definition: log-httplog.c:87
#define SCFree(a)
Definition: util-mem.h:322
#define LOG_HTTP_DEFAULT
#define MAX_SIZE_HEADER_NAME
uint16_t tx_id
void * data
Definition: tm-modules.h:81
void OutputRegisterTxModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a tx output module.
Definition: output.c:391
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1068
int Base64Encode(const unsigned char *in, unsigned long inlen, unsigned char *out, unsigned long *outlen)
Definition: util-crypt.c:272
#define XFF_DISABLED
uint8_t len
Per thread variable structure.
Definition: threadvars.h:57
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
uint32_t flags
Definition: log-httplog.c:88
struct LogHttpFileCtx_ LogHttpFileCtx
Flow data structure.
Definition: flow.h:325
#define XFF_EXTRADATA
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82