suricata
output-json-http.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  *
23  * Implements HTTP JSON logging portion of the engine.
24  */
25 
26 #include "suricata-common.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "pkt-var.h"
30 #include "conf.h"
31 
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 
36 #include "util-print.h"
37 #include "util-unittest.h"
38 
39 #include "util-debug.h"
40 
41 #include "output.h"
42 #include "app-layer-htp.h"
43 #include "app-layer-htp-file.h"
44 #include "app-layer-htp-xff.h"
45 #include "app-layer.h"
46 #include "app-layer-parser.h"
47 #include "util-privs.h"
48 #include "util-buffer.h"
49 #include "util-proto-name.h"
50 #include "util-logopenfile.h"
51 #include "util-time.h"
52 #include "util-crypt.h"
53 #include "output-json.h"
54 #include "output-json-alert.h"
55 #include "output-json-http.h"
56 #include "util-byte.h"
57 
58 typedef struct LogHttpFileCtx_ {
60  uint32_t flags; /** Store mode */
61  uint64_t fields;/** Store fields */
66 
67 typedef struct JsonHttpLogThread_ {
70  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
71  uint32_t uri_cnt;
72 
75 
76 #define MAX_SIZE_HEADER_NAME 256
77 #define MAX_SIZE_HEADER_VALUE 2048
78 
79 #define LOG_HTTP_DEFAULT 0
80 #define LOG_HTTP_EXTENDED 1
81 #define LOG_HTTP_REQUEST 2 /* request field */
82 #define LOG_HTTP_ARRAY 4 /* require array handling */
83 #define LOG_HTTP_REQ_HEADERS 8
84 #define LOG_HTTP_RES_HEADERS 16
85 
86 typedef enum {
142 
143 struct {
144  const char *config_field;
145  const char *htp_field;
146  uint32_t flags;
147 } http_fields[] = {
148  { "accept", "accept", LOG_HTTP_REQUEST },
149  { "accept_charset", "accept-charset", LOG_HTTP_REQUEST },
150  { "accept_encoding", "accept-encoding", LOG_HTTP_REQUEST },
151  { "accept_language", "accept-language", LOG_HTTP_REQUEST },
152  { "accept_datetime", "accept-datetime", LOG_HTTP_REQUEST },
153  { "authorization", "authorization", LOG_HTTP_REQUEST },
154  { "cache_control", "cache-control", LOG_HTTP_REQUEST },
155  { "cookie", "cookie", LOG_HTTP_REQUEST|LOG_HTTP_ARRAY },
156  { "from", "from", LOG_HTTP_REQUEST },
157  { "max_forwards", "max-forwards", LOG_HTTP_REQUEST },
158  { "origin", "origin", LOG_HTTP_REQUEST },
159  { "pragma", "pragma", LOG_HTTP_REQUEST },
160  { "proxy_authorization", "proxy-authorization", LOG_HTTP_REQUEST },
161  { "range", "range", LOG_HTTP_REQUEST },
162  { "te", "te", LOG_HTTP_REQUEST },
163  { "via", "via", LOG_HTTP_REQUEST },
164  { "x_requested_with", "x-requested-with", LOG_HTTP_REQUEST },
165  { "dnt", "dnt", LOG_HTTP_REQUEST },
166  { "x_forwarded_proto", "x-forwarded-proto", LOG_HTTP_REQUEST },
167  { "x_authenticated_user", "x-authenticated-user", LOG_HTTP_REQUEST },
168  { "x_flash_version", "x-flash-version", LOG_HTTP_REQUEST },
169  { "accept_range", "accept-range", 0 },
170  { "age", "age", 0 },
171  { "allow", "allow", 0 },
172  { "connection", "connection", 0 },
173  { "content_encoding", "content-encoding", 0 },
174  { "content_language", "content-language", 0 },
175  { "content_length", "content-length", 0 },
176  { "content_location", "content-location", 0 },
177  { "content_md5", "content-md5", 0 },
178  { "content_range", "content-range", 0 },
179  { "content_type", "content-type", 0 },
180  { "date", "date", 0 },
181  { "etag", "etags", 0 },
182  { "expires", "expires" , 0 },
183  { "last_modified", "last-modified", 0 },
184  { "link", "link", 0 },
185  { "location", "location", 0 },
186  { "proxy_authenticate", "proxy-authenticate", 0 },
187  { "referrer", "referrer", LOG_HTTP_EXTENDED },
188  { "refresh", "refresh", 0 },
189  { "retry_after", "retry-after", 0 },
190  { "server", "server", 0 },
191  { "set_cookie", "set-cookie", 0 },
192  { "trailer", "trailer", 0 },
193  { "transfer_encoding", "transfer-encoding", 0 },
194  { "upgrade", "upgrade", 0 },
195  { "vary", "vary", 0 },
196  { "warning", "warning", 0 },
197  { "www_authenticate", "www-authenticate", 0 },
198  { "true_client_ip", "true-client-ip", LOG_HTTP_REQUEST },
199  { "org_src_ip", "org-src-ip", LOG_HTTP_REQUEST },
200  { "x_bluecoat_via", "x-bluecoat-via", LOG_HTTP_REQUEST },
201 };
202 
203 static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx)
204 {
205  /* hostname */
206  if (tx->request_hostname != NULL) {
207  jb_set_string_from_bytes(
208  js, "hostname", bstr_ptr(tx->request_hostname), bstr_len(tx->request_hostname));
209  }
210 
211  /* port */
212  /* NOTE: this field will be set ONLY if the port is present in the
213  * hostname. It may be present in the header "Host" or in the URL.
214  * There is no connection (from the suricata point of view) between this
215  * port and the TCP destination port of the flow.
216  */
217  if (tx->request_port_number >= 0) {
218  jb_set_uint(js, "http_port", tx->request_port_number);
219  }
220 
221  /* uri */
222  if (tx->request_uri != NULL) {
223  jb_set_string_from_bytes(js, "url", bstr_ptr(tx->request_uri), bstr_len(tx->request_uri));
224  }
225 
226  if (tx->request_headers != NULL) {
227  /* user agent */
228  htp_header_t *h_user_agent = htp_table_get_c(tx->request_headers, "user-agent");
229  if (h_user_agent != NULL) {
230  jb_set_string_from_bytes(js, "http_user_agent", bstr_ptr(h_user_agent->value),
231  bstr_len(h_user_agent->value));
232  }
233 
234  /* x-forwarded-for */
235  htp_header_t *h_x_forwarded_for = htp_table_get_c(tx->request_headers, "x-forwarded-for");
236  if (h_x_forwarded_for != NULL) {
237  jb_set_string_from_bytes(js, "xff", bstr_ptr(h_x_forwarded_for->value),
238  bstr_len(h_x_forwarded_for->value));
239  }
240  }
241 
242  /* content-type */
243  if (tx->response_headers != NULL) {
244  htp_header_t *h_content_type = htp_table_get_c(tx->response_headers, "content-type");
245  if (h_content_type != NULL) {
246  const size_t size = bstr_len(h_content_type->value) * 2 + 1;
247  char string[size];
248  BytesToStringBuffer(bstr_ptr(h_content_type->value), bstr_len(h_content_type->value), string, size);
249  char *p = strchr(string, ';');
250  if (p != NULL)
251  *p = '\0';
252  jb_set_string(js, "http_content_type", string);
253  }
254  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
255  if (h_content_range != NULL) {
256  jb_open_object(js, "content_range");
257  jb_set_string_from_bytes(
258  js, "raw", bstr_ptr(h_content_range->value), bstr_len(h_content_range->value));
259  HtpContentRange crparsed;
260  if (HTPParseContentRange(h_content_range->value, &crparsed) == 0) {
261  if (crparsed.start >= 0)
262  jb_set_uint(js, "start", crparsed.start);
263  if (crparsed.end >= 0)
264  jb_set_uint(js, "end", crparsed.end);
265  if (crparsed.size >= 0)
266  jb_set_uint(js, "size", crparsed.size);
267  }
268  jb_close(js);
269  }
270  }
271 }
272 
273 static void EveHttpLogJSONCustom(LogHttpFileCtx *http_ctx, JsonBuilder *js, htp_tx_t *tx)
274 {
275  char *c;
276  HttpField f;
277 
278  for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++)
279  {
280  if ((http_ctx->fields & (1ULL<<f)) != 0)
281  {
282  /* prevent logging a field twice if extended logging is
283  enabled */
284  if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) ||
285  ((http_ctx->flags & LOG_HTTP_EXTENDED) !=
286  (http_fields[f].flags & LOG_HTTP_EXTENDED)))
287  {
288  htp_header_t *h_field = NULL;
289  if ((http_fields[f].flags & LOG_HTTP_REQUEST) != 0)
290  {
291  if (tx->request_headers != NULL) {
292  h_field = htp_table_get_c(tx->request_headers,
293  http_fields[f].htp_field);
294  }
295  } else {
296  if (tx->response_headers != NULL) {
297  h_field = htp_table_get_c(tx->response_headers,
298  http_fields[f].htp_field);
299  }
300  }
301  if (h_field != NULL) {
302  c = bstr_util_strdup_to_c(h_field->value);
303  if (c != NULL) {
304  jb_set_string(js, http_fields[f].config_field, c);
305  SCFree(c);
306  }
307  }
308  }
309  }
310  }
311 }
312 
313 static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx)
314 {
315  /* referer */
316  htp_header_t *h_referer = NULL;
317  if (tx->request_headers != NULL) {
318  h_referer = htp_table_get_c(tx->request_headers, "referer");
319  }
320  if (h_referer != NULL) {
321  jb_set_string_from_bytes(
322  js, "http_refer", bstr_ptr(h_referer->value), bstr_len(h_referer->value));
323  }
324 
325  /* method */
326  if (tx->request_method != NULL) {
327  jb_set_string_from_bytes(
328  js, "http_method", bstr_ptr(tx->request_method), bstr_len(tx->request_method));
329  }
330 
331  /* protocol */
332  if (tx->request_protocol != NULL) {
333  jb_set_string_from_bytes(
334  js, "protocol", bstr_ptr(tx->request_protocol), bstr_len(tx->request_protocol));
335  }
336 
337  /* response status */
338  if (tx->response_status != NULL) {
339  const size_t status_size = bstr_len(tx->response_status) * 2 + 1;
340  char status_string[status_size];
341  BytesToStringBuffer(bstr_ptr(tx->response_status), bstr_len(tx->response_status),
342  status_string, status_size);
343  unsigned int val = strtoul(status_string, NULL, 10);
344  jb_set_uint(js, "status", val);
345 
346  htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
347  if (h_location != NULL) {
348  jb_set_string_from_bytes(
349  js, "redirect", bstr_ptr(h_location->value), bstr_len(h_location->value));
350  }
351  }
352 
353  /* length */
354  jb_set_uint(js, "length", tx->response_message_len);
355 }
356 
357 static void EveHttpLogJSONHeaders(JsonBuilder *js, uint32_t direction, htp_tx_t *tx)
358 {
359  htp_table_t * headers = direction & LOG_HTTP_REQ_HEADERS ?
360  tx->request_headers : tx->response_headers;
361  char name[MAX_SIZE_HEADER_NAME] = {0};
362  char value[MAX_SIZE_HEADER_VALUE] = {0};
363  size_t n = htp_table_size(headers);
364  jb_open_array(js, direction & LOG_HTTP_REQ_HEADERS ? "request_headers" : "response_headers");
365  for (size_t i = 0; i < n; i++) {
366  htp_header_t * h = htp_table_get_index(headers, i, NULL);
367  if (h == NULL) {
368  continue;
369  }
370  jb_start_object(js);
371  size_t size_name = bstr_len(h->name) < MAX_SIZE_HEADER_NAME - 1 ?
372  bstr_len(h->name) : MAX_SIZE_HEADER_NAME - 1;
373  memcpy(name, bstr_ptr(h->name), size_name);
374  name[size_name] = '\0';
375  jb_set_string(js, "name", name);
376  size_t size_value = bstr_len(h->value) < MAX_SIZE_HEADER_VALUE - 1 ?
377  bstr_len(h->value) : MAX_SIZE_HEADER_VALUE - 1;
378  memcpy(value, bstr_ptr(h->value), size_value);
379  value[size_value] = '\0';
380  jb_set_string(js, "value", value);
381  jb_close(js);
382  }
383  // Close array.
384  jb_close(js);
385 }
386 
387 static void BodyPrintableBuffer(JsonBuilder *js, HtpBody *body, const char *key)
388 {
389  if (body->sb != NULL && body->sb->buf != NULL) {
390  uint32_t offset = 0;
391  const uint8_t *body_data;
392  uint32_t body_data_len;
393  uint64_t body_offset;
394 
395  if (StreamingBufferGetData(body->sb, &body_data,
396  &body_data_len, &body_offset) == 0) {
397  return;
398  }
399 
400  uint8_t printable_buf[body_data_len + 1];
401  PrintStringsToBuffer(printable_buf, &offset,
402  sizeof(printable_buf),
403  body_data, body_data_len);
404  if (offset > 0) {
405  jb_set_string(js, key, (char *)printable_buf);
406  }
407  }
408 }
409 
410 void EveHttpLogJSONBodyPrintable(JsonBuilder *js, Flow *f, uint64_t tx_id)
411 {
412  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
413  if (htp_state) {
414  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
415  if (tx) {
416  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
417  if (htud != NULL) {
418  BodyPrintableBuffer(js, &htud->request_body, "http_request_body_printable");
419  BodyPrintableBuffer(js, &htud->response_body, "http_response_body_printable");
420  }
421  }
422  }
423 }
424 
425 static void BodyBase64Buffer(JsonBuilder *js, HtpBody *body, const char *key)
426 {
427  if (body->sb != NULL && body->sb->buf != NULL) {
428  const uint8_t *body_data;
429  uint32_t body_data_len;
430  uint64_t body_offset;
431 
432  if (StreamingBufferGetData(body->sb, &body_data,
433  &body_data_len, &body_offset) == 0) {
434  return;
435  }
436 
437  unsigned long len = body_data_len * 2 + 1;
438  uint8_t encoded[len];
439  if (Base64Encode(body_data, body_data_len, encoded, &len) == SC_BASE64_OK) {
440  jb_set_string(js, key, (char *)encoded);
441  }
442  }
443 }
444 
445 void EveHttpLogJSONBodyBase64(JsonBuilder *js, Flow *f, uint64_t tx_id)
446 {
447  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
448  if (htp_state) {
449  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
450  if (tx) {
451  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
452  if (htud != NULL) {
453  BodyBase64Buffer(js, &htud->request_body, "http_request_body");
454  BodyBase64Buffer(js, &htud->response_body, "http_response_body");
455  }
456  }
457  }
458 }
459 
460 /* JSON format logging */
461 static void EveHttpLogJSON(JsonHttpLogThread *aft, JsonBuilder *js, htp_tx_t *tx, uint64_t tx_id)
462 {
463  LogHttpFileCtx *http_ctx = aft->httplog_ctx;
464  jb_open_object(js, "http");
465 
466  EveHttpLogJSONBasic(js, tx);
467  /* log custom fields if configured */
468  if (http_ctx->fields != 0)
469  EveHttpLogJSONCustom(http_ctx, js, tx);
470  if (http_ctx->flags & LOG_HTTP_EXTENDED)
471  EveHttpLogJSONExtended(js, tx);
472  if (http_ctx->flags & LOG_HTTP_REQ_HEADERS)
473  EveHttpLogJSONHeaders(js, LOG_HTTP_REQ_HEADERS, tx);
474  if (http_ctx->flags & LOG_HTTP_RES_HEADERS)
475  EveHttpLogJSONHeaders(js, LOG_HTTP_RES_HEADERS, tx);
476 
477  jb_close(js);
478 }
479 
480 static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
481 {
482  SCEnter();
483 
484  htp_tx_t *tx = txptr;
485  JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data;
486 
487  JsonBuilder *js = CreateEveHeaderWithTxId(p, LOG_DIR_FLOW, "http", NULL, tx_id);
488  if (unlikely(js == NULL))
489  return TM_ECODE_OK;
490  EveAddCommonOptions(&jhl->httplog_ctx->cfg, p, f, js);
491 
492  SCLogDebug("got a HTTP request and now logging !!");
493 
494  /* reset */
495  MemBufferReset(jhl->buffer);
496 
497  EveHttpLogJSON(jhl, js, tx, tx_id);
498  HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg != NULL ?
500 
501  /* xff header */
502  if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
503  int have_xff_ip = 0;
504  char buffer[XFF_MAXLEN];
505 
506  have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN);
507 
508  if (have_xff_ip) {
509  if (xff_cfg->flags & XFF_EXTRADATA) {
510  jb_set_string(js, "xff", buffer);
511  }
512  else if (xff_cfg->flags & XFF_OVERWRITE) {
513  if (p->flowflags & FLOW_PKT_TOCLIENT) {
514  jb_set_string(js, "dest_ip", buffer);
515  } else {
516  jb_set_string(js, "src_ip", buffer);
517  }
518  }
519  }
520  }
521 
522  OutputJsonBuilderBuffer(js, jhl->file_ctx, &jhl->buffer);
523  jb_free(js);
524 
526 }
527 
528 bool EveHttpAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js)
529 {
530  HtpState *htp_state = (HtpState *)FlowGetAppState(f);
531  if (htp_state) {
532  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
533 
534  if (tx) {
535  EveHttpLogJSONBasic(js, tx);
536  EveHttpLogJSONExtended(js, tx);
537  return true;
538  }
539  }
540 
541  return false;
542 }
543 
544 static void OutputHttpLogDeinitSub(OutputCtx *output_ctx)
545 {
546  LogHttpFileCtx *http_ctx = output_ctx->data;
547  if (http_ctx->xff_cfg) {
548  SCFree(http_ctx->xff_cfg);
549  }
550  SCFree(http_ctx);
551  SCFree(output_ctx);
552 }
553 
554 static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
555 {
556  OutputInitResult result = { NULL, false };
557  OutputJsonCtx *ojc = parent_ctx->data;
558 
559  LogHttpFileCtx *http_ctx = SCCalloc(1, sizeof(LogHttpFileCtx));
560  if (unlikely(http_ctx == NULL))
561  return result;
562  memset(http_ctx, 0x00, sizeof(*http_ctx));
563 
564  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
565  if (unlikely(output_ctx == NULL)) {
566  SCFree(http_ctx);
567  return result;
568  }
569 
570  http_ctx->file_ctx = ojc->file_ctx;
571  http_ctx->flags = LOG_HTTP_DEFAULT;
572  http_ctx->cfg = ojc->cfg;
573 
574  if (conf) {
575  const char *extended = ConfNodeLookupChildValue(conf, "extended");
576 
577  if (extended != NULL) {
578  if (ConfValIsTrue(extended)) {
579  http_ctx->flags = LOG_HTTP_EXTENDED;
580  }
581  }
582 
583  ConfNode *custom;
584  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
585  ConfNode *field;
586  TAILQ_FOREACH(field, &custom->head, next)
587  {
588  if (field != NULL)
589  {
590  HttpField f;
591  for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++)
592  {
593  if ((strcmp(http_fields[f].config_field,
594  field->val) == 0) ||
595  (strcasecmp(http_fields[f].htp_field,
596  field->val) == 0))
597  {
598  http_ctx->fields |= (1ULL<<f);
599  break;
600  }
601  }
602  }
603  }
604  }
605  const char *all_headers = ConfNodeLookupChildValue(
606  conf, "dump-all-headers");
607  if (all_headers != NULL) {
608  if (strncmp(all_headers, "both", 4) == 0) {
609  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
610  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
611  } else if (strncmp(all_headers, "request", 7) == 0) {
612  http_ctx->flags |= LOG_HTTP_REQ_HEADERS;
613  } else if (strncmp(all_headers, "response", 8) == 0) {
614  http_ctx->flags |= LOG_HTTP_RES_HEADERS;
615  }
616  }
617  }
618 
619  if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {
620  http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
621  if (http_ctx->xff_cfg != NULL) {
622  HttpXFFGetCfg(conf, http_ctx->xff_cfg);
623  }
624  } else if (ojc->xff_cfg) {
625  http_ctx->parent_xff_cfg = ojc->xff_cfg;
626  }
627 
628  output_ctx->data = http_ctx;
629  output_ctx->DeInit = OutputHttpLogDeinitSub;
630 
631  /* enable the logger for the app layer */
633 
634  result.ctx = output_ctx;
635  result.ok = true;
636  return result;
637 }
638 
639 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data)
640 {
641  JsonHttpLogThread *aft = SCCalloc(1, sizeof(JsonHttpLogThread));
642  if (unlikely(aft == NULL))
643  return TM_ECODE_FAILED;
644 
645  if(initdata == NULL)
646  {
647  SCLogDebug("Error getting context for EveLogHTTP. \"initdata\" argument NULL");
648  goto error_exit;
649  }
650 
651  /* Use the Output Context (file pointer and mutex) */
652  aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO
653 
655  if (aft->buffer == NULL) {
656  goto error_exit;
657  }
658 
660  if (!aft->file_ctx) {
661  goto error_exit;
662  }
663 
664  *data = (void *)aft;
665  return TM_ECODE_OK;
666 
667 error_exit:
668  if (aft->buffer != NULL) {
669  MemBufferFree(aft->buffer);
670  }
671  SCFree(aft);
672  return TM_ECODE_FAILED;
673 }
674 
675 static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data)
676 {
677  JsonHttpLogThread *aft = (JsonHttpLogThread *)data;
678  if (aft == NULL) {
679  return TM_ECODE_OK;
680  }
681 
682  MemBufferFree(aft->buffer);
683  /* clear memory */
684  memset(aft, 0, sizeof(JsonHttpLogThread));
685 
686  SCFree(aft);
687  return TM_ECODE_OK;
688 }
689 
691 {
692  /* register as child of eve-log */
693  OutputRegisterTxSubModule(LOGGER_JSON_HTTP, "eve-log", "JsonHttpLog",
694  "eve-log.http", OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger,
695  JsonHttpLogThreadInit, JsonHttpLogThreadDeinit, NULL);
696 }
HTPParseContentRange
int HTPParseContentRange(bstr *rawvalue, HtpContentRange *range)
Definition: app-layer-htp-file.c:161
XFF_MAXLEN
#define XFF_MAXLEN
Definition: app-layer-htp-xff.h:39
util-byte.h
tm-threads.h
HTTP_FIELD_SIZE
@ HTTP_FIELD_SIZE
Definition: output-json-http.c:140
len
uint8_t len
Definition: app-layer-dnp3.h:2
HTTP_FIELD_CONTENT_LOCATION
@ HTTP_FIELD_CONTENT_LOCATION
Definition: output-json-http.c:115
JsonHttpLogThread_
Definition: output-json-http.c:67
EveHttpLogJSONBodyPrintable
void EveHttpLogJSONBodyPrintable(JsonBuilder *js, Flow *f, uint64_t tx_id)
Definition: output-json-http.c:410
OutputJsonCtx_::xff_cfg
HttpXFFCfg * xff_cfg
Definition: output-json.h:105
offset
uint64_t offset
Definition: util-streaming-buffer.h:0
XFF_EXTRADATA
#define XFF_EXTRADATA
Definition: app-layer-htp-xff.h:31
MAX_SIZE_HEADER_NAME
#define MAX_SIZE_HEADER_NAME
Definition: output-json-http.c:76
ConfNode_::val
char * val
Definition: conf.h:34
HtpBody_::sb
StreamingBuffer * sb
Definition: app-layer-htp.h:189
OutputJsonCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json.h:104
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
HTTP_FIELD_ALLOW
@ HTTP_FIELD_ALLOW
Definition: output-json-http.c:110
HTTP_FIELD_ACCEPT_RANGES
@ HTTP_FIELD_ACCEPT_RANGES
Definition: output-json-http.c:108
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
HTTP_FIELD_X_FLASH_VERSION
@ HTTP_FIELD_X_FLASH_VERSION
Definition: output-json-http.c:107
HTTP_FIELD_REFERRER
@ HTTP_FIELD_REFERRER
Definition: output-json-http.c:126
StreamingBuffer_::buf
uint8_t * buf
Definition: util-streaming-buffer.h:99
MAX_SIZE_HEADER_VALUE
#define MAX_SIZE_HEADER_VALUE
Definition: output-json-http.c:77
JSON_OUTPUT_BUFFER_SIZE
#define JSON_OUTPUT_BUFFER_SIZE
Definition: output-json.h:63
threads.h
OutputJsonCtx_
Definition: output-json.h:101
Flow_
Flow data structure.
Definition: flow.h:347
OutputJsonCommonSettings_
Definition: output-json.h:91
LogFileCtx_
Definition: util-logopenfile.h:60
LogHttpFileCtx
struct LogHttpFileCtx_ LogHttpFileCtx
JsonHttpLogThread_::buffer
MemBuffer * buffer
Definition: output-json-http.c:73
LogHttpFileCtx_::cfg
OutputJsonCommonSettings cfg
Definition: output-json-http.c:64
LOG_HTTP_ARRAY
#define LOG_HTTP_ARRAY
Definition: output-json-http.c:82
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
HTTP_FIELD_EXPIRES
@ HTTP_FIELD_EXPIRES
Definition: output-json-http.c:121
HtpTxUserData_::request_body
HtpBody request_body
Definition: app-layer-htp.h:216
HTTP_FIELD_FROM
@ HTTP_FIELD_FROM
Definition: output-json-http.c:95
HTTP_FIELD_CONTENT_ENCODING
@ HTTP_FIELD_CONTENT_ENCODING
Definition: output-json-http.c:112
HTTP_FIELD_WWW_AUTHENTICATE
@ HTTP_FIELD_WWW_AUTHENTICATE
Definition: output-json-http.c:136
StreamingBufferGetData
int StreamingBufferGetData(const StreamingBuffer *sb, const uint8_t **data, uint32_t *data_len, uint64_t *stream_offset)
Definition: util-streaming-buffer.c:849
OutputJsonBuilderBuffer
int OutputJsonBuilderBuffer(JsonBuilder *js, LogFileCtx *file_ctx, MemBuffer **buffer)
Definition: output-json.c:978
HTTP_FIELD_WARNING
@ HTTP_FIELD_WARNING
Definition: output-json-http.c:135
HTTP_FIELD_COOKIE
@ HTTP_FIELD_COOKIE
Definition: output-json-http.c:94
HTTP_FIELD_CONTENT_LENGTH
@ HTTP_FIELD_CONTENT_LENGTH
Definition: output-json-http.c:114
util-privs.h
LogHttpFileCtx_::xff_cfg
HttpXFFCfg * xff_cfg
Definition: output-json-http.c:62
EveAddCommonOptions
void EveAddCommonOptions(const OutputJsonCommonSettings *cfg, const Packet *p, const Flow *f, JsonBuilder *js)
Definition: output-json.c:444
Packet_::flowflags
uint8_t flowflags
Definition: decode.h:445
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:81
HTTP_FIELD_ETAG
@ HTTP_FIELD_ETAG
Definition: output-json-http.c:120
HTTP_FIELD_X_BLUECOAT_VIA
@ HTTP_FIELD_X_BLUECOAT_VIA
Definition: output-json-http.c:139
SC_BASE64_OK
@ SC_BASE64_OK
Definition: util-crypt.h:37
util-unittest.h
HTTP_FIELD_AGE
@ HTTP_FIELD_AGE
Definition: output-json-http.c:109
EveHttpAddMetadata
bool EveHttpAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js)
Definition: output-json-http.c:528
HtpState_
Definition: app-layer-htp.h:243
JsonHttpLogThread_::httplog_ctx
LogHttpFileCtx * httplog_ctx
Definition: output-json-http.c:68
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:565
FlowGetAppState
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1102
HTTP_FIELD_CACHE_CONTROL
@ HTTP_FIELD_CACHE_CONTROL
Definition: output-json-http.c:93
OutputCtx_::data
void * data
Definition: tm-modules.h:81
config_field
const char * config_field
Definition: output-json-http.c:144
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
PrintStringsToBuffer
void PrintStringsToBuffer(uint8_t *dst_buf, uint32_t *dst_buf_offset_ptr, uint32_t dst_buf_size, const uint8_t *src_buf, const uint32_t src_buf_len)
Definition: util-print.c:228
OutputCtx_
Definition: tm-modules.h:78
app-layer-htp-xff.h
JsonHttpLogThread_::uri_cnt
uint32_t uri_cnt
Definition: output-json-http.c:71
app-layer-htp-file.h
HTTP_FIELD_RETRY_AFTER
@ HTTP_FIELD_RETRY_AFTER
Definition: output-json-http.c:128
HTTP_FIELD_UPGRADE
@ HTTP_FIELD_UPGRADE
Definition: output-json-http.c:133
HtpContentRange_::end
int64_t end
Definition: app-layer-htp-file.h:30
HTTP_FIELD_CONTENT_RANGE
@ HTTP_FIELD_CONTENT_RANGE
Definition: output-json-http.c:117
HTTP_FIELD_ACCEPT
@ HTTP_FIELD_ACCEPT
Definition: output-json-http.c:87
HTTP_FIELD_PRAGMA
@ HTTP_FIELD_PRAGMA
Definition: output-json-http.c:98
HTTP_FIELD_ACCEPT_LANGUAGE
@ HTTP_FIELD_ACCEPT_LANGUAGE
Definition: output-json-http.c:90
HTTP_FIELD_LOCATION
@ HTTP_FIELD_LOCATION
Definition: output-json-http.c:124
HTTP_FIELD_DATE
@ HTTP_FIELD_DATE
Definition: output-json-http.c:119
app-layer-htp.h
LOG_HTTP_DEFAULT
#define LOG_HTTP_DEFAULT
Definition: output-json-http.c:79
HTTP_FIELD_ORIGIN
@ HTTP_FIELD_ORIGIN
Definition: output-json-http.c:97
Base64Encode
int Base64Encode(const unsigned char *in, unsigned long inlen, unsigned char *out, unsigned long *outlen)
Definition: util-crypt.c:272
util-debug.h
htp_field
const char * htp_field
Definition: output-json-http.c:145
HTTP_FIELD_ORG_SRC_IP
@ HTTP_FIELD_ORG_SRC_IP
Definition: output-json-http.c:138
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:44
HTTP_FIELD_LAST_MODIFIED
@ HTTP_FIELD_LAST_MODIFIED
Definition: output-json-http.c:122
HTTP_FIELD_AUTHORIZATION
@ HTTP_FIELD_AUTHORIZATION
Definition: output-json-http.c:92
output-json.h
LOG_HTTP_REQUEST
#define LOG_HTTP_REQUEST
Definition: output-json-http.c:81
AppLayerParserRegisterLogger
void AppLayerParserRegisterLogger(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:456
util-print.h
HTTP_FIELD_RANGE
@ HTTP_FIELD_RANGE
Definition: output-json-http.c:100
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
util-crypt.h
detect.h
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
pkt-var.h
HttpField
HttpField
Definition: output-json-http.c:86
HTTP_FIELD_LINK
@ HTTP_FIELD_LINK
Definition: output-json-http.c:123
LogFileEnsureExists
LogFileCtx * LogFileEnsureExists(LogFileCtx *parent_ctx, int thread_id)
LogFileEnsureExists() Ensure a log file context for the thread exists.
Definition: util-logopenfile.c:659
JsonHttpLogThread_::file_ctx
LogFileCtx * file_ctx
Definition: output-json-http.c:69
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:45
HTTP_FIELD_ACCEPT_ENCODING
@ HTTP_FIELD_ACCEPT_ENCODING
Definition: output-json-http.c:89
EveHttpLogJSONBodyBase64
void EveHttpLogJSONBodyBase64(JsonBuilder *js, Flow *f, uint64_t tx_id)
Definition: output-json-http.c:445
app-layer-parser.h
LOGGER_JSON_HTTP
@ LOGGER_JSON_HTTP
Definition: suricata-common.h:450
HTTP_FIELD_SERVER
@ HTTP_FIELD_SERVER
Definition: output-json-http.c:129
ThreadVars_::id
int id
Definition: threadvars.h:87
HTTP_FIELD_CONNECTION
@ HTTP_FIELD_CONNECTION
Definition: output-json-http.c:111
HTTP_FIELD_TE
@ HTTP_FIELD_TE
Definition: output-json-http.c:101
HTTP_FIELD_SET_COOKIE
@ HTTP_FIELD_SET_COOKIE
Definition: output-json-http.c:130
Packet_
Definition: decode.h:414
conf.h
LogHttpFileCtx_::file_ctx
LogFileCtx * file_ctx
Definition: log-httplog.c:87
HTTP_FIELD_TRAILER
@ HTTP_FIELD_TRAILER
Definition: output-json-http.c:131
HtpContentRange_::size
int64_t size
Definition: app-layer-htp-file.h:31
XFF_OVERWRITE
#define XFF_OVERWRITE
Definition: app-layer-htp-xff.h:33
TmEcode
TmEcode
Definition: tm-threads-common.h:79
HttpXFFCfg_
Definition: app-layer-htp-xff.h:41
util-proto-name.h
MemBuffer_
Definition: util-buffer.h:27
FLOW_PKT_TOCLIENT
#define FLOW_PKT_TOCLIENT
Definition: flow.h:220
HtpTxUserData_::response_body
HtpBody response_body
Definition: app-layer-htp.h:217
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1032
http_fields
struct @134 http_fields[]
HTTP_FIELD_CONTENT_MD5
@ HTTP_FIELD_CONTENT_MD5
Definition: output-json-http.c:116
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:814
HTTP_FIELD_DNT
@ HTTP_FIELD_DNT
Definition: output-json-http.c:104
MemBufferReset
#define MemBufferReset(mem_buffer)
Reset the mem buffer.
Definition: util-buffer.h:42
OutputInitResult_
Definition: output.h:43
Packet_::flow
struct Flow_ * flow
Definition: decode.h:451
HtpBody_
Definition: app-layer-htp.h:185
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
LOG_HTTP_REQ_HEADERS
#define LOG_HTTP_REQ_HEADERS
Definition: output-json-http.c:83
MemBufferFree
void MemBufferFree(MemBuffer *buffer)
Definition: util-buffer.c:82
HTTP_FIELD_PROXY_AUTHORIZATION
@ HTTP_FIELD_PROXY_AUTHORIZATION
Definition: output-json-http.c:99
JsonHttpLogThread
struct JsonHttpLogThread_ JsonHttpLogThread
HTTP_FIELD_VARY
@ HTTP_FIELD_VARY
Definition: output-json-http.c:134
HtpTxUserData_
Definition: app-layer-htp.h:208
OutputRegisterTxSubModule
void OutputRegisterTxSubModule(LoggerId id, const char *parent_name, const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Definition: output.c:409
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
HTTP_FIELD_TRUE_CLIENT_IP
@ HTTP_FIELD_TRUE_CLIENT_IP
Definition: output-json-http.c:137
output-json-alert.h
threadvars.h
LOG_DIR_FLOW
@ LOG_DIR_FLOW
Definition: output-json.h:40
HTTP_FIELD_CONTENT_LANGUAGE
@ HTTP_FIELD_CONTENT_LANGUAGE
Definition: output-json-http.c:113
HttpXFFCfg_::flags
uint8_t flags
Definition: app-layer-htp-xff.h:42
SCFree
#define SCFree(p)
Definition: util-mem.h:61
CreateEveHeaderWithTxId
JsonBuilder * CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, uint64_t tx_id)
Definition: output-json.c:923
ConfNode_
Definition: conf.h:32
util-logopenfile.h
LogHttpFileCtx_::fields
uint64_t fields
Definition: output-json-http.c:61
HTTP_FIELD_X_FORWARDED_PROTO
@ HTTP_FIELD_X_FORWARDED_PROTO
Definition: output-json-http.c:105
util-buffer.h
ALPROTO_HTTP
@ ALPROTO_HTTP
Definition: app-layer-protos.h:30
BytesToStringBuffer
void BytesToStringBuffer(const uint8_t *bytes, size_t nbytes, char *outstr, size_t outlen)
Turn byte array into string.
Definition: util-byte.c:85
HTTP_FIELD_MAX_FORWARDS
@ HTTP_FIELD_MAX_FORWARDS
Definition: output-json-http.c:96
LOG_HTTP_EXTENDED
#define LOG_HTTP_EXTENDED
Definition: output-json-http.c:80
HTTP_FIELD_TRANSFER_ENCODING
@ HTTP_FIELD_TRANSFER_ENCODING
Definition: output-json-http.c:132
OutputJsonCtx_::file_ctx
LogFileCtx * file_ctx
Definition: output-json.h:102
JsonHttpLogRegister
void JsonHttpLogRegister(void)
Definition: output-json-http.c:690
HTTP_FIELD_ACCEPT_DATETIME
@ HTTP_FIELD_ACCEPT_DATETIME
Definition: output-json-http.c:91
HTTP_FIELD_VIA
@ HTTP_FIELD_VIA
Definition: output-json-http.c:102
HttpXFFGetCfg
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
Definition: app-layer-htp-xff.c:202
HTTP_FIELD_ACCEPT_CHARSET
@ HTTP_FIELD_ACCEPT_CHARSET
Definition: output-json-http.c:88
HttpXFFGetIPFromTx
int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
Function to return XFF IP if any in the selected transaction. The caller needs to lock the flow.
Definition: app-layer-htp-xff.c:113
HTTP_FIELD_REFRESH
@ HTTP_FIELD_REFRESH
Definition: output-json-http.c:127
HTTP_FIELD_PROXY_AUTHENTICATE
@ HTTP_FIELD_PROXY_AUTHENTICATE
Definition: output-json-http.c:125
HTTP_FIELD_CONTENT_TYPE
@ HTTP_FIELD_CONTENT_TYPE
Definition: output-json-http.c:118
output-json-http.h
LogHttpFileCtx_::flags
uint32_t flags
Definition: log-httplog.c:88
HTTP_FIELD_X_AUTHENTICATED_USER
@ HTTP_FIELD_X_AUTHENTICATED_USER
Definition: output-json-http.c:106
flags
uint32_t flags
Definition: output-json-http.c:146
LogHttpFileCtx_
Definition: log-httplog.c:86
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
XFF_DISABLED
#define XFF_DISABLED
Definition: app-layer-htp-xff.h:29
HtpContentRange_
Definition: app-layer-htp-file.h:28
HTTP_FIELD_X_REQUESTED_WITH
@ HTTP_FIELD_X_REQUESTED_WITH
Definition: output-json-http.c:103
MemBufferCreateNew
MemBuffer * MemBufferCreateNew(uint32_t size)
Definition: util-buffer.c:32
debug.h
LOG_HTTP_RES_HEADERS
#define LOG_HTTP_RES_HEADERS
Definition: output-json-http.c:84
output.h
HtpContentRange_::start
int64_t start
Definition: app-layer-htp-file.h:29
app-layer.h
LogHttpFileCtx_::parent_xff_cfg
HttpXFFCfg * parent_xff_cfg
Definition: output-json-http.c:63
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842