suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 #include "util-byte.h"
49 
50 #include "stream-tcp-private.h"
51 #include "stream-tcp-reassemble.h"
52 #include "stream-tcp.h"
53 #include "stream.h"
54 
55 #include "app-layer-protos.h"
56 #include "app-layer-parser.h"
57 
58 #include "app-layer.h"
59 #include "app-layer-htp.h"
60 #include "app-layer-htp-body.h"
61 #include "app-layer-htp-file.h"
62 #include "app-layer-htp-libhtp.h"
63 #include "app-layer-htp-xff.h"
64 
65 #include "util-spm.h"
66 #include "util-debug.h"
67 #include "util-time.h"
68 #include "util-misc.h"
69 
70 #include "util-unittest.h"
71 #include "util-unittest-helper.h"
72 #include "flow-util.h"
73 
74 #include "detect-engine.h"
75 #include "detect-engine-state.h"
76 #include "detect-parse.h"
77 
78 #include "decode-events.h"
79 
80 #include "util-memcmp.h"
81 #include "util-random.h"
82 #include "util-validate.h"
83 
84 //#define PRINT
85 
86 /** Fast lookup tree (radix) for the various HTP configurations */
87 static SCRadixTree *cfgtree;
88 /** List of HTP configurations. */
89 static HTPCfgRec cfglist;
90 
91 /** Limit to the number of libhtp messages that can be handled */
92 #define HTP_MAX_MESSAGES 512
93 
94 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
95 
96 #ifdef DEBUG
97 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
98 static uint64_t htp_state_memuse = 0;
99 static uint64_t htp_state_memcnt = 0;
100 #endif
101 
103  { "UNKNOWN_ERROR",
105  { "GZIP_DECOMPRESSION_FAILED",
107  { "REQUEST_FIELD_MISSING_COLON",
109  { "RESPONSE_FIELD_MISSING_COLON",
111  { "INVALID_REQUEST_CHUNK_LEN",
113  { "INVALID_RESPONSE_CHUNK_LEN",
115  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
117  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
119  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
121  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
123  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
125  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
127  { "100_CONTINUE_ALREADY_SEEN",
129  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
131  { "INVALID_SERVER_PORT_IN_REQUEST",
133  { "INVALID_AUTHORITY_PORT",
135  { "REQUEST_HEADER_INVALID",
137  { "RESPONSE_HEADER_INVALID",
139  { "MISSING_HOST_HEADER",
141  { "HOST_HEADER_AMBIGUOUS",
143  { "INVALID_REQUEST_FIELD_FOLDING",
145  { "INVALID_RESPONSE_FIELD_FOLDING",
147  { "REQUEST_FIELD_TOO_LONG",
149  { "RESPONSE_FIELD_TOO_LONG",
151  { "REQUEST_LINE_INVALID",
153  { "REQUEST_BODY_UNEXPECTED",
155  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
157  { "REQUEST_URI_HOST_INVALID",
159  { "REQUEST_HEADER_HOST_INVALID",
161  { "REQUEST_AUTH_UNRECOGNIZED",
163  { "REQUEST_HEADER_REPETITION",
165  { "RESPONSE_HEADER_REPETITION",
167  { "DOUBLE_ENCODED_URI",
169  { "URI_DELIM_NON_COMPLIANT",
171  { "METHOD_DELIM_NON_COMPLIANT",
173  { "REQUEST_LINE_LEADING_WHITESPACE",
175  { "TOO_MANY_ENCODING_LAYERS",
177  { "ABNORMAL_CE_HEADER",
179  { "RESPONSE_MULTIPART_BYTERANGES",
181  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
183  { "RESPONSE_CHUNKED_OLD_PROTO",
185  { "RESPONSE_INVALID_PROTOCOL",
187  { "RESPONSE_INVALID_STATUS",
189  { "REQUEST_LINE_INCOMPLETE",
191 
192  { "LZMA_MEMLIMIT_REACHED",
194  { "COMPRESSION_BOMB",
196 
197  /* suricata warnings/errors */
198  { "MULTIPART_GENERIC_ERROR",
200  { "MULTIPART_NO_FILEDATA",
202  { "MULTIPART_INVALID_HEADER",
204 
205  { "TOO_MANY_WARNINGS",
207 
208  { NULL, -1 },
209 };
210 
211 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
212 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
213 static uint64_t HTPStateGetTxCnt(void *alstate);
214 #ifdef UNITTESTS
215 static void HTPParserRegisterTests(void);
216 #endif
217 
218 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
219 {
220  uint64_t id = HTPStateGetTxCnt(s);
221  BUG_ON(id == 0);
222  return id - 1;
223 }
224 
225 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
226 {
227  return s->transaction_cnt;
228 }
229 
230 #ifdef DEBUG
231 /**
232  * \internal
233  *
234  * \brief Lookup the HTP personality string from the numeric personality.
235  *
236  * \todo This needs to be a libhtp function.
237  */
238 static const char *HTPLookupPersonalityString(int p)
239 {
240 #define CASE_HTP_PERSONALITY_STRING(p) \
241  case HTP_SERVER_ ## p: return #p
242 
243  switch (p) {
244  CASE_HTP_PERSONALITY_STRING(MINIMAL);
245  CASE_HTP_PERSONALITY_STRING(GENERIC);
246  CASE_HTP_PERSONALITY_STRING(IDS);
247  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
248  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
249  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
250  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
251  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
252  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
253  CASE_HTP_PERSONALITY_STRING(APACHE_2);
254  }
255 
256  return NULL;
257 }
258 #endif /* DEBUG */
259 
260 /**
261  * \internal
262  *
263  * \brief Lookup the numeric HTP personality from a string.
264  *
265  * \todo This needs to be a libhtp function.
266  */
267 static int HTPLookupPersonality(const char *str)
268 {
269 #define IF_HTP_PERSONALITY_NUM(p) \
270  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
271 
272  IF_HTP_PERSONALITY_NUM(MINIMAL);
273  IF_HTP_PERSONALITY_NUM(GENERIC);
275  IF_HTP_PERSONALITY_NUM(IIS_4_0);
276  IF_HTP_PERSONALITY_NUM(IIS_5_0);
277  IF_HTP_PERSONALITY_NUM(IIS_5_1);
278  IF_HTP_PERSONALITY_NUM(IIS_6_0);
279  IF_HTP_PERSONALITY_NUM(IIS_7_0);
280  IF_HTP_PERSONALITY_NUM(IIS_7_5);
281  IF_HTP_PERSONALITY_NUM(APACHE_2);
282  if (strcasecmp("TOMCAT_6_0", str) == 0) {
283  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
284  "longer supported by libhtp.", str);
285  return -1;
286  } else if ((strcasecmp("APACHE", str) == 0) ||
287  (strcasecmp("APACHE_2_2", str) == 0))
288  {
289  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
290  "longer supported by libhtp, failing back to "
291  "Apache2 personality.", str);
292  return HTP_SERVER_APACHE_2;
293  }
294 
295  return -1;
296 }
297 
298 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
299  const uint8_t dir, const uint8_t e)
300 {
301  SCLogDebug("setting event %u", e);
302 
303  if (htud) {
305  s->events++;
306  return;
307  }
308 
309  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
310  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
311 
312  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
313  if (tx == NULL && tx_id > 0)
314  tx = HTPStateGetTx(s, tx_id - 1);
315  if (tx != NULL) {
316  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
317  if (htud != NULL) {
319  s->events++;
320  return;
321  }
322  }
323  SCLogDebug("couldn't set event %u", e);
324 }
325 
326 static AppLayerDecoderEvents *HTPGetEvents(void *tx)
327 {
328  SCLogDebug("get HTTP events for TX %p", tx);
329 
330  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
331  if (htud != NULL) {
332  SCLogDebug("has htud, htud->decoder_events %p", htud->decoder_events);
333  return htud->decoder_events;
334  }
335 
336  return NULL;
337 }
338 
339 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
340  * connection parser to be used by the HTP library
341  */
342 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
343 {
344  SCEnter();
345 
346  HtpState *s = HTPMalloc(sizeof(HtpState));
347  if (unlikely(s == NULL)) {
348  SCReturnPtr(NULL, "void");
349  }
350 
351  memset(s, 0x00, sizeof(HtpState));
352 
353 #ifdef DEBUG
354  SCMutexLock(&htp_state_mem_lock);
355  htp_state_memcnt++;
356  htp_state_memuse += sizeof(HtpState);
357  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
358  SCMutexUnlock(&htp_state_mem_lock);
359 #endif
360 
361  SCReturnPtr((void *)s, "void");
362 }
363 
364 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
365 {
366  if (likely(htud)) {
367  HtpBodyFree(&htud->request_body);
368  HtpBodyFree(&htud->response_body);
369  bstr_free(htud->request_uri_normalized);
370  if (htud->request_headers_raw)
372  if (htud->response_headers_raw)
375  if (htud->boundary)
376  HTPFree(htud->boundary, htud->boundary_len);
377  if (htud->de_state != NULL) {
379  }
380  HTPFree(htud, sizeof(HtpTxUserData));
381  }
382 }
383 
384 /** \brief Function to frees the HTTP state memory and also frees the HTTP
385  * connection parser memory which was used by the HTP library
386  */
387 void HTPStateFree(void *state)
388 {
389  SCEnter();
390 
391  HtpState *s = (HtpState *)state;
392  if (s == NULL) {
393  SCReturn;
394  }
395 
396  /* free the connection parser memory used by HTP library */
397  if (s->connp != NULL) {
398  SCLogDebug("freeing HTP state");
399 
400  uint64_t tx_id;
401  uint64_t total_txs = HTPStateGetTxCnt(state);
402  /* free the list of body chunks */
403  if (s->conn != NULL) {
404  for (tx_id = 0; tx_id < total_txs; tx_id++) {
405  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
406  if (tx != NULL) {
407  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
408  HtpTxUserDataFree(s, htud);
409  htp_tx_set_user_data(tx, NULL);
410  }
411  }
412  }
413  htp_connp_destroy_all(s->connp);
414  }
415 
418  HTPFree(s, sizeof(HtpState));
419 
420 #ifdef DEBUG
421  SCMutexLock(&htp_state_mem_lock);
422  htp_state_memcnt--;
423  htp_state_memuse -= sizeof(HtpState);
424  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
425  SCMutexUnlock(&htp_state_mem_lock);
426 #endif
427 
428  SCReturn;
429 }
430 
431 /**
432  * \brief HTP transaction cleanup callback
433  *
434  * \warning We cannot actually free the transactions here. It seems that
435  * HTP only accepts freeing of transactions in the response callback.
436  */
437 static void HTPStateTransactionFree(void *state, uint64_t id)
438 {
439  SCEnter();
440 
441  HtpState *s = (HtpState *)state;
442 
443  SCLogDebug("state %p, id %"PRIu64, s, id);
444 
445  htp_tx_t *tx = HTPStateGetTx(s, id);
446  if (tx != NULL) {
447  /* This will remove obsolete body chunks */
448  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
449  HtpTxUserDataFree(s, htud);
450  htp_tx_set_user_data(tx, NULL);
451 
452  /* hack: even if libhtp considers the tx incomplete, we want to
453  * free it here. htp_tx_destroy however, will refuse to do this.
454  * As htp_tx_destroy_incomplete isn't available in the public API,
455  * we hack around it here. */
456  if (unlikely(!(
457  tx->request_progress == HTP_REQUEST_COMPLETE &&
458  tx->response_progress == HTP_RESPONSE_COMPLETE)))
459  {
460  tx->request_progress = HTP_REQUEST_COMPLETE;
461  tx->response_progress = HTP_RESPONSE_COMPLETE;
462  }
463  htp_tx_destroy(tx);
464  }
465 }
466 
467 /**
468  * \brief Sets a flag that informs the HTP app layer that some module in the
469  * engine needs the http request body data.
470  * \initonly
471  */
473 {
474  SCEnter();
475 
476  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
477  SCReturn;
478 }
479 
480 /**
481  * \brief Sets a flag that informs the HTP app layer that some module in the
482  * engine needs the http request body data.
483  * \initonly
484  */
486 {
487  SCEnter();
488 
489  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
490  SCReturn;
491 }
492 
493 /**
494  * \brief Sets a flag that informs the HTP app layer that some module in the
495  * engine needs the http request multi part header.
496  *
497  * \initonly
498  */
499 static void AppLayerHtpNeedMultipartHeader(void)
500 {
501  SCEnter();
503 
504  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
505  SCReturn;
506 }
507 
508 /**
509  * \brief Sets a flag that informs the HTP app layer that some module in the
510  * engine needs the http request file.
511  *
512  * \initonly
513  */
515 {
516  SCEnter();
517  AppLayerHtpNeedMultipartHeader();
520 
521  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
522  SCReturn;
523 }
524 
525 static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
526 {
527  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
528  if (tx_ud) {
529  if (flags & STREAM_TOCLIENT) {
530  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
531  } else {
532  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
533  }
534  }
535 }
536 
537 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
538 {
539  if (flags & HTP_STREAM_DEPTH_SET) {
540  uint32_t stream_depth = FileReassemblyDepth();
541  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
542  return true;
543  }
544  } else {
545  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
546  return true;
547  }
548  }
549  return false;
550 }
551 
552 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
553  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
554 {
555  uint32_t chunk_len = 0;
556  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
557  (content_len_so_far < (uint64_t)body_limit) &&
558  (content_len_so_far + (uint64_t)data_len) > body_limit)
559  {
560  chunk_len = body_limit - content_len_so_far;
561  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
562  (content_len_so_far < (uint64_t)stream_depth) &&
563  (content_len_so_far + (uint64_t)data_len) > stream_depth)
564  {
565  chunk_len = stream_depth - content_len_so_far;
566  }
567  SCLogDebug("len %u", chunk_len);
568  return (chunk_len == 0 ? data_len : chunk_len);
569 }
570 
571 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
572 struct {
573  const char *msg;
574  int de;
575 } htp_errors[] = {
576  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
577  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
578  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
579  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
580  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
581 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
582  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
583 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
584  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
585  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
586  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
587  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
588 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
589  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
590  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
591  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
592  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
593 };
594 
595 struct {
596  const char *msg;
597  int de;
598 } htp_warnings[] = {
599  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
600  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
601  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
602  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
603  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
604 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
605  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
606  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
607  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
608  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
609  * luckily, "Request server port=" is unique */
610 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
612  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
613  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
614  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
615  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
616  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
617  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
618  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
619  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
620  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
621  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
622  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
623  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
624  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
625  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
626  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
627  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
628  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
629  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
630 };
631 
632 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
633 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
634 
635 /**
636  * \internal
637  *
638  * \brief Get the warning id for the warning msg.
639  *
640  * \param msg warning message
641  *
642  * \retval id the id or 0 in case of not found
643  */
644 static int HTPHandleWarningGetId(const char *msg)
645 {
646  SCLogDebug("received warning \"%s\"", msg);
647  size_t idx;
648  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
649  if (strncmp(htp_warnings[idx].msg, msg,
650  strlen(htp_warnings[idx].msg)) == 0)
651  {
652  return htp_warnings[idx].de;
653  }
654  }
655 
656  return 0;
657 }
658 
659 /**
660  * \internal
661  *
662  * \brief Get the error id for the error msg.
663  *
664  * \param msg error message
665  *
666  * \retval id the id or 0 in case of not found
667  */
668 static int HTPHandleErrorGetId(const char *msg)
669 {
670  SCLogDebug("received error \"%s\"", msg);
671 
672  size_t idx;
673  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
674  if (strncmp(htp_errors[idx].msg, msg,
675  strlen(htp_errors[idx].msg)) == 0)
676  {
677  return htp_errors[idx].de;
678  }
679  }
680 
681  return 0;
682 }
683 
684 /**
685  * \internal
686  *
687  * \brief Check state for errors, warnings and add any as events
688  *
689  * \param s state
690  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
691  */
692 static void HTPHandleError(HtpState *s, const uint8_t dir)
693 {
694  if (s == NULL || s->conn == NULL ||
695  s->conn->messages == NULL) {
696  return;
697  }
698 
699  size_t size = htp_list_size(s->conn->messages);
700  size_t msg;
701  if(size >= HTP_MAX_MESSAGES) {
703  //only once per HtpState
704  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
706  //too noisy in fuzzing
707  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
708  }
709  // ignore further messages
710  return;
711  }
712 
713  for (msg = s->htp_messages_offset; msg < size; msg++) {
714  htp_log_t *log = htp_list_get(s->conn->messages, msg);
715  if (log == NULL)
716  continue;
717 
718  HtpTxUserData *htud = NULL;
719  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
720  if (tx != NULL)
721  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
722 
723  SCLogDebug("message %s", log->msg);
724 
725  int id = HTPHandleErrorGetId(log->msg);
726  if (id == 0) {
727  id = HTPHandleWarningGetId(log->msg);
728  if (id == 0)
730  }
731 
732  if (id > 0) {
733  HTPSetEvent(s, htud, dir, id);
734  }
735  }
736  s->htp_messages_offset = (uint16_t)msg;
737  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
738 }
739 
740 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
741 {
742 #ifdef DEBUG
743  BUG_ON(s == NULL || tx == NULL);
744 #endif
745  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
746  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
747  HTP_HOSTH_INVALID))
748  {
749  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
750  if (htud == NULL)
751  return;
752 
753  if (tx->flags & HTP_REQUEST_INVALID_T_E)
754  HTPSetEvent(s, htud, STREAM_TOSERVER,
756  if (tx->flags & HTP_REQUEST_INVALID_C_L)
757  HTPSetEvent(s, htud, STREAM_TOSERVER,
759  if (tx->flags & HTP_HOST_MISSING)
760  HTPSetEvent(s, htud, STREAM_TOSERVER,
762  if (tx->flags & HTP_HOST_AMBIGUOUS)
763  HTPSetEvent(s, htud, STREAM_TOSERVER,
765  if (tx->flags & HTP_HOSTU_INVALID)
766  HTPSetEvent(s, htud, STREAM_TOSERVER,
768  if (tx->flags & HTP_HOSTH_INVALID)
769  HTPSetEvent(s, htud, STREAM_TOSERVER,
771  }
772  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
773  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
774  if (htud == NULL)
775  return;
776  HTPSetEvent(s, htud, STREAM_TOSERVER,
778  }
779  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
780  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
781  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
782  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
783  if (htud == NULL)
784  return;
785  HTPSetEvent(s, htud, STREAM_TOSERVER,
787  }
788 }
789 
790 static int Setup(Flow *f, HtpState *hstate)
791 {
792  /* store flow ref in state so callbacks can access it */
793  hstate->f = f;
794 
795  HTPCfgRec *htp_cfg_rec = &cfglist;
796  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
797  void *user_data = NULL;
798 
799  if (FLOW_IS_IPV4(f)) {
800  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
801  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
802  }
803  else if (FLOW_IS_IPV6(f)) {
804  SCLogDebug("Looking up HTP config for ipv6");
805  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
806  }
807  else {
808  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
809  goto error;
810  }
811 
812  if (user_data != NULL) {
813  htp_cfg_rec = user_data;
814  htp = htp_cfg_rec->cfg;
815  SCLogDebug("LIBHTP using config: %p", htp);
816  } else {
817  SCLogDebug("Using default HTP config: %p", htp);
818  }
819 
820  if (NULL == htp) {
821 #ifdef DEBUG_VALIDATION
822  BUG_ON(htp == NULL);
823 #endif
824  /* should never happen if HTPConfigure is properly invoked */
825  goto error;
826  }
827 
828  hstate->connp = htp_connp_create(htp);
829  if (hstate->connp == NULL) {
830  goto error;
831  }
832 
833  hstate->conn = htp_connp_get_connection(hstate->connp);
834 
835  htp_connp_set_user_data(hstate->connp, (void *)hstate);
836  hstate->cfg = htp_cfg_rec;
837 
838  SCLogDebug("New hstate->connp %p", hstate->connp);
839 
840  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
841 
843  htp_cfg_rec->request.inspect_min_size);
845  htp_cfg_rec->response.inspect_min_size);
846  return 0;
847 error:
848  return -1;
849 }
850 
851 /**
852  * \brief Function to handle the reassembled data from client and feed it to
853  * the HTP library to process it.
854  *
855  * \param flow Pointer to the flow the data belong to
856  * \param htp_state Pointer the state in which the parsed value to be stored
857  * \param pstate Application layer parser state for this session
858  * \param input Pointer the received HTTP client data
859  * \param input_len Length in bytes of the received data
860  * \param output Pointer to the output (not used in this function)
861  *
862  * \retval On success returns 1 or on failure returns -1.
863  */
864 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state,
865  AppLayerParserState *pstate,
866  const uint8_t *input, uint32_t input_len,
867  void *local_data, const uint8_t flags)
868 {
869  SCEnter();
870  int ret = 0;
871  HtpState *hstate = (HtpState *)htp_state;
872 
873  /* On the first invocation, create the connection parser structure to
874  * be used by HTP library. This is looked up via IP in the radix
875  * tree. Failing that, the default HTP config is used.
876  */
877  if (NULL == hstate->conn) {
878  if (Setup(f, hstate) != 0) {
880  }
881  }
882  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
883 
884  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
885  /* pass the new data to the htp parser */
886  if (input_len > 0) {
887  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
888  switch (r) {
889  case HTP_STREAM_ERROR:
890  ret = -1;
891  break;
892  default:
893  break;
894  }
895  HTPHandleError(hstate, STREAM_TOSERVER);
896  }
897 
898  /* if the TCP connection is closed, then close the HTTP connection */
900  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
901  {
902  htp_connp_req_close(hstate->connp, &ts);
903  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
904  SCLogDebug("stream eof encountered, closing htp handle for ts");
905  }
906 
907  SCLogDebug("hstate->connp %p", hstate->connp);
908 
909  if (ret < 0) {
911  }
913 }
914 
915 /**
916  * \brief Function to handle the reassembled data from server and feed it to
917  * the HTP library to process it.
918  *
919  * \param flow Pointer to the flow the data belong to
920  * \param htp_state Pointer the state in which the parsed value to be stored
921  * \param pstate Application layer parser state for this session
922  * \param input Pointer the received HTTP server data
923  * \param input_len Length in bytes of the received data
924  * \param output Pointer to the output (not used in this function)
925  *
926  * \retval On success returns 1 or on failure returns -1
927  */
928 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state,
929  AppLayerParserState *pstate,
930  const uint8_t *input, uint32_t input_len,
931  void *local_data, const uint8_t flags)
932 {
933  SCEnter();
934  int ret = 0;
935  HtpState *hstate = (HtpState *)htp_state;
936 
937  /* On the first invocation, create the connection parser structure to
938  * be used by HTP library. This is looked up via IP in the radix
939  * tree. Failing that, the default HTP config is used.
940  */
941  if (NULL == hstate->conn) {
942  if (Setup(f, hstate) != 0) {
944  }
945  }
946  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
947 
948  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
949  htp_tx_t *tx = NULL;
950  size_t consumed = 0;
951  if (input_len > 0) {
952  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
953  switch (r) {
954  case HTP_STREAM_ERROR:
955  ret = -1;
956  break;
957  case HTP_STREAM_TUNNEL:
958  tx = htp_connp_get_out_tx(hstate->connp);
959  if (tx != NULL && tx->response_status_number == 101) {
960  htp_header_t *h =
961  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
962  if (h != NULL) {
963  if (bstr_cmp_c(h->value, "h2c") == 0) {
964  uint16_t dp = 0;
965  if (tx->request_port_number != -1) {
966  dp = (uint16_t)tx->request_port_number;
967  }
968  consumed = htp_connp_res_data_consumed(hstate->connp);
970  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
971  // and we need to parser the remaining part with HTTP2
972  if (consumed > 0 && consumed < input_len) {
974  APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
975  }
977  }
978  }
979  }
980  break;
981  default:
982  break;
983  }
984  HTPHandleError(hstate, STREAM_TOCLIENT);
985  }
986 
987  /* if we the TCP connection is closed, then close the HTTP connection */
989  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
990  {
991  htp_connp_close(hstate->connp, &ts);
992  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
993  }
994 
995  SCLogDebug("hstate->connp %p", hstate->connp);
996 
997  if (ret < 0) {
999  }
1001 }
1002 
1003 /**
1004  * \param name /Lowercase/ version of the variable name
1005  */
1006 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1007  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1008 {
1009 #ifdef PRINT
1010  printf("DATA START: \n");
1011  PrintRawDataFp(stdout, data, len);
1012  printf("DATA END: \n");
1013 #endif
1014  size_t x;
1015  int quote = 0;
1016 
1017  for (x = 0; x < len; x++) {
1018  if (!(isspace(data[x])))
1019  break;
1020  }
1021 
1022  if (x >= len)
1023  return 0;
1024 
1025  uint8_t *line = data+x;
1026  size_t line_len = len-x;
1027  size_t offset = 0;
1028 #ifdef PRINT
1029  printf("LINE START: \n");
1030  PrintRawDataFp(stdout, line, line_len);
1031  printf("LINE END: \n");
1032 #endif
1033  for (x = 0 ; x < line_len; x++) {
1034  if (x > 0) {
1035  if (line[x - 1] != '\\' && line[x] == '\"') {
1036  quote++;
1037  }
1038 
1039  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1040  uint8_t *token = line + offset;
1041  size_t token_len = x - offset;
1042 
1043  if ((x + 1) == line_len) {
1044  token_len++;
1045  }
1046 
1047  offset = x + 1;
1048 
1049  while (offset < line_len && isspace(line[offset])) {
1050  x++;
1051  offset++;
1052  }
1053 #ifdef PRINT
1054  printf("TOKEN START: \n");
1055  PrintRawDataFp(stdout, token, token_len);
1056  printf("TOKEN END: \n");
1057 #endif
1058  if (token_len > name_len) {
1059  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1060  uint8_t *value = token + name_len;
1061  size_t value_len = token_len - name_len;
1062 
1063  if (value[0] == '\"') {
1064  value++;
1065  value_len--;
1066  }
1067  if (value[value_len-1] == '\"') {
1068  value_len--;
1069  }
1070 #ifdef PRINT
1071  printf("VALUE START: \n");
1072  PrintRawDataFp(stdout, value, value_len);
1073  printf("VALUE END: \n");
1074 #endif
1075  *retptr = value;
1076  *retlen = value_len;
1077  return 1;
1078  }
1079  }
1080  }
1081  }
1082  }
1083 
1084  return 0;
1085 }
1086 
1087 /**
1088  * \param name /Lowercase/ version of the variable name
1089  */
1090 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1091  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1092 {
1093  SCEnter();
1094 #ifdef PRINT
1095  printf("DATA START: \n");
1096  PrintRawDataFp(stdout, data, len);
1097  printf("DATA END: \n");
1098 #endif
1099  size_t x;
1100  int quote = 0;
1101 
1102  for (x = 0; x < len; x++) {
1103  if (!(isspace(data[x])))
1104  break;
1105  }
1106 
1107  if (x >= len) {
1108  SCReturnInt(0);
1109  }
1110 
1111  uint8_t *line = data+x;
1112  size_t line_len = len-x;
1113  size_t offset = 0;
1114 #ifdef PRINT
1115  printf("LINE START: \n");
1116  PrintRawDataFp(stdout, line, line_len);
1117  printf("LINE END: \n");
1118 #endif
1119  for (x = 0 ; x < line_len; x++) {
1120  if (x > 0) {
1121  if (line[x - 1] != '\\' && line[x] == '\"') {
1122  quote++;
1123  }
1124 
1125  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1126  uint8_t *token = line + offset;
1127  size_t token_len = x - offset;
1128 
1129  if ((x + 1) == line_len) {
1130  token_len++;
1131  }
1132 
1133  offset = x + 1;
1134 
1135  while (offset < line_len && isspace(line[offset])) {
1136  x++;
1137  offset++;
1138  }
1139 #ifdef PRINT
1140  printf("TOKEN START: \n");
1141  PrintRawDataFp(stdout, token, token_len);
1142  printf("TOKEN END: \n");
1143 #endif
1144  if (token_len > name_len) {
1145  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1146  uint8_t *value = token + name_len;
1147  size_t value_len = token_len - name_len;
1148 
1149  if (value[0] == '\"') {
1150  value++;
1151  value_len--;
1152  }
1153  if (value[value_len-1] == '\"') {
1154  value_len--;
1155  }
1156 #ifdef PRINT
1157  printf("VALUE START: \n");
1158  PrintRawDataFp(stdout, value, value_len);
1159  printf("VALUE END: \n");
1160 #endif
1161  *retptr = value;
1162  *retlen = value_len;
1163  SCReturnInt(1);
1164  }
1165  }
1166  }
1167  }
1168  }
1169 
1170  SCReturnInt(0);
1171 }
1172 
1173 /**
1174  * \brief setup multipart parsing: extract boundary and store it
1175  *
1176  * \param d HTTP transaction
1177  * \param htud transaction userdata
1178  *
1179  * \retval 1 ok, multipart set up
1180  * \retval 0 ok, not multipart though
1181  * \retval -1 error: problem with the boundary
1182  *
1183  * If the request contains a multipart message, this function will
1184  * set the HTP_BOUNDARY_SET in the transaction.
1185  */
1186 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1187 {
1188  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1189  "Content-Type");
1190  if (h != NULL && bstr_len(h->value) > 0) {
1191  uint8_t *boundary = NULL;
1192  size_t boundary_len = 0;
1193 
1194  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1195  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1196  &boundary, &boundary_len);
1197  if (r == 1) {
1198 #ifdef PRINT
1199  printf("BOUNDARY START: \n");
1200  PrintRawDataFp(stdout, boundary, boundary_len);
1201  printf("BOUNDARY END: \n");
1202 #endif
1203  if (boundary_len < HTP_BOUNDARY_MAX) {
1204  htud->boundary = HTPMalloc(boundary_len);
1205  if (htud->boundary == NULL) {
1206  return -1;
1207  }
1208  htud->boundary_len = (uint8_t)boundary_len;
1209  memcpy(htud->boundary, boundary, boundary_len);
1210 
1211  htud->tsflags |= HTP_BOUNDARY_SET;
1212  } else {
1213  SCLogDebug("invalid boundary");
1214  return -1;
1215  }
1216  SCReturnInt(1);
1217  }
1218  //SCReturnInt(1);
1219  }
1220  SCReturnInt(0);
1221 }
1222 
1223 #define C_D_HDR "content-disposition:"
1224 #define C_D_HDR_LEN 20
1225 #define C_T_HDR "content-type:"
1226 #define C_T_HDR_LEN 13
1228 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1229  HtpTxUserData *htud,
1230  uint8_t *header, uint32_t header_len,
1231  uint8_t **filename, uint16_t *filename_len,
1232  uint8_t **filetype, uint16_t *filetype_len)
1233 {
1234  uint8_t *fn = NULL;
1235  size_t fn_len = 0;
1236  uint8_t *ft = NULL;
1237  size_t ft_len = 0;
1238 
1239 #ifdef PRINT
1240  printf("HEADER START: \n");
1241  PrintRawDataFp(stdout, header, header_len);
1242  printf("HEADER END: \n");
1243 #endif
1244 
1245  while (header_len > 0) {
1246  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1247  uint8_t *line = header;
1248  uint32_t line_len;
1249 
1250  if (next_line == NULL) {
1251  line_len = header_len;
1252  } else {
1253  line_len = next_line - header;
1254  }
1255  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1256  if (sc == NULL) {
1257  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1259  /* if the : we found is the final char, it means we have
1260  * no value */
1261  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1262  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1264  } else {
1265 #ifdef PRINT
1266  printf("LINE START: \n");
1267  PrintRawDataFp(stdout, line, line_len);
1268  printf("LINE END: \n");
1269 #endif
1270  if (line_len >= C_D_HDR_LEN &&
1271  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1272  uint8_t *value = line + C_D_HDR_LEN;
1273  uint32_t value_len = line_len - C_D_HDR_LEN;
1274 
1275  /* parse content-disposition */
1276  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1277  value, value_len, &fn, &fn_len);
1278  } else if (line_len >= C_T_HDR_LEN &&
1279  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1280  SCLogDebug("content-type line");
1281  uint8_t *value = line + C_T_HDR_LEN;
1282  uint32_t value_len = line_len - C_T_HDR_LEN;
1283 
1284  (void)HTTPParseContentTypeHeader(NULL, 0,
1285  value, value_len, &ft, &ft_len);
1286  }
1287  }
1288 
1289  if (next_line == NULL) {
1290  SCLogDebug("no next_line");
1291  break;
1292  }
1293  header_len -= ((next_line + 2) - header);
1294  header = next_line + 2;
1295  } /* while (header_len > 0) */
1296 
1297  if (fn_len > USHRT_MAX)
1298  fn_len = USHRT_MAX;
1299  if (ft_len > USHRT_MAX)
1300  ft_len = USHRT_MAX;
1301 
1302  *filename = fn;
1303  *filename_len = fn_len;
1304  *filetype = ft;
1305  *filetype_len = ft_len;
1306 }
1307 
1308 /**
1309  * \brief Create a single buffer from the HtpBodyChunks in our list
1310  *
1311  * \param htud transaction user data
1312  * \param chunks_buffers pointer to pass back the buffer to the caller
1313  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1314  */
1315 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1316  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1317 {
1319  chunks_buffer, chunks_buffer_len,
1320  htud->request_body.body_parsed);
1321 }
1322 
1323 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1324 {
1325  SCEnter();
1326  if (tx && tx->de_state) {
1327  if (dir == STREAM_TOSERVER) {
1328  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1330  } else if (STREAM_TOCLIENT) {
1331  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1333  }
1334  }
1335 }
1336 
1337 /**
1338  * \brief Setup boundary buffers
1339  */
1340 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1341  uint8_t *boundary, uint32_t boundary_len)
1342 {
1343  memset(boundary, '-', boundary_len);
1344  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1345 }
1346 
1347 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1348  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1349 {
1350  int result = 0;
1351  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1352  uint32_t expected_boundary_len = htud->boundary_len + 2;
1353  uint32_t expected_boundary_end_len = htud->boundary_len + 4;
1354  int tx_progress = 0;
1355 
1356 #ifdef PRINT
1357  printf("CHUNK START: \n");
1358  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1359  printf("CHUNK END: \n");
1360 #endif
1361 
1362  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1363 
1364  /* search for the header start, header end and form end */
1365  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1366  boundary, expected_boundary_len);
1367  /* end of the multipart form */
1368  const uint8_t *form_end = NULL;
1369  /* end marker belonging to header_start */
1370  const uint8_t *header_end = NULL;
1371  if (header_start != NULL) {
1372  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1373  (uint8_t *)"\r\n\r\n", 4);
1374  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1375  boundary, expected_boundary_end_len);
1376  }
1377 
1378  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1379  header_end, form_end);
1380 
1381  /* we currently only handle multipart for ts. When we support it for tc,
1382  * we will need to supply right direction */
1383  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1384  /* if we're in the file storage process, deal with that now */
1385  if (htud->tsflags & HTP_FILENAME_SET) {
1386  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1387  SCLogDebug("reached the end of the file");
1388 
1389  const uint8_t *filedata = chunks_buffer;
1390  uint32_t filedata_len = 0;
1391  uint8_t flags = 0;
1392 
1393  if (header_start != NULL) {
1394  if (header_start == filedata + 2) {
1395  /* last chunk had all data, but not the boundary */
1396  SCLogDebug("last chunk had all data, but not the boundary");
1397  filedata_len = 0;
1398  } else if (header_start > filedata + 2) {
1399  SCLogDebug("some data from last file before the boundary");
1400  /* some data from last file before the boundary */
1401  filedata_len = header_start - filedata - 2;
1402  }
1403  }
1404  /* body parsing done, we did not get our form end. Use all data
1405  * we still have and signal to files API we have an issue. */
1406  if (tx_progress > HTP_REQUEST_BODY) {
1407  filedata_len = chunks_buffer_len;
1409  }
1410 
1411  if (filedata_len > chunks_buffer_len) {
1412  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1414  goto end;
1415  }
1416 #ifdef PRINT
1417  printf("FILEDATA (final chunk) START: \n");
1418  PrintRawDataFp(stdout, filedata, filedata_len);
1419  printf("FILEDATA (final chunk) END: \n");
1420 #endif
1421  if (!(htud->tsflags & HTP_DONTSTORE)) {
1422  if (HTPFileClose(hstate, filedata, filedata_len, flags,
1423  STREAM_TOSERVER) == -1)
1424  {
1425  goto end;
1426  }
1427  }
1428 
1429  htud->tsflags &=~ HTP_FILENAME_SET;
1430 
1431  /* fall through */
1432  } else {
1433  SCLogDebug("not yet at the end of the file");
1434 
1435  if (chunks_buffer_len > expected_boundary_end_len) {
1436  const uint8_t *filedata = chunks_buffer;
1437  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1438 #ifdef PRINT
1439  printf("FILEDATA (part) START: \n");
1440  PrintRawDataFp(stdout, filedata, filedata_len);
1441  printf("FILEDATA (part) END: \n");
1442 #endif
1443 
1444  if (!(htud->tsflags & HTP_DONTSTORE)) {
1445  result = HTPFileStoreChunk(hstate, filedata,
1446  filedata_len, STREAM_TOSERVER);
1447  if (result == -1) {
1448  goto end;
1449  } else if (result == -2) {
1450  /* we know for sure we're not storing the file */
1451  htud->tsflags |= HTP_DONTSTORE;
1452  }
1453  }
1454 
1455  htud->request_body.body_parsed += filedata_len;
1456  } else {
1457  SCLogDebug("chunk too small to already process in part");
1458  }
1459 
1460  goto end;
1461  }
1462  }
1463 
1464  while (header_start != NULL && header_end != NULL &&
1465  header_end != form_end &&
1466  header_start < (chunks_buffer + chunks_buffer_len) &&
1467  header_end < (chunks_buffer + chunks_buffer_len) &&
1468  header_start < header_end)
1469  {
1470  uint8_t *filename = NULL;
1471  uint16_t filename_len = 0;
1472  uint8_t *filetype = NULL;
1473  uint16_t filetype_len = 0;
1474 
1475  uint32_t header_len = header_end - header_start;
1476  SCLogDebug("header_len %u", header_len);
1477  uint8_t *header = (uint8_t *)header_start;
1478 
1479  /* skip empty records */
1480  if (expected_boundary_len == header_len) {
1481  goto next;
1482  } else if ((expected_boundary_len + 2) <= header_len) {
1483  header_len -= (expected_boundary_len + 2);
1484  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1485  }
1486 
1487  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1488  &filename, &filename_len, &filetype, &filetype_len);
1489 
1490  if (filename != NULL) {
1491  const uint8_t *filedata = NULL;
1492  uint32_t filedata_len = 0;
1493 
1494  SCLogDebug("we have a filename");
1495 
1496  htud->tsflags |= HTP_FILENAME_SET;
1497  htud->tsflags &= ~HTP_DONTSTORE;
1498 
1499  SCLogDebug("header_end %p", header_end);
1500  SCLogDebug("form_end %p", form_end);
1501 
1502  /* everything until the final boundary is the file */
1503  if (form_end != NULL) {
1504  SCLogDebug("have form_end");
1505 
1506  filedata = header_end + 4;
1507  if (form_end == filedata) {
1508  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1510  goto end;
1511  } else if (form_end < filedata) {
1512  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1514  goto end;
1515  }
1516 
1517  filedata_len = form_end - (header_end + 4 + 2);
1518  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1519 
1520  /* or is it? */
1521  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1522  boundary, expected_boundary_len);
1523  if (header_next != NULL) {
1524  filedata_len -= (form_end - header_next);
1525  }
1526 
1527  if (filedata_len > chunks_buffer_len) {
1528  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1530  goto end;
1531  }
1532  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1533 #ifdef PRINT
1534  printf("FILEDATA START: \n");
1535  PrintRawDataFp(stdout, filedata, filedata_len);
1536  printf("FILEDATA END: \n");
1537 #endif
1538 
1539  result = HTPFileOpen(hstate, filename, filename_len,
1540  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1541  STREAM_TOSERVER);
1542  if (result == -1) {
1543  goto end;
1544  } else if (result == -2) {
1545  htud->tsflags |= HTP_DONTSTORE;
1546  } else {
1547  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1548  goto end;
1549  }
1550  }
1551  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1552 
1553  htud->request_body.body_parsed += (header_end - chunks_buffer);
1554  htud->tsflags &= ~HTP_FILENAME_SET;
1555  } else {
1556  SCLogDebug("chunk doesn't contain form end");
1557 
1558  filedata = header_end + 4;
1559  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1560  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1561 
1562  if (filedata_len > chunks_buffer_len) {
1563  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1565  goto end;
1566  }
1567 
1568 #ifdef PRINT
1569  printf("FILEDATA START: \n");
1570  PrintRawDataFp(stdout, filedata, filedata_len);
1571  printf("FILEDATA END: \n");
1572 #endif
1573  /* form doesn't end in this chunk, but the part might. Lets
1574  * see if have another coming up */
1575  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1576  boundary, expected_boundary_len);
1577  SCLogDebug("header_next %p", header_next);
1578  if (header_next == NULL) {
1579  SCLogDebug("more file data to come");
1580 
1581  uint32_t offset = (header_end + 4) - chunks_buffer;
1582  SCLogDebug("offset %u", offset);
1583  htud->request_body.body_parsed += offset;
1584 
1585  if (filedata_len >= (expected_boundary_len + 2)) {
1586  filedata_len -= (expected_boundary_len + 2 - 1);
1587  SCLogDebug("opening file with partial data");
1588  } else {
1589  filedata = NULL;
1590  filedata_len = 0;
1591  }
1592  result = HTPFileOpen(hstate, filename, filename_len,
1593  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1594  STREAM_TOSERVER);
1595  if (result == -1) {
1596  goto end;
1597  } else if (result == -2) {
1598  htud->tsflags |= HTP_DONTSTORE;
1599  }
1600  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1601  htud->request_body.body_parsed += filedata_len;
1602  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1603 
1604  } else if (header_next - filedata > 2) {
1605  filedata_len = header_next - filedata - 2;
1606  SCLogDebug("filedata_len %u", filedata_len);
1607 
1608  result = HTPFileOpen(hstate, filename, filename_len,
1609  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1610  STREAM_TOSERVER);
1611  if (result == -1) {
1612  goto end;
1613  } else if (result == -2) {
1614  htud->tsflags |= HTP_DONTSTORE;
1615  } else {
1616  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1617  goto end;
1618  }
1619  }
1620  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1621 
1622  htud->tsflags &= ~HTP_FILENAME_SET;
1623  htud->request_body.body_parsed += (header_end - chunks_buffer);
1624  }
1625  }
1626  }
1627 next:
1628  SCLogDebug("header_start %p, header_end %p, form_end %p",
1629  header_start, header_end, form_end);
1630 
1631  /* Search next boundary entry after the start of body */
1632  uint32_t cursizeread = header_end - chunks_buffer;
1633  header_start = Bs2bmSearch(header_end + 4,
1634  chunks_buffer_len - (cursizeread + 4),
1635  boundary, expected_boundary_len);
1636  if (header_start != NULL) {
1637  header_end = Bs2bmSearch(header_end + 4,
1638  chunks_buffer_len - (cursizeread + 4),
1639  (uint8_t *) "\r\n\r\n", 4);
1640  }
1641  }
1642 
1643  /* if we're parsing the multipart and we're not currently processing a
1644  * file, we move the body pointer forward. */
1645  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1646  if (chunks_buffer_len > expected_boundary_end_len) {
1647  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1648 
1649  htud->request_body.body_parsed += move;
1650  SCLogDebug("form not ready, file not set, parsing non-file "
1651  "record: moved %u", move);
1652  }
1653  }
1654 
1655 end:
1656  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1657  return 0;
1658 }
1659 
1660 /** \internal
1661  * \brief Handle POST or PUT, no multipart body data
1662  */
1663 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1664  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1665 {
1666  int result = 0;
1667 
1668  /* see if we need to open the file */
1669  if (!(htud->tsflags & HTP_FILENAME_SET))
1670  {
1671  uint8_t *filename = NULL;
1672  size_t filename_len = 0;
1673 
1674  /* get the name */
1675  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1676  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1677  filename_len = bstr_len(tx->parsed_uri->path);
1678  }
1679 
1680  if (filename != NULL) {
1681  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1682  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1683  if (result == -1) {
1684  goto end;
1685  } else if (result == -2) {
1686  htud->tsflags |= HTP_DONTSTORE;
1687  } else {
1688  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1689  htud->tsflags |= HTP_FILENAME_SET;
1690  htud->tsflags &= ~HTP_DONTSTORE;
1691  }
1692  }
1693  }
1694  else
1695  {
1696  /* otherwise, just store the data */
1697 
1698  if (!(htud->tsflags & HTP_DONTSTORE)) {
1699  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1700  if (result == -1) {
1701  goto end;
1702  } else if (result == -2) {
1703  /* we know for sure we're not storing the file */
1704  htud->tsflags |= HTP_DONTSTORE;
1705  }
1706  }
1707  }
1708 
1709  return 0;
1710 end:
1711  return -1;
1712 }
1713 
1714 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1715  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1716 {
1717  SCEnter();
1718 
1719  int result = 0;
1720 
1721  /* see if we need to open the file
1722  * we check for tx->response_line in case of junk
1723  * interpreted as body before response line
1724  */
1725  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1726  (tx->response_line != NULL || tx->is_protocol_0_9))
1727  {
1728  SCLogDebug("setting up file name");
1729 
1730  uint8_t *filename = NULL;
1731  size_t filename_len = 0;
1732 
1733  /* try Content-Disposition header first */
1734  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1735  "Content-Disposition");
1736  if (h != NULL && bstr_len(h->value) > 0) {
1737  /* parse content-disposition */
1738  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1739  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1740  }
1741 
1742  /* fall back to name from the uri */
1743  if (filename == NULL) {
1744  /* get the name */
1745  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1746  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1747  filename_len = bstr_len(tx->parsed_uri->path);
1748  }
1749  }
1750 
1751  if (filename != NULL) {
1752  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len,
1753  data, data_len, HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1754  SCLogDebug("result %d", result);
1755  if (result == -1) {
1756  goto end;
1757  } else if (result == -2) {
1758  htud->tcflags |= HTP_DONTSTORE;
1759  } else {
1760  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1761  htud->tcflags |= HTP_FILENAME_SET;
1762  htud->tcflags &= ~HTP_DONTSTORE;
1763  }
1764  //set range if present
1765  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1766  if (h_content_range != NULL) {
1767  HTPFileSetRange(hstate, h_content_range->value);
1768  }
1769  }
1770  }
1771  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1772  {
1773  /* otherwise, just store the data */
1774 
1775  if (!(htud->tcflags & HTP_DONTSTORE)) {
1776  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1777  SCLogDebug("result %d", result);
1778  if (result == -1) {
1779  goto end;
1780  } else if (result == -2) {
1781  /* we know for sure we're not storing the file */
1782  htud->tcflags |= HTP_DONTSTORE;
1783  }
1784  }
1785  }
1786 
1787  htud->response_body.body_parsed += data_len;
1788  return 0;
1789 end:
1790  return -1;
1791 }
1792 
1793 /**
1794  * \brief Function callback to append chunks for Requests
1795  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1796  * \retval int HTP_OK if all goes well
1797  */
1798 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1799 {
1800  SCEnter();
1801 
1802  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1803  SCReturnInt(HTP_OK);
1804 
1805  if (d->len == 0)
1806  SCReturnInt(HTP_OK);
1807 
1808 #ifdef PRINT
1809  printf("HTPBODY START: \n");
1810  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1811  printf("HTPBODY END: \n");
1812 #endif
1813 
1814  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1815  if (hstate == NULL) {
1816  SCReturnInt(HTP_ERROR);
1817  }
1818 
1819  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1820  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1821 
1822  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1823  if (tx_ud == NULL) {
1824  SCReturnInt(HTP_OK);
1825  }
1826  if (!tx_ud->response_body_init) {
1827  tx_ud->response_body_init = 1;
1828 
1829  if (d->tx->request_method_number == HTP_M_POST) {
1830  SCLogDebug("POST");
1831  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1832  if (r == 1) {
1834  } else if (r == 0) {
1836  SCLogDebug("not multipart");
1837  }
1838  } else if (d->tx->request_method_number == HTP_M_PUT) {
1840  }
1841  }
1842 
1843  /* see if we can get rid of htp body chunks */
1844  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1845 
1846  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1847  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1848 
1849  /* within limits, add the body chunk to the state. */
1850  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1851  uint32_t stream_depth = FileReassemblyDepth();
1852  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1853  hstate->cfg->request.body_limit,
1854  stream_depth,
1855  tx_ud->tsflags,
1856  (uint32_t)d->len);
1857  BUG_ON(len > (uint32_t)d->len);
1858 
1859  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1860 
1861  const uint8_t *chunks_buffer = NULL;
1862  uint32_t chunks_buffer_len = 0;
1863 
1865  /* multi-part body handling starts here */
1866  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1867  goto end;
1868  }
1869 
1870  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1871  if (chunks_buffer == NULL) {
1872  goto end;
1873  }
1874 #ifdef PRINT
1875  printf("REASSCHUNK START: \n");
1876  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1877  printf("REASSCHUNK END: \n");
1878 #endif
1879 
1880  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1881 
1882  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1884  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1885  }
1886 
1887  } else {
1888  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1889  SCLogDebug("closing file that was being stored");
1890  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1891  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1892  }
1893  }
1894 
1895 end:
1896  if (hstate->conn != NULL) {
1897  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1899  hstate->cfg->request.inspect_min_size,
1900  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1901 
1902  /* if we reach the inspect_min_size we'll trigger inspection,
1903  * so make sure that raw stream is also inspected. Set the
1904  * data to be used to the amount of raw bytes we've seen to
1905  * get here. */
1906  if (tx_ud->request_body.body_inspected == 0 &&
1908  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1909  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1910  {
1911  uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1912 
1913  /* body still in progress, but due to min inspect size we need to inspect now */
1916  }
1917  /* after the start of the body, disable the depth logic */
1918  } else if (tx_ud->request_body.body_inspected > 0) {
1920  }
1921  }
1922  SCReturnInt(HTP_OK);
1923 }
1924 
1925 /**
1926  * \brief Function callback to append chunks for Responses
1927  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1928  * \retval int HTP_OK if all goes well
1929  */
1930 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1931 {
1932  SCEnter();
1933 
1934  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1935  SCReturnInt(HTP_OK);
1936 
1937  if (d->len == 0)
1938  SCReturnInt(HTP_OK);
1939 
1940  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1941  if (hstate == NULL) {
1942  SCReturnInt(HTP_ERROR);
1943  }
1944 
1945  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1946  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1947 
1948  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1949  if (tx_ud == NULL) {
1950  SCReturnInt(HTP_OK);
1951  }
1952  if (!tx_ud->request_body_init) {
1953  tx_ud->request_body_init = 1;
1954  }
1955 
1956  /* see if we can get rid of htp body chunks */
1957  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1958 
1959  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1960  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1961 
1962  /* within limits, add the body chunk to the state. */
1963  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1964  uint32_t stream_depth = FileReassemblyDepth();
1965  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1966  hstate->cfg->response.body_limit,
1967  stream_depth,
1968  tx_ud->tcflags,
1969  (uint32_t)d->len);
1970  BUG_ON(len > (uint32_t)d->len);
1971 
1972  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1973 
1974  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1975  } else {
1976  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1977  SCLogDebug("closing file that was being stored");
1978  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1979  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1980  }
1981  }
1982 
1983  if (hstate->conn != NULL) {
1984  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1986  hstate->cfg->response.inspect_min_size,
1987  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
1988  /* if we reach the inspect_min_size we'll trigger inspection,
1989  * so make sure that raw stream is also inspected. Set the
1990  * data to be used to the amount of raw bytes we've seen to
1991  * get here. */
1992  if (tx_ud->response_body.body_inspected == 0 &&
1994  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
1995  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
1996  {
1997  uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
1998 
1999  /* body still in progress, but due to min inspect size we need to inspect now */
2002  }
2003  /* after the start of the body, disable the depth logic */
2004  } else if (tx_ud->response_body.body_inspected > 0) {
2006  }
2007  }
2008  SCReturnInt(HTP_OK);
2009 }
2010 
2011 /**
2012  * \brief Print the stats of the HTTP requests
2013  */
2015 {
2016 #ifdef DEBUG
2017  SCEnter();
2018  SCMutexLock(&htp_state_mem_lock);
2019  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2020  htp_state_memcnt, htp_state_memuse);
2021  SCMutexUnlock(&htp_state_mem_lock);
2022  SCReturn;
2023 #endif
2024 }
2025 
2026 /** \brief Clears the HTTP server configuration memory used by HTP library */
2027 void HTPFreeConfig(void)
2028 {
2029  SCEnter();
2030 
2031  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2032  !AppLayerParserConfParserEnabled("tcp", "http"))
2033  {
2034  SCReturn;
2035  }
2036 
2037  HTPCfgRec *nextrec = cfglist.next;
2038  SCRadixReleaseRadixTree(cfgtree);
2039  cfgtree = NULL;
2040  htp_config_destroy(cfglist.cfg);
2041  while (nextrec != NULL) {
2042  HTPCfgRec *htprec = nextrec;
2043  nextrec = nextrec->next;
2044 
2045  htp_config_destroy(htprec->cfg);
2046  SCFree(htprec);
2047  }
2048  SCReturn;
2049 }
2050 
2051 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2052 {
2053  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2054  if (htud != NULL) {
2055  htud->request_has_trailers = 1;
2056  }
2057  return HTP_OK;
2058 }
2059 
2060 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2061 {
2062  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2063  if (htud != NULL) {
2064  htud->response_has_trailers = 1;
2065  }
2066  return HTP_OK;
2067 }
2068 
2069 /**\internal
2070  * \brief called at start of request
2071  * Set min inspect size.
2072  */
2073 static int HTPCallbackRequestStart(htp_tx_t *tx)
2074 {
2075  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2076  if (hstate == NULL) {
2077  SCReturnInt(HTP_ERROR);
2078  }
2079 
2080  if (hstate->cfg)
2082  hstate->cfg->request.inspect_min_size);
2083 
2084  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2085  if (tx_ud == NULL) {
2086  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2087  if (unlikely(tx_ud == NULL)) {
2088  SCReturnInt(HTP_OK);
2089  }
2090  htp_tx_set_user_data(tx, tx_ud);
2091  }
2092  SCReturnInt(HTP_OK);
2093 }
2094 
2095 /**\internal
2096  * \brief called at start of response
2097  * Set min inspect size.
2098  */
2099 static int HTPCallbackResponseStart(htp_tx_t *tx)
2100 {
2101  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2102  if (hstate == NULL) {
2103  SCReturnInt(HTP_ERROR);
2104  }
2105 
2106  if (hstate->cfg)
2108  hstate->cfg->response.inspect_min_size);
2109 
2110  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2111  if (tx_ud == NULL) {
2112  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2113  if (unlikely(tx_ud == NULL)) {
2114  SCReturnInt(HTP_OK);
2115  }
2116  htp_tx_set_user_data(tx, tx_ud);
2117  }
2118  SCReturnInt(HTP_OK);
2119 }
2120 
2121 
2122 /**
2123  * \brief callback for request to store the recent incoming request
2124  in to the recent_in_tx for the given htp state
2125  * \param connp pointer to the current connection parser which has the htp
2126  * state in it as user data
2127  */
2128 static int HTPCallbackRequest(htp_tx_t *tx)
2129 {
2130  SCEnter();
2131 
2132  if (tx == NULL) {
2133  SCReturnInt(HTP_ERROR);
2134  }
2135 
2136  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2137  if (hstate == NULL) {
2138  SCReturnInt(HTP_ERROR);
2139  }
2140 
2141  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2142  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2143 
2144  SCLogDebug("HTTP request completed");
2145 
2146  HTPErrorCheckTxRequestFlags(hstate, tx);
2147 
2148  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2149  if (htud != NULL) {
2150  if (htud->tsflags & HTP_FILENAME_SET) {
2151  SCLogDebug("closing file that was being stored");
2152  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
2153  htud->tsflags &= ~HTP_FILENAME_SET;
2155  (uint32_t)hstate->conn->in_data_counter);
2156  }
2157  }
2158 
2159  hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
2160  /* request done, do raw reassembly now to inspect state and stream
2161  * at the same time. */
2163  SCReturnInt(HTP_OK);
2164 }
2165 
2166 /**
2167  * \brief callback for response to remove the recent received requests
2168  from the recent_in_tx for the given htp state
2169  * \param connp pointer to the current connection parser which has the htp
2170  * state in it as user data
2171  */
2172 static int HTPCallbackResponse(htp_tx_t *tx)
2173 {
2174  SCEnter();
2175 
2176  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2177  if (hstate == NULL) {
2178  SCReturnInt(HTP_ERROR);
2179  }
2180 
2181  /* we have one whole transaction now */
2182  hstate->transaction_cnt++;
2183 
2184  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2185  if (htud != NULL) {
2186  if (htud->tcflags & HTP_FILENAME_SET) {
2187  SCLogDebug("closing file that was being stored");
2188  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
2189  htud->tcflags &= ~HTP_FILENAME_SET;
2190  }
2191  }
2192 
2193  /* response done, do raw reassembly now to inspect state and stream
2194  * at the same time. */
2196 
2197  /* handle HTTP CONNECT */
2198  if (tx->request_method_number == HTP_M_CONNECT) {
2199  /* any 2XX status response implies that the connection will become
2200  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2201  if ((tx->response_status_number >= 200) &&
2202  (tx->response_status_number < 300) &&
2203  (hstate->transaction_cnt == 1)) {
2204  uint16_t dp = 0;
2205  if (tx->request_port_number != -1) {
2206  dp = (uint16_t)tx->request_port_number;
2207  }
2208  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2210  tx->request_progress = HTP_REQUEST_COMPLETE;
2211  tx->response_progress = HTP_RESPONSE_COMPLETE;
2212  }
2213  }
2214 
2215  hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
2216  SCReturnInt(HTP_OK);
2217 }
2218 
2219 static int HTPCallbackRequestLine(htp_tx_t *tx)
2220 {
2221  HtpTxUserData *tx_ud;
2222  bstr *request_uri_normalized;
2223  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2224  const HTPCfgRec *cfg = hstate->cfg;
2225 
2226  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2227  if (request_uri_normalized == NULL)
2228  return HTP_OK;
2229 
2230  tx_ud = htp_tx_get_user_data(tx);
2231  if (likely(tx_ud == NULL)) {
2232  tx_ud = HTPMalloc(sizeof(*tx_ud));
2233  if (unlikely(tx_ud == NULL)) {
2234  bstr_free(request_uri_normalized);
2235  return HTP_OK;
2236  }
2237  memset(tx_ud, 0, sizeof(*tx_ud));
2238  htp_tx_set_user_data(tx, tx_ud);
2239  }
2240  if (unlikely(tx_ud->request_uri_normalized != NULL))
2241  bstr_free(tx_ud->request_uri_normalized);
2242  tx_ud->request_uri_normalized = request_uri_normalized;
2243 
2244  if (tx->flags) {
2245  HTPErrorCheckTxRequestFlags(hstate, tx);
2246  }
2247  return HTP_OK;
2248 }
2249 
2250 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2251 {
2252  if (part == NULL)
2253  return HTP_OK;
2254 
2255  uint64_t flags = 0;
2256  size_t prevlen = bstr_len(part);
2257  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2258  // shorter string means that uri was encoded
2259  if (res == HTP_OK && prevlen > bstr_len(part)) {
2260  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2261  if (htud == NULL)
2262  return HTP_OK;
2263  HtpState *s = htp_connp_get_user_data(tx->connp);
2264  if (s == NULL)
2265  return HTP_OK;
2266  HTPSetEvent(s, htud, STREAM_TOSERVER,
2268  }
2269 
2270  return HTP_OK;
2271 }
2272 
2273 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2274 {
2275  if (tx->parsed_uri == NULL)
2276  return HTP_OK;
2277 
2278  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2279 }
2280 
2281 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2282 {
2283  if (tx->parsed_uri == NULL)
2284  return HTP_OK;
2285 
2286  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2287 }
2288 
2289 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2290 {
2291  void *ptmp;
2292  if (tx_data->len == 0 || tx_data->tx == NULL)
2293  return HTP_OK;
2294 
2295  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2296  if (tx_ud == NULL) {
2297  return HTP_OK;
2298  }
2299  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2300  tx_ud->request_headers_raw_len,
2301  tx_ud->request_headers_raw_len + tx_data->len);
2302  if (ptmp == NULL) {
2303  return HTP_OK;
2304  }
2305  tx_ud->request_headers_raw = ptmp;
2306 
2307  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2308  tx_data->data, tx_data->len);
2309  tx_ud->request_headers_raw_len += tx_data->len;
2310 
2311  if (tx_data->tx && tx_data->tx->flags) {
2312  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2313  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2314  }
2315  return HTP_OK;
2316 }
2317 
2318 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2319 {
2320  void *ptmp;
2321  if (tx_data->len == 0 || tx_data->tx == NULL)
2322  return HTP_OK;
2323 
2324  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2325  if (tx_ud == NULL) {
2326  return HTP_OK;
2327  }
2328  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2329  tx_ud->response_headers_raw_len,
2330  tx_ud->response_headers_raw_len + tx_data->len);
2331  if (ptmp == NULL) {
2332  return HTP_OK;
2333  }
2334  tx_ud->response_headers_raw = ptmp;
2335 
2336  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2337  tx_data->data, tx_data->len);
2338  tx_ud->response_headers_raw_len += tx_data->len;
2339 
2340  return HTP_OK;
2341 }
2342 
2343 /*
2344  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2345  */
2346 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2347 {
2348  cfg_prec->uri_include_all = FALSE;
2355 
2356  if (!g_disable_randomness) {
2358  } else {
2359  cfg_prec->randomize = 0;
2360  }
2362 
2363  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2364  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2365  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2366  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2367 
2368  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2369  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2370 
2371  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2372  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2373 
2374  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2375  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
2376 
2377  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2378  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
2379 
2380  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2381 
2382  /* don't convert + to space by default */
2383  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2384  // enables request decompression
2385  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2386 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2387  // disable by default
2388  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2389 #endif
2390 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2391  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2393 #endif
2394 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2395  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2397 #endif
2398 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2399  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2400 #endif
2401  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2402  * only the hard limit. So we set both here to the (current) htp defaults.
2403  * The reason we do this is that if the user sets the hard limit in the
2404  * config, we have to set the soft limit as well. If libhtp starts using
2405  * the soft limit in the future, we at least make sure we control what
2406  * it's value is. */
2407  htp_config_set_field_limits(cfg_prec->cfg,
2410  return;
2411 }
2412 
2413 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2414  * but we can get both <0 and >RAND_MAX values from RandomGet
2415  */
2416 static int RandomGetWrap(void)
2417 {
2418  unsigned long r;
2419 
2420  do {
2421  r = RandomGet();
2422  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2423 
2424  return r % RAND_MAX;
2425 }
2426 
2427 /*
2428  * We have this splitup so that in case double decoding has been enabled
2429  * for query and path, they would be called first on the callback queue,
2430  * before the callback set by Phase2() is called. We need this, since
2431  * the callback in Phase2() generates the normalized uri which utilizes
2432  * the query and path. */
2433 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2434 {
2435  /* randomize inspection size if needed */
2436  if (cfg_prec->randomize) {
2437  int rdrange = cfg_prec->randomize_range;
2438 
2439  long int r = RandomGetWrap();
2440  cfg_prec->request.inspect_min_size +=
2441  (int) (cfg_prec->request.inspect_min_size *
2442  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2443 
2444  r = RandomGetWrap();
2445  cfg_prec->request.inspect_window +=
2446  (int) (cfg_prec->request.inspect_window *
2447  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2448  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2449  " %d and 'request-body-inspect-window' set to %d after"
2450  " randomization.",
2451  name,
2452  cfg_prec->request.inspect_min_size,
2453  cfg_prec->request.inspect_window);
2454 
2455 
2456  r = RandomGetWrap();
2457  cfg_prec->response.inspect_min_size +=
2458  (int) (cfg_prec->response.inspect_min_size *
2459  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2460 
2461  r = RandomGetWrap();
2462  cfg_prec->response.inspect_window +=
2463  (int) (cfg_prec->response.inspect_window *
2464  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2465 
2466  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2467  " %d and 'response-body-inspect-window' set to %d after"
2468  " randomization.",
2469  name,
2470  cfg_prec->response.inspect_min_size,
2471  cfg_prec->response.inspect_window);
2472  }
2473 
2474  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2475 
2476  cfg_prec->request.sbcfg.flags = 0;
2477  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2478  cfg_prec->request.inspect_window : 256;
2479  cfg_prec->request.sbcfg.buf_slide = 0;
2480  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2481  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2482  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2483  cfg_prec->request.sbcfg.Free = HTPFree;
2484 
2485  cfg_prec->response.sbcfg.flags = 0;
2486  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2487  cfg_prec->response.inspect_window : 256;
2488  cfg_prec->response.sbcfg.buf_slide = 0;
2489  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2490  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2491  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2492  cfg_prec->response.sbcfg.Free = HTPFree;
2493  return;
2494 }
2495 
2496 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2497  SCRadixTree *tree)
2498 {
2499  if (cfg_prec == NULL || s == NULL || tree == NULL)
2500  return;
2501 
2502  ConfNode *p = NULL;
2503 
2504  /* Default Parameters */
2505  TAILQ_FOREACH(p, &s->head, next) {
2506 
2507  if (strcasecmp("address", p->name) == 0) {
2508  ConfNode *pval;
2509  /* Addresses */
2510  TAILQ_FOREACH(pval, &p->head, next) {
2511  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2512  pval->val);
2513 
2514  /* IPV6 or IPV4? */
2515  if (strchr(pval->val, ':') != NULL) {
2516  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2517  s->name, pval->val, cfg_prec->cfg);
2518  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2519  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2520  "add ipv6 server %s, ignoring", pval->val);
2521  }
2522  } else {
2523  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2524  s->name, pval->val, cfg_prec->cfg);
2525  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2526  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2527  "to add ipv4 server %s, ignoring",
2528  pval->val);
2529  }
2530  } /* else - if (strchr(pval->val, ':') != NULL) */
2531  } /* TAILQ_FOREACH(pval, &p->head, next) */
2532 
2533  } else if (strcasecmp("personality", p->name) == 0) {
2534  /* Personalities */
2535  int personality = HTPLookupPersonality(p->val);
2536  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2537  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2538 
2539  if (personality >= 0) {
2540  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2541  personality);
2542  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2543  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2544  "personality \"%s\", ignoring", p->val);
2545  } else {
2546  SCLogDebug("LIBHTP personality set to %s",
2547  HTPLookupPersonalityString(personality));
2548  }
2549 
2550  /* The IDS personality by default converts the path (and due to
2551  * our query string callback also the query string) to lowercase.
2552  * Signatures do not expect this, so override it. */
2553  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2554  } else {
2555  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2556  "\"%s\", ignoring", p->val);
2557  continue;
2558  }
2559 
2560  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2561  strcasecmp("request_body_limit", p->name) == 0) {
2562  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2563  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2564  "from conf file - %s. Killing engine", p->val);
2565  exit(EXIT_FAILURE);
2566  }
2567 
2568  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2569  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2570  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2571  "from conf file - %s. Killing engine", p->val);
2572  exit(EXIT_FAILURE);
2573  }
2574 
2575  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2576  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2577  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2578  "from conf file - %s. Killing engine", p->val);
2579  exit(EXIT_FAILURE);
2580  }
2581 
2582  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2583  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2584  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2585  "from conf file - %s. Killing engine", p->val);
2586  exit(EXIT_FAILURE);
2587  }
2588 
2589  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2590  if (ConfValIsTrue(p->val)) {
2591  htp_config_register_request_line(cfg_prec->cfg,
2592  HTPCallbackDoubleDecodeQuery);
2593  }
2594 
2595  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2596  if (ConfValIsTrue(p->val)) {
2597  htp_config_register_request_line(cfg_prec->cfg,
2598  HTPCallbackDoubleDecodePath);
2599  }
2600 
2601  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2602  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2603  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2604  "from conf file - %s. Killing engine", p->val);
2605  exit(EXIT_FAILURE);
2606  }
2607 
2608  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2609  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2610  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2611  "from conf file - %s. Killing engine", p->val);
2612  exit(EXIT_FAILURE);
2613  }
2614 
2615  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2616  uint32_t value = 2;
2617  if (ParseSizeStringU32(p->val, &value) < 0) {
2618  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2619  "from conf file - %s. Killing engine", p->val);
2620  exit(EXIT_FAILURE);
2621  }
2622 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2623  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2624 #else
2625  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2626  "to %u, libhtp version too old", value);
2627 #endif
2628  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2629  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2630  HTP_DECODER_URL_PATH,
2631  ConfValIsTrue(p->val));
2632  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2633  if (strlen(p->val) == 1) {
2634  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2635  HTP_DECODER_URL_PATH,
2636  p->val[0]);
2637  } else {
2638  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2639  "for libhtp param path-bestfit-replacement-char");
2640  }
2641  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2642  htp_config_set_convert_lowercase(cfg_prec->cfg,
2643  HTP_DECODER_URL_PATH,
2644  ConfValIsTrue(p->val));
2645  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2646  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2647  HTP_DECODER_URL_PATH,
2648  ConfValIsTrue(p->val));
2649  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2650  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2651  HTP_DECODER_URL_PATH,
2652  ConfValIsTrue(p->val));
2653  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2654  htp_config_set_path_separators_compress(cfg_prec->cfg,
2655  HTP_DECODER_URL_PATH,
2656  ConfValIsTrue(p->val));
2657  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2658  htp_config_set_path_separators_decode(cfg_prec->cfg,
2659  HTP_DECODER_URL_PATH,
2660  ConfValIsTrue(p->val));
2661  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2662  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2663  HTP_DECODER_URL_PATH,
2664  ConfValIsTrue(p->val));
2665  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2666  enum htp_url_encoding_handling_t handling;
2667  if (strcasecmp(p->val, "preserve_percent") == 0) {
2668  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2669  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2670  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2671  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2672  handling = HTP_URL_DECODE_PROCESS_INVALID;
2673  } else {
2674  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2675  "for libhtp param path-url-encoding-invalid-handling");
2676  return;
2677  }
2678  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2679  HTP_DECODER_URL_PATH,
2680  handling);
2681  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2682  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2683  HTP_DECODER_URL_PATH,
2684  ConfValIsTrue(p->val));
2685  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2686  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2687  SCLogDebug("uri-include-all %s",
2688  cfg_prec->uri_include_all ? "enabled" : "disabled");
2689  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2690  htp_config_set_plusspace_decode(cfg_prec->cfg,
2691  HTP_DECODER_URLENCODED,
2692  ConfValIsTrue(p->val));
2693  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2694  uint32_t limit = 0;
2695  if (ParseSizeStringU32(p->val, &limit) < 0) {
2696  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2697  "from conf file - %s. Killing engine", p->val);
2698  exit(EXIT_FAILURE);
2699  }
2700  if (limit == 0) {
2701  FatalError(SC_ERR_FATAL, "Error meta-field-limit "
2702  "from conf file cannot be 0. Killing engine");
2703  }
2704  /* set default soft-limit with our new hard limit */
2705  htp_config_set_field_limits(cfg_prec->cfg,
2707  (size_t)limit);
2708 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2709  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2710  uint32_t limit = 0;
2711  if (ParseSizeStringU32(p->val, &limit) < 0) {
2712  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'lzma-memlimit' "
2713  "from conf file - %s.", p->val);
2714  }
2715  if (limit == 0) {
2716  FatalError(SC_ERR_SIZE_PARSE, "'lzma-memlimit' "
2717  "from conf file cannot be 0.");
2718  }
2719  /* set default soft-limit with our new hard limit */
2720  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2721  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2722 #endif
2723 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2724  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2725  if (ConfValIsTrue(p->val)) {
2726  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2727  } else if (!ConfValIsFalse(p->val)) {
2728  int8_t limit;
2729  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2731  "failed to parse 'lzma-enabled' "
2732  "from conf file - %s.",
2733  p->val);
2734  }
2735  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2736  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2737  }
2738 #endif
2739 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2740  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2741  uint32_t limit = 0;
2742  if (ParseSizeStringU32(p->val, &limit) < 0) {
2743  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'compression-bomb-limit' "
2744  "from conf file - %s.", p->val);
2745  }
2746  if (limit == 0) {
2747  FatalError(SC_ERR_SIZE_PARSE, "'compression-bomb-limit' "
2748  "from conf file cannot be 0.");
2749  }
2750  /* set default soft-limit with our new hard limit */
2751  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2752  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2753 #endif
2754 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2755  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2756  uint32_t limit = 0;
2757  // between 1 usec and 1 second
2758  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2760  "failed to parse 'decompression-time-limit' "
2761  "from conf file - %s.",
2762  p->val);
2763  }
2764  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2765  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2766 #endif
2767  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2768  if (!g_disable_randomness) {
2769  cfg_prec->randomize = ConfValIsTrue(p->val);
2770  }
2771  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2772  uint32_t range;
2773  if (StringParseU32RangeCheck(&range, 10, 0,
2774  (const char *)p->val, 0, 100) < 0) {
2775  SCLogError(SC_ERR_INVALID_VALUE, "Invalid value for randomize"
2776  "-inspection-range setting from conf file - \"%s\"."
2777  " It should be a valid integer less than or equal to 100."
2778  " Killing engine",
2779  p->val);
2780  exit(EXIT_FAILURE);
2781  }
2782  cfg_prec->randomize_range = range;
2783  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2784  if (ConfValIsTrue(p->val)) {
2785  cfg_prec->http_body_inline = 1;
2786  } else if (ConfValIsFalse(p->val)) {
2787  cfg_prec->http_body_inline = 0;
2788  } else {
2789  if (strcmp("auto", p->val) != 0) {
2790  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2791  }
2792  if (EngineModeIsIPS()) {
2793  cfg_prec->http_body_inline = 1;
2794  } else {
2795  cfg_prec->http_body_inline = 0;
2796  }
2797  }
2798  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2799  ConfNode *pval;
2800 
2801  TAILQ_FOREACH(pval, &p->head, next) {
2802  if (strcasecmp("enabled", pval->name) == 0) {
2803  if (ConfValIsTrue(pval->val)) {
2804  cfg_prec->swf_decompression_enabled = 1;
2805  } else if (ConfValIsFalse(pval->val)) {
2806  cfg_prec->swf_decompression_enabled = 0;
2807  } else {
2808  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2809  }
2810  } else if (strcasecmp("type", pval->name) == 0) {
2811  if (strcasecmp("no", pval->val) == 0) {
2813  } else if (strcasecmp("deflate", pval->val) == 0) {
2815  } else if (strcasecmp("lzma", pval->val) == 0) {
2817  } else if (strcasecmp("both", pval->val) == 0) {
2819  } else {
2821  "Invalid entry for "
2822  "swf-decompression.type: %s - "
2823  "Killing engine", pval->val);
2824  exit(EXIT_FAILURE);
2825  }
2826  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2827  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2829  "Error parsing swf-decompression.compression-depth "
2830  "from conf file - %s. Killing engine", p->val);
2831  exit(EXIT_FAILURE);
2832  }
2833  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2834  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2836  "Error parsing swf-decompression.decompression-depth "
2837  "from conf file - %s. Killing engine", p->val);
2838  exit(EXIT_FAILURE);
2839  }
2840  } else {
2841  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2842  }
2843  }
2844  } else {
2845  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2846  "default config: %s", p->name);
2847  }
2848  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2849 
2850  return;
2851 }
2852 
2853 void HTPConfigure(void)
2854 {
2855  SCEnter();
2856 
2857  cfglist.next = NULL;
2858 
2859  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2860  if (NULL == cfgtree)
2861  exit(EXIT_FAILURE);
2862 
2863  /* Default Config */
2864  cfglist.cfg = htp_config_create();
2865  if (NULL == cfglist.cfg) {
2866  FatalError(SC_ERR_FATAL, "Failed to create HTP default config");
2867  }
2868  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2869  HTPConfigSetDefaultsPhase1(&cfglist);
2870  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2871  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2872  cfgtree);
2873  } else {
2874  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2875  }
2876  HTPConfigSetDefaultsPhase2("default", &cfglist);
2877 
2878  HTPParseMemcap();
2879 
2880  /* Read server config and create a parser for each IP in radix tree */
2881  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2882  if (server_config == NULL) {
2883  server_config = ConfGetNode("libhtp.server-config");
2884  if (server_config == NULL) {
2885  SCLogDebug("LIBHTP Configuring %p", server_config);
2886  SCReturn;
2887  }
2888  }
2889  SCLogDebug("LIBHTP Configuring %p", server_config);
2890 
2891  ConfNode *si;
2892  /* Server Nodes */
2893  TAILQ_FOREACH(si, &server_config->head, next) {
2894  /* Need the named node, not the index */
2895  ConfNode *s = TAILQ_FIRST(&si->head);
2896  if (NULL == s) {
2897  SCLogDebug("LIBHTP s NULL");
2898  continue;
2899  }
2900 
2901  SCLogDebug("LIBHTP server %s", s->name);
2902 
2903  HTPCfgRec *nextrec = cfglist.next;
2904  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2905  if (NULL == htprec)
2906  exit(EXIT_FAILURE);
2907  memset(htprec, 0x00, sizeof(*htprec));
2908 
2909  cfglist.next = htprec;
2910 
2911  cfglist.next->next = nextrec;
2912  cfglist.next->cfg = htp_config_create();
2913  if (NULL == cfglist.next->cfg) {
2914  FatalError(SC_ERR_FATAL, "Failed to create HTP server config");
2915  }
2916 
2917  HTPConfigSetDefaultsPhase1(htprec);
2918  HTPConfigParseParameters(htprec, s, cfgtree);
2919  HTPConfigSetDefaultsPhase2(s->name, htprec);
2920  }
2921 
2922  SCReturn;
2923 }
2924 
2926 {
2927 #ifdef DEBUG
2928  SCMutexLock(&htp_state_mem_lock);
2929  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2930  SCMutexUnlock(&htp_state_mem_lock);
2931 #endif
2932 }
2933 
2934 /** \internal
2935  * \brief get files callback
2936  * \param state state ptr
2937  * \param direction flow direction
2938  * \retval files files ptr
2939  */
2940 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
2941 {
2942  if (state == NULL)
2943  return NULL;
2944 
2945  HtpState *http_state = (HtpState *)state;
2946 
2947  if (direction & STREAM_TOCLIENT) {
2948  SCReturnPtr(http_state->files_tc, "FileContainer");
2949  } else {
2950  SCReturnPtr(http_state->files_ts, "FileContainer");
2951  }
2952 }
2953 
2954 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2955 {
2956  if (direction & STREAM_TOSERVER)
2957  return ((htp_tx_t *)tx)->request_progress;
2958  else
2959  return ((htp_tx_t *)tx)->response_progress;
2960 }
2961 
2962 static uint64_t HTPStateGetTxCnt(void *alstate)
2963 {
2964  HtpState *http_state = (HtpState *)alstate;
2965 
2966  if (http_state != NULL && http_state->conn != NULL) {
2967  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
2968  if (size < 0)
2969  return 0ULL;
2970  SCLogDebug("size %"PRIu64, size);
2971  return (uint64_t)size;
2972  } else {
2973  return 0ULL;
2974  }
2975 }
2976 
2977 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2978 {
2979  HtpState *http_state = (HtpState *)alstate;
2980 
2981  if (http_state != NULL && http_state->conn != NULL)
2982  return htp_list_get(http_state->conn->transactions, tx_id);
2983  else
2984  return NULL;
2985 }
2986 
2987 void *HtpGetTxForH2(void *alstate)
2988 {
2989  // gets last transaction
2990  HtpState *http_state = (HtpState *)alstate;
2991  if (http_state != NULL && http_state->conn != NULL) {
2992  size_t txid = htp_list_array_size(http_state->conn->transactions);
2993  if (txid > 0) {
2994  return htp_list_get(http_state->conn->transactions, txid - 1);
2995  }
2996  }
2997  return NULL;
2998 }
2999 
3000 static int HTPStateGetEventInfo(const char *event_name,
3001  int *event_id, AppLayerEventType *event_type)
3002 {
3003  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3004  if (*event_id == -1) {
3005  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
3006  "http's enum map table.", event_name);
3007  /* this should be treated as fatal */
3008  return -1;
3009  }
3010 
3011  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3012 
3013  return 0;
3014 }
3015 
3016 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3017  AppLayerEventType *event_type)
3018 {
3019  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3020  if (*event_name == NULL) {
3021  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
3022  "http's enum map table.", event_id);
3023  /* this should be treated as fatal */
3024  return -1;
3025  }
3026 
3027  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3028 
3029  return 0;
3030 }
3031 
3032 static void HTPStateTruncate(void *state, uint8_t direction)
3033 {
3034  FileContainer *fc = HTPStateGetFiles(state, direction);
3035  if (fc != NULL) {
3037  }
3038 }
3039 
3040 static DetectEngineState *HTPGetTxDetectState(void *vtx)
3041 {
3042  htp_tx_t *tx = (htp_tx_t *)vtx;
3043  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3044  return tx_ud ? tx_ud->de_state : NULL;
3045 }
3046 
3047 static int HTPSetTxDetectState(void *vtx, DetectEngineState *s)
3048 {
3049  htp_tx_t *tx = (htp_tx_t *)vtx;
3050  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3051  if (tx_ud == NULL) {
3052  return -ENOMEM;
3053  }
3054  tx_ud->de_state = s;
3055  return 0;
3056 }
3057 
3058 static AppLayerTxData *HTPGetTxData(void *vtx)
3059 {
3060  htp_tx_t *tx = (htp_tx_t *)vtx;
3061  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3062  if (tx_ud) {
3063  return &tx_ud->tx_data;
3064  }
3065  return NULL;
3066 }
3067 
3068 static int HTPRegisterPatternsForProtocolDetection(void)
3069 {
3070  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3071  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3072  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3073  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3074  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3075  const char *spacings[] = { "|20|", "|09|", NULL };
3076  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3077 
3078  int methods_pos;
3079  int spacings_pos;
3080  int versions_pos;
3081  int register_result;
3082  char method_buffer[32] = "";
3083 
3084  /* Loop through all the methods ands spacings and register the patterns */
3085  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3086  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3087 
3088  /* Combine the method name and the spacing */
3089  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3090 
3091  /* Register the new method+spacing pattern
3092  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3093  * but the pattern matching should only be one char
3094  */
3095  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3096  method_buffer, strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3097  if (register_result < 0) {
3098  return -1;
3099  }
3100  }
3101  }
3102 
3103  /* Loop through all the http verions patterns that are TO_CLIENT */
3104  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3105  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3106  versions[versions_pos], strlen(versions[versions_pos]), 0, STREAM_TOCLIENT);
3107  if (register_result < 0) {
3108  return -1;
3109  }
3110  }
3111 
3112  return 0;
3113 }
3114 
3115 /**
3116  * \brief Register the HTTP protocol and state handling functions to APP layer
3117  * of the engine.
3118  */
3120 {
3121  SCEnter();
3122 
3123  const char *proto_name = "http";
3124 
3125  /** HTTP */
3126  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3128  if (HTPRegisterPatternsForProtocolDetection() < 0)
3129  return;
3130  } else {
3131  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3132  proto_name);
3133  return;
3134  }
3135 
3136  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3137  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3138  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3139  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetFiles);
3141  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3142  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3143  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3144 
3146  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3147  AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetEvents);
3148  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3150  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3151 
3152  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTruncate);
3154  IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxDetectState, HTPSetTxDetectState);
3155  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3156 
3158  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3159 
3161  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3163  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3164  SC_ATOMIC_INIT(htp_config_flags);
3165  /* This parser accepts gaps. */
3170  HTPConfigure();
3171  } else {
3172  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3173  "still on.", proto_name);
3174  }
3175 #ifdef UNITTESTS
3176  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3177 #endif
3178 
3179  SCReturn;
3180 }
3181 
3182 #ifdef UNITTESTS
3183 static HTPCfgRec cfglist_backup;
3184 
3186 {
3187  cfglist_backup = cfglist;
3188 
3189  return;
3190 }
3191 
3193 {
3194  cfglist = cfglist_backup;
3195 
3196  return;
3197 }
3198 
3199 /** \test Test case where chunks are sent in smaller chunks and check the
3200  * response of the parser from HTP library. */
3201 static int HTPParserTest01(void)
3202 {
3203  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3204  " Data is c0oL!";
3205  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3206 
3207  TcpSession ssn;
3208  memset(&ssn, 0, sizeof(ssn));
3209 
3212 
3213  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3214  FAIL_IF_NULL(f);
3215  f->protoctx = &ssn;
3216  f->proto = IPPROTO_TCP;
3217  f->alproto = ALPROTO_HTTP1;
3218 
3219  StreamTcpInitConfig(true);
3220 
3221  uint32_t u;
3222  for (u = 0; u < httplen1; u++) {
3223  uint8_t flags = 0;
3224 
3225  if (u == 0)
3227  else if (u == (httplen1 - 1))
3229  else
3231 
3232  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3233  FAIL_IF(r != 0);
3234  }
3235 
3236  HtpState *htp_state = f->alstate;
3237  FAIL_IF_NULL(htp_state);
3238 
3239  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3240  FAIL_IF_NULL(tx);
3241 
3242  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3243  FAIL_IF_NULL(h);
3244 
3245  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3246  FAIL_IF(tx->request_method_number != HTP_M_POST);
3247  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3248 
3250  StreamTcpFreeConfig(true);
3251  UTHFreeFlow(f);
3252  PASS;
3253 }
3254 
3255 /** \test Test folding in 1 read case */
3256 static int HTPParserTest01b(void)
3257 {
3258  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3259  " Data is c0oL!";
3260  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3261 
3262  TcpSession ssn;
3263  memset(&ssn, 0, sizeof(ssn));
3264 
3267 
3268  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3269  FAIL_IF_NULL(f);
3270  f->protoctx = &ssn;
3271  f->proto = IPPROTO_TCP;
3272  f->alproto = ALPROTO_HTTP1;
3273 
3274  StreamTcpInitConfig(true);
3275 
3277  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3278  FAIL_IF(r != 0);
3279 
3280  HtpState *htp_state = f->alstate;
3281  FAIL_IF_NULL(htp_state);
3282 
3283  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3284  FAIL_IF_NULL(tx);
3285 
3286  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3287  FAIL_IF_NULL(h);
3288 
3289  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3290  FAIL_IF(tx->request_method_number != HTP_M_POST);
3291  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3292 
3294  StreamTcpFreeConfig(true);
3295  UTHFreeFlow(f);
3296  PASS;
3297 }
3298 
3299 /** \test Test folding in 1byte per read case */
3300 static int HTPParserTest01c(void)
3301 {
3302  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3303  " Data is c0oL!";
3304  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3305 
3306  TcpSession ssn;
3307  memset(&ssn, 0, sizeof(ssn));
3308 
3311 
3312  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3313  FAIL_IF_NULL(f);
3314  f->protoctx = &ssn;
3315  f->proto = IPPROTO_TCP;
3316  f->alproto = ALPROTO_HTTP1;
3317 
3318  StreamTcpInitConfig(true);
3319 
3320  uint32_t u;
3321  for (u = 0; u < httplen1; u++) {
3322  uint8_t flags = 0;
3323 
3324  if (u == 0)
3326  else if (u == (httplen1 - 1))
3328  else
3330 
3331  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3332  FAIL_IF(r != 0);
3333  }
3334 
3335  HtpState *htp_state = f->alstate;
3336  FAIL_IF_NULL(htp_state);
3337 
3338  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3339  FAIL_IF_NULL(tx);
3340 
3341  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3342  FAIL_IF_NULL(h);
3343 
3344  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3345  FAIL_IF(tx->request_method_number != HTP_M_POST);
3346  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3347 
3349  StreamTcpFreeConfig(true);
3350  UTHFreeFlow(f);
3351  PASS;
3352 }
3353 
3354 /** \test Test case where chunks are sent in smaller chunks and check the
3355  * response of the parser from HTP library. */
3356 static int HTPParserTest01a(void)
3357 {
3358  int result = 0;
3359  Flow *f = NULL;
3360  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3361  " Data is c0oL!";
3362  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3363  TcpSession ssn;
3364  HtpState *htp_state = NULL;
3365  int r = 0;
3367 
3368  memset(&ssn, 0, sizeof(ssn));
3369 
3370  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3371  if (f == NULL)
3372  goto end;
3373  f->protoctx = &ssn;
3374  f->proto = IPPROTO_TCP;
3375  f->alproto = ALPROTO_HTTP1;
3376 
3377  StreamTcpInitConfig(true);
3378 
3379  uint32_t u;
3380  for (u = 0; u < httplen1; u++) {
3381  uint8_t flags = 0;
3382 
3383  if (u == 0)
3385  else if (u == (httplen1 - 1))
3387  else
3389 
3390  FLOWLOCK_WRLOCK(f);
3391  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3392  if (r != 0) {
3393  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3394  " 0: ", u, r);
3395  FLOWLOCK_UNLOCK(f);
3396  goto end;
3397  }
3398  FLOWLOCK_UNLOCK(f);
3399  }
3400 
3401  htp_state = f->alstate;
3402  if (htp_state == NULL) {
3403  printf("no http state: ");
3404  goto end;
3405  }
3406 
3407  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3408  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3409  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3410  || tx->request_method_number != HTP_M_POST ||
3411  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3412  {
3413  printf("expected header value: Victor/1.0 and got %s: and expected"
3414  " method: POST and got %s, expected protocol number HTTP/1.0"
3415  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3416  bstr_util_strdup_to_c(tx->request_method),
3417  bstr_util_strdup_to_c(tx->request_protocol));
3418  goto end;
3419  }
3420  result = 1;
3421 end:
3422  if (alp_tctx != NULL)
3424  StreamTcpFreeConfig(true);
3425  UTHFreeFlow(f);
3426  return result;
3427 }
3428 
3429 /** \test See how it deals with an incomplete request. */
3430 static int HTPParserTest02(void)
3431 {
3432  int result = 0;
3433  Flow *f = NULL;
3434  uint8_t httpbuf1[] = "POST";
3435  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3436  TcpSession ssn;
3437  HtpState *http_state = NULL;
3439 
3440  memset(&ssn, 0, sizeof(ssn));
3441 
3442  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3443  if (f == NULL)
3444  goto end;
3445  f->protoctx = &ssn;
3446  f->proto = IPPROTO_TCP;
3447  f->alproto = ALPROTO_HTTP1;
3448 
3449  StreamTcpInitConfig(true);
3450 
3451  FLOWLOCK_WRLOCK(f);
3452  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3453  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3454  if (r != 0) {
3455  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3456  FLOWLOCK_UNLOCK(f);
3457  goto end;
3458  }
3459  FLOWLOCK_UNLOCK(f);
3460 
3461  http_state = f->alstate;
3462  if (http_state == NULL) {
3463  printf("no http state: ");
3464  goto end;
3465  }
3466 
3467  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3468  FAIL_IF_NULL(tx);
3469  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3470  FAIL_IF_NOT_NULL(h);
3471 
3472  FAIL_IF_NULL(tx->request_method);
3473  char *method = bstr_util_strdup_to_c(tx->request_method);
3474  FAIL_IF_NULL(method);
3475 
3476  FAIL_IF(strcmp(method, "POST") != 0);
3477  SCFree(method);
3478 
3479  result = 1;
3480 end:
3481  if (alp_tctx != NULL)
3483  StreamTcpFreeConfig(true);
3484  UTHFreeFlow(f);
3485  return result;
3486 }
3487 
3488 /** \test Test case where method is invalid and data is sent in smaller chunks
3489  * and check the response of the parser from HTP library. */
3490 static int HTPParserTest03(void)
3491 {
3492  int result = 0;
3493  Flow *f = NULL;
3494  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3495  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3496  TcpSession ssn;
3497  HtpState *htp_state = NULL;
3498  int r = 0;
3500 
3501  memset(&ssn, 0, sizeof(ssn));
3502 
3503  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3504  if (f == NULL)
3505  goto end;
3506  f->protoctx = &ssn;
3507  f->proto = IPPROTO_TCP;
3508  f->alproto = ALPROTO_HTTP1;
3509 
3510  StreamTcpInitConfig(true);
3511 
3512  uint32_t u;
3513  for (u = 0; u < httplen1; u++) {
3514  uint8_t flags = 0;
3515 
3516  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3517  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3518  else flags = STREAM_TOSERVER;
3519 
3520  FLOWLOCK_WRLOCK(f);
3521  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3522  if (r != 0) {
3523  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3524  " 0: ", u, r);
3525  FLOWLOCK_UNLOCK(f);
3526  goto end;
3527  }
3528  FLOWLOCK_UNLOCK(f);
3529  }
3530  htp_state = f->alstate;
3531  if (htp_state == NULL) {
3532  printf("no http state: ");
3533  goto end;
3534  }
3535 
3536  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3537 
3538  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3539  if (tx->request_method_number != HTP_M_UNKNOWN ||
3540  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3541  {
3542  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3543  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3544  bstr_util_strdup_to_c(tx->request_protocol));
3545  goto end;
3546  }
3547  result = 1;
3548 end:
3549  if (alp_tctx != NULL)
3551  StreamTcpFreeConfig(true);
3552  UTHFreeFlow(f);
3553  return result;
3554 }
3555 
3556 /** \test Test case where invalid data is sent and check the response of the
3557  * parser from HTP library. */
3558 static int HTPParserTest04(void)
3559 {
3560  int result = 0;
3561  Flow *f = NULL;
3562  HtpState *htp_state = NULL;
3563  uint8_t httpbuf1[] = "World!\r\n";
3564  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3565  TcpSession ssn;
3566  int r = 0;
3568 
3569  memset(&ssn, 0, sizeof(ssn));
3570 
3571  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3572  if (f == NULL)
3573  goto end;
3574  f->protoctx = &ssn;
3575  f->proto = IPPROTO_TCP;
3576  f->alproto = ALPROTO_HTTP1;
3577 
3578  StreamTcpInitConfig(true);
3579 
3580  FLOWLOCK_WRLOCK(f);
3582  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3583  if (r != 0) {
3584  FLOWLOCK_UNLOCK(f);
3585  goto end;
3586  }
3587  FLOWLOCK_UNLOCK(f);
3588 
3589  htp_state = f->alstate;
3590  if (htp_state == NULL) {
3591  printf("no http state: ");
3592  goto end;
3593  }
3594 
3595  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3596  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3597  if (tx->request_method_number != HTP_M_UNKNOWN ||
3598  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3599  {
3600  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3601  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3602  bstr_util_strdup_to_c(tx->request_protocol));
3603  goto end;
3604  }
3605  result = 1;
3606 end:
3607  if (alp_tctx != NULL)
3609  StreamTcpFreeConfig(true);
3610  UTHFreeFlow(f);
3611  return result;
3612 }
3613 
3614 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3615  * properly parsed them and also keeps them separated. */
3616 static int HTPParserTest05(void)
3617 {
3618  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3619  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3620  uint8_t httpbuf2[] = "Post D";
3621  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3622  uint8_t httpbuf3[] = "ata is c0oL!";
3623  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3624 
3625  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3626  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3627  uint8_t httpbuf5[] = "post R";
3628  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3629  uint8_t httpbuf6[] = "esults are tha bomb!";
3630  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3631 
3632  TcpSession ssn;
3633  memset(&ssn, 0, sizeof(ssn));
3634 
3637 
3638  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3639  FAIL_IF_NULL(f);
3640  f->protoctx = &ssn;
3641  f->proto = IPPROTO_TCP;
3642  f->alproto = ALPROTO_HTTP1;
3643 
3644  StreamTcpInitConfig(true);
3645 
3646  int r = AppLayerParserParse(
3647  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3648  FAIL_IF(r != 0);
3649 
3650  r = AppLayerParserParse(
3651  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3652  FAIL_IF(r != 0);
3653 
3654  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3655  FAIL_IF(r != 0);
3656 
3657  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3658  FAIL_IF(r != 0);
3659 
3660  r = AppLayerParserParse(
3661  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3662  FAIL_IF(r != 0);
3663 
3664  r = AppLayerParserParse(
3665  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3666  FAIL_IF(r != 0);
3667 
3668  HtpState *http_state = f->alstate;
3669  FAIL_IF_NULL(http_state);
3670 
3671  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3672  FAIL_IF_NULL(tx);
3673  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3674  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3675 
3676  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3677  FAIL_IF_NULL(h);
3678 
3679  FAIL_IF_NOT(tx->response_status_number == 200);
3680 
3682  StreamTcpFreeConfig(true);
3683  UTHFreeFlow(f);
3684  PASS;
3685 }
3686 
3687 /** \test Test proper chunked encoded response body
3688  */
3689 static int HTPParserTest06(void)
3690 {
3691  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3692  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3693  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3694  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3695  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3696  "GMT\r\n"
3697  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3698  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3699  "FrontPage/5.0.2.2510\r\n"
3700  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3701  "chunked\r\n"
3702  "Content-Type: text/html\r\n\r\n"
3703  "580\r\n"
3704  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3705  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3706  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3707  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3708  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3709  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3710  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3711  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3712  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3713  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3714  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3715  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3716  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3717  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3718  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3719  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3720  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3721  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3722  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3723  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3724  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3725  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3726  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3727  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3728  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3729  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3730  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3731  "aHA=\r\n0\r\n\r\n";
3732  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3733  TcpSession ssn;
3734 
3737 
3738  memset(&ssn, 0, sizeof(ssn));
3739 
3740  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3741  FAIL_IF_NULL(f);
3742  f->protoctx = &ssn;
3743  f->proto = IPPROTO_TCP;
3744  f->alproto = ALPROTO_HTTP1;
3745 
3746  StreamTcpInitConfig(true);
3747 
3748  int r = AppLayerParserParse(
3749  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3750  FAIL_IF(r != 0);
3751  r = AppLayerParserParse(
3752  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3753  FAIL_IF(r != 0);
3754 
3755  HtpState *http_state = f->alstate;
3756  FAIL_IF_NULL(http_state);
3757 
3758  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3759  FAIL_IF_NULL(tx);
3760 
3761  FAIL_IF(tx->request_method_number != HTP_M_GET);
3762  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3763 
3764  FAIL_IF(tx->response_status_number != 200);
3765  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3766 
3767  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3768  FAIL_IF_NULL(h);
3769 
3771  StreamTcpFreeConfig(true);
3772  UTHFreeFlow(f);
3773  PASS;
3774 }
3775 
3776 /** \test
3777  */
3778 static int HTPParserTest07(void)
3779 {
3780  int result = 0;
3781  Flow *f = NULL;
3782  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3783  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3784  TcpSession ssn;
3785  HtpState *htp_state = NULL;
3786  int r = 0;
3788 
3789  memset(&ssn, 0, sizeof(ssn));
3790 
3791  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3792  if (f == NULL)
3793  goto end;
3794  f->protoctx = &ssn;
3795  f->proto = IPPROTO_TCP;
3796  f->alproto = ALPROTO_HTTP1;
3797 
3798  StreamTcpInitConfig(true);
3799 
3800  uint32_t u;
3801  for (u = 0; u < httplen1; u++) {
3802  uint8_t flags = 0;
3803 
3804  if (u == 0)
3806  else if (u == (httplen1 - 1))
3808  else
3810 
3811  FLOWLOCK_WRLOCK(f);
3812  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3813  if (r != 0) {
3814  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3815  " 0: ", u, r);
3816  FLOWLOCK_UNLOCK(f);
3817  goto end;
3818  }
3819  FLOWLOCK_UNLOCK(f);
3820  }
3821 
3822  htp_state = f->alstate;
3823  if (htp_state == NULL) {
3824  printf("no http state: ");
3825  goto end;
3826  }
3827 
3828  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3829  size_t reflen = sizeof(ref) - 1;
3830 
3831  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3832  if (tx == NULL)
3833  goto end;
3834  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3835  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3836  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3837  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3838  (uintmax_t)reflen,
3839  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3840  goto end;
3841  }
3842 
3843  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3844  bstr_len(tx_ud->request_uri_normalized)) != 0)
3845  {
3846  printf("normalized uri \"");
3847  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3848  printf("\" != \"");
3849  PrintRawUriFp(stdout, ref, reflen);
3850  printf("\": ");
3851  goto end;
3852  }
3853  }
3854 
3855  result = 1;
3856 end:
3857  if (alp_tctx != NULL)
3859  StreamTcpFreeConfig(true);
3860  UTHFreeFlow(f);
3861  return result;
3862 }
3863 
3864 #include "conf-yaml-loader.h"
3865 
3866 /** \test Abort
3867  */
3868 static int HTPParserTest08(void)
3869 {
3870  int result = 0;
3871  Flow *f = NULL;
3872  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3873  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3874  TcpSession ssn;
3876 
3877  char input[] = "\
3878 %YAML 1.1\n\
3879 ---\n\
3880 libhtp:\n\
3881 \n\
3882  default-config:\n\
3883  personality: IDS\n\
3884 ";
3885 
3887  ConfInit();
3889 
3890  ConfYamlLoadString(input, strlen(input));
3891  HTPConfigure();
3892 
3893  HtpState *htp_state = NULL;
3894  int r = 0;
3895  memset(&ssn, 0, sizeof(ssn));
3896 
3897  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3898  if (f == NULL)
3899  goto end;
3900  f->protoctx = &ssn;
3901  f->proto = IPPROTO_TCP;
3902  f->alproto = ALPROTO_HTTP1;
3903 
3904  StreamTcpInitConfig(true);
3905 
3906  uint8_t flags = 0;
3908 
3909  FLOWLOCK_WRLOCK(f);
3910  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3911  if (r != 0) {
3912  printf("toserver chunk returned %" PRId32 ", expected"
3913  " 0: ", r);
3914  result = 0;
3915  FLOWLOCK_UNLOCK(f);
3916  goto end;
3917  }
3918  FLOWLOCK_UNLOCK(f);
3919 
3920  htp_state = f->alstate;
3921  if (htp_state == NULL) {
3922  printf("no http state: ");
3923  result = 0;
3924  goto end;
3925  }
3926 
3927  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3928  if (tx == NULL)
3929  goto end;
3930  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3931  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3932  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3933  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3934  bstr_len(tx_ud->request_uri_normalized));
3935  }
3936 
3937  result = 1;
3938 end:
3939  if (alp_tctx != NULL)
3941  StreamTcpFreeConfig(true);
3942  HTPFreeConfig();
3943  ConfDeInit();
3946  UTHFreeFlow(f);
3947  return result;
3948 }
3949 
3950 /** \test Abort
3951  */
3952 static int HTPParserTest09(void)
3953 {
3954  int result = 0;
3955  Flow *f = NULL;
3956  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3957  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3958  TcpSession ssn;
3960 
3961  char input[] = "\
3962 %YAML 1.1\n\
3963 ---\n\
3964 libhtp:\n\
3965 \n\
3966  default-config:\n\
3967  personality: Apache_2_2\n\
3968 ";
3969 
3971  ConfInit();
3973 
3974  ConfYamlLoadString(input, strlen(input));
3975  HTPConfigure();
3976 
3977  HtpState *htp_state = NULL;
3978  int r = 0;
3979 
3980  memset(&ssn, 0, sizeof(ssn));
3981 
3982  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3983  if (f == NULL)
3984  goto end;
3985  f->protoctx = &ssn;
3986  f->proto = IPPROTO_TCP;
3987  f->alproto = ALPROTO_HTTP1;
3988 
3989  StreamTcpInitConfig(true);
3990 
3991  uint8_t flags = 0;
3993 
3994  FLOWLOCK_WRLOCK(f);
3995  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3996  if (r != 0) {
3997  printf("toserver chunk returned %" PRId32 ", expected"
3998  " 0: ", r);
3999  FLOWLOCK_UNLOCK(f);
4000  goto end;
4001  }
4002  FLOWLOCK_UNLOCK(f);
4003 
4004  htp_state = f->alstate;
4005  if (htp_state == NULL) {
4006  printf("no http state: ");
4007  goto end;
4008  }
4009 
4010  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4011  if (tx == NULL)
4012  goto end;
4013  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4014  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4015  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4016  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4017  bstr_len(tx_ud->request_uri_normalized));
4018  }
4019 
4020  result = 1;
4021 end:
4022  if (alp_tctx != NULL)
4024  StreamTcpFreeConfig(true);
4025  HTPFreeConfig();
4026  ConfDeInit();
4029  UTHFreeFlow(f);
4030  return result;
4031 }
4032 
4033 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4034  */
4035 static int HTPParserTest10(void)
4036 {
4037  int result = 0;
4038  Flow *f = NULL;
4039  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4040  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4041  TcpSession ssn;
4042  HtpState *htp_state = NULL;
4043  int r = 0;
4045 
4046  memset(&ssn, 0, sizeof(ssn));
4047 
4048  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4049  if (f == NULL)
4050  goto end;
4051  f->protoctx = &ssn;
4052  f->proto = IPPROTO_TCP;
4053  f->alproto = ALPROTO_HTTP1;
4054 
4055  StreamTcpInitConfig(true);
4056 
4057  uint32_t u;
4058  for (u = 0; u < httplen1; u++) {
4059  uint8_t flags = 0;
4060 
4061  if (u == 0)
4063  else if (u == (httplen1 - 1))
4065  else
4067 
4068  FLOWLOCK_WRLOCK(f);
4069  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4070  if (r != 0) {
4071  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4072  " 0: ", u, r);
4073  FLOWLOCK_UNLOCK(f);
4074  goto end;
4075  }
4076  FLOWLOCK_UNLOCK(f);
4077  }
4078 
4079  htp_state = f->alstate;
4080  if (htp_state == NULL) {
4081  printf("no http state: ");
4082  goto end;
4083  }
4084 
4085  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4086  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4087  if (h == NULL) {
4088  goto end;
4089  }
4090 
4091  char *name = bstr_util_strdup_to_c(h->name);
4092  if (name == NULL) {
4093  goto end;
4094  }
4095 
4096  if (strcmp(name, "Host") != 0) {
4097  printf("header name not \"Host\", instead \"%s\": ", name);
4098  free(name);
4099  goto end;
4100  }
4101  free(name);
4102 
4103  char *value = bstr_util_strdup_to_c(h->value);
4104  if (value == NULL) {
4105  goto end;
4106  }
4107 
4108  if (strcmp(value, "www.google.com") != 0) {
4109  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4110  free(value);
4111  goto end;
4112  }
4113  free(value);
4114 
4115  result = 1;
4116 end:
4117  if (alp_tctx != NULL)
4119  StreamTcpFreeConfig(true);
4120  UTHFreeFlow(f);
4121  return result;
4122 }
4123 
4124 /** \test double encoding in path
4125  */
4126 static int HTPParserTest11(void)
4127 {
4128  int result = 0;
4129  Flow *f = NULL;
4130  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4131  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4132  TcpSession ssn;
4133  HtpState *htp_state = NULL;
4134  int r = 0;
4136 
4137  memset(&ssn, 0, sizeof(ssn));
4138 
4139  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4140  if (f == NULL)
4141  goto end;
4142  f->protoctx = &ssn;
4143  f->proto = IPPROTO_TCP;
4144  f->alproto = ALPROTO_HTTP1;
4145 
4146  StreamTcpInitConfig(true);
4147 
4148  uint32_t u;
4149  for (u = 0; u < httplen1; u++) {
4150  uint8_t flags = 0;
4151 
4152  if (u == 0)
4154  else if (u == (httplen1 - 1))
4156  else
4158 
4159  FLOWLOCK_WRLOCK(f);
4160  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4161  if (r != 0) {
4162  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4163  " 0: ", u, r);
4164  FLOWLOCK_UNLOCK(f);
4165  goto end;
4166  }
4167  FLOWLOCK_UNLOCK(f);
4168  }
4169 
4170  htp_state = f->alstate;
4171  if (htp_state == NULL) {
4172  printf("no http state: ");
4173  goto end;
4174  }
4175 
4176  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4177  if (tx == NULL)
4178  goto end;
4179  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4180  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4181  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4182  printf("normalized uri len should be 2, is %"PRIuMAX,
4183  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4184  goto end;
4185  }
4186 
4187  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4188  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4189  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4190  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4191  {
4192  printf("normalized uri \"");
4193  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4194  printf("\": ");
4195  goto end;
4196  }
4197  }
4198 
4199  result = 1;
4200 end:
4201  if (alp_tctx != NULL)
4203  StreamTcpFreeConfig(true);
4204  UTHFreeFlow(f);
4205  return result;
4206 }
4207 
4208 /** \test double encoding in query
4209  */
4210 static int HTPParserTest12(void)
4211 {
4212  int result = 0;
4213  Flow *f = NULL;
4214  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4215  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4216  TcpSession ssn;
4217  HtpState *htp_state = NULL;
4218  int r = 0;
4220 
4221  memset(&ssn, 0, sizeof(ssn));
4222 
4223  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4224  if (f == NULL)
4225  goto end;
4226  f->protoctx = &ssn;
4227  f->proto = IPPROTO_TCP;
4228  f->alproto = ALPROTO_HTTP1;
4229 
4230  StreamTcpInitConfig(true);
4231 
4232  uint32_t u;
4233  for (u = 0; u < httplen1; u++) {
4234  uint8_t flags = 0;
4235 
4236  if (u == 0)
4238  else if (u == (httplen1 - 1))
4240  else
4242 
4243  FLOWLOCK_WRLOCK(f);
4244  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4245  if (r != 0) {
4246  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4247  " 0: ", u, r);
4248  FLOWLOCK_UNLOCK(f);
4249  goto end;
4250  }
4251  FLOWLOCK_UNLOCK(f);
4252  }
4253 
4254  htp_state = f->alstate;
4255  if (htp_state == NULL) {
4256  printf("no http state: ");
4257  goto end;
4258  }
4259 
4260  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4261  if (tx == NULL)
4262  goto end;
4263  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4264  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4265  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4266  printf("normalized uri len should be 5, is %"PRIuMAX,
4267  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4268  goto end;
4269  }
4270 
4271  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4272  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4273  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4274  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4275  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4276  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4277  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4278  {
4279  printf("normalized uri \"");
4280  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4281  printf("\": ");
4282  goto end;
4283  }
4284  }
4285 
4286  result = 1;
4287  end:
4288  if (alp_tctx != NULL)
4290  StreamTcpFreeConfig(true);
4291  UTHFreeFlow(f);
4292  return result;
4293 }
4294 
4295 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4296  */
4297 static int HTPParserTest13(void)
4298 {
4299  int result = 0;
4300  Flow *f = NULL;
4301  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4302  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4303  TcpSession ssn;
4304  HtpState *htp_state = NULL;
4305  int r = 0;
4307 
4308  memset(&ssn, 0, sizeof(ssn));
4309 
4310  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4311  if (f == NULL)
4312  goto end;
4313  f->protoctx = &ssn;
4314  f->proto = IPPROTO_TCP;
4315  f->alproto = ALPROTO_HTTP1;
4316 
4317  StreamTcpInitConfig(true);
4318 
4319  uint32_t u;
4320  for (u = 0; u < httplen1; u++) {
4321  uint8_t flags = 0;
4322 
4323  if (u == 0)
4325  else if (u == (httplen1 - 1))
4327  else
4329 
4330  FLOWLOCK_WRLOCK(f);
4331  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4332  if (r != 0) {
4333  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4334  " 0: ", u, r);
4335  FLOWLOCK_UNLOCK(f);
4336  goto end;
4337  }
4338  FLOWLOCK_UNLOCK(f);
4339  }
4340 
4341  htp_state = f->alstate;
4342  if (htp_state == NULL) {
4343  printf("no http state: ");
4344  goto end;
4345  }
4346 
4347  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4348  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4349  if (h == NULL) {
4350  goto end;
4351  }
4352 
4353  char *name = bstr_util_strdup_to_c(h->name);
4354  if (name == NULL) {
4355  goto end;
4356  }
4357 
4358  if (strcmp(name, "Host") != 0) {
4359  printf("header name not \"Host\", instead \"%s\": ", name);
4360  free(name);
4361  goto end;
4362  }
4363  free(name);
4364 
4365  char *value = bstr_util_strdup_to_c(h->value);
4366  if (value == NULL) {
4367  goto end;
4368  }
4369 
4370  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4371  printf("header value not \"www.google.com\", instead \"");
4372  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4373  printf("\": ");
4374  free(value);
4375  goto end;
4376  }
4377  free(value);
4378 
4379  result = 1;
4380 end:
4381  if (alp_tctx != NULL)
4383  StreamTcpFreeConfig(true);
4384  UTHFreeFlow(f);
4385  return result;
4386 }
4387 
4388 /** \test Test basic config */
4389 static int HTPParserConfigTest01(void)
4390 {
4391  int ret = 0;
4392  char input[] = "\
4393 %YAML 1.1\n\
4394 ---\n\
4395 libhtp:\n\
4396 \n\
4397  default-config:\n\
4398  personality: IDS\n\
4399 \n\
4400  server-config:\n\
4401 \n\
4402  - apache-tomcat:\n\
4403  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4404  personality: Tomcat_6_0\n\
4405 \n\
4406  - iis7:\n\
4407  address: \n\
4408  - 192.168.0.0/24\n\
4409  - 192.168.10.0/24\n\
4410  personality: IIS_7_0\n\
4411 ";
4412 
4414  ConfInit();
4415 
4416  ConfYamlLoadString(input, strlen(input));
4417 
4418  ConfNode *outputs;
4419  outputs = ConfGetNode("libhtp.default-config.personality");
4420  if (outputs == NULL) {
4421  goto end;
4422  }
4423 
4424  outputs = ConfGetNode("libhtp.server-config");
4425  if (outputs == NULL) {
4426  goto end;
4427  }
4428 
4429  ConfNode *node = TAILQ_FIRST(&outputs->head);
4430  if (node == NULL) {
4431  goto end;
4432  }
4433  if (strcmp(node->name, "0") != 0) {
4434  goto end;
4435  }
4436  node = TAILQ_FIRST(&node->head);
4437  if (node == NULL) {
4438  goto end;
4439  }
4440  if (strcmp(node->name, "apache-tomcat") != 0) {
4441  goto end;
4442  }
4443 
4444  int i = 0;
4445  ConfNode *n;
4446 
4447  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4448  if (node2 == NULL) {
4449  goto end;
4450  }
4451  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4452  goto end;
4453  }
4454 
4455  node = ConfNodeLookupChild(node, "address");
4456  if (node == NULL) {
4457  goto end;
4458  }
4459  TAILQ_FOREACH(n, &node->head, next) {
4460  if (n == NULL) {
4461  goto end;
4462  }
4463 
4464  switch(i) {
4465  case 0:
4466  if (strcmp(n->name, "0") != 0) {
4467  goto end;
4468  }
4469  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4470  goto end;
4471  }
4472  break;
4473  case 1:
4474  if (strcmp(n->name, "1") != 0) {
4475  goto end;
4476  }
4477  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4478  goto end;
4479  }
4480  break;
4481  case 2:
4482  if (strcmp(n->name, "2") != 0) {
4483  goto end;
4484  }
4485  if (strcmp(n->val, "::1") != 0) {
4486  goto end;
4487  }
4488  break;
4489  default:
4490  goto end;
4491  }
4492  i++;
4493  }
4494 
4495  outputs = ConfGetNode("libhtp.server-config");
4496  if (outputs == NULL) {
4497  goto end;
4498  }
4499 
4500  node = TAILQ_FIRST(&outputs->head);
4501  node = TAILQ_NEXT(node, next);
4502  if (node == NULL) {
4503  goto end;
4504  }
4505  if (strcmp(node->name, "1") != 0) {
4506  goto end;
4507  }
4508  node = TAILQ_FIRST(&node->head);
4509  if (node == NULL) {
4510  goto end;
4511  }
4512  if (strcmp(node->name, "iis7") != 0) {
4513  goto end;
4514  }
4515 
4516  node2 = ConfNodeLookupChild(node, "personality");
4517  if (node2 == NULL) {
4518  goto end;
4519  }
4520  if (strcmp(node2->val, "IIS_7_0") != 0) {
4521  goto end;
4522  }
4523 
4524  node = ConfNodeLookupChild(node, "address");
4525  if (node == NULL) {
4526  goto end;
4527  }
4528 
4529  i = 0;
4530  TAILQ_FOREACH(n, &node->head, next) {
4531  if (n == NULL) {
4532  goto end;
4533  }
4534 
4535  switch(i) {
4536  case 0:
4537  if (strcmp(n->name, "0") != 0) {
4538  goto end;
4539  }
4540  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4541  goto end;
4542  }
4543  break;
4544  case 1:
4545  if (strcmp(n->name, "1") != 0) {
4546  goto end;
4547  }
4548  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4549  goto end;
4550  }
4551  break;
4552  default:
4553  goto end;
4554  }
4555  i++;
4556  }
4557 
4558  ret = 1;
4559 
4560 end:
4561  ConfDeInit();
4563 
4564  return ret;
4565 }
4566 
4567 /** \test Test config builds radix correctly */
4568 static int HTPParserConfigTest02(void)
4569 {
4570  int ret = 0;
4571  char input[] = "\
4572 %YAML 1.1\n\
4573 ---\n\
4574 libhtp:\n\
4575 \n\
4576  default-config:\n\
4577  personality: IDS\n\
4578 \n\
4579  server-config:\n\
4580 \n\
4581  - apache-tomcat:\n\
4582  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4583  personality: Tomcat_6_0\n\
4584 \n\
4585  - iis7:\n\
4586  address: \n\
4587  - 192.168.0.0/24\n\
4588  - 192.168.10.0/24\n\
4589  personality: IIS_7_0\n\
4590 ";
4591 
4593  ConfInit();
4595 
4596  ConfYamlLoadString(input, strlen(input));
4597 
4598  HTPConfigure();
4599 
4600  if (cfglist.cfg == NULL) {
4601  printf("No default config created.\n");
4602  goto end;
4603  }
4604 
4605  if (cfgtree == NULL) {
4606  printf("No config tree created.\n");
4607  goto end;
4608  }
4609 
4610  htp_cfg_t *htp = cfglist.cfg;
4611  uint8_t buf[128];
4612  const char *addr;
4613  void *user_data = NULL;
4614 
4615  addr = "192.168.10.42";
4616  if (inet_pton(AF_INET, addr, buf) == 1) {
4617  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4618  if (user_data != NULL) {
4619  HTPCfgRec *htp_cfg_rec = user_data;
4620  htp = htp_cfg_rec->cfg;
4621  SCLogDebug("LIBHTP using config: %p", htp);
4622  }
4623  if (htp == NULL) {
4624  printf("Could not get config for: %s\n", addr);
4625  goto end;
4626  }
4627  }
4628  else {
4629  printf("Failed to parse address: %s\n", addr);
4630  goto end;
4631  }
4632 
4633  user_data = NULL;
4634  addr = "::1";
4635  if (inet_pton(AF_INET6, addr, buf) == 1) {
4636  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4637  if (user_data != NULL) {
4638  HTPCfgRec *htp_cfg_rec = user_data;
4639  htp = htp_cfg_rec->cfg;
4640  SCLogDebug("LIBHTP using config: %p", htp);
4641  }
4642  if (htp == NULL) {
4643  printf("Could not get config for: %s\n", addr);
4644  goto end;
4645  }
4646  }
4647  else {
4648  printf("Failed to parse address: %s\n", addr);
4649  goto end;
4650  }
4651 
4652  ret = 1;
4653 
4654 end:
4655  HTPFreeConfig();
4656  ConfDeInit();
4659 
4660  return ret;
4661 }
4662 
4663 /** \test Test traffic is handled by the correct htp config */
4664 static int HTPParserConfigTest03(void)
4665 {
4666  int result = 1;
4667  Flow *f = NULL;
4668  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4669  " Data is c0oL!";
4670  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4671  TcpSession ssn;
4673 
4674  HtpState *htp_state = NULL;
4675  int r = 0;
4676  char input[] = "\
4677 %YAML 1.1\n\
4678 ---\n\
4679 libhtp:\n\
4680 \n\
4681  default-config:\n\
4682  personality: IDS\n\
4683 \n\
4684  server-config:\n\
4685 \n\
4686  - apache-tomcat:\n\
4687  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4688  personality: Tomcat_6_0\n\
4689 \n\
4690  - iis7:\n\
4691  address: \n\
4692  - 192.168.0.0/24\n\
4693  - 192.168.10.0/24\n\
4694  personality: IIS_7_0\n\
4695 ";
4696 
4698  ConfInit();
4700 
4701  ConfYamlLoadString(input, strlen(input));
4702 
4703  HTPConfigure();
4704 
4705  const char *addr = "192.168.10.42";
4706 
4707  memset(&ssn, 0, sizeof(ssn));
4708 
4709  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4710  if (f == NULL)
4711  goto end;
4712  f->protoctx = &ssn;
4713  f->proto = IPPROTO_TCP;
4714  f->alproto = ALPROTO_HTTP1;
4715 
4716  htp_cfg_t *htp = cfglist.cfg;
4717 
4718  void *user_data = NULL;
4719  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4720  if (user_data != NULL) {
4721  HTPCfgRec *htp_cfg_rec = user_data;
4722  htp = htp_cfg_rec->cfg;
4723  SCLogDebug("LIBHTP using config: %p", htp);
4724  }
4725  if (htp == NULL) {
4726  printf("Could not get config for: %s\n", addr);
4727  goto end;
4728  }
4729 
4730  StreamTcpInitConfig(true);
4731 
4732  uint32_t u;
4733  for (u = 0; u < httplen1; u++) {
4734  uint8_t flags = 0;
4735 
4736  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4737  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4738  else flags = STREAM_TOSERVER;
4739 
4740  FLOWLOCK_WRLOCK(f);
4741  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4742  if (r != 0) {
4743  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4744  " 0: ", u, r);
4745  result = 0;
4746  FLOWLOCK_UNLOCK(f);
4747  goto end;
4748  }
4749  FLOWLOCK_UNLOCK(f);
4750  }
4751 
4752  htp_state = f->alstate;
4753  if (htp_state == NULL) {
4754  printf("no http state: ");
4755  result = 0;
4756  goto end;
4757  }
4758 
4759  if (HTPStateGetTxCnt(htp_state) != 2) {
4760  printf("HTPStateGetTxCnt(htp_state) failure\n");
4761  goto end;
4762  }
4763 
4764  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4765  if (tx == NULL)
4766  goto end;
4767  if (tx->cfg != htp) {
4768  printf("wrong HTP config (%p instead of %p - default=%p): ",
4769  tx->cfg, htp, cfglist.cfg);
4770  goto end;
4771  }
4772  tx = HTPStateGetTx(htp_state, 1);
4773  if (tx == NULL)
4774  goto end;
4775  if (tx->cfg != htp) {
4776  printf("wrong HTP config (%p instead of %p - default=%p): ",
4777  tx->cfg, htp, cfglist.cfg);
4778  goto end;
4779  }
4780 
4781 end:
4782  if (alp_tctx != NULL)
4784  HTPFreeConfig();
4785  ConfDeInit();
4788 
4789  StreamTcpFreeConfig(true);
4790  UTHFreeFlow(f);
4791  return result;
4792 }
4793 
4794 /* disabled when we upgraded to libhtp 0.5.x */
4795 #if 0
4796 static int HTPParserConfigTest04(void)
4797 {
4798  int result = 0;
4799 
4800  char input[] = "\
4801 %YAML 1.1\n\
4802 ---\n\
4803 libhtp:\n\
4804 \n\
4805  default-config:\n\
4806  personality: IDS\n\
4807  path-control-char-handling: status_400\n\
4808  path-convert-utf8: yes\n\
4809  path-invalid-encoding-handling: remove_percent\n\
4810 \n\
4811  server-config:\n\
4812 \n\
4813  - apache-tomcat:\n\
4814  personality: Tomcat_6_0\n\
4815  path-invalid-utf8-handling: none\n\
4816  path-nul-encoded-handling: status_404\n\
4817  path-nul-raw-handling: status_400\n\
4818 \n\
4819  - iis7:\n\
4820  personality: IIS_7_0\n\
4821  path-replacement-char: o\n\
4822  path-unicode-mapping: status_400\n\
4823 ";
4824 
4826  ConfInit();
4828 
4829  ConfYamlLoadString(input, strlen(input));
4830 
4831  HTPConfigure();
4832 
4833  HTPCfgRec *cfg_rec = &cfglist;
4834  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4835  cfg_rec->cfg->path_convert_utf8 != 1 ||
4836  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4837  printf("failed 1\n");
4838  goto end;
4839  }
4840 
4841  cfg_rec = cfg_rec->next;
4842  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4843  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4844  printf("failed 2\n");
4845  goto end;
4846  }
4847 
4848  cfg_rec = cfg_rec->next;
4849  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4850  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4851  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4852  printf("failed 3\n");
4853  goto end;
4854  }
4855 
4856  result = 1;
4857 
4858 end:
4859  HTPFreeConfig();
4860  ConfDeInit();
4863 
4864  return result;
4865 }
4866 #endif
4867 
4868 /** \test Test %2f decoding in profile Apache_2_2
4869  *
4870  * %2f in path is left untouched
4871  * %2f in query string is normalized to %2F
4872  * %252f in query string is decoded/normalized to %2F
4873  */
4874 static int HTPParserDecodingTest01(void)
4875 {
4876  int result = 0;
4877  Flow *f = NULL;
4878  uint8_t httpbuf1[] =
4879  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4880  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4881  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4882  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4883  TcpSession ssn;
4885 
4886  HtpState *htp_state = NULL;
4887  int r = 0;
4888  char input[] = "\
4889 %YAML 1.1\n\
4890 ---\n\
4891 libhtp:\n\
4892 \n\
4893  default-config:\n\
4894  personality: Apache_2\n\
4895 ";
4896 
4898  ConfInit();
4900  ConfYamlLoadString(input, strlen(input));
4901  HTPConfigure();
4902  const char *addr = "4.3.2.1";
4903  memset(&ssn, 0, sizeof(ssn));
4904 
4905  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4906  if (f == NULL)
4907  goto end;
4908  f->protoctx = &ssn;
4909  f->proto = IPPROTO_TCP;
4910  f->alproto = ALPROTO_HTTP1;
4911 
4912  StreamTcpInitConfig(true);
4913 
4914  uint32_t u;
4915  for (u = 0; u < httplen1; u++) {
4916  uint8_t flags = 0;
4917 
4918  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4919  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4920  else flags = STREAM_TOSERVER;
4921 
4922  FLOWLOCK_WRLOCK(f);
4923  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4924  if (r != 0) {
4925  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4926  " 0: ", u, r);
4927  FLOWLOCK_UNLOCK(f);
4928  goto end;
4929  }
4930  FLOWLOCK_UNLOCK(f);
4931  }
4932 
4933  htp_state = f->alstate;
4934  if (htp_state == NULL) {
4935  printf("no http state: ");
4936  goto end;
4937  }
4938 
4939  uint8_t ref1[] = "/abc%2fdef";
4940  size_t reflen = sizeof(ref1) - 1;
4941 
4942  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4943  if (tx == NULL)
4944  goto end;
4945  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4946  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4947  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4948  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4949  (uintmax_t)reflen,
4950  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4951  goto end;
4952  }
4953 
4954  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4955  bstr_len(tx_ud->request_uri_normalized)) != 0)
4956  {
4957  printf("normalized uri \"");
4958  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4959  printf("\" != \"");
4960  PrintRawUriFp(stdout, ref1, reflen);
4961  printf("\": ");
4962  goto end;
4963  }
4964  }
4965 
4966  uint8_t ref2[] = "/abc/def?ghi/jkl";
4967  reflen = sizeof(ref2) - 1;
4968 
4969  tx = HTPStateGetTx(htp_state, 1);
4970  if (tx == NULL)
4971  goto end;
4972  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4973  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4974  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4975  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4976  (uintmax_t)reflen,
4977  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4978  goto end;
4979  }
4980 
4981  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4982  bstr_len(tx_ud->request_uri_normalized)) != 0)
4983  {
4984  printf("normalized uri \"");
4985  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4986  printf("\" != \"");
4987  PrintRawUriFp(stdout, ref2, reflen);
4988  printf("\": ");
4989  goto end;
4990  }
4991  }
4992 
4993  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4994  reflen = sizeof(ref3) - 1;
4995  tx = HTPStateGetTx(htp_state, 2);
4996  if (tx == NULL)
4997  goto end;
4998  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4999  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5000  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5001  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5002  (uintmax_t)reflen,
5003  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5004  goto end;
5005  }
5006 
5007  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5008  bstr_len(tx_ud->request_uri_normalized)) != 0)
5009  {
5010  printf("normalized uri \"");
5011  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5012  printf("\" != \"");
5013  PrintRawUriFp(stdout, ref3, reflen);
5014  printf("\": ");
5015  goto end;
5016  }
5017  }
5018 
5019  result = 1;
5020 
5021 end:
5022  if (alp_tctx != NULL)
5024  HTPFreeConfig();
5025  ConfDeInit();
5028 
5029  StreamTcpFreeConfig(true);
5030  UTHFreeFlow(f);
5031  return result;
5032 }
5033 
5034 /** \test Test %2f decoding in profile IDS
5035  *
5036  * %2f in path decoded to /
5037  * %2f in query string is decoded to /
5038  * %252f in query string is decoded to %2F
5039  */
5040 static int HTPParserDecodingTest02(void)
5041 {
5042  int result = 0;
5043  Flow *f = NULL;
5044  uint8_t httpbuf1[] =
5045  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5046  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5047  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5048  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5049  TcpSession ssn;
5051 
5052  HtpState *htp_state = NULL;
5053  int r = 0;
5054  char input[] = "\
5055 %YAML 1.1\n\
5056 ---\n\
5057 libhtp:\n\
5058 \n\
5059  default-config:\n\
5060  personality: IDS\n\
5061  double-decode-path: no\n\
5062  double-decode-query: no\n\
5063 ";
5064 
5066  ConfInit();
5068  ConfYamlLoadString(input, strlen(input));
5069  HTPConfigure();
5070  const char *addr = "4.3.2.1";
5071  memset(&ssn, 0, sizeof(ssn));
5072 
5073  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5074  if (f == NULL)
5075  goto end;
5076  f->protoctx = &ssn;
5077  f->proto = IPPROTO_TCP;
5078  f->alproto = ALPROTO_HTTP1;
5079 
5080  StreamTcpInitConfig(true);
5081 
5082  uint32_t u;
5083  for (u = 0; u < httplen1; u++) {
5084  uint8_t flags = 0;
5085 
5086  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5087  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5088  else flags = STREAM_TOSERVER;
5089 
5090  FLOWLOCK_WRLOCK(f);
5091  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5092  if (r != 0) {
5093  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5094  " 0: ", u, r);
5095  FLOWLOCK_UNLOCK(f);
5096  goto end;
5097  }
5098  FLOWLOCK_UNLOCK(f);
5099  }
5100 
5101  htp_state = f->alstate;
5102  if (htp_state == NULL) {
5103  printf("no http state: ");
5104  goto end;
5105  }
5106 
5107  uint8_t ref1[] = "/abc/def";
5108  size_t reflen = sizeof(ref1) - 1;
5109 
5110  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5111  if (tx == NULL)
5112  goto end;
5113  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5114  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5115  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5116  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5117  (uintmax_t)reflen,
5118  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5119  goto end;
5120  }
5121 
5122  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5123  bstr_len(tx_ud->request_uri_normalized)) != 0)
5124  {
5125  printf("normalized uri \"");
5126  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5127  printf("\" != \"");
5128  PrintRawUriFp(stdout, ref1, reflen);
5129  printf("\": ");
5130  goto end;
5131  }
5132  }
5133 
5134  uint8_t ref2[] = "/abc/def?ghi/jkl";
5135  reflen = sizeof(ref2) - 1;
5136 
5137  tx = HTPStateGetTx(htp_state, 1);
5138  if (tx == NULL)
5139  goto end;
5140  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5141  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5142  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5143  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5144  (uintmax_t)reflen,
5145  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5146  goto end;
5147  }
5148 
5149  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5150  bstr_len(tx_ud->request_uri_normalized)) != 0)
5151  {
5152  printf("normalized uri \"");
5153  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5154  printf("\" != \"");
5155  PrintRawUriFp(stdout, ref2, reflen);
5156  printf("\": ");
5157  goto end;
5158  }
5159  }
5160 
5161  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5162  reflen = sizeof(ref3) - 1;
5163  tx = HTPStateGetTx(htp_state, 2);
5164  if (tx == NULL)
5165  goto end;
5166  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5167  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5168  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5169  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5170  (uintmax_t)reflen,
5171  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5172  goto end;
5173  }
5174 
5175  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5176  bstr_len(tx_ud->request_uri_normalized)) != 0)
5177  {
5178  printf("normalized uri \"");
5179  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5180  printf("\" != \"");
5181  PrintRawUriFp(stdout, ref3, reflen);
5182  printf("\": ");
5183  goto end;
5184  }
5185  }
5186 
5187  result = 1;
5188 
5189 end:
5190  if (alp_tctx != NULL)
5192  HTPFreeConfig();
5193  ConfDeInit();
5196 
5197  StreamTcpFreeConfig(true);
5198  UTHFreeFlow(f);
5199  return result;
5200 }
5201 
5202 /** \test Test %2f decoding in profile IDS with double-decode-* options
5203  *
5204  * %252f in path decoded to /
5205  * %252f in query string is decoded to /
5206  */
5207 static int HTPParserDecodingTest03(void)
5208 {
5209  int result = 0;
5210  Flow *f = NULL;
5211  uint8_t httpbuf1[] =
5212  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5213  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5214  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5215  TcpSession ssn;
5217 
5218  HtpState *htp_state = NULL;
5219  int r = 0;
5220  char input[] = "\
5221 %YAML 1.1\n\
5222 ---\n\
5223 libhtp:\n\
5224 \n\
5225  default-config:\n\
5226  personality: IDS\n\
5227  double-decode-path: yes\n\
5228  double-decode-query: yes\n\
5229 ";
5230 
5232  ConfInit();
5234  ConfYamlLoadString(input, strlen(input));
5235  HTPConfigure();
5236  const char *addr = "4.3.2.1";
5237  memset(&ssn, 0, sizeof(ssn));
5238 
5239  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5240  if (f == NULL)
5241  goto end;
5242  f->protoctx = &ssn;
5243  f->proto = IPPROTO_TCP;
5244  f->alproto = ALPROTO_HTTP1;
5245 
5246  StreamTcpInitConfig(true);
5247 
5248  uint32_t u;
5249  for (u = 0; u < httplen1; u++) {
5250  uint8_t flags = 0;
5251 
5252  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5253  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5254  else flags = STREAM_TOSERVER;
5255 
5256  FLOWLOCK_WRLOCK(f);
5257  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5258  if (r != 0) {
5259  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5260  " 0: ", u, r);
5261  FLOWLOCK_UNLOCK(f);
5262  goto end;
5263  }
5264  FLOWLOCK_UNLOCK(f);
5265  }
5266 
5267  htp_state = f->alstate;
5268  if (htp_state == NULL) {
5269  printf("no http state: ");
5270  goto end;
5271  }
5272 
5273  uint8_t ref1[] = "/abc/def";
5274  size_t reflen = sizeof(ref1) - 1;
5275 
5276  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5277  if (tx == NULL)
5278  goto end;
5279  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5280  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5281  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5282  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5283  (uintmax_t)reflen,
5284  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5285  goto end;
5286  }
5287 
5288  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5289  bstr_len(tx_ud->request_uri_normalized)) != 0)
5290  {
5291  printf("normalized uri \"");
5292  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5293  printf("\" != \"");
5294  PrintRawUriFp(stdout, ref1, reflen);
5295  printf("\": ");
5296  goto end;
5297  }
5298  }
5299 
5300  uint8_t ref2[] = "/abc/def?ghi/jkl";
5301  reflen = sizeof(ref2) - 1;
5302 
5303  tx = HTPStateGetTx(htp_state, 1);
5304  if (tx == NULL)
5305  goto end;
5306  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5307  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5308  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5309  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5310  (uintmax_t)reflen,
5311  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5312  goto end;
5313  }
5314 
5315  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5316  bstr_len(tx_ud->request_uri_normalized)) != 0)
5317  {
5318  printf("normalized uri \"");
5319  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5320  printf("\" != \"");
5321  PrintRawUriFp(stdout, ref2, reflen);
5322  printf("\": ");
5323  goto end;
5324  }
5325  }
5326 
5327  result = 1;
5328 
5329 end:
5330  if (alp_tctx != NULL)
5332  HTPFreeConfig();
5333  ConfDeInit();
5336 
5337  StreamTcpFreeConfig(true);
5338  UTHFreeFlow(f);
5339  return result;
5340 }
5341 
5342 /** \test Test http:// in query profile IDS
5343  */
5344 static int HTPParserDecodingTest04(void)
5345 {
5346  int result = 0;
5347  Flow *f = NULL;
5348  uint8_t httpbuf1[] =
5349  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5350  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5351  TcpSession ssn;
5353 
5354  HtpState *htp_state = NULL;
5355  int r = 0;
5356  char input[] = "\
5357 %YAML 1.1\n\
5358 ---\n\
5359 libhtp:\n\
5360 \n\
5361  default-config:\n\
5362  personality: IDS\n\
5363  double-decode-path: yes\n\
5364  double-decode-query: yes\n\
5365 ";
5366 
5368  ConfInit();
5370  ConfYamlLoadString(input, strlen(input));
5371  HTPConfigure();
5372  const char *addr = "4.3.2.1";
5373  memset(&ssn, 0, sizeof(ssn));
5374 
5375  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5376  if (f == NULL)
5377  goto end;
5378  f->protoctx = &ssn;
5379  f->proto = IPPROTO_TCP;
5380  f->alproto = ALPROTO_HTTP1;
5381 
5382  StreamTcpInitConfig(true);
5383 
5384  uint32_t u;
5385  for (u = 0; u < httplen1; u++) {
5386  uint8_t flags = 0;
5387 
5388  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5389  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5390  else flags = STREAM_TOSERVER;
5391 
5392  FLOWLOCK_WRLOCK(f);
5393  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5394  if (r != 0) {
5395  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5396  " 0: ", u, r);
5397  FLOWLOCK_UNLOCK(f);
5398  goto end;
5399  }
5400  FLOWLOCK_UNLOCK(f);
5401  }
5402 
5403  htp_state = f->alstate;
5404  if (htp_state == NULL) {
5405  printf("no http state: ");
5406  goto end;
5407  }
5408 
5409  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5410  size_t reflen = sizeof(ref1) - 1;
5411 
5412  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5413  if (tx == NULL)
5414  goto end;
5415  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5416  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5417  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5418  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5419  (uintmax_t)reflen,
5420  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5421  goto end;
5422  }
5423 
5424  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5425  bstr_len(tx_ud->request_uri_normalized)) != 0)
5426  {
5427  printf("normalized uri \"");
5428  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5429  printf("\" != \"");
5430  PrintRawUriFp(stdout, ref1, reflen);
5431  printf("\": ");
5432  goto end;
5433  }
5434  }
5435 
5436  result = 1;
5437 
5438 end:
5439  if (alp_tctx != NULL)
5441  HTPFreeConfig();
5442  ConfDeInit();
5445 
5446  StreamTcpFreeConfig(true);
5447  UTHFreeFlow(f);
5448  return result;
5449 }
5450 
5451 /** \test Test \ char in query profile IDS. Bug 739
5452  */
5453 static int HTPParserDecodingTest05(void)
5454 {
5455  int result = 0;
5456  Flow *f = NULL;
5457  uint8_t httpbuf1[] =
5458  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5459  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5460  TcpSession ssn;
5462 
5463  HtpState *htp_state = NULL;
5464  int r = 0;
5465  char input[] = "\
5466 %YAML 1.1\n\
5467 ---\n\
5468 libhtp:\n\
5469 \n\
5470  default-config:\n\
5471  personality: IDS\n\
5472  double-decode-path: yes\n\
5473  double-decode-query: yes\n\
5474 ";
5475 
5477  ConfInit();
5479  ConfYamlLoadString(input, strlen(input));
5480  HTPConfigure();
5481  const char *addr = "4.3.2.1";
5482  memset(&ssn, 0, sizeof(ssn));
5483 
5484  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5485  if (f == NULL)
5486  goto end;
5487  f->protoctx = &ssn;
5488  f->proto = IPPROTO_TCP;
5489  f->alproto = ALPROTO_HTTP1;
5490 
5491  StreamTcpInitConfig(true);
5492 
5493  uint32_t u;
5494  for (u = 0; u < httplen1; u++) {
5495  uint8_t flags = 0;
5496 
5497  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5498  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5499  else flags = STREAM_TOSERVER;
5500 
5501  FLOWLOCK_WRLOCK(f);
5502  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5503  if (r != 0) {
5504  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5505  " 0: ", u, r);
5506  FLOWLOCK_UNLOCK(f);
5507  goto end;
5508  }
5509  FLOWLOCK_UNLOCK(f);
5510  }
5511 
5512  htp_state = f->alstate;
5513  if (htp_state == NULL) {
5514  printf("no http state: ");
5515  goto end;
5516  }
5517 
5518  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5519  size_t reflen = sizeof(ref1) - 1;
5520 
5521  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5522  if (tx == NULL)
5523  goto end;
5524  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5525  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5526  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5527  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5528  (uintmax_t)reflen,
5529  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5530  goto end;
5531  }
5532 
5533  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5534  bstr_len(tx_ud->request_uri_normalized)) != 0)
5535  {
5536  printf("normalized uri \"");
5537  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5538  printf("\" != \"");
5539  PrintRawUriFp(stdout, ref1, reflen);
5540  printf("\": ");
5541  goto end;
5542  }
5543  }
5544 
5545  result = 1;
5546 
5547 end:
5548  if (alp_tctx != NULL)
5550  HTPFreeConfig();
5551  ConfDeInit();
5554 
5555  StreamTcpFreeConfig(true);
5556  UTHFreeFlow(f);
5557  return result;
5558 }
5559 
5560 /** \test Test + char in query. Bug 1035
5561  */
5562 static int HTPParserDecodingTest06(void)
5563 {
5564  int result = 0;
5565  Flow *f = NULL;
5566  uint8_t httpbuf1[] =
5567  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5568  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5569  TcpSession ssn;
5571 
5572  HtpState *htp_state = NULL;
5573  int r = 0;
5574  char input[] = "\
5575 %YAML 1.1\n\
5576 ---\n\
5577 libhtp:\n\
5578 \n\
5579  default-config:\n\
5580  personality: IDS\n\
5581  double-decode-path: yes\n\
5582  double-decode-query: yes\n\
5583 ";
5584 
5586  ConfInit();
5588  ConfYamlLoadString(input, strlen(input));
5589  HTPConfigure();
5590  const char *addr = "4.3.2.1";
5591  memset(&ssn, 0, sizeof(ssn));
5592 
5593  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5594  if (f == NULL)
5595  goto end;
5596  f->protoctx = &ssn;
5597  f->proto = IPPROTO_TCP;
5598  f->alproto = ALPROTO_HTTP1;
5599 
5600  StreamTcpInitConfig(true);
5601 
5602  uint32_t u;
5603  for (u = 0; u < httplen1; u++) {
5604  uint8_t flags = 0;
5605 
5606  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5607  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5608  else flags = STREAM_TOSERVER;
5609 
5610  FLOWLOCK_WRLOCK(f);
5611  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5612  if (r != 0) {
5613  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5614  " 0: ", u, r);
5615  FLOWLOCK_UNLOCK(f);
5616  goto end;
5617  }
5618  FLOWLOCK_UNLOCK(f);
5619  }
5620 
5621  htp_state = f->alstate;
5622  if (htp_state == NULL) {
5623  printf("no http state: ");
5624  goto end;
5625  }
5626 
5627  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5628  size_t reflen = sizeof(ref1) - 1;
5629 
5630  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5631  if (tx == NULL)
5632  goto end;
5633  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5634  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5635  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5636  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5637  (uintmax_t)reflen,
5638  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5639  goto end;
5640  }
5641 
5642  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5643  bstr_len(tx_ud->request_uri_normalized)) != 0)
5644  {
5645  printf("normalized uri \"");
5646  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5647  printf("\" != \"");
5648  PrintRawUriFp(stdout, ref1, reflen);
5649  printf("\": ");
5650  goto end;
5651  }
5652  }
5653 
5654  result = 1;
5655 
5656 end:
5657  if (alp_tctx != NULL)
5659  HTPFreeConfig();
5660  ConfDeInit();
5663 
5664  StreamTcpFreeConfig(true);
5665  UTHFreeFlow(f);
5666  return result;
5667 }
5668 
5669 /** \test Test + char in query. Bug 1035
5670  */
5671 static int HTPParserDecodingTest07(void)
5672 {
5673  int result = 0;
5674  Flow *f = NULL;
5675  uint8_t httpbuf1[] =
5676  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5677  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5678  TcpSession ssn;
5680 
5681  HtpState *htp_state = NULL;
5682  int r = 0;
5683  char input[] = "\
5684 %YAML 1.1\n\
5685 ---\n\
5686 libhtp:\n\
5687 \n\
5688  default-config:\n\
5689  personality: IDS\n\
5690  double-decode-path: yes\n\
5691  double-decode-query: yes\n\
5692  query-plusspace-decode: yes\n\
5693 ";
5694 
5696  ConfInit();
5698  ConfYamlLoadString(input, strlen(input));
5699  HTPConfigure();
5700  const char *addr = "4.3.2.1";
5701  memset(&ssn, 0, sizeof(ssn));
5702 
5703  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5704  if (f == NULL)
5705  goto end;
5706  f->protoctx = &ssn;
5707  f->proto = IPPROTO_TCP;
5708  f->alproto = ALPROTO_HTTP1;
5709 
5710  StreamTcpInitConfig(true);
5711 
5712  uint32_t u;
5713  for (u = 0; u < httplen1; u++) {
5714  uint8_t flags = 0;
5715 
5716  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5717  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5718  else flags = STREAM_TOSERVER;
5719 
5720  FLOWLOCK_WRLOCK(f);
5721  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5722  if (r != 0) {
5723  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5724  " 0: ", u, r);
5725  FLOWLOCK_UNLOCK(f);
5726  goto end;
5727  }
5728  FLOWLOCK_UNLOCK(f);
5729  }
5730 
5731  htp_state = f->alstate;
5732  if (htp_state == NULL) {
5733  printf("no http state: ");
5734  goto end;
5735  }
5736 
5737  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5738  size_t reflen = sizeof(ref1) - 1;
5739 
5740  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5741  if (tx == NULL)
5742  goto end;
5743  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5744  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5745  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5746  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5747  (uintmax_t)reflen,
5748  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5749  goto end;
5750  }
5751 
5752  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5753  bstr_len(tx_ud->request_uri_normalized)) != 0)
5754  {
5755  printf("normalized uri \"");
5756  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5757  printf("\" != \"");
5758  PrintRawUriFp(stdout, ref1, reflen);
5759  printf("\": ");
5760  goto end;
5761  }
5762  }
5763 
5764  result = 1;
5765 
5766 end:
5767  if (alp_tctx != NULL)
5769  HTPFreeConfig();
5770  ConfDeInit();
5773 
5774  StreamTcpFreeConfig(true);
5775  UTHFreeFlow(f);
5776  return result;
5777 }
5778 
5779 /** \test Test 'proxy' URI normalization. Ticket 1008
5780  */