suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-htp.h"
59 #include "app-layer-htp-body.h"
60 #include "app-layer-htp-file.h"
61 #include "app-layer-htp-libhtp.h"
62 #include "app-layer-htp-xff.h"
63 
64 #include "util-spm.h"
65 #include "util-debug.h"
66 #include "util-time.h"
67 #include "util-misc.h"
68 
69 #include "util-unittest.h"
70 #include "util-unittest-helper.h"
71 #include "flow-util.h"
72 
73 #include "detect-engine.h"
74 #include "detect-engine-state.h"
75 #include "detect-parse.h"
76 
77 #include "decode-events.h"
78 
79 #include "util-memcmp.h"
80 #include "util-random.h"
81 #include "util-validate.h"
82 
83 //#define PRINT
84 
85 /** Fast lookup tree (radix) for the various HTP configurations */
86 static SCRadixTree *cfgtree;
87 /** List of HTP configurations. */
88 static HTPCfgRec cfglist;
89 
90 #ifdef DEBUG
91 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
92 static uint64_t htp_state_memuse = 0;
93 static uint64_t htp_state_memcnt = 0;
94 #endif
95 
97  { "UNKNOWN_ERROR",
99  { "GZIP_DECOMPRESSION_FAILED",
101  { "REQUEST_FIELD_MISSING_COLON",
103  { "RESPONSE_FIELD_MISSING_COLON",
105  { "INVALID_REQUEST_CHUNK_LEN",
107  { "INVALID_RESPONSE_CHUNK_LEN",
109  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
111  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
113  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
115  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
117  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
119  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
121  { "100_CONTINUE_ALREADY_SEEN",
123  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
125  { "INVALID_SERVER_PORT_IN_REQUEST",
127  { "INVALID_AUTHORITY_PORT",
129  { "REQUEST_HEADER_INVALID",
131  { "RESPONSE_HEADER_INVALID",
133  { "MISSING_HOST_HEADER",
135  { "HOST_HEADER_AMBIGUOUS",
137  { "INVALID_REQUEST_FIELD_FOLDING",
139  { "INVALID_RESPONSE_FIELD_FOLDING",
141  { "REQUEST_FIELD_TOO_LONG",
143  { "RESPONSE_FIELD_TOO_LONG",
145  { "REQUEST_LINE_INVALID",
147  { "REQUEST_BODY_UNEXPECTED",
149  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
151  { "REQUEST_URI_HOST_INVALID",
153  { "REQUEST_HEADER_HOST_INVALID",
155  { "REQUEST_AUTH_UNRECOGNIZED",
157  { "REQUEST_HEADER_REPETITION",
159  { "RESPONSE_HEADER_REPETITION",
161  { "DOUBLE_ENCODED_URI",
163  { "URI_DELIM_NON_COMPLIANT",
165  { "METHOD_DELIM_NON_COMPLIANT",
167  { "REQUEST_LINE_LEADING_WHITESPACE",
169  { "TOO_MANY_ENCODING_LAYERS",
171  { "ABNORMAL_CE_HEADER",
173  { "RESPONSE_MULTIPART_BYTERANGES",
175  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
177  { "RESPONSE_CHUNKED_OLD_PROTO",
179  { "RESPONSE_INVALID_PROTOCOL",
181  { "RESPONSE_INVALID_STATUS",
183  { "REQUEST_LINE_INCOMPLETE",
185 
186  { "LZMA_MEMLIMIT_REACHED",
188  { "COMPRESSION_BOMB",
190 
191  /* suricata warnings/errors */
192  { "MULTIPART_GENERIC_ERROR",
194  { "MULTIPART_NO_FILEDATA",
196  { "MULTIPART_INVALID_HEADER",
198 
199  { NULL, -1 },
200 };
201 
202 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
203 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
204 static uint64_t HTPStateGetTxCnt(void *alstate);
205 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction);
206 
207 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
208 {
209  uint64_t id = HTPStateGetTxCnt(s);
210  BUG_ON(id == 0);
211  return id - 1;
212 }
213 
214 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
215 {
216  return s->transaction_cnt;
217 }
218 
219 #ifdef DEBUG
220 /**
221  * \internal
222  *
223  * \brief Lookup the HTP personality string from the numeric personality.
224  *
225  * \todo This needs to be a libhtp function.
226  */
227 static const char *HTPLookupPersonalityString(int p)
228 {
229 #define CASE_HTP_PERSONALITY_STRING(p) \
230  case HTP_SERVER_ ## p: return #p
231 
232  switch (p) {
233  CASE_HTP_PERSONALITY_STRING(MINIMAL);
234  CASE_HTP_PERSONALITY_STRING(GENERIC);
235  CASE_HTP_PERSONALITY_STRING(IDS);
236  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
237  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
238  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
239  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
240  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
241  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
242  CASE_HTP_PERSONALITY_STRING(APACHE_2);
243  }
244 
245  return NULL;
246 }
247 #endif /* DEBUG */
248 
249 /**
250  * \internal
251  *
252  * \brief Lookup the numeric HTP personality from a string.
253  *
254  * \todo This needs to be a libhtp function.
255  */
256 static int HTPLookupPersonality(const char *str)
257 {
258 #define IF_HTP_PERSONALITY_NUM(p) \
259  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
260 
261  IF_HTP_PERSONALITY_NUM(MINIMAL);
262  IF_HTP_PERSONALITY_NUM(GENERIC);
264  IF_HTP_PERSONALITY_NUM(IIS_4_0);
265  IF_HTP_PERSONALITY_NUM(IIS_5_0);
266  IF_HTP_PERSONALITY_NUM(IIS_5_1);
267  IF_HTP_PERSONALITY_NUM(IIS_6_0);
268  IF_HTP_PERSONALITY_NUM(IIS_7_0);
269  IF_HTP_PERSONALITY_NUM(IIS_7_5);
270  IF_HTP_PERSONALITY_NUM(APACHE_2);
271  if (strcasecmp("TOMCAT_6_0", str) == 0) {
272  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
273  "longer supported by libhtp.", str);
274  return -1;
275  } else if ((strcasecmp("APACHE", str) == 0) ||
276  (strcasecmp("APACHE_2_2", str) == 0))
277  {
278  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
279  "longer supported by libhtp, failing back to "
280  "Apache2 personality.", str);
281  return HTP_SERVER_APACHE_2;
282  }
283 
284  return -1;
285 }
286 
287 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
288  const uint8_t dir, const uint8_t e)
289 {
290  SCLogDebug("setting event %u", e);
291 
292  if (htud) {
294  s->events++;
295  return;
296  }
297 
298  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
299  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
300 
301  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
302  if (tx == NULL && tx_id > 0)
303  tx = HTPStateGetTx(s, tx_id - 1);
304  if (tx != NULL) {
305  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
306  if (htud != NULL) {
308  s->events++;
309  return;
310  }
311  }
312  SCLogDebug("couldn't set event %u", e);
313 }
314 
315 static AppLayerDecoderEvents *HTPGetEvents(void *tx)
316 {
317  SCLogDebug("get HTTP events for TX %p", tx);
318 
319  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
320  if (htud != NULL) {
321  SCLogDebug("has htud, htud->decoder_events %p", htud->decoder_events);
322  return htud->decoder_events;
323  }
324 
325  return NULL;
326 }
327 
328 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
329  * connection parser to be used by the HTP library
330  */
331 static void *HTPStateAlloc(void)
332 {
333  SCEnter();
334 
335  HtpState *s = HTPMalloc(sizeof(HtpState));
336  if (unlikely(s == NULL)) {
337  SCReturnPtr(NULL, "void");
338  }
339 
340  memset(s, 0x00, sizeof(HtpState));
341 
342 #ifdef DEBUG
343  SCMutexLock(&htp_state_mem_lock);
344  htp_state_memcnt++;
345  htp_state_memuse += sizeof(HtpState);
346  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
347  SCMutexUnlock(&htp_state_mem_lock);
348 #endif
349 
350  SCReturnPtr((void *)s, "void");
351 }
352 
353 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
354 {
355  if (likely(htud)) {
356  HtpBodyFree(&htud->request_body);
357  HtpBodyFree(&htud->response_body);
358  bstr_free(htud->request_uri_normalized);
359  if (htud->request_headers_raw)
361  if (htud->response_headers_raw)
364  if (htud->boundary)
365  HTPFree(htud->boundary, htud->boundary_len);
366  if (htud->de_state != NULL) {
368  }
369  HTPFree(htud, sizeof(HtpTxUserData));
370  }
371 }
372 
373 /** \brief Function to frees the HTTP state memory and also frees the HTTP
374  * connection parser memory which was used by the HTP library
375  */
376 void HTPStateFree(void *state)
377 {
378  SCEnter();
379 
380  HtpState *s = (HtpState *)state;
381  if (s == NULL) {
382  SCReturn;
383  }
384 
385  /* free the connection parser memory used by HTP library */
386  if (s->connp != NULL) {
387  SCLogDebug("freeing HTP state");
388 
389  uint64_t tx_id;
390  uint64_t total_txs = HTPStateGetTxCnt(state);
391  /* free the list of body chunks */
392  if (s->conn != NULL) {
393  for (tx_id = 0; tx_id < total_txs; tx_id++) {
394  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
395  if (tx != NULL) {
396  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
397  HtpTxUserDataFree(s, htud);
398  htp_tx_set_user_data(tx, NULL);
399  }
400  }
401  }
402  htp_connp_destroy_all(s->connp);
403  }
404 
407  HTPFree(s, sizeof(HtpState));
408 
409 #ifdef DEBUG
410  SCMutexLock(&htp_state_mem_lock);
411  htp_state_memcnt--;
412  htp_state_memuse -= sizeof(HtpState);
413  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
414  SCMutexUnlock(&htp_state_mem_lock);
415 #endif
416 
417  SCReturn;
418 }
419 
420 /**
421  * \brief HTP transaction cleanup callback
422  *
423  * \warning We cannot actually free the transactions here. It seems that
424  * HTP only accepts freeing of transactions in the response callback.
425  */
426 static void HTPStateTransactionFree(void *state, uint64_t id)
427 {
428  SCEnter();
429 
430  HtpState *s = (HtpState *)state;
431 
432  SCLogDebug("state %p, id %"PRIu64, s, id);
433 
434  htp_tx_t *tx = HTPStateGetTx(s, id);
435  if (tx != NULL) {
436  /* This will remove obsolete body chunks */
437  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
438  HtpTxUserDataFree(s, htud);
439  htp_tx_set_user_data(tx, NULL);
440 
441  /* hack: even if libhtp considers the tx incomplete, we want to
442  * free it here. htp_tx_destroy however, will refuse to do this.
443  * As htp_tx_destroy_incomplete isn't available in the public API,
444  * we hack around it here. */
445  if (unlikely(!(
446  tx->request_progress == HTP_REQUEST_COMPLETE &&
447  tx->response_progress == HTP_RESPONSE_COMPLETE)))
448  {
449  tx->request_progress = HTP_REQUEST_COMPLETE;
450  tx->response_progress = HTP_RESPONSE_COMPLETE;
451  }
452  htp_tx_destroy(tx);
453  }
454 }
455 
456 /**
457  * \brief Sets a flag that informs the HTP app layer that some module in the
458  * engine needs the http request body data.
459  * \initonly
460  */
462 {
463  SCEnter();
464 
465  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
466  SCReturn;
467 }
468 
469 /**
470  * \brief Sets a flag that informs the HTP app layer that some module in the
471  * engine needs the http request body data.
472  * \initonly
473  */
475 {
476  SCEnter();
477 
478  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
479  SCReturn;
480 }
481 
482 /**
483  * \brief Sets a flag that informs the HTP app layer that some module in the
484  * engine needs the http request multi part header.
485  *
486  * \initonly
487  */
488 static void AppLayerHtpNeedMultipartHeader(void)
489 {
490  SCEnter();
492 
493  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
494  SCReturn;
495 }
496 
497 /**
498  * \brief Sets a flag that informs the HTP app layer that some module in the
499  * engine needs the http request file.
500  *
501  * \initonly
502  */
504 {
505  SCEnter();
506  AppLayerHtpNeedMultipartHeader();
509 
510  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
511  SCReturn;
512 }
513 
514 static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
515 {
516  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
517  if (tx_ud) {
518  if (flags & STREAM_TOCLIENT) {
519  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
520  } else {
521  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
522  }
523  }
524 }
525 
526 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
527 {
528  if (flags & HTP_STREAM_DEPTH_SET) {
529  uint32_t stream_depth = FileReassemblyDepth();
530  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
531  return true;
532  }
533  } else {
534  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
535  return true;
536  }
537  }
538  return false;
539 }
540 
541 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
542  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
543 {
544  uint32_t chunk_len = 0;
545  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
546  (content_len_so_far < (uint64_t)body_limit) &&
547  (content_len_so_far + (uint64_t)data_len) > body_limit)
548  {
549  chunk_len = body_limit - content_len_so_far;
550  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
551  (content_len_so_far < (uint64_t)stream_depth) &&
552  (content_len_so_far + (uint64_t)data_len) > stream_depth)
553  {
554  chunk_len = stream_depth - content_len_so_far;
555  }
556  SCLogDebug("len %u", chunk_len);
557  return (chunk_len == 0 ? data_len : chunk_len);
558 }
559 
560 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
561 struct {
562  const char *msg;
563  int de;
564 } htp_errors[] = {
565  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
566  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
567  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
568  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
569  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
570 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
571  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
572 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
573  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
574  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
575  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
576  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
577 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
578  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
579  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
580  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
581  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
582 };
583 
584 struct {
585  const char *msg;
586  int de;
587 } htp_warnings[] = {
588  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
589  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
590  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
591  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
592  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
593 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
594  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
595  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
596  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
597  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
598  * luckily, "Request server port=" is unique */
599 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
601  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
602  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
603  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
604  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
605  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
606  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
607  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
608  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
609  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
610  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
611  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
612  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
613  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
614  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
615  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
616  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
617  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
618  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
619 };
620 
621 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
622 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
623 
624 /**
625  * \internal
626  *
627  * \brief Get the warning id for the warning msg.
628  *
629  * \param msg warning message
630  *
631  * \retval id the id or 0 in case of not found
632  */
633 static int HTPHandleWarningGetId(const char *msg)
634 {
635  SCLogDebug("received warning \"%s\"", msg);
636  size_t idx;
637  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
638  if (strncmp(htp_warnings[idx].msg, msg,
639  strlen(htp_warnings[idx].msg)) == 0)
640  {
641  return htp_warnings[idx].de;
642  }
643  }
644 
645  return 0;
646 }
647 
648 /**
649  * \internal
650  *
651  * \brief Get the error id for the error msg.
652  *
653  * \param msg error message
654  *
655  * \retval id the id or 0 in case of not found
656  */
657 static int HTPHandleErrorGetId(const char *msg)
658 {
659  SCLogDebug("received error \"%s\"", msg);
660 
661  size_t idx;
662  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
663  if (strncmp(htp_errors[idx].msg, msg,
664  strlen(htp_errors[idx].msg)) == 0)
665  {
666  return htp_errors[idx].de;
667  }
668  }
669 
670  return 0;
671 }
672 
673 /**
674  * \internal
675  *
676  * \brief Check state for errors, warnings and add any as events
677  *
678  * \param s state
679  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
680  */
681 static void HTPHandleError(HtpState *s, const uint8_t dir)
682 {
683  if (s == NULL || s->conn == NULL ||
684  s->conn->messages == NULL) {
685  return;
686  }
687 
688  size_t size = htp_list_size(s->conn->messages);
689  size_t msg;
690 
691  for (msg = s->htp_messages_offset; msg < size; msg++) {
692  htp_log_t *log = htp_list_get(s->conn->messages, msg);
693  if (log == NULL)
694  continue;
695 
696  HtpTxUserData *htud = NULL;
697  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
698  if (tx != NULL)
699  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
700 
701  SCLogDebug("message %s", log->msg);
702 
703  int id = HTPHandleErrorGetId(log->msg);
704  if (id == 0) {
705  id = HTPHandleWarningGetId(log->msg);
706  if (id == 0)
708  }
709 
710  if (id > 0) {
711  HTPSetEvent(s, htud, dir, id);
712  }
713  }
714  s->htp_messages_offset = (uint16_t)msg;
715  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
716 }
717 
718 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
719 {
720 #ifdef DEBUG
721  BUG_ON(s == NULL || tx == NULL);
722 #endif
723  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
724  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
725  HTP_HOSTH_INVALID))
726  {
727  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
728  if (htud == NULL)
729  return;
730 
731  if (tx->flags & HTP_REQUEST_INVALID_T_E)
732  HTPSetEvent(s, htud, STREAM_TOSERVER,
734  if (tx->flags & HTP_REQUEST_INVALID_C_L)
735  HTPSetEvent(s, htud, STREAM_TOSERVER,
737  if (tx->flags & HTP_HOST_MISSING)
738  HTPSetEvent(s, htud, STREAM_TOSERVER,
740  if (tx->flags & HTP_HOST_AMBIGUOUS)
741  HTPSetEvent(s, htud, STREAM_TOSERVER,
743  if (tx->flags & HTP_HOSTU_INVALID)
744  HTPSetEvent(s, htud, STREAM_TOSERVER,
746  if (tx->flags & HTP_HOSTH_INVALID)
747  HTPSetEvent(s, htud, STREAM_TOSERVER,
749  }
750  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
751  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
752  if (htud == NULL)
753  return;
754  HTPSetEvent(s, htud, STREAM_TOSERVER,
756  }
757  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
758  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
759  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
760  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
761  if (htud == NULL)
762  return;
763  HTPSetEvent(s, htud, STREAM_TOSERVER,
765  }
766 }
767 
768 static int Setup(Flow *f, HtpState *hstate)
769 {
770  /* store flow ref in state so callbacks can access it */
771  hstate->f = f;
772 
773  HTPCfgRec *htp_cfg_rec = &cfglist;
774  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
775  void *user_data = NULL;
776 
777  if (FLOW_IS_IPV4(f)) {
778  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
779  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
780  }
781  else if (FLOW_IS_IPV6(f)) {
782  SCLogDebug("Looking up HTP config for ipv6");
783  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
784  }
785  else {
786  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
787  goto error;
788  }
789 
790  if (user_data != NULL) {
791  htp_cfg_rec = user_data;
792  htp = htp_cfg_rec->cfg;
793  SCLogDebug("LIBHTP using config: %p", htp);
794  } else {
795  SCLogDebug("Using default HTP config: %p", htp);
796  }
797 
798  if (NULL == htp) {
799 #ifdef DEBUG_VALIDATION
800  BUG_ON(htp == NULL);
801 #endif
802  /* should never happen if HTPConfigure is properly invoked */
803  goto error;
804  }
805 
806  hstate->connp = htp_connp_create(htp);
807  if (hstate->connp == NULL) {
808  goto error;
809  }
810 
811  hstate->conn = htp_connp_get_connection(hstate->connp);
812 
813  htp_connp_set_user_data(hstate->connp, (void *)hstate);
814  hstate->cfg = htp_cfg_rec;
815 
816  SCLogDebug("New hstate->connp %p", hstate->connp);
817 
818  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
819 
821  htp_cfg_rec->request.inspect_min_size);
823  htp_cfg_rec->response.inspect_min_size);
824  return 0;
825 error:
826  return -1;
827 }
828 
829 /**
830  * \brief Function to handle the reassembled data from client and feed it to
831  * the HTP library to process it.
832  *
833  * \param flow Pointer to the flow the data belong to
834  * \param htp_state Pointer the state in which the parsed value to be stored
835  * \param pstate Application layer parser state for this session
836  * \param input Pointer the received HTTP client data
837  * \param input_len Length in bytes of the received data
838  * \param output Pointer to the output (not used in this function)
839  *
840  * \retval On success returns 1 or on failure returns -1.
841  */
842 static int HTPHandleRequestData(Flow *f, void *htp_state,
843  AppLayerParserState *pstate,
844  const uint8_t *input, uint32_t input_len,
845  void *local_data, const uint8_t flags)
846 {
847  SCEnter();
848  int ret = 1;
849  HtpState *hstate = (HtpState *)htp_state;
850 
851  /* On the first invocation, create the connection parser structure to
852  * be used by HTP library. This is looked up via IP in the radix
853  * tree. Failing that, the default HTP config is used.
854  */
855  if (NULL == hstate->conn) {
856  if (Setup(f, hstate) != 0) {
857  goto error;
858  }
859  }
860  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
861 
862  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
863  /* pass the new data to the htp parser */
864  if (input_len > 0) {
865  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
866  switch (r) {
867  case HTP_STREAM_ERROR:
868  ret = -1;
869  break;
870  default:
871  break;
872  }
873  HTPHandleError(hstate, STREAM_TOSERVER);
874  }
875 
876  /* if the TCP connection is closed, then close the HTTP connection */
878  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
879  {
880  htp_connp_close(hstate->connp, &ts);
881  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
882  SCLogDebug("stream eof encountered, closing htp handle for ts");
883  }
884 
885  SCLogDebug("hstate->connp %p", hstate->connp);
886  SCReturnInt(ret);
887 
888 error:
889  SCReturnInt(-1);
890 }
891 
892 /**
893  * \brief Function to handle the reassembled data from server and feed it to
894  * the HTP library to process it.
895  *
896  * \param flow Pointer to the flow the data belong to
897  * \param htp_state Pointer the state in which the parsed value to be stored
898  * \param pstate Application layer parser state for this session
899  * \param input Pointer the received HTTP server data
900  * \param input_len Length in bytes of the received data
901  * \param output Pointer to the output (not used in this function)
902  *
903  * \retval On success returns 1 or on failure returns -1
904  */
905 static int HTPHandleResponseData(Flow *f, void *htp_state,
906  AppLayerParserState *pstate,
907  const uint8_t *input, uint32_t input_len,
908  void *local_data, const uint8_t flags)
909 {
910  SCEnter();
911  int ret = 1;
912  HtpState *hstate = (HtpState *)htp_state;
913 
914  /* On the first invocation, create the connection parser structure to
915  * be used by HTP library. This is looked up via IP in the radix
916  * tree. Failing that, the default HTP config is used.
917  */
918  if (NULL == hstate->conn) {
919  if (Setup(f, hstate) != 0) {
920  goto error;
921  }
922  }
923  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
924 
925  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
926  if (input_len > 0) {
927  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
928  switch (r) {
929  case HTP_STREAM_ERROR:
930  ret = -1;
931  break;
932  default:
933  break;
934  }
935  HTPHandleError(hstate, STREAM_TOCLIENT);
936  }
937 
938  /* if we the TCP connection is closed, then close the HTTP connection */
940  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
941  {
942  htp_connp_close(hstate->connp, &ts);
943  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
944  }
945 
946  SCLogDebug("hstate->connp %p", hstate->connp);
947  SCReturnInt(ret);
948 error:
949  SCReturnInt(-1);
950 }
951 
952 /**
953  * \param name /Lowercase/ version of the variable name
954  */
955 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
956  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
957 {
958 #ifdef PRINT
959  printf("DATA START: \n");
960  PrintRawDataFp(stdout, data, len);
961  printf("DATA END: \n");
962 #endif
963  size_t x;
964  int quote = 0;
965 
966  for (x = 0; x < len; x++) {
967  if (!(isspace(data[x])))
968  break;
969  }
970 
971  if (x >= len)
972  return 0;
973 
974  uint8_t *line = data+x;
975  size_t line_len = len-x;
976  size_t offset = 0;
977 #ifdef PRINT
978  printf("LINE START: \n");
979  PrintRawDataFp(stdout, line, line_len);
980  printf("LINE END: \n");
981 #endif
982  for (x = 0 ; x < line_len; x++) {
983  if (x > 0) {
984  if (line[x - 1] != '\\' && line[x] == '\"') {
985  quote++;
986  }
987 
988  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
989  uint8_t *token = line + offset;
990  size_t token_len = x - offset;
991 
992  if ((x + 1) == line_len) {
993  token_len++;
994  }
995 
996  offset = x + 1;
997 
998  while (offset < line_len && isspace(line[offset])) {
999  x++;
1000  offset++;
1001  }
1002 #ifdef PRINT
1003  printf("TOKEN START: \n");
1004  PrintRawDataFp(stdout, token, token_len);
1005  printf("TOKEN END: \n");
1006 #endif
1007  if (token_len > name_len) {
1008  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1009  uint8_t *value = token + name_len;
1010  size_t value_len = token_len - name_len;
1011 
1012  if (value[0] == '\"') {
1013  value++;
1014  value_len--;
1015  }
1016  if (value[value_len-1] == '\"') {
1017  value_len--;
1018  }
1019 #ifdef PRINT
1020  printf("VALUE START: \n");
1021  PrintRawDataFp(stdout, value, value_len);
1022  printf("VALUE END: \n");
1023 #endif
1024  *retptr = value;
1025  *retlen = value_len;
1026  return 1;
1027  }
1028  }
1029  }
1030  }
1031  }
1032 
1033  return 0;
1034 }
1035 
1036 /**
1037  * \param name /Lowercase/ version of the variable name
1038  */
1039 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1040  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1041 {
1042  SCEnter();
1043 #ifdef PRINT
1044  printf("DATA START: \n");
1045  PrintRawDataFp(stdout, data, len);
1046  printf("DATA END: \n");
1047 #endif
1048  size_t x;
1049  int quote = 0;
1050 
1051  for (x = 0; x < len; x++) {
1052  if (!(isspace(data[x])))
1053  break;
1054  }
1055 
1056  if (x >= len) {
1057  SCReturnInt(0);
1058  }
1059 
1060  uint8_t *line = data+x;
1061  size_t line_len = len-x;
1062  size_t offset = 0;
1063 #ifdef PRINT
1064  printf("LINE START: \n");
1065  PrintRawDataFp(stdout, line, line_len);
1066  printf("LINE END: \n");
1067 #endif
1068  for (x = 0 ; x < line_len; x++) {
1069  if (x > 0) {
1070  if (line[x - 1] != '\\' && line[x] == '\"') {
1071  quote++;
1072  }
1073 
1074  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1075  uint8_t *token = line + offset;
1076  size_t token_len = x - offset;
1077 
1078  if ((x + 1) == line_len) {
1079  token_len++;
1080  }
1081 
1082  offset = x + 1;
1083 
1084  while (offset < line_len && isspace(line[offset])) {
1085  x++;
1086  offset++;
1087  }
1088 #ifdef PRINT
1089  printf("TOKEN START: \n");
1090  PrintRawDataFp(stdout, token, token_len);
1091  printf("TOKEN END: \n");
1092 #endif
1093  if (token_len > name_len) {
1094  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1095  uint8_t *value = token + name_len;
1096  size_t value_len = token_len - name_len;
1097 
1098  if (value[0] == '\"') {
1099  value++;
1100  value_len--;
1101  }
1102  if (value[value_len-1] == '\"') {
1103  value_len--;
1104  }
1105 #ifdef PRINT
1106  printf("VALUE START: \n");
1107  PrintRawDataFp(stdout, value, value_len);
1108  printf("VALUE END: \n");
1109 #endif
1110  *retptr = value;
1111  *retlen = value_len;
1112  SCReturnInt(1);
1113  }
1114  }
1115  }
1116  }
1117  }
1118 
1119  SCReturnInt(0);
1120 }
1121 
1122 /**
1123  * \brief setup multipart parsing: extract boundary and store it
1124  *
1125  * \param d HTTP transaction
1126  * \param htud transaction userdata
1127  *
1128  * \retval 1 ok, multipart set up
1129  * \retval 0 ok, not multipart though
1130  * \retval -1 error: problem with the boundary
1131  *
1132  * If the request contains a multipart message, this function will
1133  * set the HTP_BOUNDARY_SET in the transaction.
1134  */
1135 static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud)
1136 {
1137  htp_header_t *h = (htp_header_t *)htp_table_get_c(d->tx->request_headers,
1138  "Content-Type");
1139  if (h != NULL && bstr_len(h->value) > 0) {
1140  uint8_t *boundary = NULL;
1141  size_t boundary_len = 0;
1142 
1143  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1144  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1145  &boundary, &boundary_len);
1146  if (r == 1) {
1147 #ifdef PRINT
1148  printf("BOUNDARY START: \n");
1149  PrintRawDataFp(stdout, boundary, boundary_len);
1150  printf("BOUNDARY END: \n");
1151 #endif
1152  if (boundary_len < HTP_BOUNDARY_MAX) {
1153  htud->boundary = HTPMalloc(boundary_len);
1154  if (htud->boundary == NULL) {
1155  return -1;
1156  }
1157  htud->boundary_len = (uint8_t)boundary_len;
1158  memcpy(htud->boundary, boundary, boundary_len);
1159 
1160  htud->tsflags |= HTP_BOUNDARY_SET;
1161  } else {
1162  SCLogDebug("invalid boundary");
1163  return -1;
1164  }
1165  SCReturnInt(1);
1166  }
1167  //SCReturnInt(1);
1168  }
1169  SCReturnInt(0);
1170 }
1171 
1172 #define C_D_HDR "content-disposition:"
1173 #define C_D_HDR_LEN 20
1174 #define C_T_HDR "content-type:"
1175 #define C_T_HDR_LEN 13
1176 
1177 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1178  HtpTxUserData *htud,
1179  uint8_t *header, uint32_t header_len,
1180  uint8_t **filename, uint16_t *filename_len,
1181  uint8_t **filetype, uint16_t *filetype_len)
1182 {
1183  uint8_t *fn = NULL;
1184  size_t fn_len = 0;
1185  uint8_t *ft = NULL;
1186  size_t ft_len = 0;
1187 
1188 #ifdef PRINT
1189  printf("HEADER START: \n");
1190  PrintRawDataFp(stdout, header, header_len);
1191  printf("HEADER END: \n");
1192 #endif
1193 
1194  while (header_len > 0) {
1195  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1196  uint8_t *line = header;
1197  uint32_t line_len;
1198 
1199  if (next_line == NULL) {
1200  line_len = header_len;
1201  } else {
1202  line_len = next_line - header;
1203  }
1204  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1205  if (sc == NULL) {
1206  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1208  /* if the : we found is the final char, it means we have
1209  * no value */
1210  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1211  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1213  } else {
1214 #ifdef PRINT
1215  printf("LINE START: \n");
1216  PrintRawDataFp(stdout, line, line_len);
1217  printf("LINE END: \n");
1218 #endif
1219  if (line_len >= C_D_HDR_LEN &&
1220  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1221  uint8_t *value = line + C_D_HDR_LEN;
1222  uint32_t value_len = line_len - C_D_HDR_LEN;
1223 
1224  /* parse content-disposition */
1225  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1226  value, value_len, &fn, &fn_len);
1227  } else if (line_len >= C_T_HDR_LEN &&
1228  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1229  SCLogDebug("content-type line");
1230  uint8_t *value = line + C_T_HDR_LEN;
1231  uint32_t value_len = line_len - C_T_HDR_LEN;
1232 
1233  (void)HTTPParseContentTypeHeader(NULL, 0,
1234  value, value_len, &ft, &ft_len);
1235  }
1236  }
1237 
1238  if (next_line == NULL) {
1239  SCLogDebug("no next_line");
1240  break;
1241  }
1242  header_len -= ((next_line + 2) - header);
1243  header = next_line + 2;
1244  } /* while (header_len > 0) */
1245 
1246  if (fn_len > USHRT_MAX)
1247  fn_len = USHRT_MAX;
1248  if (ft_len > USHRT_MAX)
1249  ft_len = USHRT_MAX;
1250 
1251  *filename = fn;
1252  *filename_len = fn_len;
1253  *filetype = ft;
1254  *filetype_len = ft_len;
1255 }
1256 
1257 /**
1258  * \brief Create a single buffer from the HtpBodyChunks in our list
1259  *
1260  * \param htud transaction user data
1261  * \param chunks_buffers pointer to pass back the buffer to the caller
1262  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1263  */
1264 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1265  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1266 {
1268  chunks_buffer, chunks_buffer_len,
1269  htud->request_body.body_parsed);
1270 }
1271 
1272 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1273 {
1274  SCEnter();
1275  if (tx && tx->de_state) {
1276  if (dir == STREAM_TOSERVER) {
1277  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1279  } else if (STREAM_TOCLIENT) {
1280  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1282  }
1283  }
1284 }
1285 
1286 /**
1287  * \brief Setup boundary buffers
1288  */
1289 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1290  uint8_t *boundary, uint32_t boundary_len)
1291 {
1292  memset(boundary, '-', boundary_len);
1293  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1294 }
1295 
1296 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1297  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1298 {
1299  int result = 0;
1300  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1301  uint8_t expected_boundary_len = htud->boundary_len + 2;
1302  uint8_t expected_boundary_end_len = htud->boundary_len + 4;
1303  int tx_progress = 0;
1304 
1305 #ifdef PRINT
1306  printf("CHUNK START: \n");
1307  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1308  printf("CHUNK END: \n");
1309 #endif
1310 
1311  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1312 
1313  /* search for the header start, header end and form end */
1314  uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1315  boundary, expected_boundary_len);
1316  uint8_t *header_end = NULL;
1317  if (header_start != NULL) {
1318  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1319  (uint8_t *)"\r\n\r\n", 4);
1320  }
1321  uint8_t *form_end = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1322  boundary, expected_boundary_end_len);
1323 
1324  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1325  header_end, form_end);
1326 
1327  /* we currently only handle multipart for ts. When we support it for tc,
1328  * we will need to supply right direction */
1329  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, STREAM_TOSERVER);
1330  /* if we're in the file storage process, deal with that now */
1331  if (htud->tsflags & HTP_FILENAME_SET) {
1332  if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1333  SCLogDebug("reached the end of the file");
1334 
1335  const uint8_t *filedata = chunks_buffer;
1336  uint32_t filedata_len = 0;
1337  uint8_t flags = 0;
1338 
1339  if (header_start < form_end || (header_start != NULL && form_end == NULL)) {
1340  filedata_len = header_start - filedata - 2; /* 0d 0a */
1341  } else if (form_end != NULL && form_end < header_start) {
1342  filedata_len = form_end - filedata;
1343  } else if (form_end != NULL && form_end == header_start) {
1344  filedata_len = form_end - filedata - 2; /* 0d 0a */
1345  } else if (tx_progress > HTP_REQUEST_BODY) {
1346  filedata_len = chunks_buffer_len;
1347  flags = FILE_TRUNCATED;
1348  }
1349 
1350  if (filedata_len > chunks_buffer_len) {
1351  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1353  goto end;
1354  }
1355 #ifdef PRINT
1356  printf("FILEDATA (final chunk) START: \n");
1357  PrintRawDataFp(stdout, filedata, filedata_len);
1358  printf("FILEDATA (final chunk) END: \n");
1359 #endif
1360  if (!(htud->tsflags & HTP_DONTSTORE)) {
1361  if (HTPFileClose(hstate, filedata, filedata_len, flags,
1362  STREAM_TOSERVER) == -1)
1363  {
1364  goto end;
1365  }
1366  }
1367 
1368  htud->tsflags &=~ HTP_FILENAME_SET;
1369 
1370  /* fall through */
1371  } else {
1372  SCLogDebug("not yet at the end of the file");
1373 
1374  if (chunks_buffer_len > expected_boundary_end_len) {
1375  const uint8_t *filedata = chunks_buffer;
1376  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1377 #ifdef PRINT
1378  printf("FILEDATA (part) START: \n");
1379  PrintRawDataFp(stdout, filedata, filedata_len);
1380  printf("FILEDATA (part) END: \n");
1381 #endif
1382 
1383  if (!(htud->tsflags & HTP_DONTSTORE)) {
1384  result = HTPFileStoreChunk(hstate, filedata,
1385  filedata_len, STREAM_TOSERVER);
1386  if (result == -1) {
1387  goto end;
1388  } else if (result == -2) {
1389  /* we know for sure we're not storing the file */
1390  htud->tsflags |= HTP_DONTSTORE;
1391  }
1392  }
1393 
1394  htud->request_body.body_parsed += filedata_len;
1395  } else {
1396  SCLogDebug("chunk too small to already process in part");
1397  }
1398 
1399  goto end;
1400  }
1401  }
1402 
1403  while (header_start != NULL && header_end != NULL &&
1404  header_end != form_end &&
1405  header_start < (chunks_buffer + chunks_buffer_len) &&
1406  header_end < (chunks_buffer + chunks_buffer_len) &&
1407  header_start < header_end)
1408  {
1409  uint8_t *filename = NULL;
1410  uint16_t filename_len = 0;
1411  uint8_t *filetype = NULL;
1412  uint16_t filetype_len = 0;
1413 
1414  uint32_t header_len = header_end - header_start;
1415  SCLogDebug("header_len %u", header_len);
1416  uint8_t *header = header_start;
1417 
1418  /* skip empty records */
1419  if (expected_boundary_len == header_len) {
1420  goto next;
1421  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1422  header_len -= (expected_boundary_len + 2);
1423  header = header_start + (expected_boundary_len + 2); // + for 0d 0a
1424  }
1425 
1426  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1427  &filename, &filename_len, &filetype, &filetype_len);
1428 
1429  if (filename != NULL) {
1430  uint8_t *filedata = NULL;
1431  uint32_t filedata_len = 0;
1432 
1433  SCLogDebug("we have a filename");
1434 
1435  htud->tsflags |= HTP_FILENAME_SET;
1436  htud->tsflags &= ~HTP_DONTSTORE;
1437 
1438  SCLogDebug("header_end %p", header_end);
1439  SCLogDebug("form_end %p", form_end);
1440 
1441  /* everything until the final boundary is the file */
1442  if (form_end != NULL) {
1443  filedata = header_end + 4;
1444  if (form_end == filedata) {
1445  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1447  goto end;
1448  } else if (form_end < filedata) {
1449  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1451  goto end;
1452  }
1453 
1454  filedata_len = form_end - (header_end + 4 + 2);
1455  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1456 
1457  /* or is it? */
1458  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1459  boundary, expected_boundary_len);
1460  if (header_next != NULL) {
1461  filedata_len -= (form_end - header_next);
1462  }
1463 
1464  if (filedata_len > chunks_buffer_len) {
1465  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1467  goto end;
1468  }
1469  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1470 #ifdef PRINT
1471  printf("FILEDATA START: \n");
1472  PrintRawDataFp(stdout, filedata, filedata_len);
1473  printf("FILEDATA END: \n");
1474 #endif
1475 
1476  result = HTPFileOpen(hstate, filename, filename_len,
1477  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1478  STREAM_TOSERVER);
1479  if (result == -1) {
1480  goto end;
1481  } else if (result == -2) {
1482  htud->tsflags |= HTP_DONTSTORE;
1483  } else {
1484  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1485  goto end;
1486  }
1487  }
1488  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1489 
1490  htud->request_body.body_parsed += (header_end - chunks_buffer);
1491  htud->tsflags &= ~HTP_FILENAME_SET;
1492  } else {
1493  SCLogDebug("chunk doesn't contain form end");
1494 
1495  filedata = header_end + 4;
1496  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1497  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1498 
1499  if (filedata_len > chunks_buffer_len) {
1500  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1502  goto end;
1503  }
1504 
1505 #ifdef PRINT
1506  printf("FILEDATA START: \n");
1507  PrintRawDataFp(stdout, filedata, filedata_len);
1508  printf("FILEDATA END: \n");
1509 #endif
1510  /* form doesn't end in this chunk, but part might. Lets
1511  * see if have another coming up */
1512  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1513  boundary, expected_boundary_len);
1514  SCLogDebug("header_next %p", header_next);
1515  if (header_next == NULL) {
1516  /* no, but we'll handle the file data when we see the
1517  * form_end */
1518 
1519  SCLogDebug("more file data to come");
1520 
1521  uint32_t offset = (header_end + 4) - chunks_buffer;
1522  SCLogDebug("offset %u", offset);
1523  htud->request_body.body_parsed += offset;
1524 
1525  result = HTPFileOpen(hstate, filename, filename_len,
1526  NULL, 0, HtpGetActiveRequestTxID(hstate),
1527  STREAM_TOSERVER);
1528  if (result == -1) {
1529  goto end;
1530  } else if (result == -2) {
1531  htud->tsflags |= HTP_DONTSTORE;
1532  }
1533  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1534 
1535  } else if (header_next - filedata > 2) {
1536  filedata_len = header_next - filedata - 2;
1537  SCLogDebug("filedata_len %u", filedata_len);
1538 
1539  result = HTPFileOpen(hstate, filename, filename_len,
1540  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1541  STREAM_TOSERVER);
1542  if (result == -1) {
1543  goto end;
1544  } else if (result == -2) {
1545  htud->tsflags |= HTP_DONTSTORE;
1546  } else {
1547  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1548  goto end;
1549  }
1550  }
1551  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1552 
1553  htud->tsflags &= ~HTP_FILENAME_SET;
1554  htud->request_body.body_parsed += (header_end - chunks_buffer);
1555  }
1556  }
1557  }
1558 next:
1559  SCLogDebug("header_start %p, header_end %p, form_end %p",
1560  header_start, header_end, form_end);
1561 
1562  /* Search next boundary entry after the start of body */
1563  uint32_t cursizeread = header_end - chunks_buffer;
1564  header_start = Bs2bmSearch(header_end + 4,
1565  chunks_buffer_len - (cursizeread + 4),
1566  boundary, expected_boundary_len);
1567  if (header_start != NULL) {
1568  header_end = Bs2bmSearch(header_end + 4,
1569  chunks_buffer_len - (cursizeread + 4),
1570  (uint8_t *) "\r\n\r\n", 4);
1571  }
1572  }
1573 
1574  /* if we're parsing the multipart and we're not currently processing a
1575  * file, we move the body pointer forward. */
1576  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1577  if (chunks_buffer_len > expected_boundary_end_len) {
1578  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1579 
1580  htud->request_body.body_parsed += move;
1581  SCLogDebug("form not ready, file not set, parsing non-file "
1582  "record: moved %u", move);
1583  }
1584  }
1585 
1586 end:
1587  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1588  return 0;
1589 }
1590 
1591 /** \brief setup things for put request
1592  * \todo really needed? */
1593 static int HtpRequestBodySetupPUT(htp_tx_data_t *d, HtpTxUserData *htud)
1594 {
1595 // if (d->tx->parsed_uri == NULL || d->tx->parsed_uri->path == NULL) {
1596 // return -1;
1597 // }
1598 
1599  /* filename is d->tx->parsed_uri->path */
1600 
1601  return 0;
1602 }
1603 
1604 /** \internal
1605  * \brief Handle POST, no multipart body data
1606  */
1607 static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
1608  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1609 {
1610  int result = 0;
1611 
1612  /* see if we need to open the file */
1613  if (!(htud->tsflags & HTP_FILENAME_SET))
1614  {
1615  uint8_t *filename = NULL;
1616  size_t filename_len = 0;
1617 
1618  /* get the name */
1619  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1620  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1621  filename_len = bstr_len(tx->parsed_uri->path);
1622  }
1623 
1624  if (filename != NULL) {
1625  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1626  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1627  if (result == -1) {
1628  goto end;
1629  } else if (result == -2) {
1630  htud->tsflags |= HTP_DONTSTORE;
1631  } else {
1632  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1633  htud->tsflags |= HTP_FILENAME_SET;
1634  htud->tsflags &= ~HTP_DONTSTORE;
1635  }
1636  }
1637  }
1638  else
1639  {
1640  /* otherwise, just store the data */
1641 
1642  if (!(htud->tsflags & HTP_DONTSTORE)) {
1643  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1644  if (result == -1) {
1645  goto end;
1646  } else if (result == -2) {
1647  /* we know for sure we're not storing the file */
1648  htud->tsflags |= HTP_DONTSTORE;
1649  }
1650  }
1651  }
1652 
1653  return 0;
1654 end:
1655  return -1;
1656 }
1657 
1658 /** \internal
1659  * \brief Handle PUT body data
1660  */
1661 static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
1662  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1663 {
1664  int result = 0;
1665 
1666  /* see if we need to open the file */
1667  if (!(htud->tsflags & HTP_FILENAME_SET))
1668  {
1669  uint8_t *filename = NULL;
1670  size_t filename_len = 0;
1671 
1672  /* get the name */
1673  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1674  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1675  filename_len = bstr_len(tx->parsed_uri->path);
1676  }
1677 
1678  if (filename != NULL) {
1679  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1680  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1681  if (result == -1) {
1682  goto end;
1683  } else if (result == -2) {
1684  htud->tsflags |= HTP_DONTSTORE;
1685  } else {
1686  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1687  htud->tsflags |= HTP_FILENAME_SET;
1688  htud->tsflags &= ~HTP_DONTSTORE;
1689  }
1690  }
1691  }
1692  else
1693  {
1694  /* otherwise, just store the data */
1695 
1696  if (!(htud->tsflags & HTP_DONTSTORE)) {
1697  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1698  if (result == -1) {
1699  goto end;
1700  } else if (result == -2) {
1701  /* we know for sure we're not storing the file */
1702  htud->tsflags |= HTP_DONTSTORE;
1703  }
1704  }
1705  }
1706 
1707  return 0;
1708 end:
1709  return -1;
1710 }
1711 
1712 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1713  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1714 {
1715  SCEnter();
1716 
1717  int result = 0;
1718 
1719  /* see if we need to open the file
1720  * we check for tx->response_line in case of junk
1721  * interpreted as body before response line
1722  */
1723  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1724  (tx->response_line != NULL || tx->is_protocol_0_9))
1725  {
1726  SCLogDebug("setting up file name");
1727 
1728  uint8_t *filename = NULL;
1729  size_t filename_len = 0;
1730 
1731  /* try Content-Disposition header first */
1732  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1733  "Content-Disposition");
1734  if (h != NULL && bstr_len(h->value) > 0) {
1735  /* parse content-disposition */
1736  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1737  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1738  }
1739 
1740  /* fall back to name from the uri */
1741  if (filename == NULL) {
1742  /* get the name */
1743  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1744  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1745  filename_len = bstr_len(tx->parsed_uri->path);
1746  }
1747  }
1748 
1749  if (filename != NULL) {
1750  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len,
1751  data, data_len, HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1752  SCLogDebug("result %d", result);
1753  if (result == -1) {
1754  goto end;
1755  } else if (result == -2) {
1756  htud->tcflags |= HTP_DONTSTORE;
1757  } else {
1758  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1759  htud->tcflags |= HTP_FILENAME_SET;
1760  htud->tcflags &= ~HTP_DONTSTORE;
1761  }
1762  //set range if present
1763  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1764  if (h_content_range != NULL) {
1765  HTPFileSetRange(hstate, h_content_range->value);
1766  }
1767  }
1768  }
1769  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1770  {
1771  /* otherwise, just store the data */
1772 
1773  if (!(htud->tcflags & HTP_DONTSTORE)) {
1774  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1775  SCLogDebug("result %d", result);
1776  if (result == -1) {
1777  goto end;
1778  } else if (result == -2) {
1779  /* we know for sure we're not storing the file */
1780  htud->tcflags |= HTP_DONTSTORE;
1781  }
1782  }
1783  }
1784 
1785  htud->response_body.body_parsed += data_len;
1786  return 0;
1787 end:
1788  return -1;
1789 }
1790 
1791 /**
1792  * \brief Function callback to append chunks for Requests
1793  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1794  * \retval int HTP_OK if all goes well
1795  */
1796 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1797 {
1798  SCEnter();
1799 
1800  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1801  SCReturnInt(HTP_OK);
1802 
1803  if (d->data == NULL || d->len == 0)
1804  SCReturnInt(HTP_OK);
1805 
1806 #ifdef PRINT
1807  printf("HTPBODY START: \n");
1808  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1809  printf("HTPBODY END: \n");
1810 #endif
1811 
1812  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1813  if (hstate == NULL) {
1814  SCReturnInt(HTP_ERROR);
1815  }
1816 
1817  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1818  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1819 
1820  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1821  if (tx_ud == NULL) {
1822  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1823  if (unlikely(tx_ud == NULL)) {
1824  SCReturnInt(HTP_OK);
1825  }
1826  memset(tx_ud, 0, sizeof(HtpTxUserData));
1827 
1828  /* Set the user data for handling body chunks on this transaction */
1829  htp_tx_set_user_data(d->tx, tx_ud);
1830  }
1831  if (!tx_ud->response_body_init) {
1832  tx_ud->response_body_init = 1;
1833 
1834  if (d->tx->request_method_number == HTP_M_POST) {
1835  SCLogDebug("POST");
1836  int r = HtpRequestBodySetupMultipart(d, tx_ud);
1837  if (r == 1) {
1839  } else if (r == 0) {
1841  SCLogDebug("not multipart");
1842  }
1843  } else if (d->tx->request_method_number == HTP_M_PUT) {
1844  if (HtpRequestBodySetupPUT(d, tx_ud) == 0) {
1846  }
1847  }
1848  }
1849 
1850  /* see if we can get rid of htp body chunks */
1851  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1852 
1853  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1854  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1855 
1856  /* within limits, add the body chunk to the state. */
1857  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1858  uint32_t stream_depth = FileReassemblyDepth();
1859  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1860  hstate->cfg->request.body_limit,
1861  stream_depth,
1862  tx_ud->tsflags,
1863  (uint32_t)d->len);
1864  BUG_ON(len > (uint32_t)d->len);
1865 
1866  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1867 
1868  const uint8_t *chunks_buffer = NULL;
1869  uint32_t chunks_buffer_len = 0;
1870 
1872  /* multi-part body handling starts here */
1873  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1874  goto end;
1875  }
1876 
1877  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1878  if (chunks_buffer == NULL) {
1879  goto end;
1880  }
1881 #ifdef PRINT
1882  printf("REASSCHUNK START: \n");
1883  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1884  printf("REASSCHUNK END: \n");
1885 #endif
1886 
1887  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1888 
1889  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
1890  HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1891  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
1892  HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1893  }
1894 
1895  } else {
1896  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1897  SCLogDebug("closing file that was being stored");
1898  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1899  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1900  }
1901  }
1902 
1903 end:
1904  if (hstate->conn != NULL) {
1905  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1907  hstate->cfg->request.inspect_min_size,
1908  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1909 
1910  /* if we reach the inspect_min_size we'll trigger inspection,
1911  * so make sure that raw stream is also inspected. Set the
1912  * data to be used to the amount of raw bytes we've seen to
1913  * get here. */
1914  if (tx_ud->request_body.body_inspected == 0 &&
1916  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1917  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1918  {
1919  uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1920 
1921  /* body still in progress, but due to min inspect size we need to inspect now */
1924  }
1925  /* after the start of the body, disable the depth logic */
1926  } else if (tx_ud->request_body.body_inspected > 0) {
1928  }
1929  }
1930  SCReturnInt(HTP_OK);
1931 }
1932 
1933 /**
1934  * \brief Function callback to append chunks for Responses
1935  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1936  * \retval int HTP_OK if all goes well
1937  */
1938 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1939 {
1940  SCEnter();
1941 
1942  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1943  SCReturnInt(HTP_OK);
1944 
1945  if (d->data == NULL || d->len == 0)
1946  SCReturnInt(HTP_OK);
1947 
1948  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1949  if (hstate == NULL) {
1950  SCReturnInt(HTP_ERROR);
1951  }
1952 
1953  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1954  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1955 
1956  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1957  if (tx_ud == NULL) {
1958  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1959  if (unlikely(tx_ud == NULL)) {
1960  SCReturnInt(HTP_OK);
1961  }
1962  memset(tx_ud, 0, sizeof(HtpTxUserData));
1963 
1964  /* Set the user data for handling body chunks on this transaction */
1965  htp_tx_set_user_data(d->tx, tx_ud);
1966  }
1967  if (!tx_ud->request_body_init) {
1968  tx_ud->request_body_init = 1;
1969  }
1970 
1971  /* see if we can get rid of htp body chunks */
1972  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1973 
1974  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1975  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1976 
1977  /* within limits, add the body chunk to the state. */
1978  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1979  uint32_t stream_depth = FileReassemblyDepth();
1980  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1981  hstate->cfg->response.body_limit,
1982  stream_depth,
1983  tx_ud->tcflags,
1984  (uint32_t)d->len);
1985  BUG_ON(len > (uint32_t)d->len);
1986 
1987  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1988 
1989  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1990  } else {
1991  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1992  SCLogDebug("closing file that was being stored");
1993  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1994  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1995  }
1996  }
1997 
1998  if (hstate->conn != NULL) {
1999  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
2001  hstate->cfg->response.inspect_min_size,
2002  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
2003  /* if we reach the inspect_min_size we'll trigger inspection,
2004  * so make sure that raw stream is also inspected. Set the
2005  * data to be used to the amount of raw bytes we've seen to
2006  * get here. */
2007  if (tx_ud->response_body.body_inspected == 0 &&
2009  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2010  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2011  {
2012  uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
2013 
2014  /* body still in progress, but due to min inspect size we need to inspect now */
2017  }
2018  /* after the start of the body, disable the depth logic */
2019  } else if (tx_ud->response_body.body_inspected > 0) {
2021  }
2022  }
2023  SCReturnInt(HTP_OK);
2024 }
2025 
2026 /**
2027  * \brief Print the stats of the HTTP requests
2028  */
2030 {
2031 #ifdef DEBUG
2032  SCEnter();
2033  SCMutexLock(&htp_state_mem_lock);
2034  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2035  htp_state_memcnt, htp_state_memuse);
2036  SCMutexUnlock(&htp_state_mem_lock);
2037  SCReturn;
2038 #endif
2039 }
2040 
2041 /** \brief Clears the HTTP server configuration memory used by HTP library */
2042 void HTPFreeConfig(void)
2043 {
2044  SCEnter();
2045 
2046  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2047  !AppLayerParserConfParserEnabled("tcp", "http"))
2048  {
2049  SCReturn;
2050  }
2051 
2052  HTPCfgRec *nextrec = cfglist.next;
2053  SCRadixReleaseRadixTree(cfgtree);
2054  cfgtree = NULL;
2055  htp_config_destroy(cfglist.cfg);
2056  while (nextrec != NULL) {
2057  HTPCfgRec *htprec = nextrec;
2058  nextrec = nextrec->next;
2059 
2060  htp_config_destroy(htprec->cfg);
2061  SCFree(htprec);
2062  }
2063  HTPDestroyMemcap();
2064  SCReturn;
2065 }
2066 
2067 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2068 {
2069  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2070  if (htud != NULL) {
2071  htud->request_has_trailers = 1;
2072  }
2073  return HTP_OK;
2074 }
2075 
2076 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2077 {
2078  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2079  if (htud != NULL) {
2080  htud->response_has_trailers = 1;
2081  }
2082  return HTP_OK;
2083 }
2084 
2085 /**\internal
2086  * \brief called at start of request
2087  * Set min inspect size.
2088  */
2089 static int HTPCallbackRequestStart(htp_tx_t *tx)
2090 {
2091  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2092  if (hstate == NULL) {
2093  SCReturnInt(HTP_ERROR);
2094  }
2095 
2096  if (hstate->cfg)
2098  hstate->cfg->request.inspect_min_size);
2099  SCReturnInt(HTP_OK);
2100 }
2101 
2102 /**\internal
2103  * \brief called at start of response
2104  * Set min inspect size.
2105  */
2106 static int HTPCallbackResponseStart(htp_tx_t *tx)
2107 {
2108  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2109  if (hstate == NULL) {
2110  SCReturnInt(HTP_ERROR);
2111  }
2112 
2113  if (hstate->cfg)
2115  hstate->cfg->response.inspect_min_size);
2116  SCReturnInt(HTP_OK);
2117 }
2118 
2119 
2120 /**
2121  * \brief callback for request to store the recent incoming request
2122  in to the recent_in_tx for the given htp state
2123  * \param connp pointer to the current connection parser which has the htp
2124  * state in it as user data
2125  */
2126 static int HTPCallbackRequest(htp_tx_t *tx)
2127 {
2128  SCEnter();
2129 
2130  if (tx == NULL) {
2131  SCReturnInt(HTP_ERROR);
2132  }
2133 
2134  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2135  if (hstate == NULL) {
2136  SCReturnInt(HTP_ERROR);
2137  }
2138 
2139  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2140  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2141 
2142  SCLogDebug("HTTP request completed");
2143 
2144  HTPErrorCheckTxRequestFlags(hstate, tx);
2145 
2146  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2147  if (htud != NULL) {
2148  if (htud->tsflags & HTP_FILENAME_SET) {
2149  SCLogDebug("closing file that was being stored");
2150  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
2151  htud->tsflags &= ~HTP_FILENAME_SET;
2153  (uint32_t)hstate->conn->in_data_counter);
2154  }
2155  }
2156 
2157  hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
2158  /* request done, do raw reassembly now to inspect state and stream
2159  * at the same time. */
2161  SCReturnInt(HTP_OK);
2162 }
2163 
2164 /**
2165  * \brief callback for response to remove the recent received requests
2166  from the recent_in_tx for the given htp state
2167  * \param connp pointer to the current connection parser which has the htp
2168  * state in it as user data
2169  */
2170 static int HTPCallbackResponse(htp_tx_t *tx)
2171 {
2172  SCEnter();
2173 
2174  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2175  if (hstate == NULL) {
2176  SCReturnInt(HTP_ERROR);
2177  }
2178 
2179  /* we have one whole transaction now */
2180  hstate->transaction_cnt++;
2181 
2182  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2183  if (htud != NULL) {
2184  if (htud->tcflags & HTP_FILENAME_SET) {
2185  SCLogDebug("closing file that was being stored");
2186  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
2187  htud->tcflags &= ~HTP_FILENAME_SET;
2188  }
2189  }
2190 
2191  /* response done, do raw reassembly now to inspect state and stream
2192  * at the same time. */
2194 
2195  /* handle HTTP CONNECT */
2196  if (tx->request_method_number == HTP_M_CONNECT) {
2197  /* any 2XX status response implies that the connection will become
2198  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2199  if ((tx->response_status_number >= 200) &&
2200  (tx->response_status_number < 300) &&
2201  (hstate->transaction_cnt == 1)) {
2202  uint16_t dp = 0;
2203  if (tx->request_port_number != -1) {
2204  dp = (uint16_t)tx->request_port_number;
2205  }
2206  // both ALPROTO_HTTP and ALPROTO_TLS are normal options
2208  tx->request_progress = HTP_REQUEST_COMPLETE;
2209  tx->response_progress = HTP_RESPONSE_COMPLETE;
2210  }
2211  }
2212 
2213  hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
2214  SCReturnInt(HTP_OK);
2215 }
2216 
2217 static int HTPCallbackRequestLine(htp_tx_t *tx)
2218 {
2219  HtpTxUserData *tx_ud;
2220  bstr *request_uri_normalized;
2221  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2222  const HTPCfgRec *cfg = hstate->cfg;
2223 
2224  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2225  if (request_uri_normalized == NULL)
2226  return HTP_OK;
2227 
2228  tx_ud = htp_tx_get_user_data(tx);
2229  if (likely(tx_ud == NULL)) {
2230  tx_ud = HTPMalloc(sizeof(*tx_ud));
2231  if (unlikely(tx_ud == NULL)) {
2232  bstr_free(request_uri_normalized);
2233  return HTP_OK;
2234  }
2235  memset(tx_ud, 0, sizeof(*tx_ud));
2236  htp_tx_set_user_data(tx, tx_ud);
2237  }
2238  if (unlikely(tx_ud->request_uri_normalized != NULL))
2239  bstr_free(tx_ud->request_uri_normalized);
2240  tx_ud->request_uri_normalized = request_uri_normalized;
2241 
2242  if (tx->flags) {
2243  HTPErrorCheckTxRequestFlags(hstate, tx);
2244  }
2245  return HTP_OK;
2246 }
2247 
2248 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2249 {
2250  if (part == NULL)
2251  return HTP_OK;
2252 
2253  uint64_t flags = 0;
2254  size_t prevlen = bstr_len(part);
2255  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2256  // shorter string means that uri was encoded
2257  if (res == HTP_OK && prevlen > bstr_len(part)) {
2258  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2259  if (likely(htud == NULL)) {
2260  htud = HTPCalloc(1, sizeof(*htud));
2261  if (unlikely(htud == NULL))
2262  return HTP_OK;
2263  htp_tx_set_user_data(tx, htud);
2264  }
2265  HtpState *s = htp_connp_get_user_data(tx->connp);
2266  if (s == NULL)
2267  return HTP_OK;
2268  HTPSetEvent(s, htud, STREAM_TOSERVER,
2270  }
2271 
2272  return HTP_OK;
2273 }
2274 
2275 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2276 {
2277  if (tx->parsed_uri == NULL)
2278  return HTP_OK;
2279 
2280  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2281 }
2282 
2283 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2284 {
2285  if (tx->parsed_uri == NULL)
2286  return HTP_OK;
2287 
2288  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2289 }
2290 
2291 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2292 {
2293  void *ptmp;
2294  if (tx_data->len == 0 || tx_data->tx == NULL)
2295  return HTP_OK;
2296 
2297  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2298  if (tx_ud == NULL) {
2299  tx_ud = HTPMalloc(sizeof(*tx_ud));
2300  if (unlikely(tx_ud == NULL))
2301  return HTP_OK;
2302  memset(tx_ud, 0, sizeof(*tx_ud));
2303  htp_tx_set_user_data(tx_data->tx, tx_ud);
2304  }
2305  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2306  tx_ud->request_headers_raw_len,
2307  tx_ud->request_headers_raw_len + tx_data->len);
2308  if (ptmp == NULL) {
2309  /* error: we're freeing the entire user data */
2310  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2311  HtpTxUserDataFree(hstate, tx_ud);
2312  htp_tx_set_user_data(tx_data->tx, NULL);
2313  return HTP_OK;
2314  }
2315  tx_ud->request_headers_raw = ptmp;
2316 
2317  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2318  tx_data->data, tx_data->len);
2319  tx_ud->request_headers_raw_len += tx_data->len;
2320 
2321  if (tx_data->tx && tx_data->tx->flags) {
2322  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2323  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2324  }
2325  return HTP_OK;
2326 }
2327 
2328 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2329 {
2330  void *ptmp;
2331  if (tx_data->len == 0 || tx_data->tx == NULL)
2332  return HTP_OK;
2333 
2334  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2335  if (tx_ud == NULL) {
2336  tx_ud = HTPMalloc(sizeof(*tx_ud));
2337  if (unlikely(tx_ud == NULL))
2338  return HTP_OK;
2339  memset(tx_ud, 0, sizeof(*tx_ud));
2340  htp_tx_set_user_data(tx_data->tx, tx_ud);
2341  }
2342  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2343  tx_ud->response_headers_raw_len,
2344  tx_ud->response_headers_raw_len + tx_data->len);
2345  if (ptmp == NULL) {
2346  /* error: we're freeing the entire user data */
2347  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2348  HtpTxUserDataFree(hstate, tx_ud);
2349  htp_tx_set_user_data(tx_data->tx, NULL);
2350  return HTP_OK;
2351  }
2352  tx_ud->response_headers_raw = ptmp;
2353 
2354  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2355  tx_data->data, tx_data->len);
2356  tx_ud->response_headers_raw_len += tx_data->len;
2357 
2358  return HTP_OK;
2359 }
2360 
2361 /*
2362  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2363  */
2364 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2365 {
2366  cfg_prec->uri_include_all = FALSE;
2373 
2374  if (!g_disable_randomness) {
2376  } else {
2377  cfg_prec->randomize = 0;
2378  }
2380 
2381  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2382  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2383  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2384  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2385 
2386  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2387  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2388 
2389  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2390  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2391 
2392  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2393  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
2394 
2395  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2396  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
2397 
2398  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2399 
2400  /* don't convert + to space by default */
2401  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2402 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2403  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2405 #endif
2406 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2407  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2409 #endif
2410  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2411  * only the hard limit. So we set both here to the (current) htp defaults.
2412  * The reason we do this is that if the user sets the hard limit in the
2413  * config, we have to set the soft limit as well. If libhtp starts using
2414  * the soft limit in the future, we at least make sure we control what
2415  * it's value is. */
2416  htp_config_set_field_limits(cfg_prec->cfg,
2419  return;
2420 }
2421 
2422 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2423  * but we can get both <0 and >RAND_MAX values from RandomGet
2424  */
2425 static int RandomGetWrap(void)
2426 {
2427  unsigned long r;
2428 
2429  do {
2430  r = RandomGet();
2431  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2432 
2433  return r % RAND_MAX;
2434 }
2435 
2436 /*
2437  * We have this splitup so that in case double decoding has been enabled
2438  * for query and path, they would be called first on the callback queue,
2439  * before the callback set by Phase2() is called. We need this, since
2440  * the callback in Phase2() generates the normalized uri which utilizes
2441  * the query and path. */
2442 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2443 {
2444  /* randomize inspection size if needed */
2445  if (cfg_prec->randomize) {
2446  int rdrange = cfg_prec->randomize_range;
2447 
2448  long int r = RandomGetWrap();
2449  cfg_prec->request.inspect_min_size +=
2450  (int) (cfg_prec->request.inspect_min_size *
2451  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2452 
2453  r = RandomGetWrap();
2454  cfg_prec->request.inspect_window +=
2455  (int) (cfg_prec->request.inspect_window *
2456  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2457  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2458  " %d and 'request-body-inspect-window' set to %d after"
2459  " randomization.",
2460  name,
2461  cfg_prec->request.inspect_min_size,
2462  cfg_prec->request.inspect_window);
2463 
2464 
2465  r = RandomGetWrap();
2466  cfg_prec->response.inspect_min_size +=
2467  (int) (cfg_prec->response.inspect_min_size *
2468  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2469 
2470  r = RandomGetWrap();
2471  cfg_prec->response.inspect_window +=
2472  (int) (cfg_prec->response.inspect_window *
2473  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2474 
2475  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2476  " %d and 'response-body-inspect-window' set to %d after"
2477  " randomization.",
2478  name,
2479  cfg_prec->response.inspect_min_size,
2480  cfg_prec->response.inspect_window);
2481  }
2482 
2483  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2484 
2485  cfg_prec->request.sbcfg.flags = 0;
2486  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2487  cfg_prec->request.inspect_window : 256;
2488  cfg_prec->request.sbcfg.buf_slide = 0;
2489  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2490  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2491  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2492  cfg_prec->request.sbcfg.Free = HTPFree;
2493 
2494  cfg_prec->response.sbcfg.flags = 0;
2495  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2496  cfg_prec->response.inspect_window : 256;
2497  cfg_prec->response.sbcfg.buf_slide = 0;
2498  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2499  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2500  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2501  cfg_prec->response.sbcfg.Free = HTPFree;
2502  return;
2503 }
2504 
2505 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2506  SCRadixTree *tree)
2507 {
2508  if (cfg_prec == NULL || s == NULL || tree == NULL)
2509  return;
2510 
2511  ConfNode *p = NULL;
2512 
2513  /* Default Parameters */
2514  TAILQ_FOREACH(p, &s->head, next) {
2515 
2516  if (strcasecmp("address", p->name) == 0) {
2517  ConfNode *pval;
2518  /* Addresses */
2519  TAILQ_FOREACH(pval, &p->head, next) {
2520  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2521  pval->val);
2522 
2523  /* IPV6 or IPV4? */
2524  if (strchr(pval->val, ':') != NULL) {
2525  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2526  s->name, pval->val, cfg_prec->cfg);
2527  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2528  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2529  "add ipv6 server %s, ignoring", pval->val);
2530  }
2531  } else {
2532  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2533  s->name, pval->val, cfg_prec->cfg);
2534  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2535  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2536  "to add ipv4 server %s, ignoring",
2537  pval->val);
2538  }
2539  } /* else - if (strchr(pval->val, ':') != NULL) */
2540  } /* TAILQ_FOREACH(pval, &p->head, next) */
2541 
2542  } else if (strcasecmp("personality", p->name) == 0) {
2543  /* Personalities */
2544  int personality = HTPLookupPersonality(p->val);
2545  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2546  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2547 
2548  if (personality >= 0) {
2549  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2550  personality);
2551  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2552  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2553  "personality \"%s\", ignoring", p->val);
2554  } else {
2555  SCLogDebug("LIBHTP personality set to %s",
2556  HTPLookupPersonalityString(personality));
2557  }
2558 
2559  /* The IDS personality by default converts the path (and due to
2560  * our query string callback also the query string) to lowercase.
2561  * Signatures do not expect this, so override it. */
2562  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2563  } else {
2564  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2565  "\"%s\", ignoring", p->val);
2566  continue;
2567  }
2568 
2569  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2570  strcasecmp("request_body_limit", p->name) == 0) {
2571  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2572  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2573  "from conf file - %s. Killing engine", p->val);
2574  exit(EXIT_FAILURE);
2575  }
2576 
2577  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2578  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2579  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2580  "from conf file - %s. Killing engine", p->val);
2581  exit(EXIT_FAILURE);
2582  }
2583 
2584  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2585  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2586  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2587  "from conf file - %s. Killing engine", p->val);
2588  exit(EXIT_FAILURE);
2589  }
2590 
2591  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2592  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2593  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2594  "from conf file - %s. Killing engine", p->val);
2595  exit(EXIT_FAILURE);
2596  }
2597 
2598  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2599  if (ConfValIsTrue(p->val)) {
2600  htp_config_register_request_line(cfg_prec->cfg,
2601  HTPCallbackDoubleDecodeQuery);
2602  }
2603 
2604  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2605  if (ConfValIsTrue(p->val)) {
2606  htp_config_register_request_line(cfg_prec->cfg,
2607  HTPCallbackDoubleDecodePath);
2608  }
2609 
2610  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2611  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2612  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2613  "from conf file - %s. Killing engine", p->val);
2614  exit(EXIT_FAILURE);
2615  }
2616 
2617  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2618  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2619  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2620  "from conf file - %s. Killing engine", p->val);
2621  exit(EXIT_FAILURE);
2622  }
2623 
2624  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2625  uint32_t value = 2;
2626  if (ParseSizeStringU32(p->val, &value) < 0) {
2627  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2628  "from conf file - %s. Killing engine", p->val);
2629  exit(EXIT_FAILURE);
2630  }
2631 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2632  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2633 #else
2634  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2635  "to %u, libhtp version too old", value);
2636 #endif
2637  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2638  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2639  HTP_DECODER_URL_PATH,
2640  ConfValIsTrue(p->val));
2641  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2642  if (strlen(p->val) == 1) {
2643  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2644  HTP_DECODER_URL_PATH,
2645  p->val[0]);
2646  } else {
2647  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2648  "for libhtp param path-bestfit-replacement-char");
2649  }
2650  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2651  htp_config_set_convert_lowercase(cfg_prec->cfg,
2652  HTP_DECODER_URL_PATH,
2653  ConfValIsTrue(p->val));
2654  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2655  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2656  HTP_DECODER_URL_PATH,
2657  ConfValIsTrue(p->val));
2658  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2659  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2660  HTP_DECODER_URL_PATH,
2661  ConfValIsTrue(p->val));
2662  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2663  htp_config_set_path_separators_compress(cfg_prec->cfg,
2664  HTP_DECODER_URL_PATH,
2665  ConfValIsTrue(p->val));
2666  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2667  htp_config_set_path_separators_decode(cfg_prec->cfg,
2668  HTP_DECODER_URL_PATH,
2669  ConfValIsTrue(p->val));
2670  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2671  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2672  HTP_DECODER_URL_PATH,
2673  ConfValIsTrue(p->val));
2674  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2675  enum htp_url_encoding_handling_t handling;
2676  if (strcasecmp(p->val, "preserve_percent") == 0) {
2677  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2678  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2679  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2680  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2681  handling = HTP_URL_DECODE_PROCESS_INVALID;
2682  } else {
2683  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2684  "for libhtp param path-url-encoding-invalid-handling");
2685  return;
2686  }
2687  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2688  HTP_DECODER_URL_PATH,
2689  handling);
2690  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2691  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2692  HTP_DECODER_URL_PATH,
2693  ConfValIsTrue(p->val));
2694  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2695  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2696  SCLogDebug("uri-include-all %s",
2697  cfg_prec->uri_include_all ? "enabled" : "disabled");
2698  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2699  htp_config_set_plusspace_decode(cfg_prec->cfg,
2700  HTP_DECODER_URLENCODED,
2701  ConfValIsTrue(p->val));
2702  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2703  uint32_t limit = 0;
2704  if (ParseSizeStringU32(p->val, &limit) < 0) {
2705  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2706  "from conf file - %s. Killing engine", p->val);
2707  exit(EXIT_FAILURE);
2708  }
2709  if (limit == 0) {
2710  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2711  "from conf file cannot be 0. Killing engine");
2712  exit(EXIT_FAILURE);
2713  }
2714  /* set default soft-limit with our new hard limit */
2715  htp_config_set_field_limits(cfg_prec->cfg,
2717  (size_t)limit);
2718 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2719  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2720  uint32_t limit = 0;
2721  if (ParseSizeStringU32(p->val, &limit) < 0) {
2722  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'lzma-memlimit' "
2723  "from conf file - %s.", p->val);
2724  }
2725  if (limit == 0) {
2726  FatalError(SC_ERR_SIZE_PARSE, "'lzma-memlimit' "
2727  "from conf file cannot be 0.");
2728  }
2729  /* set default soft-limit with our new hard limit */
2730  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2731  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2732 #endif
2733 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2734  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2735  if (ConfValIsFalse(p->val)) {
2736  htp_config_set_lzma_memlimit(cfg_prec->cfg, 0);
2737  }
2738 #endif
2739 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2740  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2741  uint32_t limit = 0;
2742  if (ParseSizeStringU32(p->val, &limit) < 0) {
2743  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'compression-bomb-limit' "
2744  "from conf file - %s.", p->val);
2745  }
2746  if (limit == 0) {
2747  FatalError(SC_ERR_SIZE_PARSE, "'compression-bomb-limit' "
2748  "from conf file cannot be 0.");
2749  }
2750  /* set default soft-limit with our new hard limit */
2751  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2752  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2753 #endif
2754  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2755  if (!g_disable_randomness) {
2756  cfg_prec->randomize = ConfValIsTrue(p->val);
2757  }
2758  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2759  uint32_t range = atoi(p->val);
2760  if (range > 100) {
2761  SCLogError(SC_ERR_SIZE_PARSE, "Invalid value for randomize"
2762  " inspection range setting from conf file - %s."
2763  " It should be inferior to 100."
2764  " Killing engine",
2765  p->val);
2766  exit(EXIT_FAILURE);
2767  }
2768  cfg_prec->randomize_range = range;
2769  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2770  if (ConfValIsTrue(p->val)) {
2771  cfg_prec->http_body_inline = 1;
2772  } else if (ConfValIsFalse(p->val)) {
2773  cfg_prec->http_body_inline = 0;
2774  } else {
2775  if (strcmp("auto", p->val) != 0) {
2776  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2777  }
2778  if (EngineModeIsIPS()) {
2779  cfg_prec->http_body_inline = 1;
2780  } else {
2781  cfg_prec->http_body_inline = 0;
2782  }
2783  }
2784  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2785  ConfNode *pval;
2786 
2787  TAILQ_FOREACH(pval, &p->head, next) {
2788  if (strcasecmp("enabled", pval->name) == 0) {
2789  if (ConfValIsTrue(pval->val)) {
2790  cfg_prec->swf_decompression_enabled = 1;
2791  } else if (ConfValIsFalse(pval->val)) {
2792  cfg_prec->swf_decompression_enabled = 0;
2793  } else {
2794  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2795  }
2796  } else if (strcasecmp("type", pval->name) == 0) {
2797  if (strcasecmp("no", pval->val) == 0) {
2799  } else if (strcasecmp("deflate", pval->val) == 0) {
2801  } else if (strcasecmp("lzma", pval->val) == 0) {
2803  } else if (strcasecmp("both", pval->val) == 0) {
2805  } else {
2807  "Invalid entry for "
2808  "swf-decompression.type: %s - "
2809  "Killing engine", pval->val);
2810  exit(EXIT_FAILURE);
2811  }
2812  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2813  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2815  "Error parsing swf-decompression.compression-depth "
2816  "from conf file - %s. Killing engine", p->val);
2817  exit(EXIT_FAILURE);
2818  }
2819  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2820  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2822  "Error parsing swf-decompression.decompression-depth "
2823  "from conf file - %s. Killing engine", p->val);
2824  exit(EXIT_FAILURE);
2825  }
2826  } else {
2827  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2828  }
2829  }
2830  } else {
2831  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2832  "default config: %s", p->name);
2833  }
2834  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2835 
2836  return;
2837 }
2838 
2839 void HTPConfigure(void)
2840 {
2841  SCEnter();
2842 
2843  cfglist.next = NULL;
2844 
2845  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2846  if (NULL == cfgtree)
2847  exit(EXIT_FAILURE);
2848 
2849  /* Default Config */
2850  cfglist.cfg = htp_config_create();
2851  if (NULL == cfglist.cfg) {
2852  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP default config");
2853  exit(EXIT_FAILURE);
2854  }
2855  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2856  HTPConfigSetDefaultsPhase1(&cfglist);
2857  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2858  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2859  cfgtree);
2860  } else {
2861  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2862  }
2863  HTPConfigSetDefaultsPhase2("default", &cfglist);
2864 
2865  HTPParseMemcap();
2866 
2867  /* Read server config and create a parser for each IP in radix tree */
2868  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2869  if (server_config == NULL) {
2870  server_config = ConfGetNode("libhtp.server-config");
2871  if (server_config == NULL) {
2872  SCLogDebug("LIBHTP Configuring %p", server_config);
2873  SCReturn;
2874  }
2875  }
2876  SCLogDebug("LIBHTP Configuring %p", server_config);
2877 
2878  ConfNode *si;
2879  /* Server Nodes */
2880  TAILQ_FOREACH(si, &server_config->head, next) {
2881  /* Need the named node, not the index */
2882  ConfNode *s = TAILQ_FIRST(&si->head);
2883  if (NULL == s) {
2884  SCLogDebug("LIBHTP s NULL");
2885  continue;
2886  }
2887 
2888  SCLogDebug("LIBHTP server %s", s->name);
2889 
2890  HTPCfgRec *nextrec = cfglist.next;
2891  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2892  if (NULL == htprec)
2893  exit(EXIT_FAILURE);
2894  memset(htprec, 0x00, sizeof(*htprec));
2895 
2896  cfglist.next = htprec;
2897 
2898  cfglist.next->next = nextrec;
2899  cfglist.next->cfg = htp_config_create();
2900  if (NULL == cfglist.next->cfg) {
2901  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP server config");
2902  exit(EXIT_FAILURE);
2903  }
2904 
2905  HTPConfigSetDefaultsPhase1(htprec);
2906  HTPConfigParseParameters(htprec, s, cfgtree);
2907  HTPConfigSetDefaultsPhase2(s->name, htprec);
2908  }
2909 
2910  SCReturn;
2911 }
2912 
2914 {
2915 #ifdef DEBUG
2916  SCMutexLock(&htp_state_mem_lock);
2917  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2918  SCMutexUnlock(&htp_state_mem_lock);
2919 #endif
2920 }
2921 
2922 /** \internal
2923  * \brief get files callback
2924  * \param state state ptr
2925  * \param direction flow direction
2926  * \retval files files ptr
2927  */
2928 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
2929 {
2930  if (state == NULL)
2931  return NULL;
2932 
2933  HtpState *http_state = (HtpState *)state;
2934 
2935  if (direction & STREAM_TOCLIENT) {
2936  SCReturnPtr(http_state->files_tc, "FileContainer");
2937  } else {
2938  SCReturnPtr(http_state->files_ts, "FileContainer");
2939  }
2940 }
2941 
2942 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2943 {
2944  if (direction & STREAM_TOSERVER)
2945  return ((htp_tx_t *)tx)->request_progress;
2946  else
2947  return ((htp_tx_t *)tx)->response_progress;
2948 }
2949 
2950 static uint64_t HTPStateGetTxCnt(void *alstate)
2951 {
2952  HtpState *http_state = (HtpState *)alstate;
2953 
2954  if (http_state != NULL && http_state->conn != NULL) {
2955  const uint64_t size = (uint64_t)htp_list_size(http_state->conn->transactions);
2956  SCLogDebug("size %"PRIu64, size);
2957  return size;
2958  } else {
2959  return 0ULL;
2960  }
2961 }
2962 
2963 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2964 {
2965  HtpState *http_state = (HtpState *)alstate;
2966 
2967  if (http_state != NULL && http_state->conn != NULL)
2968  return htp_list_get(http_state->conn->transactions, tx_id);
2969  else
2970  return NULL;
2971 }
2972 
2973 static void HTPStateSetTxLogged(void *alstate, void *vtx, LoggerId bits)
2974 {
2975  htp_tx_t *tx = (htp_tx_t *)vtx;
2976  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2977  if (tx_ud)
2978  tx_ud->logged = bits;
2979 }
2980 
2981 static LoggerId HTPStateGetTxLogged(void *alstate, void *vtx)
2982 {
2983  htp_tx_t *tx = (htp_tx_t *)vtx;
2984  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2985  if (tx_ud != NULL)
2986  return tx_ud->logged;
2987 
2988  return 0;
2989 }
2990 
2991 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction)
2992 {
2993  return (direction & STREAM_TOSERVER) ? HTP_REQUEST_COMPLETE : HTP_RESPONSE_COMPLETE;
2994 }
2995 
2996 static int HTPStateGetEventInfo(const char *event_name,
2997  int *event_id, AppLayerEventType *event_type)
2998 {
2999  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3000  if (*event_id == -1) {
3001  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
3002  "http's enum map table.", event_name);
3003  /* this should be treated as fatal */
3004  return -1;
3005  }
3006 
3007  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3008 
3009  return 0;
3010 }
3011 
3012 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3014 {
3015  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3016  if (*event_name == NULL) {
3017  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
3018  "http's enum map table.", event_id);
3019  /* this should be treated as fatal */
3020  return -1;
3021  }
3022 
3023  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3024 
3025  return 0;
3026 }
3027 
3028 static void HTPStateTruncate(void *state, uint8_t direction)
3029 {
3030  FileContainer *fc = HTPStateGetFiles(state, direction);
3031  if (fc != NULL) {
3033  }
3034 }
3035 
3036 static DetectEngineState *HTPGetTxDetectState(void *vtx)
3037 {
3038  htp_tx_t *tx = (htp_tx_t *)vtx;
3039  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3040  return tx_ud ? tx_ud->de_state : NULL;
3041 }
3042 
3043 static int HTPSetTxDetectState(void *vtx, DetectEngineState *s)
3044 {
3045  htp_tx_t *tx = (htp_tx_t *)vtx;
3046  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3047  if (tx_ud == NULL) {
3048  tx_ud = HTPMalloc(sizeof(*tx_ud));
3049  if (unlikely(tx_ud == NULL))
3050  return -ENOMEM;
3051  memset(tx_ud, 0, sizeof(*tx_ud));
3052  htp_tx_set_user_data(tx, tx_ud);
3053  }
3054  tx_ud->de_state = s;
3055  return 0;
3056 }
3057 
3058 static uint64_t HTPGetTxDetectFlags(void *vtx, uint8_t dir)
3059 {
3060  htp_tx_t *tx = (htp_tx_t *)vtx;
3061  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3062  if (tx_ud) {
3063  if (dir & STREAM_TOSERVER) {
3064  return tx_ud->detect_flags_ts;
3065  } else {
3066  return tx_ud->detect_flags_tc;
3067  }
3068  }
3069  return 0;
3070 }
3071 
3072 static void HTPSetTxDetectFlags(void *vtx, uint8_t dir, uint64_t detect_flags)
3073 {
3074  htp_tx_t *tx = (htp_tx_t *)vtx;
3075  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3076  if (tx_ud == NULL) {
3077  tx_ud = HTPMalloc(sizeof(*tx_ud));
3078  if (unlikely(tx_ud == NULL))
3079  return;
3080  memset(tx_ud, 0, sizeof(*tx_ud));
3081  htp_tx_set_user_data(tx, tx_ud);
3082  }
3083  if (dir & STREAM_TOSERVER) {
3084  tx_ud->detect_flags_ts = detect_flags;
3085  } else {
3086  tx_ud->detect_flags_tc = detect_flags;
3087  }
3088  return;
3089 }
3090 
3091 static int HTPRegisterPatternsForProtocolDetection(void)
3092 {
3093  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3094  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3095  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3096  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3097  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3098  const char *spacings[] = { "|20|", "|09|", NULL };
3099  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3100 
3101  int methods_pos;
3102  int spacings_pos;
3103  int versions_pos;
3104  int register_result;
3105  char method_buffer[32] = "";
3106 
3107  /* Loop through all the methods ands spacings and register the patterns */
3108  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3109  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3110 
3111  /* Combine the method name and the spacing */
3112  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3113 
3114  /* Register the new method+spacing pattern
3115  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3116  * but the pattern matching should only be one char
3117  */
3118  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
3119  ALPROTO_HTTP, method_buffer, strlen(method_buffer)-3, 0, STREAM_TOSERVER);
3120  if (register_result < 0) {
3121  return -1;
3122  }
3123  }
3124  }
3125 
3126  /* Loop through all the http verions patterns that are TO_CLIENT */
3127  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3128  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
3129  ALPROTO_HTTP, versions[versions_pos], strlen(versions[versions_pos]),
3130  0, STREAM_TOCLIENT);
3131  if (register_result < 0) {
3132  return -1;
3133  }
3134  }
3135 
3136  return 0;
3137 }
3138 
3139 /**
3140  * \brief Register the HTTP protocol and state handling functions to APP layer
3141  * of the engine.
3142  */
3144 {
3145  SCEnter();
3146 
3147  const char *proto_name = "http";
3148 
3149  /** HTTP */
3150  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3152  if (HTPRegisterPatternsForProtocolDetection() < 0)
3153  return;
3154  } else {
3155  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3156  proto_name);
3157  return;
3158  }
3159 
3160  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3161  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateAlloc, HTPStateFree);
3162  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTransactionFree);
3163  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetFiles);
3164  AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetAlstateProgress);
3165  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxCnt);
3166  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTx);
3167  AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxLogged,
3168  HTPStateSetTxLogged);
3170  HTPStateGetAlstateProgressCompletionStatus);
3171  AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPGetEvents);
3172  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfo);
3173  AppLayerParserRegisterGetEventInfoById(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfoById);
3174 
3175  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTruncate);
3177  HTPGetTxDetectState, HTPSetTxDetectState);
3179  HTPGetTxDetectFlags, HTPSetTxDetectFlags);
3180 
3182  AppLayerHtpSetStreamDepthFlag);
3183 
3185  HTPHandleRequestData);
3187  HTPHandleResponseData);
3188  SC_ATOMIC_INIT(htp_config_flags);
3191  HTPConfigure();
3192  } else {
3193  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3194  "still on.", proto_name);
3195  }
3196 #ifdef UNITTESTS
3198 #endif
3199 
3200  SCReturn;
3201 }
3202 
3203 #ifdef UNITTESTS
3204 static HTPCfgRec cfglist_backup;
3205 
3207 {
3208  cfglist_backup = cfglist;
3209 
3210  return;
3211 }
3212 
3214 {
3215  cfglist = cfglist_backup;
3216 
3217  return;
3218 }
3219 
3220 /** \test Test case where chunks are sent in smaller chunks and check the
3221  * response of the parser from HTP library. */
3222 static int HTPParserTest01(void)
3223 {
3224  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3225  " Data is c0oL!";
3226  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3227 
3228  TcpSession ssn;
3229  memset(&ssn, 0, sizeof(ssn));
3230 
3232  FAIL_IF_NULL(alp_tctx);
3233 
3234  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3235  FAIL_IF_NULL(f);
3236  f->protoctx = &ssn;
3237  f->proto = IPPROTO_TCP;
3238  f->alproto = ALPROTO_HTTP;
3239 
3241 
3242  uint32_t u;
3243  for (u = 0; u < httplen1; u++) {
3244  uint8_t flags = 0;
3245 
3246  if (u == 0)
3247  flags = STREAM_TOSERVER|STREAM_START;
3248  else if (u == (httplen1 - 1))
3249  flags = STREAM_TOSERVER|STREAM_EOF;
3250  else
3251  flags = STREAM_TOSERVER;
3252 
3253  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3254  &httpbuf1[u], 1);
3255  FAIL_IF(r != 0);
3256  }
3257 
3258  HtpState *htp_state = f->alstate;
3259  FAIL_IF_NULL(htp_state);
3260 
3261  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3262  FAIL_IF_NULL(tx);
3263 
3264  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3265  FAIL_IF_NULL(h);
3266 
3267  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3268  FAIL_IF(tx->request_method_number != HTP_M_POST);
3269  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3270 
3271  AppLayerParserThreadCtxFree(alp_tctx);
3273  UTHFreeFlow(f);
3274  PASS;
3275 }
3276 
3277 /** \test Test folding in 1 read case */
3278 static int HTPParserTest01b(void)
3279 {
3280  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3281  " Data is c0oL!";
3282  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3283 
3284  TcpSession ssn;
3285  memset(&ssn, 0, sizeof(ssn));
3286 
3288  FAIL_IF_NULL(alp_tctx);
3289 
3290  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3291  FAIL_IF_NULL(f);
3292  f->protoctx = &ssn;
3293  f->proto = IPPROTO_TCP;
3294  f->alproto = ALPROTO_HTTP;
3295 
3297 
3299  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3300  httpbuf1, httplen1);
3301  FAIL_IF(r != 0);
3302 
3303  HtpState *htp_state = f->alstate;
3304  FAIL_IF_NULL(htp_state);
3305 
3306  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3307  FAIL_IF_NULL(tx);
3308 
3309  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3310  FAIL_IF_NULL(h);
3311 
3312  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3313  FAIL_IF(tx->request_method_number != HTP_M_POST);
3314  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3315 
3316  AppLayerParserThreadCtxFree(alp_tctx);
3318  UTHFreeFlow(f);
3319  PASS;
3320 }
3321 
3322 /** \test Test folding in 1byte per read case */
3323 static int HTPParserTest01c(void)
3324 {
3325  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3326  " Data is c0oL!";
3327  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3328 
3329  TcpSession ssn;
3330  memset(&ssn, 0, sizeof(ssn));
3331 
3333  FAIL_IF_NULL(alp_tctx);
3334 
3335  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3336  FAIL_IF_NULL(f);
3337  f->protoctx = &ssn;
3338  f->proto = IPPROTO_TCP;
3339  f->alproto = ALPROTO_HTTP;
3340 
3342 
3343  uint32_t u;
3344  for (u = 0; u < httplen1; u++) {
3345  uint8_t flags = 0;
3346 
3347  if (u == 0)
3348  flags = STREAM_TOSERVER|STREAM_START;
3349  else if (u == (httplen1 - 1))
3350  flags = STREAM_TOSERVER|STREAM_EOF;
3351  else
3352  flags = STREAM_TOSERVER;
3353 
3354  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3355  &httpbuf1[u], 1);
3356  FAIL_IF(r != 0);
3357  }
3358 
3359  HtpState *htp_state = f->alstate;
3360  FAIL_IF_NULL(htp_state);
3361 
3362  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3363  FAIL_IF_NULL(tx);
3364 
3365  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3366  FAIL_IF_NULL(h);
3367 
3368  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3369  FAIL_IF(tx->request_method_number != HTP_M_POST);
3370  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3371 
3372  AppLayerParserThreadCtxFree(alp_tctx);
3374  UTHFreeFlow(f);
3375  PASS;
3376 }
3377 
3378 /** \test Test case where chunks are sent in smaller chunks and check the
3379  * response of the parser from HTP library. */
3380 static int HTPParserTest01a(void)
3381 {
3382  int result = 0;
3383  Flow *f = NULL;
3384  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3385  " Data is c0oL!";
3386  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3387  TcpSession ssn;
3388  HtpState *htp_state = NULL;
3389  int r = 0;
3391 
3392  memset(&ssn, 0, sizeof(ssn));
3393 
3394  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3395  if (f == NULL)
3396  goto end;
3397  f->protoctx = &ssn;
3398  f->proto = IPPROTO_TCP;
3399  f->alproto = ALPROTO_HTTP;
3400 
3402 
3403  uint32_t u;
3404  for (u = 0; u < httplen1; u++) {
3405  uint8_t flags = 0;
3406 
3407  if (u == 0)
3408  flags = STREAM_TOSERVER|STREAM_START;
3409  else if (u == (httplen1 - 1))
3410  flags = STREAM_TOSERVER|STREAM_EOF;
3411  else
3412  flags = STREAM_TOSERVER;
3413 
3414  FLOWLOCK_WRLOCK(f);
3415  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3416  &httpbuf1[u], 1);
3417  if (r != 0) {
3418  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3419  " 0: ", u, r);
3420  FLOWLOCK_UNLOCK(f);
3421  goto end;
3422  }
3423  FLOWLOCK_UNLOCK(f);
3424  }
3425 
3426  htp_state = f->alstate;
3427  if (htp_state == NULL) {
3428  printf("no http state: ");
3429  goto end;
3430  }
3431 
3432  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3433  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3434  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3435  || tx->request_method_number != HTP_M_POST ||
3436  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3437  {
3438  printf("expected header value: Victor/1.0 and got %s: and expected"
3439  " method: POST and got %s, expected protocol number HTTP/1.0"
3440  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3441  bstr_util_strdup_to_c(tx->request_method),
3442  bstr_util_strdup_to_c(tx->request_protocol));
3443  goto end;
3444  }
3445  result = 1;
3446 end:
3447  if (alp_tctx != NULL)
3448  AppLayerParserThreadCtxFree(alp_tctx);
3450  UTHFreeFlow(f);
3451  return result;
3452 }
3453 
3454 /** \test See how it deals with an incomplete request. */
3455 static int HTPParserTest02(void)
3456 {
3457  int result = 0;
3458  Flow *f = NULL;
3459  uint8_t httpbuf1[] = "POST";
3460  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3461  TcpSession ssn;
3462  HtpState *http_state = NULL;
3464 
3465  memset(&ssn, 0, sizeof(ssn));
3466 
3467  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3468  if (f == NULL)
3469  goto end;
3470  f->protoctx = &ssn;
3471  f->proto = IPPROTO_TCP;
3472  f->alproto = ALPROTO_HTTP;
3473 
3475 
3476  FLOWLOCK_WRLOCK(f);
3477  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3479  httpbuf1,
3480  httplen1);
3481  if (r != 0) {
3482  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3483  FLOWLOCK_UNLOCK(f);
3484  goto end;
3485  }
3486  FLOWLOCK_UNLOCK(f);
3487 
3488  http_state = f->alstate;
3489  if (http_state == NULL) {
3490  printf("no http state: ");
3491  goto end;
3492  }
3493 
3494  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3495  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3496  if ((tx->request_method) != NULL || h != NULL)
3497  {
3498  printf("expected method NULL, got %s \n", bstr_util_strdup_to_c(tx->request_method));
3499  goto end;
3500  }
3501  result = 1;
3502 end:
3503  if (alp_tctx != NULL)
3504  AppLayerParserThreadCtxFree(alp_tctx);
3506  UTHFreeFlow(f);
3507  return result;
3508 }
3509 
3510 /** \test Test case where method is invalid and data is sent in smaller chunks
3511  * and check the response of the parser from HTP library. */
3512 static int HTPParserTest03(void)
3513 {
3514  int result = 0;
3515  Flow *f = NULL;
3516  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3517  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3518  TcpSession ssn;
3519  HtpState *htp_state = NULL;
3520  int r = 0;
3522 
3523  memset(&ssn, 0, sizeof(ssn));
3524 
3525  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3526  if (f == NULL)
3527  goto end;
3528  f->protoctx = &ssn;
3529  f->proto = IPPROTO_TCP;
3530  f->alproto = ALPROTO_HTTP;
3531 
3533 
3534  uint32_t u;
3535  for (u = 0; u < httplen1; u++) {
3536  uint8_t flags = 0;
3537 
3538  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3539  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3540  else flags = STREAM_TOSERVER;
3541 
3542  FLOWLOCK_WRLOCK(f);
3543  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3544  &httpbuf1[u], 1);
3545  if (r != 0) {
3546  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3547  " 0: ", u, r);
3548  FLOWLOCK_UNLOCK(f);
3549  goto end;
3550  }
3551  FLOWLOCK_UNLOCK(f);
3552  }
3553  htp_state = f->alstate;
3554  if (htp_state == NULL) {
3555  printf("no http state: ");
3556  goto end;
3557  }
3558 
3559  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3560 
3561  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3562  if (tx->request_method_number != HTP_M_UNKNOWN ||
3563  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3564  {
3565  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3566  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3567  bstr_util_strdup_to_c(tx->request_protocol));
3568  goto end;
3569  }
3570  result = 1;
3571 end:
3572  if (alp_tctx != NULL)
3573  AppLayerParserThreadCtxFree(alp_tctx);
3575  UTHFreeFlow(f);
3576  return result;
3577 }
3578 
3579 /** \test Test case where invalid data is sent and check the response of the
3580  * parser from HTP library. */
3581 static int HTPParserTest04(void)
3582 {
3583  int result = 0;
3584  Flow *f = NULL;
3585  HtpState *htp_state = NULL;
3586  uint8_t httpbuf1[] = "World!\r\n";
3587  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3588  TcpSession ssn;
3589  int r = 0;
3591 
3592  memset(&ssn, 0, sizeof(ssn));
3593 
3594  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3595  if (f == NULL)
3596  goto end;
3597  f->protoctx = &ssn;
3598  f->proto = IPPROTO_TCP;
3599  f->alproto = ALPROTO_HTTP;
3600 
3602 
3603  FLOWLOCK_WRLOCK(f);
3604  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3606  httpbuf1,
3607  httplen1);
3608  if (r != 0) {
3609  FLOWLOCK_UNLOCK(f);
3610  goto end;
3611  }
3612  FLOWLOCK_UNLOCK(f);
3613 
3614  htp_state = f->alstate;
3615  if (htp_state == NULL) {
3616  printf("no http state: ");
3617  goto end;
3618  }
3619 
3620  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3621  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3622  if (tx->request_method_number != HTP_M_UNKNOWN ||
3623  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3624  {
3625  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3626  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3627  bstr_util_strdup_to_c(tx->request_protocol));
3628  goto end;
3629  }
3630  result = 1;
3631 end:
3632  if (alp_tctx != NULL)
3633  AppLayerParserThreadCtxFree(alp_tctx);
3635  UTHFreeFlow(f);
3636  return result;
3637 }
3638 
3639 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3640  * properly parsed them and also keeps them separated. */
3641 static int HTPParserTest05(void)
3642 {
3643  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3644  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3645  uint8_t httpbuf2[] = "Post D";
3646  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3647  uint8_t httpbuf3[] = "ata is c0oL!";
3648  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3649 
3650  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3651  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3652  uint8_t httpbuf5[] = "post R";
3653  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3654  uint8_t httpbuf6[] = "esults are tha bomb!";
3655  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3656 
3657  TcpSession ssn;
3658  memset(&ssn, 0, sizeof(ssn));
3659 
3661  FAIL_IF_NULL(alp_tctx);
3662 
3663  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3664  FAIL_IF_NULL(f);
3665  f->protoctx = &ssn;
3666  f->proto = IPPROTO_TCP;
3667  f->alproto = ALPROTO_HTTP;
3668 
3670 
3671  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3672  STREAM_TOSERVER | STREAM_START, httpbuf1,
3673  httplen1);
3674  FAIL_IF(r != 0);
3675 
3676  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3677  STREAM_TOCLIENT | STREAM_START, httpbuf4,
3678  httplen4);
3679  FAIL_IF(r != 0);
3680 
3681  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOCLIENT,
3682  httpbuf5, httplen5);
3683  FAIL_IF(r != 0);
3684 
3685  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOSERVER,
3686  httpbuf2, httplen2);
3687  FAIL_IF(r != 0);
3688 
3689  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3690  STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3691  FAIL_IF(r != 0);
3692 
3693  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3694  STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3695  FAIL_IF(r != 0);
3696 
3697  HtpState *http_state = f->alstate;
3698  FAIL_IF_NULL(http_state);
3699 
3700  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3701  FAIL_IF_NULL(tx);
3702  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3703  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3704 
3705  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3706  FAIL_IF_NULL(h);
3707 
3708  FAIL_IF_NOT(tx->response_status_number == 200);
3709 
3710  AppLayerParserThreadCtxFree(alp_tctx);
3712  UTHFreeFlow(f);
3713  PASS;
3714 }
3715 
3716 /** \test Test proper chunked encoded response body
3717  */
3718 static int HTPParserTest06(void)
3719 {
3720  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3721  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3722  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3723  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3724  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3725  "GMT\r\n"
3726  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3727  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3728  "FrontPage/5.0.2.2510\r\n"
3729  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3730  "chunked\r\n"
3731  "Content-Type: text/html\r\n\r\n"
3732  "580\r\n"
3733  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3734  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3735  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3736  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3737  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3738  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3739  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3740  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3741  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3742  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3743  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3744  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3745  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3746  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3747  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3748  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3749  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3750  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3751  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3752  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3753  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3754  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3755  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3756  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3757  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3758  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3759  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3760  "aHA=\r\n0\r\n\r\n";
3761  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3762  TcpSession ssn;
3763 
3765  FAIL_IF_NULL(alp_tctx);
3766 
3767  memset(&ssn, 0, sizeof(ssn));
3768 
3769  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3770  FAIL_IF_NULL(f);
3771  f->protoctx = &ssn;
3772  f->proto = IPPROTO_TCP;
3773  f->alproto = ALPROTO_HTTP;
3774 
3776 
3777  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3778  STREAM_TOSERVER | STREAM_START, httpbuf1,
3779  httplen1);
3780  FAIL_IF(r != 0);
3781  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3782  STREAM_TOCLIENT | STREAM_START, httpbuf2,
3783  httplen2);
3784  FAIL_IF(r != 0);
3785 
3786  HtpState *http_state = f->alstate;
3787  FAIL_IF_NULL(http_state);
3788 
3789  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3790  FAIL_IF_NULL(tx);
3791 
3792  FAIL_IF(tx->request_method_number != HTP_M_GET);
3793  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3794 
3795  FAIL_IF(tx->response_status_number != 200);
3796  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3797 
3798  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3799  FAIL_IF_NULL(h);
3800 
3801  AppLayerParserThreadCtxFree(alp_tctx);
3803  UTHFreeFlow(f);
3804  PASS;
3805 }
3806 
3807 /** \test
3808  */
3809 static int HTPParserTest07(void)
3810 {
3811  int result = 0;
3812  Flow *f = NULL;
3813  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3814  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3815  TcpSession ssn;
3816  HtpState *htp_state = NULL;
3817  int r = 0;
3819 
3820  memset(&ssn, 0, sizeof(ssn));
3821 
3822  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3823  if (f == NULL)
3824  goto end;
3825  f->protoctx = &ssn;
3826  f->proto = IPPROTO_TCP;
3827  f->alproto = ALPROTO_HTTP;
3828 
3830 
3831  uint32_t u;
3832  for (u = 0; u < httplen1; u++) {
3833  uint8_t flags = 0;
3834 
3835  if (u == 0)
3836  flags = STREAM_TOSERVER|STREAM_START;
3837  else if (u == (httplen1 - 1))
3838  flags = STREAM_TOSERVER|STREAM_EOF;
3839  else
3840  flags = STREAM_TOSERVER;
3841 
3842  FLOWLOCK_WRLOCK(f);
3843  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3844  &httpbuf1[u], 1);
3845  if (r != 0) {
3846  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3847  " 0: ", u, r);
3848  FLOWLOCK_UNLOCK(f);
3849  goto end;
3850  }
3851  FLOWLOCK_UNLOCK(f);
3852  }
3853 
3854  htp_state = f->alstate;
3855  if (htp_state == NULL) {
3856  printf("no http state: ");
3857  goto end;
3858  }
3859 
3860  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3861  size_t reflen = sizeof(ref) - 1;
3862 
3863  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3864  if (tx == NULL)
3865  goto end;
3866  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3867  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3868  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3869  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3870  (uintmax_t)reflen,
3871  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3872  goto end;
3873  }
3874 
3875  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3876  bstr_len(tx_ud->request_uri_normalized)) != 0)
3877  {
3878  printf("normalized uri \"");
3879  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3880  printf("\" != \"");
3881  PrintRawUriFp(stdout, ref, reflen);
3882  printf("\": ");
3883  goto end;
3884  }
3885  }
3886 
3887  result = 1;
3888 end:
3889  if (alp_tctx != NULL)
3890  AppLayerParserThreadCtxFree(alp_tctx);
3892  UTHFreeFlow(f);
3893  return result;
3894 }
3895 
3896 #include "conf-yaml-loader.h"
3897 
3898 /** \test Abort
3899  */
3900 static int HTPParserTest08(void)
3901 {
3902  int result = 0;
3903  Flow *f = NULL;
3904  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3905  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3906  TcpSession ssn;
3908 
3909  char input[] = "\
3910 %YAML 1.1\n\
3911 ---\n\
3912 libhtp:\n\
3913 \n\
3914  default-config:\n\
3915  personality: IDS\n\
3916 ";
3917 
3919  ConfInit();
3921 
3922  ConfYamlLoadString(input, strlen(input));
3923  HTPConfigure();
3924 
3925  HtpState *htp_state = NULL;
3926  int r = 0;
3927  memset(&ssn, 0, sizeof(ssn));
3928 
3929  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3930  if (f == NULL)
3931  goto end;
3932  f->protoctx = &ssn;
3933  f->proto = IPPROTO_TCP;
3934  f->alproto = ALPROTO_HTTP;
3935 
3937 
3938  uint8_t flags = 0;
3940 
3941  FLOWLOCK_WRLOCK(f);
3942  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
3943  httplen1);
3944  if (r != 0) {
3945  printf("toserver chunk returned %" PRId32 ", expected"
3946  " 0: ", r);
3947  result = 0;
3948  FLOWLOCK_UNLOCK(f);
3949  goto end;
3950  }
3951  FLOWLOCK_UNLOCK(f);
3952 
3953  htp_state = f->alstate;
3954  if (htp_state == NULL) {
3955  printf("no http state: ");
3956  result = 0;
3957  goto end;
3958  }
3959 
3960  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3961  if (tx == NULL)
3962  goto end;
3963  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3964  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3965  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3966  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3967  bstr_len(tx_ud->request_uri_normalized));
3968  }
3969 
3970  result = 1;
3971 end:
3972  if (alp_tctx != NULL)
3973  AppLayerParserThreadCtxFree(alp_tctx);
3975  HTPFreeConfig();
3976  ConfDeInit();
3979  UTHFreeFlow(f);
3980  return result;
3981 }
3982 
3983 /** \test Abort
3984  */
3985 static int HTPParserTest09(void)
3986 {
3987  int result = 0;
3988  Flow *f = NULL;
3989  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3990  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3991  TcpSession ssn;
3993 
3994  char input[] = "\
3995 %YAML 1.1\n\
3996 ---\n\
3997 libhtp:\n\
3998 \n\
3999  default-config:\n\
4000  personality: Apache_2_2\n\
4001 ";
4002 
4004  ConfInit();
4006 
4007  ConfYamlLoadString(input, strlen(input));
4008  HTPConfigure();
4009 
4010  HtpState *htp_state = NULL;
4011  int r = 0;
4012 
4013  memset(&ssn, 0, sizeof(ssn));
4014 
4015  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4016  if (f == NULL)
4017  goto end;
4018  f->protoctx = &ssn;
4019  f->proto = IPPROTO_TCP;
4020  f->alproto = ALPROTO_HTTP;
4021 
4023 
4024  uint8_t flags = 0;
4026 
4027  FLOWLOCK_WRLOCK(f);
4028  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
4029  httplen1);
4030  if (r != 0) {
4031  printf("toserver chunk returned %" PRId32 ", expected"
4032  " 0: ", r);
4033  FLOWLOCK_UNLOCK(f);
4034  goto end;
4035  }
4036  FLOWLOCK_UNLOCK(f);
4037 
4038  htp_state = f->alstate;
4039  if (htp_state == NULL) {
4040  printf("no http state: ");
4041  goto end;
4042  }
4043 
4044  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4045  if (tx == NULL)
4046  goto end;
4047  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4048  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4049  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4050  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4051  bstr_len(tx_ud->request_uri_normalized));
4052  }
4053 
4054  result = 1;
4055 end:
4056  if (alp_tctx != NULL)
4057  AppLayerParserThreadCtxFree(alp_tctx);
4059  HTPFreeConfig();
4060  ConfDeInit();
4063  UTHFreeFlow(f);
4064  return result;
4065 }
4066 
4067 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4068  */
4069 static int HTPParserTest10(void)
4070 {
4071  int result = 0;
4072  Flow *f = NULL;
4073  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4074  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4075  TcpSession ssn;
4076  HtpState *htp_state = NULL;
4077  int r = 0;
4079 
4080  memset(&ssn, 0, sizeof(ssn));
4081 
4082  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4083  if (f == NULL)
4084  goto end;
4085  f->protoctx = &ssn;
4086  f->proto = IPPROTO_TCP;
4087  f->alproto = ALPROTO_HTTP;
4088 
4090 
4091  uint32_t u;
4092  for (u = 0; u < httplen1; u++) {
4093  uint8_t flags = 0;
4094 
4095  if (u == 0)
4096  flags = STREAM_TOSERVER|STREAM_START;
4097  else if (u == (httplen1 - 1))
4098  flags = STREAM_TOSERVER|STREAM_EOF;
4099  else
4100  flags = STREAM_TOSERVER;
4101 
4102  FLOWLOCK_WRLOCK(f);
4103  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4104  &httpbuf1[u], 1);
4105  if (r != 0) {
4106  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4107  " 0: ", u, r);
4108  FLOWLOCK_UNLOCK(f);
4109  goto end;
4110  }
4111  FLOWLOCK_UNLOCK(f);
4112  }
4113 
4114  htp_state = f->alstate;
4115  if (htp_state == NULL) {
4116  printf("no http state: ");
4117  goto end;
4118  }
4119 
4120  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4121  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4122  if (h == NULL) {
4123  goto end;
4124  }
4125 
4126  char *name = bstr_util_strdup_to_c(h->name);
4127  if (name == NULL) {
4128  goto end;
4129  }
4130 
4131  if (strcmp(name, "Host") != 0) {
4132  printf("header name not \"Host\", instead \"%s\": ", name);
4133  free(name);
4134  goto end;
4135  }
4136  free(name);
4137 
4138  char *value = bstr_util_strdup_to_c(h->value);
4139  if (value == NULL) {
4140  goto end;
4141  }
4142 
4143  if (strcmp(value, "www.google.com") != 0) {
4144  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4145  free(value);
4146  goto end;
4147  }
4148  free(value);
4149 
4150  result = 1;
4151 end:
4152  if (alp_tctx != NULL)
4153  AppLayerParserThreadCtxFree(alp_tctx);
4155  UTHFreeFlow(f);
4156  return result;
4157 }
4158 
4159 /** \test double encoding in path
4160  */
4161 static int HTPParserTest11(void)
4162 {
4163  int result = 0;
4164  Flow *f = NULL;
4165  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4166  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4167  TcpSession ssn;
4168  HtpState *htp_state = NULL;
4169  int r = 0;
4171 
4172  memset(&ssn, 0, sizeof(ssn));
4173 
4174  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4175  if (f == NULL)
4176  goto end;
4177  f->protoctx = &ssn;
4178  f->proto = IPPROTO_TCP;
4179  f->alproto = ALPROTO_HTTP;
4180 
4182 
4183  uint32_t u;
4184  for (u = 0; u < httplen1; u++) {
4185  uint8_t flags = 0;
4186 
4187  if (u == 0)
4188  flags = STREAM_TOSERVER|STREAM_START;
4189  else if (u == (httplen1 - 1))
4190  flags = STREAM_TOSERVER|STREAM_EOF;
4191  else
4192  flags = STREAM_TOSERVER;
4193 
4194  FLOWLOCK_WRLOCK(f);
4195  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4196  &httpbuf1[u], 1);
4197  if (r != 0) {
4198  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4199  " 0: ", u, r);
4200  FLOWLOCK_UNLOCK(f);
4201  goto end;
4202  }
4203  FLOWLOCK_UNLOCK(f);
4204  }
4205 
4206  htp_state = f->alstate;
4207  if (htp_state == NULL) {
4208  printf("no http state: ");
4209  goto end;
4210  }
4211 
4212  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4213  if (tx == NULL)
4214  goto end;
4215  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4216  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4217  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4218  printf("normalized uri len should be 2, is %"PRIuMAX,
4219  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4220  goto end;
4221  }
4222 
4223  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4224  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4225  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4226  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4227  {
4228  printf("normalized uri \"");
4229  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4230  printf("\": ");
4231  goto end;
4232  }
4233  }
4234 
4235  result = 1;
4236 end:
4237  if (alp_tctx != NULL)
4238  AppLayerParserThreadCtxFree(alp_tctx);
4240  UTHFreeFlow(f);
4241  return result;
4242 }
4243 
4244 /** \test double encoding in query
4245  */
4246 static int HTPParserTest12(void)
4247 {
4248  int result = 0;
4249  Flow *f = NULL;
4250  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4251  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4252  TcpSession ssn;
4253  HtpState *htp_state = NULL;
4254  int r = 0;
4256 
4257  memset(&ssn, 0, sizeof(ssn));
4258 
4259  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4260  if (f == NULL)
4261  goto end;
4262  f->protoctx = &ssn;
4263  f->proto = IPPROTO_TCP;
4264  f->alproto = ALPROTO_HTTP;
4265 
4267 
4268  uint32_t u;
4269  for (u = 0; u < httplen1; u++) {
4270  uint8_t flags = 0;
4271 
4272  if (u == 0)
4273  flags = STREAM_TOSERVER|STREAM_START;
4274  else if (u == (httplen1 - 1))
4275  flags = STREAM_TOSERVER|STREAM_EOF;
4276  else
4277  flags = STREAM_TOSERVER;
4278 
4279  FLOWLOCK_WRLOCK(f);
4280  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4281  &httpbuf1[u], 1);
4282  if (r != 0) {
4283  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4284  " 0: ", u, r);
4285  FLOWLOCK_UNLOCK(f);
4286  goto end;
4287  }
4288  FLOWLOCK_UNLOCK(f);
4289  }
4290 
4291  htp_state = f->alstate;
4292  if (htp_state == NULL) {
4293  printf("no http state: ");
4294  goto end;
4295  }
4296 
4297  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4298  if (tx == NULL)
4299  goto end;
4300  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4301  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4302  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4303  printf("normalized uri len should be 5, is %"PRIuMAX,
4304  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4305  goto end;
4306  }
4307 
4308  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4309  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4310  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4311  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4312  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4313  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4314  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4315  {
4316  printf("normalized uri \"");
4317  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4318  printf("\": ");
4319  goto end;
4320  }
4321  }
4322 
4323  result = 1;
4324  end:
4325  if (alp_tctx != NULL)
4326  AppLayerParserThreadCtxFree(alp_tctx);
4328  UTHFreeFlow(f);
4329  return result;
4330 }
4331 
4332 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4333  */
4334 static int HTPParserTest13(void)
4335 {
4336  int result = 0;
4337  Flow *f = NULL;
4338  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4339  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4340  TcpSession ssn;
4341  HtpState *htp_state = NULL;
4342  int r = 0;
4344 
4345  memset(&ssn, 0, sizeof(ssn));
4346 
4347  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4348  if (f == NULL)
4349  goto end;
4350  f->protoctx = &ssn;
4351  f->proto = IPPROTO_TCP;
4352  f->alproto = ALPROTO_HTTP;
4353 
4355 
4356  uint32_t u;
4357  for (u = 0; u < httplen1; u++) {
4358  uint8_t flags = 0;
4359 
4360  if (u == 0)
4361  flags = STREAM_TOSERVER|STREAM_START;
4362  else if (u == (httplen1 - 1))
4363  flags = STREAM_TOSERVER|STREAM_EOF;
4364  else
4365  flags = STREAM_TOSERVER;
4366 
4367  FLOWLOCK_WRLOCK(f);
4368  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4369  &httpbuf1[u], 1);
4370  if (r != 0) {
4371  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4372  " 0: ", u, r);
4373  FLOWLOCK_UNLOCK(f);
4374  goto end;
4375  }
4376  FLOWLOCK_UNLOCK(f);
4377  }
4378 
4379  htp_state = f->alstate;
4380  if (htp_state == NULL) {
4381  printf("no http state: ");
4382  goto end;
4383  }
4384 
4385  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4386  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4387  if (h == NULL) {
4388  goto end;
4389  }
4390 
4391  char *name = bstr_util_strdup_to_c(h->name);
4392  if (name == NULL) {
4393  goto end;
4394  }
4395 
4396  if (strcmp(name, "Host") != 0) {
4397  printf("header name not \"Host\", instead \"%s\": ", name);
4398  free(name);
4399  goto end;
4400  }
4401  free(name);
4402 
4403  char *value = bstr_util_strdup_to_c(h->value);
4404  if (value == NULL) {
4405  goto end;
4406  }
4407 
4408  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4409  printf("header value not \"www.google.com\", instead \"");
4410  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4411  printf("\": ");
4412  free(value);
4413  goto end;
4414  }
4415  free(value);
4416 
4417  result = 1;
4418 end:
4419  if (alp_tctx != NULL)
4420  AppLayerParserThreadCtxFree(alp_tctx);
4422  UTHFreeFlow(f);
4423  return result;
4424 }
4425 
4426 /** \test Test basic config */
4427 static int HTPParserConfigTest01(void)
4428 {
4429  int ret = 0;
4430  char input[] = "\
4431 %YAML 1.1\n\
4432 ---\n\
4433 libhtp:\n\
4434 \n\
4435  default-config:\n\
4436  personality: IDS\n\
4437 \n\
4438  server-config:\n\
4439 \n\
4440  - apache-tomcat:\n\
4441  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4442  personality: Tomcat_6_0\n\
4443 \n\
4444  - iis7:\n\
4445  address: \n\
4446  - 192.168.0.0/24\n\
4447  - 192.168.10.0/24\n\
4448  personality: IIS_7_0\n\
4449 ";
4450 
4452  ConfInit();
4453 
4454  ConfYamlLoadString(input, strlen(input));
4455 
4456  ConfNode *outputs;
4457  outputs = ConfGetNode("libhtp.default-config.personality");
4458  if (outputs == NULL) {
4459  goto end;
4460  }
4461 
4462  outputs = ConfGetNode("libhtp.server-config");
4463  if (outputs == NULL) {
4464  goto end;
4465  }
4466 
4467  ConfNode *node = TAILQ_FIRST(&outputs->head);
4468  if (node == NULL) {
4469  goto end;
4470  }
4471  if (strcmp(node->name, "0") != 0) {
4472  goto end;
4473  }
4474  node = TAILQ_FIRST(&node->head);
4475  if (node == NULL) {
4476  goto end;
4477  }
4478  if (strcmp(node->name, "apache-tomcat") != 0) {
4479  goto end;
4480  }
4481 
4482  int i = 0;
4483  ConfNode *n;
4484 
4485  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4486  if (node2 == NULL) {
4487  goto end;
4488  }
4489  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4490  goto end;
4491  }
4492 
4493  node = ConfNodeLookupChild(node, "address");
4494  if (node == NULL) {
4495  goto end;
4496  }
4497  TAILQ_FOREACH(n, &node->head, next) {
4498  if (n == NULL) {
4499  goto end;
4500  }
4501 
4502  switch(i) {
4503  case 0:
4504  if (strcmp(n->name, "0") != 0) {
4505  goto end;
4506  }
4507  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4508  goto end;
4509  }
4510  break;
4511  case 1:
4512  if (strcmp(n->name, "1") != 0) {
4513  goto end;
4514  }
4515  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4516  goto end;
4517  }
4518  break;
4519  case 2:
4520  if (strcmp(n->name, "2") != 0) {
4521  goto end;
4522  }
4523  if (strcmp(n->val, "::1") != 0) {
4524  goto end;
4525  }
4526  break;
4527  default:
4528  goto end;
4529  }
4530  i++;
4531  }
4532 
4533  outputs = ConfGetNode("libhtp.server-config");
4534  if (outputs == NULL) {
4535  goto end;
4536  }
4537 
4538  node = TAILQ_FIRST(&outputs->head);
4539  node = TAILQ_NEXT(node, next);
4540  if (node == NULL) {
4541  goto end;
4542  }
4543  if (strcmp(node->name, "1") != 0) {
4544  goto end;
4545  }
4546  node = TAILQ_FIRST(&node->head);
4547  if (node == NULL) {
4548  goto end;
4549  }
4550  if (strcmp(node->name, "iis7") != 0) {
4551  goto end;
4552  }
4553 
4554  node2 = ConfNodeLookupChild(node, "personality");
4555  if (node2 == NULL) {
4556  goto end;
4557  }
4558  if (strcmp(node2->val, "IIS_7_0") != 0) {
4559  goto end;
4560  }
4561 
4562  node = ConfNodeLookupChild(node, "address");
4563  if (node == NULL) {
4564  goto end;
4565  }
4566 
4567  i = 0;
4568  TAILQ_FOREACH(n, &node->head, next) {
4569  if (n == NULL) {
4570  goto end;
4571  }
4572 
4573  switch(i) {
4574  case 0:
4575  if (strcmp(n->name, "0") != 0) {
4576  goto end;
4577  }
4578  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4579  goto end;
4580  }
4581  break;
4582  case 1:
4583  if (strcmp(n->name, "1") != 0) {
4584  goto end;
4585  }
4586  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4587  goto end;
4588  }
4589  break;
4590  default:
4591  goto end;
4592  }
4593  i++;
4594  }
4595 
4596  ret = 1;
4597 
4598 end:
4599  ConfDeInit();
4601 
4602  return ret;
4603 }
4604 
4605 /** \test Test config builds radix correctly */
4606 static int HTPParserConfigTest02(void)
4607 {
4608  int ret = 0;
4609  char input[] = "\
4610 %YAML 1.1\n\
4611 ---\n\
4612 libhtp:\n\
4613 \n\
4614  default-config:\n\
4615  personality: IDS\n\
4616 \n\
4617  server-config:\n\
4618 \n\
4619  - apache-tomcat:\n\
4620  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4621  personality: Tomcat_6_0\n\
4622 \n\
4623  - iis7:\n\
4624  address: \n\
4625  - 192.168.0.0/24\n\
4626  - 192.168.10.0/24\n\
4627  personality: IIS_7_0\n\
4628 ";
4629 
4631  ConfInit();
4633 
4634  ConfYamlLoadString(input, strlen(input));
4635 
4636  HTPConfigure();
4637 
4638  if (cfglist.cfg == NULL) {
4639  printf("No default config created.\n");
4640  goto end;
4641  }
4642 
4643  if (cfgtree == NULL) {
4644  printf("No config tree created.\n");
4645  goto end;
4646  }
4647 
4648  htp_cfg_t *htp = cfglist.cfg;
4649  uint8_t buf[128];
4650  const char *addr;
4651  void *user_data = NULL;
4652 
4653  addr = "192.168.10.42";
4654  if (inet_pton(AF_INET, addr, buf) == 1) {
4655  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4656  if (user_data != NULL) {
4657  HTPCfgRec *htp_cfg_rec = user_data;
4658  htp = htp_cfg_rec->cfg;
4659  SCLogDebug("LIBHTP using config: %p", htp);
4660  }
4661  if (htp == NULL) {
4662  printf("Could not get config for: %s\n", addr);
4663  goto end;
4664  }
4665  }
4666  else {
4667  printf("Failed to parse address: %s\n", addr);
4668  goto end;
4669  }
4670 
4671  user_data = NULL;
4672  addr = "::1";
4673  if (inet_pton(AF_INET6, addr, buf) == 1) {
4674  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4675  if (user_data != NULL) {
4676  HTPCfgRec *htp_cfg_rec = user_data;
4677  htp = htp_cfg_rec->cfg;
4678  SCLogDebug("LIBHTP using config: %p", htp);
4679  }
4680  if (htp == NULL) {
4681  printf("Could not get config for: %s\n", addr);
4682  goto end;
4683  }
4684  }
4685  else {
4686  printf("Failed to parse address: %s\n", addr);
4687  goto end;
4688  }
4689 
4690  ret = 1;
4691 
4692 end:
4693  HTPFreeConfig();
4694  ConfDeInit();
4697 
4698  return ret;
4699 }
4700 
4701 /** \test Test traffic is handled by the correct htp config */
4702 static int HTPParserConfigTest03(void)
4703 {
4704  int result = 1;
4705  Flow *f = NULL;
4706  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4707  " Data is c0oL!";
4708  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4709  TcpSession ssn;
4711 
4712  HtpState *htp_state = NULL;
4713  int r = 0;
4714  char input[] = "\
4715 %YAML 1.1\n\
4716 ---\n\
4717 libhtp:\n\
4718 \n\
4719  default-config:\n\
4720  personality: IDS\n\
4721 \n\
4722  server-config:\n\
4723 \n\
4724  - apache-tomcat:\n\
4725  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4726  personality: Tomcat_6_0\n\
4727 \n\
4728  - iis7:\n\
4729  address: \n\
4730  - 192.168.0.0/24\n\
4731  - 192.168.10.0/24\n\
4732  personality: IIS_7_0\n\
4733 ";
4734 
4736  ConfInit();
4738 
4739  ConfYamlLoadString(input, strlen(input));
4740 
4741  HTPConfigure();
4742 
4743  const char *addr = "192.168.10.42";
4744 
4745  memset(&ssn, 0, sizeof(ssn));
4746 
4747  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4748  if (f == NULL)
4749  goto end;
4750  f->protoctx = &ssn;
4751  f->proto = IPPROTO_TCP;
4752  f->alproto = ALPROTO_HTTP;
4753 
4754  htp_cfg_t *htp = cfglist.cfg;
4755 
4756  void *user_data = NULL;
4757  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4758  if (user_data != NULL) {
4759  HTPCfgRec *htp_cfg_rec = user_data;
4760  htp = htp_cfg_rec->cfg;
4761  SCLogDebug("LIBHTP using config: %p", htp);
4762  }
4763  if (htp == NULL) {
4764  printf("Could not get config for: %s\n", addr);
4765  goto end;
4766  }
4767 
4769 
4770  uint32_t u;
4771  for (u = 0; u < httplen1; u++) {
4772  uint8_t flags = 0;
4773 
4774  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4775  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4776  else flags = STREAM_TOSERVER;
4777 
4778  FLOWLOCK_WRLOCK(f);
4779  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4780  &httpbuf1[u], 1);
4781  if (r != 0) {
4782  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4783  " 0: ", u, r);
4784  result = 0;
4785  FLOWLOCK_UNLOCK(f);
4786  goto end;
4787  }
4788  FLOWLOCK_UNLOCK(f);
4789  }
4790 
4791  htp_state = f->alstate;
4792  if (htp_state == NULL) {
4793  printf("no http state: ");
4794  result = 0;
4795  goto end;
4796  }
4797 
4798  if (HTPStateGetTxCnt(htp_state) != 2) {
4799  printf("HTPStateGetTxCnt(htp_state) failure\n");
4800  goto end;
4801  }
4802 
4803  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4804  if (tx == NULL)
4805  goto end;
4806  if (tx->cfg != htp) {
4807  printf("wrong HTP config (%p instead of %p - default=%p): ",
4808  tx->cfg, htp, cfglist.cfg);
4809  goto end;
4810  }
4811  tx = HTPStateGetTx(htp_state, 1);
4812  if (tx == NULL)
4813  goto end;
4814  if (tx->cfg != htp) {
4815  printf("wrong HTP config (%p instead of %p - default=%p): ",
4816  tx->cfg, htp, cfglist.cfg);
4817  goto end;
4818  }
4819 
4820 end:
4821  if (alp_tctx != NULL)
4822  AppLayerParserThreadCtxFree(alp_tctx);
4823  HTPFreeConfig();
4824  ConfDeInit();
4827 
4829  UTHFreeFlow(f);
4830  return result;
4831 }
4832 
4833 /* disabled when we upgraded to libhtp 0.5.x */
4834 #if 0
4835 static int HTPParserConfigTest04(void)
4836 {
4837  int result = 0;
4838 
4839  char input[] = "\
4840 %YAML 1.1\n\
4841 ---\n\
4842 libhtp:\n\
4843 \n\
4844  default-config:\n\
4845  personality: IDS\n\
4846  path-control-char-handling: status_400\n\
4847  path-convert-utf8: yes\n\
4848  path-invalid-encoding-handling: remove_percent\n\
4849 \n\
4850  server-config:\n\
4851 \n\
4852  - apache-tomcat:\n\
4853  personality: Tomcat_6_0\n\
4854  path-invalid-utf8-handling: none\n\
4855  path-nul-encoded-handling: status_404\n\
4856  path-nul-raw-handling: status_400\n\
4857 \n\
4858  - iis7:\n\
4859  personality: IIS_7_0\n\
4860  path-replacement-char: o\n\
4861  path-unicode-mapping: status_400\n\
4862 ";
4863 
4865  ConfInit();
4867 
4868  ConfYamlLoadString(input, strlen(input));
4869 
4870  HTPConfigure();
4871 
4872  HTPCfgRec *cfg_rec = &cfglist;
4873  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4874  cfg_rec->cfg->path_convert_utf8 != 1 ||
4875  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4876  printf("failed 1\n");
4877  goto end;
4878  }
4879 
4880  cfg_rec = cfg_rec->next;
4881  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4882  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4883  printf("failed 2\n");
4884  goto end;
4885  }
4886 
4887  cfg_rec = cfg_rec->next;
4888  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4889  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4890  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4891  printf("failed 3\n");
4892  goto end;
4893  }
4894 
4895  result = 1;
4896 
4897 end:
4898  HTPFreeConfig();
4899  ConfDeInit();
4902 
4903  return result;
4904 }
4905 #endif
4906 
4907 /** \test Test %2f decoding in profile Apache_2_2
4908  *
4909  * %2f in path is left untouched
4910  * %2f in query string is normalized to %2F
4911  * %252f in query string is decoded/normalized to %2F
4912  */
4913 static int HTPParserDecodingTest01(void)
4914 {
4915  int result = 0;
4916  Flow *f = NULL;
4917  uint8_t httpbuf1[] =
4918  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4919  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4920  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4921  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4922  TcpSession ssn;
4924 
4925  HtpState *htp_state = NULL;
4926  int r = 0;
4927  char input[] = "\
4928 %YAML 1.1\n\
4929 ---\n\
4930 libhtp:\n\
4931 \n\
4932  default-config:\n\
4933  personality: Apache_2\n\
4934 ";
4935 
4937  ConfInit();
4939  ConfYamlLoadString(input, strlen(input));
4940  HTPConfigure();
4941  const char *addr = "4.3.2.1";
4942  memset(&ssn, 0, sizeof(ssn));
4943 
4944  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4945  if (f == NULL)
4946  goto end;
4947  f->protoctx = &ssn;
4948  f->proto = IPPROTO_TCP;
4949  f->alproto = ALPROTO_HTTP;
4950 
4952 
4953  uint32_t u;
4954  for (u = 0; u < httplen1; u++) {
4955  uint8_t flags = 0;
4956 
4957  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4958  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4959  else flags = STREAM_TOSERVER;
4960 
4961  FLOWLOCK_WRLOCK(f);
4962  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4963  &httpbuf1[u], 1);
4964  if (r != 0) {
4965  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4966  " 0: ", u, r);
4967  FLOWLOCK_UNLOCK(f);
4968  goto end;
4969  }
4970  FLOWLOCK_UNLOCK(f);
4971  }
4972 
4973  htp_state = f->alstate;
4974  if (htp_state == NULL) {
4975  printf("no http state: ");
4976  goto end;
4977  }
4978 
4979  uint8_t ref1[] = "/abc%2fdef";
4980  size_t reflen = sizeof(ref1) - 1;
4981 
4982  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4983  if (tx == NULL)
4984  goto end;
4985  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4986  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4987  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4988  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4989  (uintmax_t)reflen,
4990  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4991  goto end;
4992  }
4993 
4994  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4995  bstr_len(tx_ud->request_uri_normalized)) != 0)
4996  {
4997  printf("normalized uri \"");
4998  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4999  printf("\" != \"");
5000  PrintRawUriFp(stdout, ref1, reflen);
5001  printf("\": ");
5002  goto end;
5003  }
5004  }
5005 
5006  uint8_t ref2[] = "/abc/def?ghi/jkl";
5007  reflen = sizeof(ref2) - 1;
5008 
5009  tx = HTPStateGetTx(htp_state, 1);
5010  if (tx == NULL)
5011  goto end;
5012  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5013  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5014  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5015  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5016  (uintmax_t)reflen,
5017  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5018  goto end;
5019  }
5020 
5021  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5022  bstr_len(tx_ud->request_uri_normalized)) != 0)
5023  {
5024  printf("normalized uri \"");
5025  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5026  printf("\" != \"");
5027  PrintRawUriFp(stdout, ref2, reflen);
5028  printf("\": ");
5029  goto end;
5030  }
5031  }
5032 
5033  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5034  reflen = sizeof(ref3) - 1;
5035  tx = HTPStateGetTx(htp_state, 2);
5036  if (tx == NULL)
5037  goto end;
5038  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5039  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5040  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5041  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5042  (uintmax_t)reflen,
5043  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5044  goto end;
5045  }
5046 
5047  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5048  bstr_len(tx_ud->request_uri_normalized)) != 0)
5049  {
5050  printf("normalized uri \"");
5051  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5052  printf("\" != \"");
5053  PrintRawUriFp(stdout, ref3, reflen);
5054  printf("\": ");
5055  goto end;
5056  }
5057  }
5058 
5059  result = 1;
5060 
5061 end:
5062  if (alp_tctx != NULL)
5063  AppLayerParserThreadCtxFree(alp_tctx);
5064  HTPFreeConfig();
5065  ConfDeInit();
5068 
5070  UTHFreeFlow(f);
5071  return result;
5072 }
5073 
5074 /** \test Test %2f decoding in profile IDS
5075  *
5076  * %2f in path decoded to /
5077  * %2f in query string is decoded to /
5078  * %252f in query string is decoded to %2F
5079  */
5080 static int HTPParserDecodingTest02(void)
5081 {
5082  int result = 0;
5083  Flow *f = NULL;
5084  uint8_t httpbuf1[] =
5085  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5086  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5087  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5088  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5089  TcpSession ssn;
5091 
5092  HtpState *htp_state = NULL;
5093  int r = 0;
5094  char input[] = "\
5095 %YAML 1.1\n\
5096 ---\n\
5097 libhtp:\n\
5098 \n\
5099  default-config:\n\
5100  personality: IDS\n\
5101  double-decode-path: no\n\
5102  double-decode-query: no\n\
5103 ";
5104 
5106  ConfInit();
5108  ConfYamlLoadString(input, strlen(input));
5109  HTPConfigure();
5110  const char *addr = "4.3.2.1";
5111  memset(&ssn, 0, sizeof(ssn));
5112 
5113  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5114  if (f == NULL)
5115  goto end;
5116  f->protoctx = &ssn;
5117  f->proto = IPPROTO_TCP;
5118  f->alproto = ALPROTO_HTTP;
5119 
5121 
5122  uint32_t u;
5123  for (u = 0; u < httplen1; u++) {
5124  uint8_t flags = 0;
5125 
5126  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5127  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5128  else flags = STREAM_TOSERVER;
5129 
5130  FLOWLOCK_WRLOCK(f);
5131  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5132  &httpbuf1[u], 1);
5133  if (r != 0) {
5134  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5135  " 0: ", u, r);
5136  FLOWLOCK_UNLOCK(f);
5137  goto end;
5138  }
5139  FLOWLOCK_UNLOCK(f);
5140  }
5141 
5142  htp_state = f->alstate;
5143  if (htp_state == NULL) {
5144  printf("no http state: ");
5145  goto end;
5146  }
5147 
5148  uint8_t ref1[] = "/abc/def";
5149  size_t reflen = sizeof(ref1) - 1;
5150 
5151  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5152  if (tx == NULL)
5153  goto end;
5154  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5155  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5156  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5157  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5158  (uintmax_t)reflen,
5159  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5160  goto end;
5161  }
5162 
5163  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5164  bstr_len(tx_ud->request_uri_normalized)) != 0)
5165  {
5166  printf("normalized uri \"");
5167  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5168  printf("\" != \"");
5169  PrintRawUriFp(stdout, ref1, reflen);
5170  printf("\": ");
5171  goto end;
5172  }
5173  }
5174 
5175  uint8_t ref2[] = "/abc/def?ghi/jkl";
5176  reflen = sizeof(ref2) - 1;
5177 
5178  tx = HTPStateGetTx(htp_state, 1);
5179  if (tx == NULL)
5180  goto end;
5181  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5182  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5183  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5184  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5185  (uintmax_t)reflen,
5186  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5187  goto end;
5188  }
5189 
5190  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5191  bstr_len(tx_ud->request_uri_normalized)) != 0)
5192  {
5193  printf("normalized uri \"");
5194  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5195  printf("\" != \"");
5196  PrintRawUriFp(stdout, ref2, reflen);
5197  printf("\": ");
5198  goto end;
5199  }
5200  }
5201 
5202  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5203  reflen = sizeof(ref3) - 1;
5204  tx = HTPStateGetTx(htp_state, 2);
5205  if (tx == NULL)
5206  goto end;
5207  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5208  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5209  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5210  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5211  (uintmax_t)reflen,
5212  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5213  goto end;
5214  }
5215 
5216  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5217  bstr_len(tx_ud->request_uri_normalized)) != 0)
5218  {
5219  printf("normalized uri \"");
5220  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5221  printf("\" != \"");
5222  PrintRawUriFp(stdout, ref3, reflen);
5223  printf("\": ");
5224  goto end;
5225  }
5226  }
5227 
5228  result = 1;
5229 
5230 end:
5231  if (alp_tctx != NULL)
5232  AppLayerParserThreadCtxFree(alp_tctx);
5233  HTPFreeConfig();
5234  ConfDeInit();
5237 
5239  UTHFreeFlow(f);
5240  return result;
5241 }
5242 
5243 /** \test Test %2f decoding in profile IDS with double-decode-* options
5244  *
5245  * %252f in path decoded to /
5246  * %252f in query string is decoded to /
5247  */
5248 static int HTPParserDecodingTest03(void)
5249 {
5250  int result = 0;
5251  Flow *f = NULL;
5252  uint8_t httpbuf1[] =
5253  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5254  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5255  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5256  TcpSession ssn;
5258 
5259  HtpState *htp_state = NULL;
5260  int r = 0;
5261  char input[] = "\
5262 %YAML 1.1\n\
5263 ---\n\
5264 libhtp:\n\
5265 \n\
5266  default-config:\n\
5267  personality: IDS\n\
5268  double-decode-path: yes\n\
5269  double-decode-query: yes\n\
5270 ";
5271 
5273  ConfInit();
5275  ConfYamlLoadString(input, strlen(input));
5276  HTPConfigure();
5277  const char *addr = "4.3.2.1";
5278  memset(&ssn, 0, sizeof(ssn));
5279 
5280  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5281  if (f == NULL)
5282  goto end;
5283  f->protoctx = &ssn;
5284  f->proto = IPPROTO_TCP;
5285  f->alproto = ALPROTO_HTTP;
5286 
5288 
5289  uint32_t u;
5290  for (u = 0; u < httplen1; u++) {
5291  uint8_t flags = 0;
5292 
5293  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5294  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5295  else flags = STREAM_TOSERVER;
5296 
5297  FLOWLOCK_WRLOCK(f);
5298  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5299  &httpbuf1[u], 1);
5300  if (r != 0) {
5301  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5302  " 0: ", u, r);
5303  FLOWLOCK_UNLOCK(f);
5304  goto end;
5305  }
5306  FLOWLOCK_UNLOCK(f);
5307  }
5308 
5309  htp_state = f->alstate;
5310  if (htp_state == NULL) {
5311  printf("no http state: ");
5312  goto end;
5313  }
5314 
5315  uint8_t ref1[] = "/abc/def";
5316  size_t reflen = sizeof(ref1) - 1;
5317 
5318  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5319  if (tx == NULL)
5320  goto end;
5321  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5322  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5323  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5324  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5325  (uintmax_t)reflen,
5326  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5327  goto end;
5328  }
5329 
5330  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5331  bstr_len(tx_ud->request_uri_normalized)) != 0)
5332  {
5333  printf("normalized uri \"");
5334  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5335  printf("\" != \"");
5336  PrintRawUriFp(stdout, ref1, reflen);
5337  printf("\": ");
5338  goto end;
5339  }
5340  }
5341 
5342  uint8_t ref2[] = "/abc/def?ghi/jkl";
5343  reflen = sizeof(ref2) - 1;
5344 
5345  tx = HTPStateGetTx(htp_state, 1);
5346  if (tx == NULL)
5347  goto end;
5348  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5349  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5350  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5351  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5352  (uintmax_t)reflen,
5353  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5354  goto end;
5355  }
5356 
5357  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5358  bstr_len(tx_ud->request_uri_normalized)) != 0)
5359  {
5360  printf("normalized uri \"");
5361  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5362  printf("\" != \"");
5363  PrintRawUriFp(stdout, ref2, reflen);
5364  printf("\": ");
5365  goto end;
5366  }
5367  }
5368 
5369  result = 1;
5370 
5371 end:
5372  if (alp_tctx != NULL)
5373  AppLayerParserThreadCtxFree(alp_tctx);
5374  HTPFreeConfig();
5375  ConfDeInit();
5378 
5380  UTHFreeFlow(f);
5381  return result;
5382 }
5383 
5384 /** \test Test http:// in query profile IDS
5385  */
5386 static int HTPParserDecodingTest04(void)
5387 {
5388  int result = 0;
5389  Flow *f = NULL;
5390  uint8_t httpbuf1[] =
5391  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5392  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5393  TcpSession ssn;
5395 
5396  HtpState *htp_state = NULL;
5397  int r = 0;
5398  char input[] = "\
5399 %YAML 1.1\n\
5400 ---\n\
5401 libhtp:\n\
5402 \n\
5403  default-config:\n\
5404  personality: IDS\n\
5405  double-decode-path: yes\n\
5406  double-decode-query: yes\n\
5407 ";
5408 
5410  ConfInit();
5412  ConfYamlLoadString(input, strlen(input));
5413  HTPConfigure();
5414  const char *addr = "4.3.2.1";
5415  memset(&ssn, 0, sizeof(ssn));
5416 
5417  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5418  if (f == NULL)
5419  goto end;
5420  f->protoctx = &ssn;
5421  f->proto = IPPROTO_TCP;
5422  f->alproto = ALPROTO_HTTP;
5423 
5425 
5426  uint32_t u;
5427  for (u = 0; u < httplen1; u++) {
5428  uint8_t flags = 0;
5429 
5430  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5431  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5432  else flags = STREAM_TOSERVER;
5433 
5434  FLOWLOCK_WRLOCK(f);
5435  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5436  &httpbuf1[u], 1);
5437  if (r != 0) {
5438  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5439  " 0: ", u, r);
5440  FLOWLOCK_UNLOCK(f);
5441  goto end;
5442  }
5443  FLOWLOCK_UNLOCK(f);
5444  }
5445 
5446  htp_state = f->alstate;
5447  if (htp_state == NULL) {
5448  printf("no http state: ");
5449  goto end;
5450  }
5451 
5452  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5453  size_t reflen = sizeof(ref1) - 1;
5454 
5455  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5456  if (tx == NULL)
5457  goto end;
5458  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5459  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5460  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5461  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5462  (uintmax_t)reflen,
5463  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5464  goto end;
5465  }
5466 
5467  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5468  bstr_len(tx_ud->request_uri_normalized)) != 0)
5469  {
5470  printf("normalized uri \"");
5471  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5472  printf("\" != \"");
5473  PrintRawUriFp(stdout, ref1, reflen);
5474  printf("\": ");
5475  goto end;
5476  }
5477  }
5478 
5479  result = 1;
5480 
5481 end:
5482  if (alp_tctx != NULL)
5483  AppLayerParserThreadCtxFree(alp_tctx);
5484  HTPFreeConfig();
5485  ConfDeInit();
5488 
5490  UTHFreeFlow(f);
5491  return result;
5492 }
5493 
5494 /** \test Test \ char in query profile IDS. Bug 739
5495  */
5496 static int HTPParserDecodingTest05(void)
5497 {
5498  int result = 0;
5499  Flow *f = NULL;
5500  uint8_t httpbuf1[] =
5501  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5502  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5503  TcpSession ssn;
5505 
5506  HtpState *htp_state = NULL;
5507  int r = 0;
5508  char input[] = "\
5509 %YAML 1.1\n\
5510 ---\n\
5511 libhtp:\n\
5512 \n\
5513  default-config:\n\
5514  personality: IDS\n\
5515  double-decode-path: yes\n\
5516  double-decode-query: yes\n\
5517 ";
5518 
5520  ConfInit();
5522  ConfYamlLoadString(input, strlen(input));
5523  HTPConfigure();
5524  const char *addr = "4.3.2.1";
5525  memset(&ssn, 0, sizeof(ssn));
5526 
5527  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5528  if (f == NULL)
5529  goto end;
5530  f->protoctx = &ssn;
5531  f->proto = IPPROTO_TCP;
5532  f->alproto = ALPROTO_HTTP;
5533 
5535 
5536  uint32_t u;
5537  for (u = 0; u < httplen1; u++) {
5538  uint8_t flags = 0;
5539 
5540  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5541  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5542  else flags = STREAM_TOSERVER;
5543 
5544  FLOWLOCK_WRLOCK(f);
5545  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5546  &httpbuf1[u], 1);
5547  if (r != 0) {
5548  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5549  " 0: ", u, r);
5550  FLOWLOCK_UNLOCK(f);
5551  goto end;
5552  }
5553  FLOWLOCK_UNLOCK(f);
5554  }
5555 
5556  htp_state = f->alstate;
5557  if (htp_state == NULL) {
5558  printf("no http state: ");
5559  goto end;
5560  }
5561 
5562  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5563  size_t reflen = sizeof(ref1) - 1;
5564 
5565  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5566  if (tx == NULL)
5567  goto end;
5568  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5569  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5570  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5571  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5572  (uintmax_t)reflen,
5573  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5574  goto end;
5575  }
5576 
5577  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5578  bstr_len(tx_ud->request_uri_normalized)) != 0)
5579  {
5580  printf("normalized uri \"");
5581  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5582  printf("\" != \"");
5583  PrintRawUriFp(stdout, ref1, reflen);
5584  printf("\": ");
5585  goto end;
5586  }
5587  }
5588 
5589  result = 1;
5590 
5591 end:
5592  if (alp_tctx != NULL)
5593  AppLayerParserThreadCtxFree(alp_tctx);
5594  HTPFreeConfig();
5595  ConfDeInit();
5598 
5600  UTHFreeFlow(f);
5601  return result;
5602 }
5603 
5604 /** \test Test + char in query. Bug 1035
5605  */
5606 static int HTPParserDecodingTest06(void)
5607 {
5608  int result = 0;
5609  Flow *f = NULL;
5610  uint8_t httpbuf1[] =
5611  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5612  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5613  TcpSession ssn;
5615 
5616  HtpState *htp_state = NULL;
5617  int r = 0;
5618  char input[] = "\
5619 %YAML 1.1\n\
5620 ---\n\
5621 libhtp:\n\
5622 \n\
5623  default-config:\n\
5624  personality: IDS\n\
5625  double-decode-path: yes\n\
5626  double-decode-query: yes\n\
5627 ";
5628 
5630  ConfInit();
5632  ConfYamlLoadString(input, strlen(input));
5633  HTPConfigure();
5634  const char *addr = "4.3.2.1";
5635  memset(&ssn, 0, sizeof(ssn));
5636 
5637  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5638  if (f == NULL)
5639  goto end;
5640  f->protoctx = &ssn;
5641  f->proto = IPPROTO_TCP;
5642  f->alproto = ALPROTO_HTTP;
5643 
5645 
5646  uint32_t u;
5647  for (u = 0; u < httplen1; u++) {
5648  uint8_t flags = 0;
5649 
5650  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5651  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5652  else flags = STREAM_TOSERVER;
5653 
5654  FLOWLOCK_WRLOCK(f);
5655  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5656  &httpbuf1[u], 1);
5657  if (r != 0) {
5658  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5659  " 0: ", u, r);
5660  FLOWLOCK_UNLOCK(f);
5661  goto end;
5662  }
5663  FLOWLOCK_UNLOCK(f);
5664  }
5665 
5666  htp_state = f->alstate;
5667  if (htp_state == NULL) {
5668  printf("no http state: ");
5669  goto end;
5670  }
5671 
5672  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5673  size_t reflen = sizeof(ref1) - 1;
5674 
5675  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5676  if (tx == NULL)
5677  goto end;
5678  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5679  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5680  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5681  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5682  (uintmax_t)reflen,
5683  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5684  goto end;
5685  }
5686 
5687  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5688  bstr_len(tx_ud->request_uri_normalized)) != 0)
5689  {
5690  printf("normalized uri \"");
5691  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5692  printf("\" != \"");
5693  PrintRawUriFp(stdout, ref1, reflen);
5694  printf("\": ");
5695  goto end;
5696  }
5697  }
5698 
5699  result = 1;
5700 
5701 end:
5702  if (alp_tctx != NULL)
5703  AppLayerParserThreadCtxFree(alp_tctx);
5704  HTPFreeConfig();
5705  ConfDeInit();
5708 
5710  UTHFreeFlow(f);
5711  return result;
5712 }
5713 
5714 /** \test Test + char in query. Bug 1035
5715  */
5716 static int HTPParserDecodingTest07(void)
5717 {
5718  int result = 0;
5719  Flow *f = NULL;
5720  uint8_t httpbuf1[] =
5721  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5722  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5723  TcpSession ssn;
5725 
5726  HtpState *htp_state = NULL;
5727  int r = 0;
5728  char input[] = "\
5729 %YAML 1.1\n\
5730 ---\n\
5731 libhtp:\n\
5732 \n\
5733  default-config:\n\
5734  personality: IDS\n\
5735  double-decode-path: yes\n\
5736  double-decode-query: yes\n\
5737  query-plusspace-decode: yes\n\
5738 ";
5739 
5741  ConfInit();
5743  ConfYamlLoadString(input, strlen(input));
5744  HTPConfigure();
5745  const char *addr = "4.3.2.1";
5746  memset(&ssn, 0, sizeof(ssn));
5747 
5748  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5749  if (f == NULL)
5750  goto end;
5751  f->protoctx = &ssn;
5752  f->proto = IPPROTO_TCP;
5753  f->alproto = ALPROTO_HTTP;
5754 
5756 
5757  uint32_t u;
5758  for (u = 0; u < httplen1; u++) {
5759  uint8_t flags = 0;
5760 
5761  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5762  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5763  else flags = STREAM_TOSERVER;
5764 
5765  FLOWLOCK_WRLOCK(f);
5766  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5767  &httpbuf1[u], 1);
5768  if (r != 0) {
5769  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5770  " 0: ", u, r);
5771  FLOWLOCK_UNLOCK(f);
5772  goto end;
5773  }
5774  FLOWLOCK_UNLOCK(f);
5775  }
5776 
5777  htp_state = f->alstate;
5778  if (htp_state == NULL) {
5779  printf("no http state: ");
5780  goto end;
5781  }
5782 
5783  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5784  size_t reflen = sizeof(ref1) - 1;
5785 
5786  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5787  if (tx == NULL)
5788  goto end;
5789  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5790  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5791  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5792  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5793  (uintmax_t)reflen,
5794  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5795  goto end;
5796  }
5797 
5798  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5799  bstr_len(tx_ud->request_uri_normalized)) != 0)
5800  {
5801  printf("normalized uri \"");
5802  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5803  printf("\" != \"");
5804  PrintRawUriFp(stdout, ref1, reflen);
5805  printf("\": ");
5806  goto end;
5807  }
5808  }
5809 
5810  result = 1;
5811 
5812 end:
5813  if (alp_tctx != NULL)
5814  AppLayerParserThreadCtxFree(alp_tctx);
5815  HTPFreeConfig();
5816  ConfDeInit();
5819 
5821  UTHFreeFlow(f);
5822  return result;
5823 }
5824 
5825 /** \test Test 'proxy' URI normalization. Ticket 1008
5826  */
5827 static int HTPParserDecodingTest08(void)
5828 {
5829  int result = 0;
5830  Flow *f = NULL;
5831  uint8_t httpbuf1[] =
5832  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5833  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5834  TcpSession ssn;
5836 
5837  HtpState *htp_state = NULL;
5838  int r = 0;
5839  char input[] = "\
5840 %YAML 1.1\n\
5841 ---\n\
5842 libhtp:\n\
5843 \n\
5844  default-config:\n\
5845  personality: IDS\n\
5846 ";
5847 
5849  ConfInit();
5851  ConfYamlLoadString(input, strlen(input));
5852  HTPConfigure();
5853  const char *addr = "4.3.2.1";
5854  memset(&ssn, 0, sizeof(ssn));
5855 
5856  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5857  if (f == NULL)
5858  goto end;
5859  f->protoctx = &ssn;
5860  f->proto = IPPROTO_TCP;
5861  f->alproto = ALPROTO_HTTP;
5862 
5864 
5865  uint32_t u;
5866  for (u = 0; u < httplen1; u++) {
5867  uint8_t flags = 0;
5868 
5869  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5870  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5871  else flags = STREAM_TOSERVER;
5872 
5873  FLOWLOCK_WRLOCK(f);
5874  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5875  &httpbuf1[u], 1);
5876  if (r != 0) {
5877  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5878  " 0: ", u, r);
5879  FLOWLOCK_UNLOCK(f);
5880  goto end;
5881  }
5882  FLOWLOCK_UNLOCK(f);
5883  }
5884 
5885  htp_state = f->alstate;
5886  if (htp_state == NULL) {
5887  printf("no http state: ");
5888  goto end;
5889  }
5890 
5891  uint8_t ref1[] = "/blah/";
5892  size_t reflen = sizeof(ref1) - 1;
5893 
5894  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5895  if (tx == NULL)
5896  goto end;
5897  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5898  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5899  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5900  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5901  (uintmax_t)reflen,
5902  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5903  goto end;
5904  }
5905 
5906  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5907  bstr_len(tx_ud->request_uri_normalized)) != 0)
5908  {
5909  printf("normalized uri \"");
5910  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5911  printf("\" != \"");
5912  PrintRawUriFp(stdout, ref1, reflen);
5913  printf("\": ");
5914  goto end;
5915  }
5916  }
5917 
5918  result = 1;
5919 
5920 end:
5921  if (alp_tctx != NULL)
5922  AppLayerParserThreadCtxFree(alp_tctx);
5923  HTPFreeConfig();
5924  ConfDeInit();
5927 
5929  UTHFreeFlow(f);
5930  return result;
5931 }
5932 
5933 /** \test Test 'proxy' URI normalization. Ticket 1008
5934  */
5935 static int HTPParserDecodingTest09(void)
5936 {
5937  int result = 0;
5938  Flow *f = NULL;
5939  uint8_t httpbuf1[] =
5940  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5941  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5942  TcpSession ssn;
5944 
5945  HtpState *htp_state = NULL;
5946  int r = 0;
5947  char input[] = "\
5948 %YAML 1.1\n\