suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-htp.h"
59 #include "app-layer-htp-body.h"
60 #include "app-layer-htp-file.h"
61 #include "app-layer-htp-libhtp.h"
62 #include "app-layer-htp-xff.h"
63 
64 #include "util-spm.h"
65 #include "util-debug.h"
66 #include "util-time.h"
67 #include "util-misc.h"
68 
69 #include "util-unittest.h"
70 #include "util-unittest-helper.h"
71 #include "flow-util.h"
72 
73 #include "detect-engine.h"
74 #include "detect-engine-state.h"
75 #include "detect-parse.h"
76 
77 #include "decode-events.h"
78 
79 #include "util-memcmp.h"
80 #include "util-random.h"
81 #include "util-validate.h"
82 
83 //#define PRINT
84 
85 /** Fast lookup tree (radix) for the various HTP configurations */
86 static SCRadixTree *cfgtree;
87 /** List of HTP configurations. */
88 static HTPCfgRec cfglist;
89 
90 #ifdef DEBUG
91 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
92 static uint64_t htp_state_memuse = 0;
93 static uint64_t htp_state_memcnt = 0;
94 #endif
95 
97  { "UNKNOWN_ERROR",
99  { "GZIP_DECOMPRESSION_FAILED",
101  { "REQUEST_FIELD_MISSING_COLON",
103  { "RESPONSE_FIELD_MISSING_COLON",
105  { "INVALID_REQUEST_CHUNK_LEN",
107  { "INVALID_RESPONSE_CHUNK_LEN",
109  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
111  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
113  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
115  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
117  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
119  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
121  { "100_CONTINUE_ALREADY_SEEN",
123  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
125  { "INVALID_SERVER_PORT_IN_REQUEST",
127  { "INVALID_AUTHORITY_PORT",
129  { "REQUEST_HEADER_INVALID",
131  { "RESPONSE_HEADER_INVALID",
133  { "MISSING_HOST_HEADER",
135  { "HOST_HEADER_AMBIGUOUS",
137  { "INVALID_REQUEST_FIELD_FOLDING",
139  { "INVALID_RESPONSE_FIELD_FOLDING",
141  { "REQUEST_FIELD_TOO_LONG",
143  { "RESPONSE_FIELD_TOO_LONG",
145  { "REQUEST_LINE_INVALID",
147  { "REQUEST_BODY_UNEXPECTED",
149  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
151  { "REQUEST_URI_HOST_INVALID",
153  { "REQUEST_HEADER_HOST_INVALID",
155  { "REQUEST_AUTH_UNRECOGNIZED",
157  { "REQUEST_HEADER_REPETITION",
159  { "RESPONSE_HEADER_REPETITION",
161  { "DOUBLE_ENCODED_URI",
163  { "URI_DELIM_NON_COMPLIANT",
165  { "METHOD_DELIM_NON_COMPLIANT",
167  { "REQUEST_LINE_LEADING_WHITESPACE",
169  { "TOO_MANY_ENCODING_LAYERS",
171  { "ABNORMAL_CE_HEADER",
173  { "RESPONSE_MULTIPART_BYTERANGES",
175  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
177  { "RESPONSE_CHUNKED_OLD_PROTO",
179  { "RESPONSE_INVALID_PROTOCOL",
181  { "RESPONSE_INVALID_STATUS",
183  { "REQUEST_LINE_INCOMPLETE",
185 
186  { "LZMA_MEMLIMIT_REACHED",
188  { "COMPRESSION_BOMB",
190 
191  /* suricata warnings/errors */
192  { "MULTIPART_GENERIC_ERROR",
194  { "MULTIPART_NO_FILEDATA",
196  { "MULTIPART_INVALID_HEADER",
198 
199  { NULL, -1 },
200 };
201 
202 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
203 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
204 static uint64_t HTPStateGetTxCnt(void *alstate);
205 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction);
206 
207 #ifdef DEBUG
208 /**
209  * \internal
210  *
211  * \brief Lookup the HTP personality string from the numeric personality.
212  *
213  * \todo This needs to be a libhtp function.
214  */
215 static const char *HTPLookupPersonalityString(int p)
216 {
217 #define CASE_HTP_PERSONALITY_STRING(p) \
218  case HTP_SERVER_ ## p: return #p
219 
220  switch (p) {
221  CASE_HTP_PERSONALITY_STRING(MINIMAL);
222  CASE_HTP_PERSONALITY_STRING(GENERIC);
223  CASE_HTP_PERSONALITY_STRING(IDS);
224  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
225  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
226  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
227  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
228  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
229  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
230  CASE_HTP_PERSONALITY_STRING(APACHE_2);
231  }
232 
233  return NULL;
234 }
235 #endif /* DEBUG */
236 
237 /**
238  * \internal
239  *
240  * \brief Lookup the numeric HTP personality from a string.
241  *
242  * \todo This needs to be a libhtp function.
243  */
244 static int HTPLookupPersonality(const char *str)
245 {
246 #define IF_HTP_PERSONALITY_NUM(p) \
247  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
248 
249  IF_HTP_PERSONALITY_NUM(MINIMAL);
250  IF_HTP_PERSONALITY_NUM(GENERIC);
252  IF_HTP_PERSONALITY_NUM(IIS_4_0);
253  IF_HTP_PERSONALITY_NUM(IIS_5_0);
254  IF_HTP_PERSONALITY_NUM(IIS_5_1);
255  IF_HTP_PERSONALITY_NUM(IIS_6_0);
256  IF_HTP_PERSONALITY_NUM(IIS_7_0);
257  IF_HTP_PERSONALITY_NUM(IIS_7_5);
258  IF_HTP_PERSONALITY_NUM(APACHE_2);
259  if (strcasecmp("TOMCAT_6_0", str) == 0) {
260  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
261  "longer supported by libhtp.", str);
262  return -1;
263  } else if ((strcasecmp("APACHE", str) == 0) ||
264  (strcasecmp("APACHE_2_2", str) == 0))
265  {
266  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
267  "longer supported by libhtp, failing back to "
268  "Apache2 personality.", str);
269  return HTP_SERVER_APACHE_2;
270  }
271 
272  return -1;
273 }
274 
275 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud, uint8_t e)
276 {
277  SCLogDebug("setting event %u", e);
278 
279  if (htud) {
281  s->events++;
282  return;
283  }
284 
285  htp_tx_t *tx = HTPStateGetTx(s, s->transaction_cnt);
286  if (tx == NULL && s->transaction_cnt > 0)
287  tx = HTPStateGetTx(s, s->transaction_cnt - 1);
288  if (tx != NULL) {
289  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
290  if (htud != NULL) {
292  s->events++;
293  return;
294  }
295  }
296  SCLogDebug("couldn't set event %u", e);
297 }
298 
299 static AppLayerDecoderEvents *HTPGetEvents(void *tx)
300 {
301  SCLogDebug("get HTTP events for TX %p", tx);
302 
303  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
304  if (htud != NULL) {
305  SCLogDebug("has htud, htud->decoder_events %p", htud->decoder_events);
306  return htud->decoder_events;
307  }
308 
309  return NULL;
310 }
311 
312 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
313  * connection parser to be used by the HTP library
314  */
315 static void *HTPStateAlloc(void)
316 {
317  SCEnter();
318 
319  HtpState *s = HTPMalloc(sizeof(HtpState));
320  if (unlikely(s == NULL)) {
321  SCReturnPtr(NULL, "void");
322  }
323 
324  memset(s, 0x00, sizeof(HtpState));
325 
326 #ifdef DEBUG
327  SCMutexLock(&htp_state_mem_lock);
328  htp_state_memcnt++;
329  htp_state_memuse += sizeof(HtpState);
330  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
331  SCMutexUnlock(&htp_state_mem_lock);
332 #endif
333 
334  SCReturnPtr((void *)s, "void");
335 }
336 
337 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
338 {
339  if (likely(htud)) {
340  HtpBodyFree(&htud->request_body);
341  HtpBodyFree(&htud->response_body);
342  bstr_free(htud->request_uri_normalized);
343  if (htud->request_headers_raw)
345  if (htud->response_headers_raw)
348  if (htud->boundary)
349  HTPFree(htud->boundary, htud->boundary_len);
350  if (htud->de_state != NULL) {
352  }
353  HTPFree(htud, sizeof(HtpTxUserData));
354  }
355 }
356 
357 /** \brief Function to frees the HTTP state memory and also frees the HTTP
358  * connection parser memory which was used by the HTP library
359  */
360 void HTPStateFree(void *state)
361 {
362  SCEnter();
363 
364  HtpState *s = (HtpState *)state;
365  if (s == NULL) {
366  SCReturn;
367  }
368 
369  /* free the connection parser memory used by HTP library */
370  if (s->connp != NULL) {
371  SCLogDebug("freeing HTP state");
372 
373  uint64_t tx_id;
374  uint64_t total_txs = HTPStateGetTxCnt(state);
375  /* free the list of body chunks */
376  if (s->conn != NULL) {
377  for (tx_id = 0; tx_id < total_txs; tx_id++) {
378  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
379  if (tx != NULL) {
380  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
381  HtpTxUserDataFree(s, htud);
382  htp_tx_set_user_data(tx, NULL);
383  }
384  }
385  }
386  htp_connp_destroy_all(s->connp);
387  }
388 
391  HTPFree(s, sizeof(HtpState));
392 
393 #ifdef DEBUG
394  SCMutexLock(&htp_state_mem_lock);
395  htp_state_memcnt--;
396  htp_state_memuse -= sizeof(HtpState);
397  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
398  SCMutexUnlock(&htp_state_mem_lock);
399 #endif
400 
401  SCReturn;
402 }
403 
404 /**
405  * \brief HTP transaction cleanup callback
406  *
407  * \warning We cannot actually free the transactions here. It seems that
408  * HTP only accepts freeing of transactions in the response callback.
409  */
410 static void HTPStateTransactionFree(void *state, uint64_t id)
411 {
412  SCEnter();
413 
414  HtpState *s = (HtpState *)state;
415 
416  SCLogDebug("state %p, id %"PRIu64, s, id);
417 
418  htp_tx_t *tx = HTPStateGetTx(s, id);
419  if (tx != NULL) {
420  /* This will remove obsolete body chunks */
421  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
422  HtpTxUserDataFree(s, htud);
423  htp_tx_set_user_data(tx, NULL);
424 
425  /* hack: even if libhtp considers the tx incomplete, we want to
426  * free it here. htp_tx_destroy however, will refuse to do this.
427  * As htp_tx_destroy_incomplete isn't available in the public API,
428  * we hack around it here. */
429  if (unlikely(!(
430  tx->request_progress == HTP_REQUEST_COMPLETE &&
431  tx->response_progress == HTP_RESPONSE_COMPLETE)))
432  {
433  tx->request_progress = HTP_REQUEST_COMPLETE;
434  tx->response_progress = HTP_RESPONSE_COMPLETE;
435  }
436  htp_tx_destroy(tx);
437  }
438 }
439 
440 /**
441  * \brief Sets a flag that informs the HTP app layer that some module in the
442  * engine needs the http request body data.
443  * \initonly
444  */
446 {
447  SCEnter();
448 
449  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
450  SCReturn;
451 }
452 
453 /**
454  * \brief Sets a flag that informs the HTP app layer that some module in the
455  * engine needs the http request body data.
456  * \initonly
457  */
459 {
460  SCEnter();
461 
462  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
463  SCReturn;
464 }
465 
466 /**
467  * \brief Sets a flag that informs the HTP app layer that some module in the
468  * engine needs the http request multi part header.
469  *
470  * \initonly
471  */
472 static void AppLayerHtpNeedMultipartHeader(void)
473 {
474  SCEnter();
476 
477  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
478  SCReturn;
479 }
480 
481 /**
482  * \brief Sets a flag that informs the HTP app layer that some module in the
483  * engine needs the http request file.
484  *
485  * \initonly
486  */
488 {
489  SCEnter();
490  AppLayerHtpNeedMultipartHeader();
493 
494  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
495  SCReturn;
496 }
497 
498 static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
499 {
500  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
501  if (tx_ud) {
502  if (flags & STREAM_TOCLIENT) {
503  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
504  } else {
505  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
506  }
507  }
508 }
509 
510 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
511 {
512  if (flags & HTP_STREAM_DEPTH_SET) {
513  uint32_t stream_depth = FileReassemblyDepth();
514  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
515  return true;
516  }
517  } else {
518  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
519  return true;
520  }
521  }
522  return false;
523 }
524 
525 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
526  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
527 {
528  uint32_t chunk_len = 0;
529  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
530  (content_len_so_far < (uint64_t)body_limit) &&
531  (content_len_so_far + (uint64_t)data_len) > body_limit)
532  {
533  chunk_len = body_limit - content_len_so_far;
534  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
535  (content_len_so_far < (uint64_t)stream_depth) &&
536  (content_len_so_far + (uint64_t)data_len) > stream_depth)
537  {
538  chunk_len = stream_depth - content_len_so_far;
539  }
540  SCLogDebug("len %u", chunk_len);
541  return (chunk_len == 0 ? data_len : chunk_len);
542 }
543 
544 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
545 struct {
546  const char *msg;
547  int de;
548 } htp_errors[] = {
549  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
550  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
551  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
552  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
553  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
554 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
555  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
556 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
557  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
558  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
559  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
560  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
561 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
562  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
563  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
564  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
565  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
566 };
567 
568 struct {
569  const char *msg;
570  int de;
571 } htp_warnings[] = {
572  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
573  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
574  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
575  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
576  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
577 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
578  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
579  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
580  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
581  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
582  * luckily, "Request server port=" is unique */
583 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
585  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
586  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
587  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
588  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
589  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
590  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
591  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
592  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
593  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
594  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
595  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
596  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
597  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
598  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
599  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
600  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
601  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
602  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
603 };
604 
605 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
606 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
607 
608 /**
609  * \internal
610  *
611  * \brief Get the warning id for the warning msg.
612  *
613  * \param msg warning message
614  *
615  * \retval id the id or 0 in case of not found
616  */
617 static int HTPHandleWarningGetId(const char *msg)
618 {
619  SCLogDebug("received warning \"%s\"", msg);
620  size_t idx;
621  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
622  if (strncmp(htp_warnings[idx].msg, msg,
623  strlen(htp_warnings[idx].msg)) == 0)
624  {
625  return htp_warnings[idx].de;
626  }
627  }
628 
629  return 0;
630 }
631 
632 /**
633  * \internal
634  *
635  * \brief Get the error id for the error msg.
636  *
637  * \param msg error message
638  *
639  * \retval id the id or 0 in case of not found
640  */
641 static int HTPHandleErrorGetId(const char *msg)
642 {
643  SCLogDebug("received error \"%s\"", msg);
644 
645  size_t idx;
646  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
647  if (strncmp(htp_errors[idx].msg, msg,
648  strlen(htp_errors[idx].msg)) == 0)
649  {
650  return htp_errors[idx].de;
651  }
652  }
653 
654  return 0;
655 }
656 
657 /**
658  * \internal
659  *
660  * \brief Check state for errors, warnings and add any as events
661  *
662  * \param s state
663  */
664 static void HTPHandleError(HtpState *s)
665 {
666  if (s == NULL || s->conn == NULL ||
667  s->conn->messages == NULL) {
668  return;
669  }
670 
671  size_t size = htp_list_size(s->conn->messages);
672  size_t msg;
673 
674  for (msg = s->htp_messages_offset; msg < size; msg++) {
675  htp_log_t *log = htp_list_get(s->conn->messages, msg);
676  if (log == NULL)
677  continue;
678 
679  HtpTxUserData *htud = NULL;
680  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
681  if (tx != NULL)
682  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
683 
684  SCLogDebug("message %s", log->msg);
685 
686  int id = HTPHandleErrorGetId(log->msg);
687  if (id == 0) {
688  id = HTPHandleWarningGetId(log->msg);
689  if (id == 0)
691  }
692 
693  if (id > 0) {
694  HTPSetEvent(s, htud, id);
695  }
696  }
697  s->htp_messages_offset = (uint16_t)msg;
698  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
699 }
700 
701 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
702 {
703 #ifdef DEBUG
704  BUG_ON(s == NULL || tx == NULL);
705 #endif
706  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
707  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
708  HTP_HOSTH_INVALID))
709  {
710  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
711  if (htud == NULL)
712  return;
713 
714  if (tx->flags & HTP_REQUEST_INVALID_T_E)
715  HTPSetEvent(s, htud,
717  if (tx->flags & HTP_REQUEST_INVALID_C_L)
718  HTPSetEvent(s, htud,
720  if (tx->flags & HTP_HOST_MISSING)
721  HTPSetEvent(s, htud,
723  if (tx->flags & HTP_HOST_AMBIGUOUS)
724  HTPSetEvent(s, htud,
726  if (tx->flags & HTP_HOSTU_INVALID)
727  HTPSetEvent(s, htud,
729  if (tx->flags & HTP_HOSTH_INVALID)
730  HTPSetEvent(s, htud,
732  }
733  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
734  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
735  if (htud == NULL)
736  return;
737  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED);
738  }
739  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
740  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
741  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
742  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
743  if (htud == NULL)
744  return;
745  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_REQUEST_LINE_INVALID);
746  }
747 }
748 
749 static int Setup(Flow *f, HtpState *hstate)
750 {
751  /* store flow ref in state so callbacks can access it */
752  hstate->f = f;
753 
754  HTPCfgRec *htp_cfg_rec = &cfglist;
755  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
756  void *user_data = NULL;
757 
758  if (FLOW_IS_IPV4(f)) {
759  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
760  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
761  }
762  else if (FLOW_IS_IPV6(f)) {
763  SCLogDebug("Looking up HTP config for ipv6");
764  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
765  }
766  else {
767  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
768  goto error;
769  }
770 
771  if (user_data != NULL) {
772  htp_cfg_rec = user_data;
773  htp = htp_cfg_rec->cfg;
774  SCLogDebug("LIBHTP using config: %p", htp);
775  } else {
776  SCLogDebug("Using default HTP config: %p", htp);
777  }
778 
779  if (NULL == htp) {
780 #ifdef DEBUG_VALIDATION
781  BUG_ON(htp == NULL);
782 #endif
783  /* should never happen if HTPConfigure is properly invoked */
784  goto error;
785  }
786 
787  hstate->connp = htp_connp_create(htp);
788  if (hstate->connp == NULL) {
789  goto error;
790  }
791 
792  hstate->conn = htp_connp_get_connection(hstate->connp);
793 
794  htp_connp_set_user_data(hstate->connp, (void *)hstate);
795  hstate->cfg = htp_cfg_rec;
796 
797  SCLogDebug("New hstate->connp %p", hstate->connp);
798 
799  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
800 
802  htp_cfg_rec->request.inspect_min_size);
804  htp_cfg_rec->response.inspect_min_size);
805  return 0;
806 error:
807  return -1;
808 }
809 
810 /**
811  * \brief Function to handle the reassembled data from client and feed it to
812  * the HTP library to process it.
813  *
814  * \param flow Pointer to the flow the data belong to
815  * \param htp_state Pointer the state in which the parsed value to be stored
816  * \param pstate Application layer parser state for this session
817  * \param input Pointer the received HTTP client data
818  * \param input_len Length in bytes of the received data
819  * \param output Pointer to the output (not used in this function)
820  *
821  * \retval On success returns 1 or on failure returns -1.
822  */
823 static int HTPHandleRequestData(Flow *f, void *htp_state,
824  AppLayerParserState *pstate,
825  const uint8_t *input, uint32_t input_len,
826  void *local_data, const uint8_t flags)
827 {
828  SCEnter();
829  int ret = 1;
830  HtpState *hstate = (HtpState *)htp_state;
831 
832  /* On the first invocation, create the connection parser structure to
833  * be used by HTP library. This is looked up via IP in the radix
834  * tree. Failing that, the default HTP config is used.
835  */
836  if (NULL == hstate->conn) {
837  if (Setup(f, hstate) != 0) {
838  goto error;
839  }
840  }
841  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
842 
843  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
844  /* pass the new data to the htp parser */
845  if (input_len > 0) {
846  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
847  switch (r) {
848  case HTP_STREAM_ERROR:
849  ret = -1;
850  break;
851  default:
852  break;
853  }
854  HTPHandleError(hstate);
855  }
856 
857  /* if the TCP connection is closed, then close the HTTP connection */
859  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
860  {
861  htp_connp_close(hstate->connp, &ts);
862  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
863  SCLogDebug("stream eof encountered, closing htp handle for ts");
864  }
865 
866  SCLogDebug("hstate->connp %p", hstate->connp);
867  SCReturnInt(ret);
868 
869 error:
870  SCReturnInt(-1);
871 }
872 
873 /**
874  * \brief Function to handle the reassembled data from server and feed it to
875  * the HTP library to process it.
876  *
877  * \param flow Pointer to the flow the data belong to
878  * \param htp_state Pointer the state in which the parsed value to be stored
879  * \param pstate Application layer parser state for this session
880  * \param input Pointer the received HTTP server data
881  * \param input_len Length in bytes of the received data
882  * \param output Pointer to the output (not used in this function)
883  *
884  * \retval On success returns 1 or on failure returns -1
885  */
886 static int HTPHandleResponseData(Flow *f, void *htp_state,
887  AppLayerParserState *pstate,
888  const uint8_t *input, uint32_t input_len,
889  void *local_data, const uint8_t flags)
890 {
891  SCEnter();
892  int ret = 1;
893  HtpState *hstate = (HtpState *)htp_state;
894 
895  /* On the first invocation, create the connection parser structure to
896  * be used by HTP library. This is looked up via IP in the radix
897  * tree. Failing that, the default HTP config is used.
898  */
899  if (NULL == hstate->conn) {
900  if (Setup(f, hstate) != 0) {
901  goto error;
902  }
903  }
904  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
905 
906  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
907  if (input_len > 0) {
908  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
909  switch (r) {
910  case HTP_STREAM_ERROR:
911  ret = -1;
912  break;
913  default:
914  break;
915  }
916  HTPHandleError(hstate);
917  }
918 
919  /* if we the TCP connection is closed, then close the HTTP connection */
921  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
922  {
923  htp_connp_close(hstate->connp, &ts);
924  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
925  }
926 
927  SCLogDebug("hstate->connp %p", hstate->connp);
928  SCReturnInt(ret);
929 error:
930  SCReturnInt(-1);
931 }
932 
933 /**
934  * \param name /Lowercase/ version of the variable name
935  */
936 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
937  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
938 {
939 #ifdef PRINT
940  printf("DATA START: \n");
941  PrintRawDataFp(stdout, data, len);
942  printf("DATA END: \n");
943 #endif
944  size_t x;
945  int quote = 0;
946 
947  for (x = 0; x < len; x++) {
948  if (!(isspace(data[x])))
949  break;
950  }
951 
952  if (x >= len)
953  return 0;
954 
955  uint8_t *line = data+x;
956  size_t line_len = len-x;
957  size_t offset = 0;
958 #ifdef PRINT
959  printf("LINE START: \n");
960  PrintRawDataFp(stdout, line, line_len);
961  printf("LINE END: \n");
962 #endif
963  for (x = 0 ; x < line_len; x++) {
964  if (x > 0) {
965  if (line[x - 1] != '\\' && line[x] == '\"') {
966  quote++;
967  }
968 
969  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
970  uint8_t *token = line + offset;
971  size_t token_len = x - offset;
972 
973  if ((x + 1) == line_len) {
974  token_len++;
975  }
976 
977  offset = x + 1;
978 
979  while (offset < line_len && isspace(line[offset])) {
980  x++;
981  offset++;
982  }
983 #ifdef PRINT
984  printf("TOKEN START: \n");
985  PrintRawDataFp(stdout, token, token_len);
986  printf("TOKEN END: \n");
987 #endif
988  if (token_len > name_len) {
989  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
990  uint8_t *value = token + name_len;
991  size_t value_len = token_len - name_len;
992 
993  if (value[0] == '\"') {
994  value++;
995  value_len--;
996  }
997  if (value[value_len-1] == '\"') {
998  value_len--;
999  }
1000 #ifdef PRINT
1001  printf("VALUE START: \n");
1002  PrintRawDataFp(stdout, value, value_len);
1003  printf("VALUE END: \n");
1004 #endif
1005  *retptr = value;
1006  *retlen = value_len;
1007  return 1;
1008  }
1009  }
1010  }
1011  }
1012  }
1013 
1014  return 0;
1015 }
1016 
1017 /**
1018  * \param name /Lowercase/ version of the variable name
1019  */
1020 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1021  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1022 {
1023  SCEnter();
1024 #ifdef PRINT
1025  printf("DATA START: \n");
1026  PrintRawDataFp(stdout, data, len);
1027  printf("DATA END: \n");
1028 #endif
1029  size_t x;
1030  int quote = 0;
1031 
1032  for (x = 0; x < len; x++) {
1033  if (!(isspace(data[x])))
1034  break;
1035  }
1036 
1037  if (x >= len) {
1038  SCReturnInt(0);
1039  }
1040 
1041  uint8_t *line = data+x;
1042  size_t line_len = len-x;
1043  size_t offset = 0;
1044 #ifdef PRINT
1045  printf("LINE START: \n");
1046  PrintRawDataFp(stdout, line, line_len);
1047  printf("LINE END: \n");
1048 #endif
1049  for (x = 0 ; x < line_len; x++) {
1050  if (x > 0) {
1051  if (line[x - 1] != '\\' && line[x] == '\"') {
1052  quote++;
1053  }
1054 
1055  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1056  uint8_t *token = line + offset;
1057  size_t token_len = x - offset;
1058 
1059  if ((x + 1) == line_len) {
1060  token_len++;
1061  }
1062 
1063  offset = x + 1;
1064 
1065  while (offset < line_len && isspace(line[offset])) {
1066  x++;
1067  offset++;
1068  }
1069 #ifdef PRINT
1070  printf("TOKEN START: \n");
1071  PrintRawDataFp(stdout, token, token_len);
1072  printf("TOKEN END: \n");
1073 #endif
1074  if (token_len > name_len) {
1075  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1076  uint8_t *value = token + name_len;
1077  size_t value_len = token_len - name_len;
1078 
1079  if (value[0] == '\"') {
1080  value++;
1081  value_len--;
1082  }
1083  if (value[value_len-1] == '\"') {
1084  value_len--;
1085  }
1086 #ifdef PRINT
1087  printf("VALUE START: \n");
1088  PrintRawDataFp(stdout, value, value_len);
1089  printf("VALUE END: \n");
1090 #endif
1091  *retptr = value;
1092  *retlen = value_len;
1093  SCReturnInt(1);
1094  }
1095  }
1096  }
1097  }
1098  }
1099 
1100  SCReturnInt(0);
1101 }
1102 
1103 /**
1104  * \brief setup multipart parsing: extract boundary and store it
1105  *
1106  * \param d HTTP transaction
1107  * \param htud transaction userdata
1108  *
1109  * \retval 1 ok, multipart set up
1110  * \retval 0 ok, not multipart though
1111  * \retval -1 error: problem with the boundary
1112  *
1113  * If the request contains a multipart message, this function will
1114  * set the HTP_BOUNDARY_SET in the transaction.
1115  */
1116 static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud)
1117 {
1118  htp_header_t *h = (htp_header_t *)htp_table_get_c(d->tx->request_headers,
1119  "Content-Type");
1120  if (h != NULL && bstr_len(h->value) > 0) {
1121  uint8_t *boundary = NULL;
1122  size_t boundary_len = 0;
1123 
1124  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1125  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1126  &boundary, &boundary_len);
1127  if (r == 1) {
1128 #ifdef PRINT
1129  printf("BOUNDARY START: \n");
1130  PrintRawDataFp(stdout, boundary, boundary_len);
1131  printf("BOUNDARY END: \n");
1132 #endif
1133  if (boundary_len < HTP_BOUNDARY_MAX) {
1134  htud->boundary = HTPMalloc(boundary_len);
1135  if (htud->boundary == NULL) {
1136  return -1;
1137  }
1138  htud->boundary_len = (uint8_t)boundary_len;
1139  memcpy(htud->boundary, boundary, boundary_len);
1140 
1141  htud->tsflags |= HTP_BOUNDARY_SET;
1142  } else {
1143  SCLogDebug("invalid boundary");
1144  return -1;
1145  }
1146  SCReturnInt(1);
1147  }
1148  //SCReturnInt(1);
1149  }
1150  SCReturnInt(0);
1151 }
1152 
1153 #define C_D_HDR "content-disposition:"
1154 #define C_D_HDR_LEN 20
1155 #define C_T_HDR "content-type:"
1156 #define C_T_HDR_LEN 13
1157 
1158 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1159  HtpTxUserData *htud,
1160  uint8_t *header, uint32_t header_len,
1161  uint8_t **filename, uint16_t *filename_len,
1162  uint8_t **filetype, uint16_t *filetype_len)
1163 {
1164  uint8_t *fn = NULL;
1165  size_t fn_len = 0;
1166  uint8_t *ft = NULL;
1167  size_t ft_len = 0;
1168 
1169 #ifdef PRINT
1170  printf("HEADER START: \n");
1171  PrintRawDataFp(stdout, header, header_len);
1172  printf("HEADER END: \n");
1173 #endif
1174 
1175  while (header_len > 0) {
1176  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1177  uint8_t *line = header;
1178  uint32_t line_len;
1179 
1180  if (next_line == NULL) {
1181  line_len = header_len;
1182  } else {
1183  line_len = next_line - header;
1184  }
1185  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1186  if (sc == NULL) {
1187  HTPSetEvent(hstate, htud,
1189  /* if the : we found is the final char, it means we have
1190  * no value */
1191  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1192  HTPSetEvent(hstate, htud,
1194  } else {
1195 #ifdef PRINT
1196  printf("LINE START: \n");
1197  PrintRawDataFp(stdout, line, line_len);
1198  printf("LINE END: \n");
1199 #endif
1200  if (line_len >= C_D_HDR_LEN &&
1201  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1202  uint8_t *value = line + C_D_HDR_LEN;
1203  uint32_t value_len = line_len - C_D_HDR_LEN;
1204 
1205  /* parse content-disposition */
1206  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1207  value, value_len, &fn, &fn_len);
1208  } else if (line_len >= C_T_HDR_LEN &&
1209  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1210  SCLogDebug("content-type line");
1211  uint8_t *value = line + C_T_HDR_LEN;
1212  uint32_t value_len = line_len - C_T_HDR_LEN;
1213 
1214  (void)HTTPParseContentTypeHeader(NULL, 0,
1215  value, value_len, &ft, &ft_len);
1216  }
1217  }
1218 
1219  if (next_line == NULL) {
1220  SCLogDebug("no next_line");
1221  break;
1222  }
1223  header_len -= ((next_line + 2) - header);
1224  header = next_line + 2;
1225  } /* while (header_len > 0) */
1226 
1227  if (fn_len > USHRT_MAX)
1228  fn_len = USHRT_MAX;
1229  if (ft_len > USHRT_MAX)
1230  ft_len = USHRT_MAX;
1231 
1232  *filename = fn;
1233  *filename_len = fn_len;
1234  *filetype = ft;
1235  *filetype_len = ft_len;
1236 }
1237 
1238 /**
1239  * \brief Create a single buffer from the HtpBodyChunks in our list
1240  *
1241  * \param htud transaction user data
1242  * \param chunks_buffers pointer to pass back the buffer to the caller
1243  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1244  */
1245 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1246  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1247 {
1249  chunks_buffer, chunks_buffer_len,
1250  htud->request_body.body_parsed);
1251 }
1252 
1253 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1254 {
1255  SCEnter();
1256  if (tx && tx->de_state) {
1257  if (dir == STREAM_TOSERVER) {
1258  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1260  } else if (STREAM_TOCLIENT) {
1261  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1263  }
1264  }
1265 }
1266 
1267 /**
1268  * \brief Setup boundary buffers
1269  */
1270 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1271  uint8_t *boundary, uint32_t boundary_len)
1272 {
1273  memset(boundary, '-', boundary_len);
1274  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1275 }
1276 
1277 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1278  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1279 {
1280  int result = 0;
1281  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1282  uint8_t expected_boundary_len = htud->boundary_len + 2;
1283  uint8_t expected_boundary_end_len = htud->boundary_len + 4;
1284  int tx_progress = 0;
1285 
1286 #ifdef PRINT
1287  printf("CHUNK START: \n");
1288  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1289  printf("CHUNK END: \n");
1290 #endif
1291 
1292  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1293 
1294  /* search for the header start, header end and form end */
1295  uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1296  boundary, expected_boundary_len);
1297  uint8_t *header_end = NULL;
1298  if (header_start != NULL) {
1299  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1300  (uint8_t *)"\r\n\r\n", 4);
1301  }
1302  uint8_t *form_end = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1303  boundary, expected_boundary_end_len);
1304 
1305  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1306  header_end, form_end);
1307 
1308  /* we currently only handle multipart for ts. When we support it for tc,
1309  * we will need to supply right direction */
1310  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, STREAM_TOSERVER);
1311  /* if we're in the file storage process, deal with that now */
1312  if (htud->tsflags & HTP_FILENAME_SET) {
1313  if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1314  SCLogDebug("reached the end of the file");
1315 
1316  const uint8_t *filedata = chunks_buffer;
1317  uint32_t filedata_len = 0;
1318  uint8_t flags = 0;
1319 
1320  if (header_start < form_end || (header_start != NULL && form_end == NULL)) {
1321  filedata_len = header_start - filedata - 2; /* 0d 0a */
1322  } else if (form_end != NULL && form_end < header_start) {
1323  filedata_len = form_end - filedata;
1324  } else if (form_end != NULL && form_end == header_start) {
1325  filedata_len = form_end - filedata - 2; /* 0d 0a */
1326  } else if (tx_progress > HTP_RESPONSE_BODY) {
1327  filedata_len = chunks_buffer_len;
1328  flags = FILE_TRUNCATED;
1329  }
1330 
1331  if (filedata_len > chunks_buffer_len) {
1332  HTPSetEvent(hstate, htud,
1334  goto end;
1335  }
1336 #ifdef PRINT
1337  printf("FILEDATA (final chunk) START: \n");
1338  PrintRawDataFp(stdout, filedata, filedata_len);
1339  printf("FILEDATA (final chunk) END: \n");
1340 #endif
1341  if (!(htud->tsflags & HTP_DONTSTORE)) {
1342  if (HTPFileClose(hstate, filedata, filedata_len, flags,
1343  STREAM_TOSERVER) == -1)
1344  {
1345  goto end;
1346  }
1347  }
1348 
1349  htud->tsflags &=~ HTP_FILENAME_SET;
1350 
1351  /* fall through */
1352  } else {
1353  SCLogDebug("not yet at the end of the file");
1354 
1355  if (chunks_buffer_len > expected_boundary_end_len) {
1356  const uint8_t *filedata = chunks_buffer;
1357  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1358 #ifdef PRINT
1359  printf("FILEDATA (part) START: \n");
1360  PrintRawDataFp(stdout, filedata, filedata_len);
1361  printf("FILEDATA (part) END: \n");
1362 #endif
1363 
1364  if (!(htud->tsflags & HTP_DONTSTORE)) {
1365  result = HTPFileStoreChunk(hstate, filedata,
1366  filedata_len, STREAM_TOSERVER);
1367  if (result == -1) {
1368  goto end;
1369  } else if (result == -2) {
1370  /* we know for sure we're not storing the file */
1371  htud->tsflags |= HTP_DONTSTORE;
1372  }
1373  }
1374 
1375  htud->request_body.body_parsed += filedata_len;
1376  } else {
1377  SCLogDebug("chunk too small to already process in part");
1378  }
1379 
1380  goto end;
1381  }
1382  }
1383 
1384  while (header_start != NULL && header_end != NULL &&
1385  header_end != form_end &&
1386  header_start < (chunks_buffer + chunks_buffer_len) &&
1387  header_end < (chunks_buffer + chunks_buffer_len) &&
1388  header_start < header_end)
1389  {
1390  uint8_t *filename = NULL;
1391  uint16_t filename_len = 0;
1392  uint8_t *filetype = NULL;
1393  uint16_t filetype_len = 0;
1394 
1395  uint32_t header_len = header_end - header_start;
1396  SCLogDebug("header_len %u", header_len);
1397  uint8_t *header = header_start;
1398 
1399  /* skip empty records */
1400  if (expected_boundary_len == header_len) {
1401  goto next;
1402  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1403  header_len -= (expected_boundary_len + 2);
1404  header = header_start + (expected_boundary_len + 2); // + for 0d 0a
1405  }
1406 
1407  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1408  &filename, &filename_len, &filetype, &filetype_len);
1409 
1410  if (filename != NULL) {
1411  uint8_t *filedata = NULL;
1412  uint32_t filedata_len = 0;
1413 
1414  SCLogDebug("we have a filename");
1415 
1416  htud->tsflags |= HTP_FILENAME_SET;
1417  htud->tsflags &= ~HTP_DONTSTORE;
1418 
1419  SCLogDebug("header_end %p", header_end);
1420  SCLogDebug("form_end %p", form_end);
1421 
1422  /* everything until the final boundary is the file */
1423  if (form_end != NULL) {
1424  filedata = header_end + 4;
1425  if (form_end == filedata) {
1426  HTPSetEvent(hstate, htud,
1428  goto end;
1429  } else if (form_end < filedata) {
1430  HTPSetEvent(hstate, htud,
1432  goto end;
1433  }
1434 
1435  filedata_len = form_end - (header_end + 4 + 2);
1436  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1437 
1438  /* or is it? */
1439  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1440  boundary, expected_boundary_len);
1441  if (header_next != NULL) {
1442  filedata_len -= (form_end - header_next);
1443  }
1444 
1445  if (filedata_len > chunks_buffer_len) {
1446  HTPSetEvent(hstate, htud,
1448  goto end;
1449  }
1450  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1451 #ifdef PRINT
1452  printf("FILEDATA START: \n");
1453  PrintRawDataFp(stdout, filedata, filedata_len);
1454  printf("FILEDATA END: \n");
1455 #endif
1456 
1457  result = HTPFileOpen(hstate, filename, filename_len,
1458  filedata, filedata_len, hstate->transaction_cnt,
1459  STREAM_TOSERVER);
1460  if (result == -1) {
1461  goto end;
1462  } else if (result == -2) {
1463  htud->tsflags |= HTP_DONTSTORE;
1464  } else {
1465  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1466  goto end;
1467  }
1468  }
1469  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1470 
1471  htud->request_body.body_parsed += (header_end - chunks_buffer);
1472  htud->tsflags &= ~HTP_FILENAME_SET;
1473  } else {
1474  SCLogDebug("chunk doesn't contain form end");
1475 
1476  filedata = header_end + 4;
1477  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1478  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1479 
1480  if (filedata_len > chunks_buffer_len) {
1481  HTPSetEvent(hstate, htud,
1483  goto end;
1484  }
1485 
1486 #ifdef PRINT
1487  printf("FILEDATA START: \n");
1488  PrintRawDataFp(stdout, filedata, filedata_len);
1489  printf("FILEDATA END: \n");
1490 #endif
1491  /* form doesn't end in this chunk, but part might. Lets
1492  * see if have another coming up */
1493  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1494  boundary, expected_boundary_len);
1495  SCLogDebug("header_next %p", header_next);
1496  if (header_next == NULL) {
1497  /* no, but we'll handle the file data when we see the
1498  * form_end */
1499 
1500  SCLogDebug("more file data to come");
1501 
1502  uint32_t offset = (header_end + 4) - chunks_buffer;
1503  SCLogDebug("offset %u", offset);
1504  htud->request_body.body_parsed += offset;
1505 
1506  result = HTPFileOpen(hstate, filename, filename_len,
1507  NULL, 0, hstate->transaction_cnt,
1508  STREAM_TOSERVER);
1509  if (result == -1) {
1510  goto end;
1511  } else if (result == -2) {
1512  htud->tsflags |= HTP_DONTSTORE;
1513  }
1514  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1515 
1516  } else if (header_next - filedata > 2) {
1517  filedata_len = header_next - filedata - 2;
1518  SCLogDebug("filedata_len %u", filedata_len);
1519 
1520  result = HTPFileOpen(hstate, filename, filename_len,
1521  filedata, filedata_len, hstate->transaction_cnt,
1522  STREAM_TOSERVER);
1523  if (result == -1) {
1524  goto end;
1525  } else if (result == -2) {
1526  htud->tsflags |= HTP_DONTSTORE;
1527  } else {
1528  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1529  goto end;
1530  }
1531  }
1532  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1533 
1534  htud->tsflags &= ~HTP_FILENAME_SET;
1535  htud->request_body.body_parsed += (header_end - chunks_buffer);
1536  }
1537  }
1538  }
1539 next:
1540  SCLogDebug("header_start %p, header_end %p, form_end %p",
1541  header_start, header_end, form_end);
1542 
1543  /* Search next boundary entry after the start of body */
1544  uint32_t cursizeread = header_end - chunks_buffer;
1545  header_start = Bs2bmSearch(header_end + 4,
1546  chunks_buffer_len - (cursizeread + 4),
1547  boundary, expected_boundary_len);
1548  if (header_start != NULL) {
1549  header_end = Bs2bmSearch(header_end + 4,
1550  chunks_buffer_len - (cursizeread + 4),
1551  (uint8_t *) "\r\n\r\n", 4);
1552  }
1553  }
1554 
1555  /* if we're parsing the multipart and we're not currently processing a
1556  * file, we move the body pointer forward. */
1557  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1558  if (chunks_buffer_len > expected_boundary_end_len) {
1559  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1560 
1561  htud->request_body.body_parsed += move;
1562  SCLogDebug("form not ready, file not set, parsing non-file "
1563  "record: moved %u", move);
1564  }
1565  }
1566 
1567 end:
1568  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1569  return 0;
1570 }
1571 
1572 /** \brief setup things for put request
1573  * \todo really needed? */
1574 static int HtpRequestBodySetupPUT(htp_tx_data_t *d, HtpTxUserData *htud)
1575 {
1576 // if (d->tx->parsed_uri == NULL || d->tx->parsed_uri->path == NULL) {
1577 // return -1;
1578 // }
1579 
1580  /* filename is d->tx->parsed_uri->path */
1581 
1582  return 0;
1583 }
1584 
1585 /** \internal
1586  * \brief Handle POST, no multipart body data
1587  */
1588 static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
1589  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1590 {
1591  int result = 0;
1592 
1593  /* see if we need to open the file */
1594  if (!(htud->tsflags & HTP_FILENAME_SET))
1595  {
1596  uint8_t *filename = NULL;
1597  size_t filename_len = 0;
1598 
1599  /* get the name */
1600  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1601  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1602  filename_len = bstr_len(tx->parsed_uri->path);
1603  }
1604 
1605  if (filename != NULL) {
1606  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1607  hstate->transaction_cnt, STREAM_TOSERVER);
1608  if (result == -1) {
1609  goto end;
1610  } else if (result == -2) {
1611  htud->tsflags |= HTP_DONTSTORE;
1612  } else {
1613  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1614  htud->tsflags |= HTP_FILENAME_SET;
1615  htud->tsflags &= ~HTP_DONTSTORE;
1616  }
1617  }
1618  }
1619  else
1620  {
1621  /* otherwise, just store the data */
1622 
1623  if (!(htud->tsflags & HTP_DONTSTORE)) {
1624  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1625  if (result == -1) {
1626  goto end;
1627  } else if (result == -2) {
1628  /* we know for sure we're not storing the file */
1629  htud->tsflags |= HTP_DONTSTORE;
1630  }
1631  }
1632  }
1633 
1634  return 0;
1635 end:
1636  return -1;
1637 }
1638 
1639 /** \internal
1640  * \brief Handle PUT body data
1641  */
1642 static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
1643  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1644 {
1645  int result = 0;
1646 
1647  /* see if we need to open the file */
1648  if (!(htud->tsflags & HTP_FILENAME_SET))
1649  {
1650  uint8_t *filename = NULL;
1651  size_t filename_len = 0;
1652 
1653  /* get the name */
1654  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1655  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1656  filename_len = bstr_len(tx->parsed_uri->path);
1657  }
1658 
1659  if (filename != NULL) {
1660  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1661  hstate->transaction_cnt, STREAM_TOSERVER);
1662  if (result == -1) {
1663  goto end;
1664  } else if (result == -2) {
1665  htud->tsflags |= HTP_DONTSTORE;
1666  } else {
1667  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1668  htud->tsflags |= HTP_FILENAME_SET;
1669  htud->tsflags &= ~HTP_DONTSTORE;
1670  }
1671  }
1672  }
1673  else
1674  {
1675  /* otherwise, just store the data */
1676 
1677  if (!(htud->tsflags & HTP_DONTSTORE)) {
1678  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1679  if (result == -1) {
1680  goto end;
1681  } else if (result == -2) {
1682  /* we know for sure we're not storing the file */
1683  htud->tsflags |= HTP_DONTSTORE;
1684  }
1685  }
1686  }
1687 
1688  return 0;
1689 end:
1690  return -1;
1691 }
1692 
1693 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1694  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1695 {
1696  SCEnter();
1697 
1698  int result = 0;
1699 
1700  /* see if we need to open the file
1701  * we check for tx->response_line in case of junk
1702  * interpreted as body before response line
1703  */
1704  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1705  (tx->response_line != NULL || tx->is_protocol_0_9))
1706  {
1707  SCLogDebug("setting up file name");
1708 
1709  uint8_t *filename = NULL;
1710  size_t filename_len = 0;
1711 
1712  /* try Content-Disposition header first */
1713  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1714  "Content-Disposition");
1715  if (h != NULL && bstr_len(h->value) > 0) {
1716  /* parse content-disposition */
1717  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1718  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1719  }
1720 
1721  /* fall back to name from the uri */
1722  if (filename == NULL) {
1723  /* get the name */
1724  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1725  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1726  filename_len = bstr_len(tx->parsed_uri->path);
1727  }
1728  }
1729 
1730  if (filename != NULL) {
1731  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len,
1732  data, data_len, hstate->transaction_cnt, STREAM_TOCLIENT);
1733  SCLogDebug("result %d", result);
1734  if (result == -1) {
1735  goto end;
1736  } else if (result == -2) {
1737  htud->tcflags |= HTP_DONTSTORE;
1738  } else {
1739  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1740  htud->tcflags |= HTP_FILENAME_SET;
1741  htud->tcflags &= ~HTP_DONTSTORE;
1742  }
1743  //set range if present
1744  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1745  if (h_content_range != NULL) {
1746  HTPFileSetRange(hstate, h_content_range->value);
1747  }
1748  }
1749  }
1750  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1751  {
1752  /* otherwise, just store the data */
1753 
1754  if (!(htud->tcflags & HTP_DONTSTORE)) {
1755  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1756  SCLogDebug("result %d", result);
1757  if (result == -1) {
1758  goto end;
1759  } else if (result == -2) {
1760  /* we know for sure we're not storing the file */
1761  htud->tcflags |= HTP_DONTSTORE;
1762  }
1763  }
1764  }
1765 
1766  htud->response_body.body_parsed += data_len;
1767  return 0;
1768 end:
1769  return -1;
1770 }
1771 
1772 /**
1773  * \brief Function callback to append chunks for Requests
1774  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1775  * \retval int HTP_OK if all goes well
1776  */
1777 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1778 {
1779  SCEnter();
1780 
1781  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1782  SCReturnInt(HTP_OK);
1783 
1784  if (d->data == NULL || d->len == 0)
1785  SCReturnInt(HTP_OK);
1786 
1787 #ifdef PRINT
1788  printf("HTPBODY START: \n");
1789  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1790  printf("HTPBODY END: \n");
1791 #endif
1792 
1793  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1794  if (hstate == NULL) {
1795  SCReturnInt(HTP_ERROR);
1796  }
1797 
1798  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1799  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1800 
1801  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1802  if (tx_ud == NULL) {
1803  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1804  if (unlikely(tx_ud == NULL)) {
1805  SCReturnInt(HTP_OK);
1806  }
1807  memset(tx_ud, 0, sizeof(HtpTxUserData));
1808 
1809  /* Set the user data for handling body chunks on this transaction */
1810  htp_tx_set_user_data(d->tx, tx_ud);
1811  }
1812  if (!tx_ud->response_body_init) {
1813  tx_ud->response_body_init = 1;
1814 
1815  if (d->tx->request_method_number == HTP_M_POST) {
1816  SCLogDebug("POST");
1817  int r = HtpRequestBodySetupMultipart(d, tx_ud);
1818  if (r == 1) {
1820  } else if (r == 0) {
1822  SCLogDebug("not multipart");
1823  }
1824  } else if (d->tx->request_method_number == HTP_M_PUT) {
1825  if (HtpRequestBodySetupPUT(d, tx_ud) == 0) {
1827  }
1828  }
1829  }
1830 
1831  /* see if we can get rid of htp body chunks */
1832  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1833 
1834  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1835  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1836 
1837  /* within limits, add the body chunk to the state. */
1838  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1839  uint32_t stream_depth = FileReassemblyDepth();
1840  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1841  hstate->cfg->request.body_limit,
1842  stream_depth,
1843  tx_ud->tsflags,
1844  (uint32_t)d->len);
1845  BUG_ON(len > (uint32_t)d->len);
1846 
1847  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1848 
1849  const uint8_t *chunks_buffer = NULL;
1850  uint32_t chunks_buffer_len = 0;
1851 
1853  /* multi-part body handling starts here */
1854  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1855  goto end;
1856  }
1857 
1858  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1859  if (chunks_buffer == NULL) {
1860  goto end;
1861  }
1862 #ifdef PRINT
1863  printf("REASSCHUNK START: \n");
1864  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1865  printf("REASSCHUNK END: \n");
1866 #endif
1867 
1868  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1869 
1870  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
1871  HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1872  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
1873  HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1874  }
1875 
1876  } else {
1877  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1878  SCLogDebug("closing file that was being stored");
1879  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1880  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1881  }
1882  }
1883 
1884 end:
1885  if (hstate->conn != NULL) {
1886  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1888  hstate->cfg->request.inspect_min_size,
1889  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1890 
1891  /* if we reach the inspect_min_size we'll trigger inspection,
1892  * so make sure that raw stream is also inspected. Set the
1893  * data to be used to the amount of raw bytes we've seen to
1894  * get here. */
1895  if (tx_ud->request_body.body_inspected == 0 &&
1897  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1898  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1899  {
1900  uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1901 
1902  /* body still in progress, but due to min inspect size we need to inspect now */
1905  }
1906  /* after the start of the body, disable the depth logic */
1907  } else if (tx_ud->request_body.body_inspected > 0) {
1909  }
1910  }
1911  SCReturnInt(HTP_OK);
1912 }
1913 
1914 /**
1915  * \brief Function callback to append chunks for Responses
1916  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1917  * \retval int HTP_OK if all goes well
1918  */
1919 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1920 {
1921  SCEnter();
1922 
1923  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1924  SCReturnInt(HTP_OK);
1925 
1926  if (d->data == NULL || d->len == 0)
1927  SCReturnInt(HTP_OK);
1928 
1929  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1930  if (hstate == NULL) {
1931  SCReturnInt(HTP_ERROR);
1932  }
1933 
1934  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1935  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1936 
1937  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1938  if (tx_ud == NULL) {
1939  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1940  if (unlikely(tx_ud == NULL)) {
1941  SCReturnInt(HTP_OK);
1942  }
1943  memset(tx_ud, 0, sizeof(HtpTxUserData));
1944 
1945  /* Set the user data for handling body chunks on this transaction */
1946  htp_tx_set_user_data(d->tx, tx_ud);
1947  }
1948  if (!tx_ud->request_body_init) {
1949  tx_ud->request_body_init = 1;
1950  }
1951 
1952  /* see if we can get rid of htp body chunks */
1953  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1954 
1955  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1956  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1957 
1958  /* within limits, add the body chunk to the state. */
1959  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1960  uint32_t stream_depth = FileReassemblyDepth();
1961  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1962  hstate->cfg->response.body_limit,
1963  stream_depth,
1964  tx_ud->tcflags,
1965  (uint32_t)d->len);
1966  BUG_ON(len > (uint32_t)d->len);
1967 
1968  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1969 
1970  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1971  } else {
1972  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1973  SCLogDebug("closing file that was being stored");
1974  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1975  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1976  }
1977  }
1978 
1979  if (hstate->conn != NULL) {
1980  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1982  hstate->cfg->response.inspect_min_size,
1983  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
1984  /* if we reach the inspect_min_size we'll trigger inspection,
1985  * so make sure that raw stream is also inspected. Set the
1986  * data to be used to the amount of raw bytes we've seen to
1987  * get here. */
1988  if (tx_ud->response_body.body_inspected == 0 &&
1990  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
1991  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
1992  {
1993  uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
1994 
1995  /* body still in progress, but due to min inspect size we need to inspect now */
1998  }
1999  /* after the start of the body, disable the depth logic */
2000  } else if (tx_ud->response_body.body_inspected > 0) {
2002  }
2003  }
2004  SCReturnInt(HTP_OK);
2005 }
2006 
2007 /**
2008  * \brief Print the stats of the HTTP requests
2009  */
2011 {
2012 #ifdef DEBUG
2013  SCEnter();
2014  SCMutexLock(&htp_state_mem_lock);
2015  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2016  htp_state_memcnt, htp_state_memuse);
2017  SCMutexUnlock(&htp_state_mem_lock);
2018  SCReturn;
2019 #endif
2020 }
2021 
2022 /** \brief Clears the HTTP server configuration memory used by HTP library */
2023 void HTPFreeConfig(void)
2024 {
2025  SCEnter();
2026 
2027  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2028  !AppLayerParserConfParserEnabled("tcp", "http"))
2029  {
2030  SCReturn;
2031  }
2032 
2033  HTPCfgRec *nextrec = cfglist.next;
2034  SCRadixReleaseRadixTree(cfgtree);
2035  cfgtree = NULL;
2036  htp_config_destroy(cfglist.cfg);
2037  while (nextrec != NULL) {
2038  HTPCfgRec *htprec = nextrec;
2039  nextrec = nextrec->next;
2040 
2041  htp_config_destroy(htprec->cfg);
2042  SCFree(htprec);
2043  }
2044  HTPDestroyMemcap();
2045  SCReturn;
2046 }
2047 
2048 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2049 {
2050  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2051  if (htud != NULL) {
2052  htud->request_has_trailers = 1;
2053  }
2054  return HTP_OK;
2055 }
2056 
2057 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2058 {
2059  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2060  if (htud != NULL) {
2061  htud->response_has_trailers = 1;
2062  }
2063  return HTP_OK;
2064 }
2065 
2066 /**\internal
2067  * \brief called at start of request
2068  * Set min inspect size.
2069  */
2070 static int HTPCallbackRequestStart(htp_tx_t *tx)
2071 {
2072  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2073  if (hstate == NULL) {
2074  SCReturnInt(HTP_ERROR);
2075  }
2076 
2077  if (hstate->cfg)
2079  hstate->cfg->request.inspect_min_size);
2080  SCReturnInt(HTP_OK);
2081 }
2082 
2083 /**\internal
2084  * \brief called at start of response
2085  * Set min inspect size.
2086  */
2087 static int HTPCallbackResponseStart(htp_tx_t *tx)
2088 {
2089  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2090  if (hstate == NULL) {
2091  SCReturnInt(HTP_ERROR);
2092  }
2093 
2094  if (hstate->cfg)
2096  hstate->cfg->response.inspect_min_size);
2097  SCReturnInt(HTP_OK);
2098 }
2099 
2100 
2101 /**
2102  * \brief callback for request to store the recent incoming request
2103  in to the recent_in_tx for the given htp state
2104  * \param connp pointer to the current connection parser which has the htp
2105  * state in it as user data
2106  */
2107 static int HTPCallbackRequest(htp_tx_t *tx)
2108 {
2109  SCEnter();
2110 
2111  if (tx == NULL) {
2112  SCReturnInt(HTP_ERROR);
2113  }
2114 
2115  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2116  if (hstate == NULL) {
2117  SCReturnInt(HTP_ERROR);
2118  }
2119 
2120  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2121  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2122 
2123  SCLogDebug("HTTP request completed");
2124 
2125  HTPErrorCheckTxRequestFlags(hstate, tx);
2126 
2127  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2128  if (htud != NULL) {
2129  if (htud->tsflags & HTP_FILENAME_SET) {
2130  SCLogDebug("closing file that was being stored");
2131  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
2132  htud->tsflags &= ~HTP_FILENAME_SET;
2134  (uint32_t)hstate->conn->in_data_counter);
2135  }
2136  }
2137 
2138  hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
2139  /* request done, do raw reassembly now to inspect state and stream
2140  * at the same time. */
2142  SCReturnInt(HTP_OK);
2143 }
2144 
2145 /**
2146  * \brief callback for response to remove the recent received requests
2147  from the recent_in_tx for the given htp state
2148  * \param connp pointer to the current connection parser which has the htp
2149  * state in it as user data
2150  */
2151 static int HTPCallbackResponse(htp_tx_t *tx)
2152 {
2153  SCEnter();
2154 
2155  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2156  if (hstate == NULL) {
2157  SCReturnInt(HTP_ERROR);
2158  }
2159 
2160  /* we have one whole transaction now */
2161  hstate->transaction_cnt++;
2162 
2163  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2164  if (htud != NULL) {
2165  if (htud->tcflags & HTP_FILENAME_SET) {
2166  SCLogDebug("closing file that was being stored");
2167  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
2168  htud->tcflags &= ~HTP_FILENAME_SET;
2169  }
2170  }
2171 
2172  /* response done, do raw reassembly now to inspect state and stream
2173  * at the same time. */
2175 
2176  /* handle HTTP CONNECT */
2177  if (tx->request_method_number == HTP_M_CONNECT) {
2178  /* any 2XX status response implies that the connection will become
2179  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2180  if ((tx->response_status_number >= 200) &&
2181  (tx->response_status_number < 300) &&
2182  (hstate->transaction_cnt == 1)) {
2183  uint16_t dp = 0;
2184  if (tx->request_port_number != -1) {
2185  dp = (uint16_t)tx->request_port_number;
2186  }
2187  // both ALPROTO_HTTP and ALPROTO_TLS are normal options
2189  tx->request_progress = HTP_REQUEST_COMPLETE;
2190  tx->response_progress = HTP_RESPONSE_COMPLETE;
2191  }
2192  }
2193 
2194  hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
2195  SCReturnInt(HTP_OK);
2196 }
2197 
2198 static int HTPCallbackRequestLine(htp_tx_t *tx)
2199 {
2200  HtpTxUserData *tx_ud;
2201  bstr *request_uri_normalized;
2202  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2203  const HTPCfgRec *cfg = hstate->cfg;
2204 
2205  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2206  if (request_uri_normalized == NULL)
2207  return HTP_OK;
2208 
2209  tx_ud = htp_tx_get_user_data(tx);
2210  if (likely(tx_ud == NULL)) {
2211  tx_ud = HTPMalloc(sizeof(*tx_ud));
2212  if (unlikely(tx_ud == NULL)) {
2213  bstr_free(request_uri_normalized);
2214  return HTP_OK;
2215  }
2216  memset(tx_ud, 0, sizeof(*tx_ud));
2217  htp_tx_set_user_data(tx, tx_ud);
2218  }
2219  if (unlikely(tx_ud->request_uri_normalized != NULL))
2220  bstr_free(tx_ud->request_uri_normalized);
2221  tx_ud->request_uri_normalized = request_uri_normalized;
2222 
2223  if (tx->flags) {
2224  HTPErrorCheckTxRequestFlags(hstate, tx);
2225  }
2226  return HTP_OK;
2227 }
2228 
2229 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2230 {
2231  if (part == NULL)
2232  return HTP_OK;
2233 
2234  uint64_t flags = 0;
2235  size_t prevlen = bstr_len(part);
2236  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2237  // shorter string means that uri was encoded
2238  if (res == HTP_OK && prevlen > bstr_len(part)) {
2239  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2240  if (likely(htud == NULL)) {
2241  htud = HTPCalloc(1, sizeof(*htud));
2242  if (unlikely(htud == NULL))
2243  return HTP_OK;
2244  htp_tx_set_user_data(tx, htud);
2245  }
2246  HtpState *s = htp_connp_get_user_data(tx->connp);
2247  if (s == NULL)
2248  return HTP_OK;
2249  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI);
2250  }
2251 
2252  return HTP_OK;
2253 }
2254 
2255 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2256 {
2257  if (tx->parsed_uri == NULL)
2258  return HTP_OK;
2259 
2260  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2261 }
2262 
2263 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2264 {
2265  if (tx->parsed_uri == NULL)
2266  return HTP_OK;
2267 
2268  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2269 }
2270 
2271 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2272 {
2273  void *ptmp;
2274  if (tx_data->len == 0 || tx_data->tx == NULL)
2275  return HTP_OK;
2276 
2277  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2278  if (tx_ud == NULL) {
2279  tx_ud = HTPMalloc(sizeof(*tx_ud));
2280  if (unlikely(tx_ud == NULL))
2281  return HTP_OK;
2282  memset(tx_ud, 0, sizeof(*tx_ud));
2283  htp_tx_set_user_data(tx_data->tx, tx_ud);
2284  }
2285  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2286  tx_ud->request_headers_raw_len,
2287  tx_ud->request_headers_raw_len + tx_data->len);
2288  if (ptmp == NULL) {
2289  /* error: we're freeing the entire user data */
2290  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2291  HtpTxUserDataFree(hstate, tx_ud);
2292  htp_tx_set_user_data(tx_data->tx, NULL);
2293  return HTP_OK;
2294  }
2295  tx_ud->request_headers_raw = ptmp;
2296 
2297  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2298  tx_data->data, tx_data->len);
2299  tx_ud->request_headers_raw_len += tx_data->len;
2300 
2301  if (tx_data->tx && tx_data->tx->flags) {
2302  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2303  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2304  }
2305  return HTP_OK;
2306 }
2307 
2308 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2309 {
2310  void *ptmp;
2311  if (tx_data->len == 0 || tx_data->tx == NULL)
2312  return HTP_OK;
2313 
2314  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2315  if (tx_ud == NULL) {
2316  tx_ud = HTPMalloc(sizeof(*tx_ud));
2317  if (unlikely(tx_ud == NULL))
2318  return HTP_OK;
2319  memset(tx_ud, 0, sizeof(*tx_ud));
2320  htp_tx_set_user_data(tx_data->tx, tx_ud);
2321  }
2322  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2323  tx_ud->response_headers_raw_len,
2324  tx_ud->response_headers_raw_len + tx_data->len);
2325  if (ptmp == NULL) {
2326  /* error: we're freeing the entire user data */
2327  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2328  HtpTxUserDataFree(hstate, tx_ud);
2329  htp_tx_set_user_data(tx_data->tx, NULL);
2330  return HTP_OK;
2331  }
2332  tx_ud->response_headers_raw = ptmp;
2333 
2334  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2335  tx_data->data, tx_data->len);
2336  tx_ud->response_headers_raw_len += tx_data->len;
2337 
2338  return HTP_OK;
2339 }
2340 
2341 /*
2342  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2343  */
2344 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2345 {
2346  cfg_prec->uri_include_all = FALSE;
2353 
2354  if (!g_disable_randomness) {
2356  } else {
2357  cfg_prec->randomize = 0;
2358  }
2360 
2361  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2362  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2363  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2364  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2365 
2366  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2367  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2368 
2369  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2370  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2371 
2372  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2373  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
2374 
2375  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2376  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
2377 
2378  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2379 
2380  /* don't convert + to space by default */
2381  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2382 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2383  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2385 #endif
2386 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2387  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2389 #endif
2390  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2391  * only the hard limit. So we set both here to the (current) htp defaults.
2392  * The reason we do this is that if the user sets the hard limit in the
2393  * config, we have to set the soft limit as well. If libhtp starts using
2394  * the soft limit in the future, we at least make sure we control what
2395  * it's value is. */
2396  htp_config_set_field_limits(cfg_prec->cfg,
2399  return;
2400 }
2401 
2402 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2403  * but we can get both <0 and >RAND_MAX values from RandomGet
2404  */
2405 static int RandomGetWrap(void)
2406 {
2407  unsigned long r;
2408 
2409  do {
2410  r = RandomGet();
2411  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2412 
2413  return r % RAND_MAX;
2414 }
2415 
2416 /*
2417  * We have this splitup so that in case double decoding has been enabled
2418  * for query and path, they would be called first on the callback queue,
2419  * before the callback set by Phase2() is called. We need this, since
2420  * the callback in Phase2() generates the normalized uri which utilizes
2421  * the query and path. */
2422 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2423 {
2424  /* randomize inspection size if needed */
2425  if (cfg_prec->randomize) {
2426  int rdrange = cfg_prec->randomize_range;
2427 
2428  long int r = RandomGetWrap();
2429  cfg_prec->request.inspect_min_size +=
2430  (int) (cfg_prec->request.inspect_min_size *
2431  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2432 
2433  r = RandomGetWrap();
2434  cfg_prec->request.inspect_window +=
2435  (int) (cfg_prec->request.inspect_window *
2436  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2437  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2438  " %d and 'request-body-inspect-window' set to %d after"
2439  " randomization.",
2440  name,
2441  cfg_prec->request.inspect_min_size,
2442  cfg_prec->request.inspect_window);
2443 
2444 
2445  r = RandomGetWrap();
2446  cfg_prec->response.inspect_min_size +=
2447  (int) (cfg_prec->response.inspect_min_size *
2448  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2449 
2450  r = RandomGetWrap();
2451  cfg_prec->response.inspect_window +=
2452  (int) (cfg_prec->response.inspect_window *
2453  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2454 
2455  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2456  " %d and 'response-body-inspect-window' set to %d after"
2457  " randomization.",
2458  name,
2459  cfg_prec->response.inspect_min_size,
2460  cfg_prec->response.inspect_window);
2461  }
2462 
2463  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2464 
2465  cfg_prec->request.sbcfg.flags = 0;
2466  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2467  cfg_prec->request.inspect_window : 256;
2468  cfg_prec->request.sbcfg.buf_slide = 0;
2469  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2470  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2471  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2472  cfg_prec->request.sbcfg.Free = HTPFree;
2473 
2474  cfg_prec->response.sbcfg.flags = 0;
2475  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2476  cfg_prec->response.inspect_window : 256;
2477  cfg_prec->response.sbcfg.buf_slide = 0;
2478  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2479  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2480  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2481  cfg_prec->response.sbcfg.Free = HTPFree;
2482  return;
2483 }
2484 
2485 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2486  SCRadixTree *tree)
2487 {
2488  if (cfg_prec == NULL || s == NULL || tree == NULL)
2489  return;
2490 
2491  ConfNode *p = NULL;
2492 
2493  /* Default Parameters */
2494  TAILQ_FOREACH(p, &s->head, next) {
2495 
2496  if (strcasecmp("address", p->name) == 0) {
2497  ConfNode *pval;
2498  /* Addresses */
2499  TAILQ_FOREACH(pval, &p->head, next) {
2500  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2501  pval->val);
2502 
2503  /* IPV6 or IPV4? */
2504  if (strchr(pval->val, ':') != NULL) {
2505  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2506  s->name, pval->val, cfg_prec->cfg);
2507  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2508  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2509  "add ipv6 server %s, ignoring", pval->val);
2510  }
2511  } else {
2512  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2513  s->name, pval->val, cfg_prec->cfg);
2514  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2515  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2516  "to add ipv4 server %s, ignoring",
2517  pval->val);
2518  }
2519  } /* else - if (strchr(pval->val, ':') != NULL) */
2520  } /* TAILQ_FOREACH(pval, &p->head, next) */
2521 
2522  } else if (strcasecmp("personality", p->name) == 0) {
2523  /* Personalities */
2524  int personality = HTPLookupPersonality(p->val);
2525  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2526  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2527 
2528  if (personality >= 0) {
2529  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2530  personality);
2531  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2532  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2533  "personality \"%s\", ignoring", p->val);
2534  } else {
2535  SCLogDebug("LIBHTP personality set to %s",
2536  HTPLookupPersonalityString(personality));
2537  }
2538 
2539  /* The IDS personality by default converts the path (and due to
2540  * our query string callback also the query string) to lowercase.
2541  * Signatures do not expect this, so override it. */
2542  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2543  } else {
2544  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2545  "\"%s\", ignoring", p->val);
2546  continue;
2547  }
2548 
2549  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2550  strcasecmp("request_body_limit", p->name) == 0) {
2551  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2552  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2553  "from conf file - %s. Killing engine", p->val);
2554  exit(EXIT_FAILURE);
2555  }
2556 
2557  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2558  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2559  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2560  "from conf file - %s. Killing engine", p->val);
2561  exit(EXIT_FAILURE);
2562  }
2563 
2564  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2565  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2566  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2567  "from conf file - %s. Killing engine", p->val);
2568  exit(EXIT_FAILURE);
2569  }
2570 
2571  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2572  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2573  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2574  "from conf file - %s. Killing engine", p->val);
2575  exit(EXIT_FAILURE);
2576  }
2577 
2578  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2579  if (ConfValIsTrue(p->val)) {
2580  htp_config_register_request_line(cfg_prec->cfg,
2581  HTPCallbackDoubleDecodeQuery);
2582  }
2583 
2584  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2585  if (ConfValIsTrue(p->val)) {
2586  htp_config_register_request_line(cfg_prec->cfg,
2587  HTPCallbackDoubleDecodePath);
2588  }
2589 
2590  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2591  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2592  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2593  "from conf file - %s. Killing engine", p->val);
2594  exit(EXIT_FAILURE);
2595  }
2596 
2597  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2598  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2599  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2600  "from conf file - %s. Killing engine", p->val);
2601  exit(EXIT_FAILURE);
2602  }
2603 
2604  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2605  uint32_t value = 2;
2606  if (ParseSizeStringU32(p->val, &value) < 0) {
2607  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2608  "from conf file - %s. Killing engine", p->val);
2609  exit(EXIT_FAILURE);
2610  }
2611 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2612  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2613 #else
2614  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2615  "to %u, libhtp version too old", value);
2616 #endif
2617  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2618  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2619  HTP_DECODER_URL_PATH,
2620  ConfValIsTrue(p->val));
2621  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2622  if (strlen(p->val) == 1) {
2623  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2624  HTP_DECODER_URL_PATH,
2625  p->val[0]);
2626  } else {
2627  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2628  "for libhtp param path-bestfit-replacement-char");
2629  }
2630  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2631  htp_config_set_convert_lowercase(cfg_prec->cfg,
2632  HTP_DECODER_URL_PATH,
2633  ConfValIsTrue(p->val));
2634  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2635  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2636  HTP_DECODER_URL_PATH,
2637  ConfValIsTrue(p->val));
2638  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2639  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2640  HTP_DECODER_URL_PATH,
2641  ConfValIsTrue(p->val));
2642  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2643  htp_config_set_path_separators_compress(cfg_prec->cfg,
2644  HTP_DECODER_URL_PATH,
2645  ConfValIsTrue(p->val));
2646  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2647  htp_config_set_path_separators_decode(cfg_prec->cfg,
2648  HTP_DECODER_URL_PATH,
2649  ConfValIsTrue(p->val));
2650  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2651  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2652  HTP_DECODER_URL_PATH,
2653  ConfValIsTrue(p->val));
2654  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2655  enum htp_url_encoding_handling_t handling;
2656  if (strcasecmp(p->val, "preserve_percent") == 0) {
2657  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2658  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2659  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2660  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2661  handling = HTP_URL_DECODE_PROCESS_INVALID;
2662  } else {
2663  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2664  "for libhtp param path-url-encoding-invalid-handling");
2665  return;
2666  }
2667  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2668  HTP_DECODER_URL_PATH,
2669  handling);
2670  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2671  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2672  HTP_DECODER_URL_PATH,
2673  ConfValIsTrue(p->val));
2674  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2675  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2676  SCLogDebug("uri-include-all %s",
2677  cfg_prec->uri_include_all ? "enabled" : "disabled");
2678  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2679  htp_config_set_plusspace_decode(cfg_prec->cfg,
2680  HTP_DECODER_URLENCODED,
2681  ConfValIsTrue(p->val));
2682  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2683  uint32_t limit = 0;
2684  if (ParseSizeStringU32(p->val, &limit) < 0) {
2685  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2686  "from conf file - %s. Killing engine", p->val);
2687  exit(EXIT_FAILURE);
2688  }
2689  if (limit == 0) {
2690  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2691  "from conf file cannot be 0. Killing engine");
2692  exit(EXIT_FAILURE);
2693  }
2694  /* set default soft-limit with our new hard limit */
2695  htp_config_set_field_limits(cfg_prec->cfg,
2697  (size_t)limit);
2698 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2699  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2700  uint32_t limit = 0;
2701  if (ParseSizeStringU32(p->val, &limit) < 0) {
2702  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'lzma-memlimit' "
2703  "from conf file - %s.", p->val);
2704  }
2705  if (limit == 0) {
2706  FatalError(SC_ERR_SIZE_PARSE, "'lzma-memlimit' "
2707  "from conf file cannot be 0.");
2708  }
2709  /* set default soft-limit with our new hard limit */
2710  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2711  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2712 #endif
2713 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2714  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2715  if (ConfValIsFalse(p->val)) {
2716  htp_config_set_lzma_memlimit(cfg_prec->cfg, 0);
2717  }
2718 #endif
2719 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2720  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2721  uint32_t limit = 0;
2722  if (ParseSizeStringU32(p->val, &limit) < 0) {
2723  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'compression-bomb-limit' "
2724  "from conf file - %s.", p->val);
2725  }
2726  if (limit == 0) {
2727  FatalError(SC_ERR_SIZE_PARSE, "'compression-bomb-limit' "
2728  "from conf file cannot be 0.");
2729  }
2730  /* set default soft-limit with our new hard limit */
2731  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2732  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2733 #endif
2734  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2735  if (!g_disable_randomness) {
2736  cfg_prec->randomize = ConfValIsTrue(p->val);
2737  }
2738  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2739  uint32_t range = atoi(p->val);
2740  if (range > 100) {
2741  SCLogError(SC_ERR_SIZE_PARSE, "Invalid value for randomize"
2742  " inspection range setting from conf file - %s."
2743  " It should be inferior to 100."
2744  " Killing engine",
2745  p->val);
2746  exit(EXIT_FAILURE);
2747  }
2748  cfg_prec->randomize_range = range;
2749  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2750  if (ConfValIsTrue(p->val)) {
2751  cfg_prec->http_body_inline = 1;
2752  } else if (ConfValIsFalse(p->val)) {
2753  cfg_prec->http_body_inline = 0;
2754  } else {
2755  if (strcmp("auto", p->val) != 0) {
2756  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2757  }
2758  if (EngineModeIsIPS()) {
2759  cfg_prec->http_body_inline = 1;
2760  } else {
2761  cfg_prec->http_body_inline = 0;
2762  }
2763  }
2764  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2765  ConfNode *pval;
2766 
2767  TAILQ_FOREACH(pval, &p->head, next) {
2768  if (strcasecmp("enabled", pval->name) == 0) {
2769  if (ConfValIsTrue(pval->val)) {
2770  cfg_prec->swf_decompression_enabled = 1;
2771  } else if (ConfValIsFalse(pval->val)) {
2772  cfg_prec->swf_decompression_enabled = 0;
2773  } else {
2774  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2775  }
2776  } else if (strcasecmp("type", pval->name) == 0) {
2777  if (strcasecmp("no", pval->val) == 0) {
2779  } else if (strcasecmp("deflate", pval->val) == 0) {
2781  } else if (strcasecmp("lzma", pval->val) == 0) {
2783  } else if (strcasecmp("both", pval->val) == 0) {
2785  } else {
2787  "Invalid entry for "
2788  "swf-decompression.type: %s - "
2789  "Killing engine", pval->val);
2790  exit(EXIT_FAILURE);
2791  }
2792  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2793  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2795  "Error parsing swf-decompression.compression-depth "
2796  "from conf file - %s. Killing engine", p->val);
2797  exit(EXIT_FAILURE);
2798  }
2799  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2800  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2802  "Error parsing swf-decompression.decompression-depth "
2803  "from conf file - %s. Killing engine", p->val);
2804  exit(EXIT_FAILURE);
2805  }
2806  } else {
2807  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2808  }
2809  }
2810  } else {
2811  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2812  "default config: %s", p->name);
2813  }
2814  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2815 
2816  return;
2817 }
2818 
2819 void HTPConfigure(void)
2820 {
2821  SCEnter();
2822 
2823  cfglist.next = NULL;
2824 
2825  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2826  if (NULL == cfgtree)
2827  exit(EXIT_FAILURE);
2828 
2829  /* Default Config */
2830  cfglist.cfg = htp_config_create();
2831  if (NULL == cfglist.cfg) {
2832  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP default config");
2833  exit(EXIT_FAILURE);
2834  }
2835  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2836  HTPConfigSetDefaultsPhase1(&cfglist);
2837  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2838  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2839  cfgtree);
2840  } else {
2841  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2842  }
2843  HTPConfigSetDefaultsPhase2("default", &cfglist);
2844 
2845  HTPParseMemcap();
2846 
2847  /* Read server config and create a parser for each IP in radix tree */
2848  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2849  if (server_config == NULL) {
2850  server_config = ConfGetNode("libhtp.server-config");
2851  if (server_config == NULL) {
2852  SCLogDebug("LIBHTP Configuring %p", server_config);
2853  SCReturn;
2854  }
2855  }
2856  SCLogDebug("LIBHTP Configuring %p", server_config);
2857 
2858  ConfNode *si;
2859  /* Server Nodes */
2860  TAILQ_FOREACH(si, &server_config->head, next) {
2861  /* Need the named node, not the index */
2862  ConfNode *s = TAILQ_FIRST(&si->head);
2863  if (NULL == s) {
2864  SCLogDebug("LIBHTP s NULL");
2865  continue;
2866  }
2867 
2868  SCLogDebug("LIBHTP server %s", s->name);
2869 
2870  HTPCfgRec *nextrec = cfglist.next;
2871  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2872  if (NULL == htprec)
2873  exit(EXIT_FAILURE);
2874  memset(htprec, 0x00, sizeof(*htprec));
2875 
2876  cfglist.next = htprec;
2877 
2878  cfglist.next->next = nextrec;
2879  cfglist.next->cfg = htp_config_create();
2880  if (NULL == cfglist.next->cfg) {
2881  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP server config");
2882  exit(EXIT_FAILURE);
2883  }
2884 
2885  HTPConfigSetDefaultsPhase1(htprec);
2886  HTPConfigParseParameters(htprec, s, cfgtree);
2887  HTPConfigSetDefaultsPhase2(s->name, htprec);
2888  }
2889 
2890  SCReturn;
2891 }
2892 
2894 {
2895 #ifdef DEBUG
2896  SCMutexLock(&htp_state_mem_lock);
2897  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2898  SCMutexUnlock(&htp_state_mem_lock);
2899 #endif
2900 }
2901 
2902 /** \internal
2903  * \brief get files callback
2904  * \param state state ptr
2905  * \param direction flow direction
2906  * \retval files files ptr
2907  */
2908 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
2909 {
2910  if (state == NULL)
2911  return NULL;
2912 
2913  HtpState *http_state = (HtpState *)state;
2914 
2915  if (direction & STREAM_TOCLIENT) {
2916  SCReturnPtr(http_state->files_tc, "FileContainer");
2917  } else {
2918  SCReturnPtr(http_state->files_ts, "FileContainer");
2919  }
2920 }
2921 
2922 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2923 {
2924  if (direction & STREAM_TOSERVER)
2925  return ((htp_tx_t *)tx)->request_progress;
2926  else
2927  return ((htp_tx_t *)tx)->response_progress;
2928 }
2929 
2930 static uint64_t HTPStateGetTxCnt(void *alstate)
2931 {
2932  HtpState *http_state = (HtpState *)alstate;
2933 
2934  if (http_state != NULL && http_state->conn != NULL) {
2935  const uint64_t size = (uint64_t)htp_list_size(http_state->conn->transactions);
2936  SCLogDebug("size %"PRIu64, size);
2937  return size;
2938  } else {
2939  return 0ULL;
2940  }
2941 }
2942 
2943 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2944 {
2945  HtpState *http_state = (HtpState *)alstate;
2946 
2947  if (http_state != NULL && http_state->conn != NULL)
2948  return htp_list_get(http_state->conn->transactions, tx_id);
2949  else
2950  return NULL;
2951 }
2952 
2953 static void HTPStateSetTxLogged(void *alstate, void *vtx, LoggerId bits)
2954 {
2955  htp_tx_t *tx = (htp_tx_t *)vtx;
2956  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2957  if (tx_ud)
2958  tx_ud->logged = bits;
2959 }
2960 
2961 static LoggerId HTPStateGetTxLogged(void *alstate, void *vtx)
2962 {
2963  htp_tx_t *tx = (htp_tx_t *)vtx;
2964  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2965  if (tx_ud != NULL)
2966  return tx_ud->logged;
2967 
2968  return 0;
2969 }
2970 
2971 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction)
2972 {
2973  return (direction & STREAM_TOSERVER) ? HTP_REQUEST_COMPLETE : HTP_RESPONSE_COMPLETE;
2974 }
2975 
2976 static int HTPStateGetEventInfo(const char *event_name,
2977  int *event_id, AppLayerEventType *event_type)
2978 {
2979  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
2980  if (*event_id == -1) {
2981  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
2982  "http's enum map table.", event_name);
2983  /* this should be treated as fatal */
2984  return -1;
2985  }
2986 
2987  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2988 
2989  return 0;
2990 }
2991 
2992 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
2994 {
2995  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
2996  if (*event_name == NULL) {
2997  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
2998  "http's enum map table.", event_id);
2999  /* this should be treated as fatal */
3000  return -1;
3001  }
3002 
3003  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3004 
3005  return 0;
3006 }
3007 
3008 static void HTPStateTruncate(void *state, uint8_t direction)
3009 {
3010  FileContainer *fc = HTPStateGetFiles(state, direction);
3011  if (fc != NULL) {
3013  }
3014 }
3015 
3016 static DetectEngineState *HTPGetTxDetectState(void *vtx)
3017 {
3018  htp_tx_t *tx = (htp_tx_t *)vtx;
3019  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3020  return tx_ud ? tx_ud->de_state : NULL;
3021 }
3022 
3023 static int HTPSetTxDetectState(void *vtx, DetectEngineState *s)
3024 {
3025  htp_tx_t *tx = (htp_tx_t *)vtx;
3026  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3027  if (tx_ud == NULL) {
3028  tx_ud = HTPMalloc(sizeof(*tx_ud));
3029  if (unlikely(tx_ud == NULL))
3030  return -ENOMEM;
3031  memset(tx_ud, 0, sizeof(*tx_ud));
3032  htp_tx_set_user_data(tx, tx_ud);
3033  }
3034  tx_ud->de_state = s;
3035  return 0;
3036 }
3037 
3038 static uint64_t HTPGetTxDetectFlags(void *vtx, uint8_t dir)
3039 {
3040  htp_tx_t *tx = (htp_tx_t *)vtx;
3041  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3042  if (tx_ud) {
3043  if (dir & STREAM_TOSERVER) {
3044  return tx_ud->detect_flags_ts;
3045  } else {
3046  return tx_ud->detect_flags_tc;
3047  }
3048  }
3049  return 0;
3050 }
3051 
3052 static void HTPSetTxDetectFlags(void *vtx, uint8_t dir, uint64_t detect_flags)
3053 {
3054  htp_tx_t *tx = (htp_tx_t *)vtx;
3055  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3056  if (tx_ud == NULL) {
3057  tx_ud = HTPMalloc(sizeof(*tx_ud));
3058  if (unlikely(tx_ud == NULL))
3059  return;
3060  memset(tx_ud, 0, sizeof(*tx_ud));
3061  htp_tx_set_user_data(tx, tx_ud);
3062  }
3063  if (dir & STREAM_TOSERVER) {
3064  tx_ud->detect_flags_ts = detect_flags;
3065  } else {
3066  tx_ud->detect_flags_tc = detect_flags;
3067  }
3068  return;
3069 }
3070 
3071 static int HTPRegisterPatternsForProtocolDetection(void)
3072 {
3073  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3074  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3075  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3076  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3077  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3078  const char *spacings[] = { "|20|", "|09|", NULL };
3079  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3080 
3081  int methods_pos;
3082  int spacings_pos;
3083  int versions_pos;
3084  int register_result;
3085  char method_buffer[32] = "";
3086 
3087  /* Loop through all the methods ands spacings and register the patterns */
3088  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3089  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3090 
3091  /* Combine the method name and the spacing */
3092  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3093 
3094  /* Register the new method+spacing pattern
3095  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3096  * but the pattern matching should only be one char
3097  */
3098  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
3099  ALPROTO_HTTP, method_buffer, strlen(method_buffer)-3, 0, STREAM_TOSERVER);
3100  if (register_result < 0) {
3101  return -1;
3102  }
3103  }
3104  }
3105 
3106  /* Loop through all the http verions patterns that are TO_CLIENT */
3107  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3108  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
3109  ALPROTO_HTTP, versions[versions_pos], strlen(versions[versions_pos]),
3110  0, STREAM_TOCLIENT);
3111  if (register_result < 0) {
3112  return -1;
3113  }
3114  }
3115 
3116  return 0;
3117 }
3118 
3119 /**
3120  * \brief Register the HTTP protocol and state handling functions to APP layer
3121  * of the engine.
3122  */
3124 {
3125  SCEnter();
3126 
3127  const char *proto_name = "http";
3128 
3129  /** HTTP */
3130  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3132  if (HTPRegisterPatternsForProtocolDetection() < 0)
3133  return;
3134  } else {
3135  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3136  proto_name);
3137  return;
3138  }
3139 
3140  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3141  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateAlloc, HTPStateFree);
3142  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTransactionFree);
3143  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetFiles);
3144  AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetAlstateProgress);
3145  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxCnt);
3146  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTx);
3147  AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxLogged,
3148  HTPStateSetTxLogged);
3150  HTPStateGetAlstateProgressCompletionStatus);
3151  AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPGetEvents);
3152  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfo);
3153  AppLayerParserRegisterGetEventInfoById(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfoById);
3154 
3155  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTruncate);
3157  HTPGetTxDetectState, HTPSetTxDetectState);
3159  HTPGetTxDetectFlags, HTPSetTxDetectFlags);
3160 
3162  AppLayerHtpSetStreamDepthFlag);
3163 
3165  HTPHandleRequestData);
3167  HTPHandleResponseData);
3168  SC_ATOMIC_INIT(htp_config_flags);
3171  HTPConfigure();
3172  } else {
3173  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3174  "still on.", proto_name);
3175  }
3176 #ifdef UNITTESTS
3178 #endif
3179 
3180  SCReturn;
3181 }
3182 
3183 #ifdef UNITTESTS
3184 static HTPCfgRec cfglist_backup;
3185 
3187 {
3188  cfglist_backup = cfglist;
3189 
3190  return;
3191 }
3192 
3194 {
3195  cfglist = cfglist_backup;
3196 
3197  return;
3198 }
3199 
3200 /** \test Test case where chunks are sent in smaller chunks and check the
3201  * response of the parser from HTP library. */
3202 static int HTPParserTest01(void)
3203 {
3204  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3205  " Data is c0oL!";
3206  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3207 
3208  TcpSession ssn;
3209  memset(&ssn, 0, sizeof(ssn));
3210 
3212  FAIL_IF_NULL(alp_tctx);
3213 
3214  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3215  FAIL_IF_NULL(f);
3216  f->protoctx = &ssn;
3217  f->proto = IPPROTO_TCP;
3218  f->alproto = ALPROTO_HTTP;
3219 
3221 
3222  uint32_t u;
3223  for (u = 0; u < httplen1; u++) {
3224  uint8_t flags = 0;
3225 
3226  if (u == 0)
3227  flags = STREAM_TOSERVER|STREAM_START;
3228  else if (u == (httplen1 - 1))
3229  flags = STREAM_TOSERVER|STREAM_EOF;
3230  else
3231  flags = STREAM_TOSERVER;
3232 
3233  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3234  &httpbuf1[u], 1);
3235  FAIL_IF(r != 0);
3236  }
3237 
3238  HtpState *htp_state = f->alstate;
3239  FAIL_IF_NULL(htp_state);
3240 
3241  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3242  FAIL_IF_NULL(tx);
3243 
3244  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3245  FAIL_IF_NULL(h);
3246 
3247  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3248  FAIL_IF(tx->request_method_number != HTP_M_POST);
3249  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3250 
3251  AppLayerParserThreadCtxFree(alp_tctx);
3253  UTHFreeFlow(f);
3254  PASS;
3255 }
3256 
3257 /** \test Test folding in 1 read case */
3258 static int HTPParserTest01b(void)
3259 {
3260  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3261  " Data is c0oL!";
3262  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3263 
3264  TcpSession ssn;
3265  memset(&ssn, 0, sizeof(ssn));
3266 
3268  FAIL_IF_NULL(alp_tctx);
3269 
3270  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3271  FAIL_IF_NULL(f);
3272  f->protoctx = &ssn;
3273  f->proto = IPPROTO_TCP;
3274  f->alproto = ALPROTO_HTTP;
3275 
3277 
3279  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3280  httpbuf1, httplen1);
3281  FAIL_IF(r != 0);
3282 
3283  HtpState *htp_state = f->alstate;
3284  FAIL_IF_NULL(htp_state);
3285 
3286  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3287  FAIL_IF_NULL(tx);
3288 
3289  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3290  FAIL_IF_NULL(h);
3291 
3292  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3293  FAIL_IF(tx->request_method_number != HTP_M_POST);
3294  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3295 
3296  AppLayerParserThreadCtxFree(alp_tctx);
3298  UTHFreeFlow(f);
3299  PASS;
3300 }
3301 
3302 /** \test Test folding in 1byte per read case */
3303 static int HTPParserTest01c(void)
3304 {
3305  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3306  " Data is c0oL!";
3307  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3308 
3309  TcpSession ssn;
3310  memset(&ssn, 0, sizeof(ssn));
3311 
3313  FAIL_IF_NULL(alp_tctx);
3314 
3315  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3316  FAIL_IF_NULL(f);
3317  f->protoctx = &ssn;
3318  f->proto = IPPROTO_TCP;
3319  f->alproto = ALPROTO_HTTP;
3320 
3322 
3323  uint32_t u;
3324  for (u = 0; u < httplen1; u++) {
3325  uint8_t flags = 0;
3326 
3327  if (u == 0)
3328  flags = STREAM_TOSERVER|STREAM_START;
3329  else if (u == (httplen1 - 1))
3330  flags = STREAM_TOSERVER|STREAM_EOF;
3331  else
3332  flags = STREAM_TOSERVER;
3333 
3334  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3335  &httpbuf1[u], 1);
3336  FAIL_IF(r != 0);
3337  }
3338 
3339  HtpState *htp_state = f->alstate;
3340  FAIL_IF_NULL(htp_state);
3341 
3342  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3343  FAIL_IF_NULL(tx);
3344 
3345  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3346  FAIL_IF_NULL(h);
3347 
3348  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3349  FAIL_IF(tx->request_method_number != HTP_M_POST);
3350  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3351 
3352  AppLayerParserThreadCtxFree(alp_tctx);
3354  UTHFreeFlow(f);
3355  PASS;
3356 }
3357 
3358 /** \test Test case where chunks are sent in smaller chunks and check the
3359  * response of the parser from HTP library. */
3360 static int HTPParserTest01a(void)
3361 {
3362  int result = 0;
3363  Flow *f = NULL;
3364  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3365  " Data is c0oL!";
3366  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3367  TcpSession ssn;
3368  HtpState *htp_state = NULL;
3369  int r = 0;
3371 
3372  memset(&ssn, 0, sizeof(ssn));
3373 
3374  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3375  if (f == NULL)
3376  goto end;
3377  f->protoctx = &ssn;
3378  f->proto = IPPROTO_TCP;
3379  f->alproto = ALPROTO_HTTP;
3380 
3382 
3383  uint32_t u;
3384  for (u = 0; u < httplen1; u++) {
3385  uint8_t flags = 0;
3386 
3387  if (u == 0)
3388  flags = STREAM_TOSERVER|STREAM_START;
3389  else if (u == (httplen1 - 1))
3390  flags = STREAM_TOSERVER|STREAM_EOF;
3391  else
3392  flags = STREAM_TOSERVER;
3393 
3394  FLOWLOCK_WRLOCK(f);
3395  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3396  &httpbuf1[u], 1);
3397  if (r != 0) {
3398  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3399  " 0: ", u, r);
3400  FLOWLOCK_UNLOCK(f);
3401  goto end;
3402  }
3403  FLOWLOCK_UNLOCK(f);
3404  }
3405 
3406  htp_state = f->alstate;
3407  if (htp_state == NULL) {
3408  printf("no http state: ");
3409  goto end;
3410  }
3411 
3412  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3413  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3414  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3415  || tx->request_method_number != HTP_M_POST ||
3416  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3417  {
3418  printf("expected header value: Victor/1.0 and got %s: and expected"
3419  " method: POST and got %s, expected protocol number HTTP/1.0"
3420  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3421  bstr_util_strdup_to_c(tx->request_method),
3422  bstr_util_strdup_to_c(tx->request_protocol));
3423  goto end;
3424  }
3425  result = 1;
3426 end:
3427  if (alp_tctx != NULL)
3428  AppLayerParserThreadCtxFree(alp_tctx);
3430  UTHFreeFlow(f);
3431  return result;
3432 }
3433 
3434 /** \test See how it deals with an incomplete request. */
3435 static int HTPParserTest02(void)
3436 {
3437  int result = 0;
3438  Flow *f = NULL;
3439  uint8_t httpbuf1[] = "POST";
3440  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3441  TcpSession ssn;
3442  HtpState *http_state = NULL;
3444 
3445  memset(&ssn, 0, sizeof(ssn));
3446 
3447  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3448  if (f == NULL)
3449  goto end;
3450  f->protoctx = &ssn;
3451  f->proto = IPPROTO_TCP;
3452  f->alproto = ALPROTO_HTTP;
3453 
3455 
3456  FLOWLOCK_WRLOCK(f);
3457  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3459  httpbuf1,
3460  httplen1);
3461  if (r != 0) {
3462  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3463  FLOWLOCK_UNLOCK(f);
3464  goto end;
3465  }
3466  FLOWLOCK_UNLOCK(f);
3467 
3468  http_state = f->alstate;
3469  if (http_state == NULL) {
3470  printf("no http state: ");
3471  goto end;
3472  }
3473 
3474  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3475  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3476  if ((tx->request_method) != NULL || h != NULL)
3477  {
3478  printf("expected method NULL, got %s \n", bstr_util_strdup_to_c(tx->request_method));
3479  goto end;
3480  }
3481  result = 1;
3482 end:
3483  if (alp_tctx != NULL)
3484  AppLayerParserThreadCtxFree(alp_tctx);
3486  UTHFreeFlow(f);
3487  return result;
3488 }
3489 
3490 /** \test Test case where method is invalid and data is sent in smaller chunks
3491  * and check the response of the parser from HTP library. */
3492 static int HTPParserTest03(void)
3493 {
3494  int result = 0;
3495  Flow *f = NULL;
3496  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3497  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3498  TcpSession ssn;
3499  HtpState *htp_state = NULL;
3500  int r = 0;
3502 
3503  memset(&ssn, 0, sizeof(ssn));
3504 
3505  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3506  if (f == NULL)
3507  goto end;
3508  f->protoctx = &ssn;
3509  f->proto = IPPROTO_TCP;
3510  f->alproto = ALPROTO_HTTP;
3511 
3513 
3514  uint32_t u;
3515  for (u = 0; u < httplen1; u++) {
3516  uint8_t flags = 0;
3517 
3518  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3519  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3520  else flags = STREAM_TOSERVER;
3521 
3522  FLOWLOCK_WRLOCK(f);
3523  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3524  &httpbuf1[u], 1);
3525  if (r != 0) {
3526  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3527  " 0: ", u, r);
3528  FLOWLOCK_UNLOCK(f);
3529  goto end;
3530  }
3531  FLOWLOCK_UNLOCK(f);
3532  }
3533  htp_state = f->alstate;
3534  if (htp_state == NULL) {
3535  printf("no http state: ");
3536  goto end;
3537  }
3538 
3539  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3540 
3541  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3542  if (tx->request_method_number != HTP_M_UNKNOWN ||
3543  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3544  {
3545  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3546  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3547  bstr_util_strdup_to_c(tx->request_protocol));
3548  goto end;
3549  }
3550  result = 1;
3551 end:
3552  if (alp_tctx != NULL)
3553  AppLayerParserThreadCtxFree(alp_tctx);
3555  UTHFreeFlow(f);
3556  return result;
3557 }
3558 
3559 /** \test Test case where invalid data is sent and check the response of the
3560  * parser from HTP library. */
3561 static int HTPParserTest04(void)
3562 {
3563  int result = 0;
3564  Flow *f = NULL;
3565  HtpState *htp_state = NULL;
3566  uint8_t httpbuf1[] = "World!\r\n";
3567  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3568  TcpSession ssn;
3569  int r = 0;
3571 
3572  memset(&ssn, 0, sizeof(ssn));
3573 
3574  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3575  if (f == NULL)
3576  goto end;
3577  f->protoctx = &ssn;
3578  f->proto = IPPROTO_TCP;
3579  f->alproto = ALPROTO_HTTP;
3580 
3582 
3583  FLOWLOCK_WRLOCK(f);
3584  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3586  httpbuf1,
3587  httplen1);
3588  if (r != 0) {
3589  FLOWLOCK_UNLOCK(f);
3590  goto end;
3591  }
3592  FLOWLOCK_UNLOCK(f);
3593 
3594  htp_state = f->alstate;
3595  if (htp_state == NULL) {
3596  printf("no http state: ");
3597  goto end;
3598  }
3599 
3600  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3601  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3602  if (tx->request_method_number != HTP_M_UNKNOWN ||
3603  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3604  {
3605  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3606  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3607  bstr_util_strdup_to_c(tx->request_protocol));
3608  goto end;
3609  }
3610  result = 1;
3611 end:
3612  if (alp_tctx != NULL)
3613  AppLayerParserThreadCtxFree(alp_tctx);
3615  UTHFreeFlow(f);
3616  return result;
3617 }
3618 
3619 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3620  * properly parsed them and also keeps them separated. */
3621 static int HTPParserTest05(void)
3622 {
3623  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3624  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3625  uint8_t httpbuf2[] = "Post D";
3626  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3627  uint8_t httpbuf3[] = "ata is c0oL!";
3628  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3629 
3630  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3631  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3632  uint8_t httpbuf5[] = "post R";
3633  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3634  uint8_t httpbuf6[] = "esults are tha bomb!";
3635  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3636 
3637  TcpSession ssn;
3638  memset(&ssn, 0, sizeof(ssn));
3639 
3641  FAIL_IF_NULL(alp_tctx);
3642 
3643  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3644  FAIL_IF_NULL(f);
3645  f->protoctx = &ssn;
3646  f->proto = IPPROTO_TCP;
3647  f->alproto = ALPROTO_HTTP;
3648 
3650 
3651  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3652  STREAM_TOSERVER | STREAM_START, httpbuf1,
3653  httplen1);
3654  FAIL_IF(r != 0);
3655 
3656  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3657  STREAM_TOCLIENT | STREAM_START, httpbuf4,
3658  httplen4);
3659  FAIL_IF(r != 0);
3660 
3661  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOCLIENT,
3662  httpbuf5, httplen5);
3663  FAIL_IF(r != 0);
3664 
3665  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOSERVER,
3666  httpbuf2, httplen2);
3667  FAIL_IF(r != 0);
3668 
3669  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3670  STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3671  FAIL_IF(r != 0);
3672 
3673  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3674  STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3675  FAIL_IF(r != 0);
3676 
3677  HtpState *http_state = f->alstate;
3678  FAIL_IF_NULL(http_state);
3679 
3680  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3681  FAIL_IF_NULL(tx);
3682  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3683  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3684 
3685  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3686  FAIL_IF_NULL(h);
3687 
3688  FAIL_IF_NOT(tx->response_status_number == 200);
3689 
3690  AppLayerParserThreadCtxFree(alp_tctx);
3692  UTHFreeFlow(f);
3693  PASS;
3694 }
3695 
3696 /** \test Test proper chunked encoded response body
3697  */
3698 static int HTPParserTest06(void)
3699 {
3700  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3701  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3702  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3703  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3704  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3705  "GMT\r\n"
3706  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3707  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3708  "FrontPage/5.0.2.2510\r\n"
3709  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3710  "chunked\r\n"
3711  "Content-Type: text/html\r\n\r\n"
3712  "580\r\n"
3713  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3714  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3715  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3716  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3717  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3718  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3719  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3720  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3721  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3722  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3723  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3724  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3725  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3726  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3727  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3728  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3729  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3730  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3731  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3732  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3733  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3734  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3735  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3736  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3737  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3738  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3739  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3740  "aHA=\r\n0\r\n\r\n";
3741  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3742  TcpSession ssn;
3743 
3745  FAIL_IF_NULL(alp_tctx);
3746 
3747  memset(&ssn, 0, sizeof(ssn));
3748 
3749  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3750  FAIL_IF_NULL(f);
3751  f->protoctx = &ssn;
3752  f->proto = IPPROTO_TCP;
3753  f->alproto = ALPROTO_HTTP;
3754 
3756 
3757  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3758  STREAM_TOSERVER | STREAM_START, httpbuf1,
3759  httplen1);
3760  FAIL_IF(r != 0);
3761  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3762  STREAM_TOCLIENT | STREAM_START, httpbuf2,
3763  httplen2);
3764  FAIL_IF(r != 0);
3765 
3766  HtpState *http_state = f->alstate;
3767  FAIL_IF_NULL(http_state);
3768 
3769  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3770  FAIL_IF_NULL(tx);
3771 
3772  FAIL_IF(tx->request_method_number != HTP_M_GET);
3773  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3774 
3775  FAIL_IF(tx->response_status_number != 200);
3776  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3777 
3778  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3779  FAIL_IF_NULL(h);
3780 
3781  AppLayerParserThreadCtxFree(alp_tctx);
3783  UTHFreeFlow(f);
3784  PASS;
3785 }
3786 
3787 /** \test
3788  */
3789 static int HTPParserTest07(void)
3790 {
3791  int result = 0;
3792  Flow *f = NULL;
3793  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3794  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3795  TcpSession ssn;
3796  HtpState *htp_state = NULL;
3797  int r = 0;
3799 
3800  memset(&ssn, 0, sizeof(ssn));
3801 
3802  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3803  if (f == NULL)
3804  goto end;
3805  f->protoctx = &ssn;
3806  f->proto = IPPROTO_TCP;
3807  f->alproto = ALPROTO_HTTP;
3808 
3810 
3811  uint32_t u;
3812  for (u = 0; u < httplen1; u++) {
3813  uint8_t flags = 0;
3814 
3815  if (u == 0)
3816  flags = STREAM_TOSERVER|STREAM_START;
3817  else if (u == (httplen1 - 1))
3818  flags = STREAM_TOSERVER|STREAM_EOF;
3819  else
3820  flags = STREAM_TOSERVER;
3821 
3822  FLOWLOCK_WRLOCK(f);
3823  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3824  &httpbuf1[u], 1);
3825  if (r != 0) {
3826  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3827  " 0: ", u, r);
3828  FLOWLOCK_UNLOCK(f);
3829  goto end;
3830  }
3831  FLOWLOCK_UNLOCK(f);
3832  }
3833 
3834  htp_state = f->alstate;
3835  if (htp_state == NULL) {
3836  printf("no http state: ");
3837  goto end;
3838  }
3839 
3840  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3841  size_t reflen = sizeof(ref) - 1;
3842 
3843  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3844  if (tx == NULL)
3845  goto end;
3846  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3847  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3848  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3849  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3850  (uintmax_t)reflen,
3851  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3852  goto end;
3853  }
3854 
3855  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3856  bstr_len(tx_ud->request_uri_normalized)) != 0)
3857  {
3858  printf("normalized uri \"");
3859  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3860  printf("\" != \"");
3861  PrintRawUriFp(stdout, ref, reflen);
3862  printf("\": ");
3863  goto end;
3864  }
3865  }
3866 
3867  result = 1;
3868 end:
3869  if (alp_tctx != NULL)
3870  AppLayerParserThreadCtxFree(alp_tctx);
3872  UTHFreeFlow(f);
3873  return result;
3874 }
3875 
3876 #include "conf-yaml-loader.h"
3877 
3878 /** \test Abort
3879  */
3880 static int HTPParserTest08(void)
3881 {
3882  int result = 0;
3883  Flow *f = NULL;
3884  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3885  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3886  TcpSession ssn;
3888 
3889  char input[] = "\
3890 %YAML 1.1\n\
3891 ---\n\
3892 libhtp:\n\
3893 \n\
3894  default-config:\n\
3895  personality: IDS\n\
3896 ";
3897 
3899  ConfInit();
3901 
3902  ConfYamlLoadString(input, strlen(input));
3903  HTPConfigure();
3904 
3905  HtpState *htp_state = NULL;
3906  int r = 0;
3907  memset(&ssn, 0, sizeof(ssn));
3908 
3909  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3910  if (f == NULL)
3911  goto end;
3912  f->protoctx = &ssn;
3913  f->proto = IPPROTO_TCP;
3914  f->alproto = ALPROTO_HTTP;
3915 
3917 
3918  uint8_t flags = 0;
3920 
3921  FLOWLOCK_WRLOCK(f);
3922  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
3923  httplen1);
3924  if (r != 0) {
3925  printf("toserver chunk returned %" PRId32 ", expected"
3926  " 0: ", r);
3927  result = 0;
3928  FLOWLOCK_UNLOCK(f);
3929  goto end;
3930  }
3931  FLOWLOCK_UNLOCK(f);
3932 
3933  htp_state = f->alstate;
3934  if (htp_state == NULL) {
3935  printf("no http state: ");
3936  result = 0;
3937  goto end;
3938  }
3939 
3940  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3941  if (tx == NULL)
3942  goto end;
3943  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3944  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3945  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3946  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3947  bstr_len(tx_ud->request_uri_normalized));
3948  }
3949 
3950  result = 1;
3951 end:
3952  if (alp_tctx != NULL)
3953  AppLayerParserThreadCtxFree(alp_tctx);
3955  HTPFreeConfig();
3956  ConfDeInit();
3959  UTHFreeFlow(f);
3960  return result;
3961 }
3962 
3963 /** \test Abort
3964  */
3965 static int HTPParserTest09(void)
3966 {
3967  int result = 0;
3968  Flow *f = NULL;
3969  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3970  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3971  TcpSession ssn;
3973 
3974  char input[] = "\
3975 %YAML 1.1\n\
3976 ---\n\
3977 libhtp:\n\
3978 \n\
3979  default-config:\n\
3980  personality: Apache_2_2\n\
3981 ";
3982 
3984  ConfInit();
3986 
3987  ConfYamlLoadString(input, strlen(input));
3988  HTPConfigure();
3989 
3990  HtpState *htp_state = NULL;
3991  int r = 0;
3992 
3993  memset(&ssn, 0, sizeof(ssn));
3994 
3995  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3996  if (f == NULL)
3997  goto end;
3998  f->protoctx = &ssn;
3999  f->proto = IPPROTO_TCP;
4000  f->alproto = ALPROTO_HTTP;
4001 
4003 
4004  uint8_t flags = 0;
4006 
4007  FLOWLOCK_WRLOCK(f);
4008  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
4009  httplen1);
4010  if (r != 0) {
4011  printf("toserver chunk returned %" PRId32 ", expected"
4012  " 0: ", r);
4013  FLOWLOCK_UNLOCK(f);
4014  goto end;
4015  }
4016  FLOWLOCK_UNLOCK(f);
4017 
4018  htp_state = f->alstate;
4019  if (htp_state == NULL) {
4020  printf("no http state: ");
4021  goto end;
4022  }
4023 
4024  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4025  if (tx == NULL)
4026  goto end;
4027  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4028  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4029  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4030  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4031  bstr_len(tx_ud->request_uri_normalized));
4032  }
4033 
4034  result = 1;
4035 end:
4036  if (alp_tctx != NULL)
4037  AppLayerParserThreadCtxFree(alp_tctx);
4039  HTPFreeConfig();
4040  ConfDeInit();
4043  UTHFreeFlow(f);
4044  return result;
4045 }
4046 
4047 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4048  */
4049 static int HTPParserTest10(void)
4050 {
4051  int result = 0;
4052  Flow *f = NULL;
4053  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4054  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4055  TcpSession ssn;
4056  HtpState *htp_state = NULL;
4057  int r = 0;
4059 
4060  memset(&ssn, 0, sizeof(ssn));
4061 
4062  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4063  if (f == NULL)
4064  goto end;
4065  f->protoctx = &ssn;
4066  f->proto = IPPROTO_TCP;
4067  f->alproto = ALPROTO_HTTP;
4068 
4070 
4071  uint32_t u;
4072  for (u = 0; u < httplen1; u++) {
4073  uint8_t flags = 0;
4074 
4075  if (u == 0)
4076  flags = STREAM_TOSERVER|STREAM_START;
4077  else if (u == (httplen1 - 1))
4078  flags = STREAM_TOSERVER|STREAM_EOF;
4079  else
4080  flags = STREAM_TOSERVER;
4081 
4082  FLOWLOCK_WRLOCK(f);
4083  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4084  &httpbuf1[u], 1);
4085  if (r != 0) {
4086  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4087  " 0: ", u, r);
4088  FLOWLOCK_UNLOCK(f);
4089  goto end;
4090  }
4091  FLOWLOCK_UNLOCK(f);
4092  }
4093 
4094  htp_state = f->alstate;
4095  if (htp_state == NULL) {
4096  printf("no http state: ");
4097  goto end;
4098  }
4099 
4100  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4101  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4102  if (h == NULL) {
4103  goto end;
4104  }
4105 
4106  char *name = bstr_util_strdup_to_c(h->name);
4107  if (name == NULL) {
4108  goto end;
4109  }
4110 
4111  if (strcmp(name, "Host") != 0) {
4112  printf("header name not \"Host\", instead \"%s\": ", name);
4113  free(name);
4114  goto end;
4115  }
4116  free(name);
4117 
4118  char *value = bstr_util_strdup_to_c(h->value);
4119  if (value == NULL) {
4120  goto end;
4121  }
4122 
4123  if (strcmp(value, "www.google.com") != 0) {
4124  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4125  free(value);
4126  goto end;
4127  }
4128  free(value);
4129 
4130  result = 1;
4131 end:
4132  if (alp_tctx != NULL)
4133  AppLayerParserThreadCtxFree(alp_tctx);
4135  UTHFreeFlow(f);
4136  return result;
4137 }
4138 
4139 /** \test double encoding in path
4140  */
4141 static int HTPParserTest11(void)
4142 {
4143  int result = 0;
4144  Flow *f = NULL;
4145  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4146  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4147  TcpSession ssn;
4148  HtpState *htp_state = NULL;
4149  int r = 0;
4151 
4152  memset(&ssn, 0, sizeof(ssn));
4153 
4154  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4155  if (f == NULL)
4156  goto end;
4157  f->protoctx = &ssn;
4158  f->proto = IPPROTO_TCP;
4159  f->alproto = ALPROTO_HTTP;
4160 
4162 
4163  uint32_t u;
4164  for (u = 0; u < httplen1; u++) {
4165  uint8_t flags = 0;
4166 
4167  if (u == 0)
4168  flags = STREAM_TOSERVER|STREAM_START;
4169  else if (u == (httplen1 - 1))
4170  flags = STREAM_TOSERVER|STREAM_EOF;
4171  else
4172  flags = STREAM_TOSERVER;
4173 
4174  FLOWLOCK_WRLOCK(f);
4175  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4176  &httpbuf1[u], 1);
4177  if (r != 0) {
4178  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4179  " 0: ", u, r);
4180  FLOWLOCK_UNLOCK(f);
4181  goto end;
4182  }
4183  FLOWLOCK_UNLOCK(f);
4184  }
4185 
4186  htp_state = f->alstate;
4187  if (htp_state == NULL) {
4188  printf("no http state: ");
4189  goto end;
4190  }
4191 
4192  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4193  if (tx == NULL)
4194  goto end;
4195  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4196  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4197  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4198  printf("normalized uri len should be 2, is %"PRIuMAX,
4199  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4200  goto end;
4201  }
4202 
4203  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4204  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4205  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4206  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4207  {
4208  printf("normalized uri \"");
4209  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4210  printf("\": ");
4211  goto end;
4212  }
4213  }
4214 
4215  result = 1;
4216 end:
4217  if (alp_tctx != NULL)
4218  AppLayerParserThreadCtxFree(alp_tctx);
4220  UTHFreeFlow(f);
4221  return result;
4222 }
4223 
4224 /** \test double encoding in query
4225  */
4226 static int HTPParserTest12(void)
4227 {
4228  int result = 0;
4229  Flow *f = NULL;
4230  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4231  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4232  TcpSession ssn;
4233  HtpState *htp_state = NULL;
4234  int r = 0;
4236 
4237  memset(&ssn, 0, sizeof(ssn));
4238 
4239  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4240  if (f == NULL)
4241  goto end;
4242  f->protoctx = &ssn;
4243  f->proto = IPPROTO_TCP;
4244  f->alproto = ALPROTO_HTTP;
4245 
4247 
4248  uint32_t u;
4249  for (u = 0; u < httplen1; u++) {
4250  uint8_t flags = 0;
4251 
4252  if (u == 0)
4253  flags = STREAM_TOSERVER|STREAM_START;
4254  else if (u == (httplen1 - 1))
4255  flags = STREAM_TOSERVER|STREAM_EOF;
4256  else
4257  flags = STREAM_TOSERVER;
4258 
4259  FLOWLOCK_WRLOCK(f);
4260  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4261  &httpbuf1[u], 1);
4262  if (r != 0) {
4263  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4264  " 0: ", u, r);
4265  FLOWLOCK_UNLOCK(f);
4266  goto end;
4267  }
4268  FLOWLOCK_UNLOCK(f);
4269  }
4270 
4271  htp_state = f->alstate;
4272  if (htp_state == NULL) {
4273  printf("no http state: ");
4274  goto end;
4275  }
4276 
4277  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4278  if (tx == NULL)
4279  goto end;
4280  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4281  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4282  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4283  printf("normalized uri len should be 5, is %"PRIuMAX,
4284  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4285  goto end;
4286  }
4287 
4288  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4289  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4290  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4291  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4292  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4293  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4294  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4295  {
4296  printf("normalized uri \"");
4297  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4298  printf("\": ");
4299  goto end;
4300  }
4301  }
4302 
4303  result = 1;
4304  end:
4305  if (alp_tctx != NULL)
4306  AppLayerParserThreadCtxFree(alp_tctx);
4308  UTHFreeFlow(f);
4309  return result;
4310 }
4311 
4312 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4313  */
4314 static int HTPParserTest13(void)
4315 {
4316  int result = 0;
4317  Flow *f = NULL;
4318  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4319  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4320  TcpSession ssn;
4321  HtpState *htp_state = NULL;
4322  int r = 0;
4324 
4325  memset(&ssn, 0, sizeof(ssn));
4326 
4327  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4328  if (f == NULL)
4329  goto end;
4330  f->protoctx = &ssn;
4331  f->proto = IPPROTO_TCP;
4332  f->alproto = ALPROTO_HTTP;
4333 
4335 
4336  uint32_t u;
4337  for (u = 0; u < httplen1; u++) {
4338  uint8_t flags = 0;
4339 
4340  if (u == 0)
4341  flags = STREAM_TOSERVER|STREAM_START;
4342  else if (u == (httplen1 - 1))
4343  flags = STREAM_TOSERVER|STREAM_EOF;
4344  else
4345  flags = STREAM_TOSERVER;
4346 
4347  FLOWLOCK_WRLOCK(f);
4348  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4349  &httpbuf1[u], 1);
4350  if (r != 0) {
4351  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4352  " 0: ", u, r);
4353  FLOWLOCK_UNLOCK(f);
4354  goto end;
4355  }
4356  FLOWLOCK_UNLOCK(f);
4357  }
4358 
4359  htp_state = f->alstate;
4360  if (htp_state == NULL) {
4361  printf("no http state: ");
4362  goto end;
4363  }
4364 
4365  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4366  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4367  if (h == NULL) {
4368  goto end;
4369  }
4370 
4371  char *name = bstr_util_strdup_to_c(h->name);
4372  if (name == NULL) {
4373  goto end;
4374  }
4375 
4376  if (strcmp(name, "Host") != 0) {
4377  printf("header name not \"Host\", instead \"%s\": ", name);
4378  free(name);
4379  goto end;
4380  }
4381  free(name);
4382 
4383  char *value = bstr_util_strdup_to_c(h->value);
4384  if (value == NULL) {
4385  goto end;
4386  }
4387 
4388  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4389  printf("header value not \"www.google.com\", instead \"");
4390  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4391  printf("\": ");
4392  free(value);
4393  goto end;
4394  }
4395  free(value);
4396 
4397  result = 1;
4398 end:
4399  if (alp_tctx != NULL)
4400  AppLayerParserThreadCtxFree(alp_tctx);
4402  UTHFreeFlow(f);
4403  return result;
4404 }
4405 
4406 /** \test Test basic config */
4407 static int HTPParserConfigTest01(void)
4408 {
4409  int ret = 0;
4410  char input[] = "\
4411 %YAML 1.1\n\
4412 ---\n\
4413 libhtp:\n\
4414 \n\
4415  default-config:\n\
4416  personality: IDS\n\
4417 \n\
4418  server-config:\n\
4419 \n\
4420  - apache-tomcat:\n\
4421  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4422  personality: Tomcat_6_0\n\
4423 \n\
4424  - iis7:\n\
4425  address: \n\
4426  - 192.168.0.0/24\n\
4427  - 192.168.10.0/24\n\
4428  personality: IIS_7_0\n\
4429 ";
4430 
4432  ConfInit();
4433 
4434  ConfYamlLoadString(input, strlen(input));
4435 
4436  ConfNode *outputs;
4437  outputs = ConfGetNode("libhtp.default-config.personality");
4438  if (outputs == NULL) {
4439  goto end;
4440  }
4441 
4442  outputs = ConfGetNode("libhtp.server-config");
4443  if (outputs == NULL) {
4444  goto end;
4445  }
4446 
4447  ConfNode *node = TAILQ_FIRST(&outputs->head);
4448  if (node == NULL) {
4449  goto end;
4450  }
4451  if (strcmp(node->name, "0") != 0) {
4452  goto end;
4453  }
4454  node = TAILQ_FIRST(&node->head);
4455  if (node == NULL) {
4456  goto end;
4457  }
4458  if (strcmp(node->name, "apache-tomcat") != 0) {
4459  goto end;
4460  }
4461 
4462  int i = 0;
4463  ConfNode *n;
4464 
4465  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4466  if (node2 == NULL) {
4467  goto end;
4468  }
4469  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4470  goto end;
4471  }
4472 
4473  node = ConfNodeLookupChild(node, "address");
4474  if (node == NULL) {
4475  goto end;
4476  }
4477  TAILQ_FOREACH(n, &node->head, next) {
4478  if (n == NULL) {
4479  goto end;
4480  }
4481 
4482  switch(i) {
4483  case 0:
4484  if (strcmp(n->name, "0") != 0) {
4485  goto end;
4486  }
4487  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4488  goto end;
4489  }
4490  break;
4491  case 1:
4492  if (strcmp(n->name, "1") != 0) {
4493  goto end;
4494  }
4495  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4496  goto end;
4497  }
4498  break;
4499  case 2:
4500  if (strcmp(n->name, "2") != 0) {
4501  goto end;
4502  }
4503  if (strcmp(n->val, "::1") != 0) {
4504  goto end;
4505  }
4506  break;
4507  default:
4508  goto end;
4509  }
4510  i++;
4511  }
4512 
4513  outputs = ConfGetNode("libhtp.server-config");
4514  if (outputs == NULL) {
4515  goto end;
4516  }
4517 
4518  node = TAILQ_FIRST(&outputs->head);
4519  node = TAILQ_NEXT(node, next);
4520  if (node == NULL) {
4521  goto end;
4522  }
4523  if (strcmp(node->name, "1") != 0) {
4524  goto end;
4525  }
4526  node = TAILQ_FIRST(&node->head);
4527  if (node == NULL) {
4528  goto end;
4529  }
4530  if (strcmp(node->name, "iis7") != 0) {
4531  goto end;
4532  }
4533 
4534  node2 = ConfNodeLookupChild(node, "personality");
4535  if (node2 == NULL) {
4536  goto end;
4537  }
4538  if (strcmp(node2->val, "IIS_7_0") != 0) {
4539  goto end;
4540  }
4541 
4542  node = ConfNodeLookupChild(node, "address");
4543  if (node == NULL) {
4544  goto end;
4545  }
4546 
4547  i = 0;
4548  TAILQ_FOREACH(n, &node->head, next) {
4549  if (n == NULL) {
4550  goto end;
4551  }
4552 
4553  switch(i) {
4554  case 0:
4555  if (strcmp(n->name, "0") != 0) {
4556  goto end;
4557  }
4558  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4559  goto end;
4560  }
4561  break;
4562  case 1:
4563  if (strcmp(n->name, "1") != 0) {
4564  goto end;
4565  }
4566  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4567  goto end;
4568  }
4569  break;
4570  default:
4571  goto end;
4572  }
4573  i++;
4574  }
4575 
4576  ret = 1;
4577 
4578 end:
4579  ConfDeInit();
4581 
4582  return ret;
4583 }
4584 
4585 /** \test Test config builds radix correctly */
4586 static int HTPParserConfigTest02(void)
4587 {
4588  int ret = 0;
4589  char input[] = "\
4590 %YAML 1.1\n\
4591 ---\n\
4592 libhtp:\n\
4593 \n\
4594  default-config:\n\
4595  personality: IDS\n\
4596 \n\
4597  server-config:\n\
4598 \n\
4599  - apache-tomcat:\n\
4600  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4601  personality: Tomcat_6_0\n\
4602 \n\
4603  - iis7:\n\
4604  address: \n\
4605  - 192.168.0.0/24\n\
4606  - 192.168.10.0/24\n\
4607  personality: IIS_7_0\n\
4608 ";
4609 
4611  ConfInit();
4613 
4614  ConfYamlLoadString(input, strlen(input));
4615 
4616  HTPConfigure();
4617 
4618  if (cfglist.cfg == NULL) {
4619  printf("No default config created.\n");
4620  goto end;
4621  }
4622 
4623  if (cfgtree == NULL) {
4624  printf("No config tree created.\n");
4625  goto end;
4626  }
4627 
4628  htp_cfg_t *htp = cfglist.cfg;
4629  uint8_t buf[128];
4630  const char *addr;
4631  void *user_data = NULL;
4632 
4633  addr = "192.168.10.42";
4634  if (inet_pton(AF_INET, addr, buf) == 1) {
4635  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4636  if (user_data != NULL) {
4637  HTPCfgRec *htp_cfg_rec = user_data;
4638  htp = htp_cfg_rec->cfg;
4639  SCLogDebug("LIBHTP using config: %p", htp);
4640  }
4641  if (htp == NULL) {
4642  printf("Could not get config for: %s\n", addr);
4643  goto end;
4644  }
4645  }
4646  else {
4647  printf("Failed to parse address: %s\n", addr);
4648  goto end;
4649  }
4650 
4651  user_data = NULL;
4652  addr = "::1";
4653  if (inet_pton(AF_INET6, addr, buf) == 1) {
4654  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4655  if (user_data != NULL) {
4656  HTPCfgRec *htp_cfg_rec = user_data;
4657  htp = htp_cfg_rec->cfg;
4658  SCLogDebug("LIBHTP using config: %p", htp);
4659  }
4660  if (htp == NULL) {
4661  printf("Could not get config for: %s\n", addr);
4662  goto end;
4663  }
4664  }
4665  else {
4666  printf("Failed to parse address: %s\n", addr);
4667  goto end;
4668  }
4669 
4670  ret = 1;
4671 
4672 end:
4673  HTPFreeConfig();
4674  ConfDeInit();
4677 
4678  return ret;
4679 }
4680 
4681 /** \test Test traffic is handled by the correct htp config */
4682 static int HTPParserConfigTest03(void)
4683 {
4684  int result = 1;
4685  Flow *f = NULL;
4686  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4687  " Data is c0oL!";
4688  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4689  TcpSession ssn;
4691 
4692  HtpState *htp_state = NULL;
4693  int r = 0;
4694  char input[] = "\
4695 %YAML 1.1\n\
4696 ---\n\
4697 libhtp:\n\
4698 \n\
4699  default-config:\n\
4700  personality: IDS\n\
4701 \n\
4702  server-config:\n\
4703 \n\
4704  - apache-tomcat:\n\
4705  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4706  personality: Tomcat_6_0\n\
4707 \n\
4708  - iis7:\n\
4709  address: \n\
4710  - 192.168.0.0/24\n\
4711  - 192.168.10.0/24\n\
4712  personality: IIS_7_0\n\
4713 ";
4714 
4716  ConfInit();
4718 
4719  ConfYamlLoadString(input, strlen(input));
4720 
4721  HTPConfigure();
4722 
4723  const char *addr = "192.168.10.42";
4724 
4725  memset(&ssn, 0, sizeof(ssn));
4726 
4727  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4728  if (f == NULL)
4729  goto end;
4730  f->protoctx = &ssn;
4731  f->proto = IPPROTO_TCP;
4732  f->alproto = ALPROTO_HTTP;
4733 
4734  htp_cfg_t *htp = cfglist.cfg;
4735 
4736  void *user_data = NULL;
4737  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4738  if (user_data != NULL) {
4739  HTPCfgRec *htp_cfg_rec = user_data;
4740  htp = htp_cfg_rec->cfg;
4741  SCLogDebug("LIBHTP using config: %p", htp);
4742  }
4743  if (htp == NULL) {
4744  printf("Could not get config for: %s\n", addr);
4745  goto end;
4746  }
4747 
4749 
4750  uint32_t u;
4751  for (u = 0; u < httplen1; u++) {
4752  uint8_t flags = 0;
4753 
4754  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4755  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4756  else flags = STREAM_TOSERVER;
4757 
4758  FLOWLOCK_WRLOCK(f);
4759  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4760  &httpbuf1[u], 1);
4761  if (r != 0) {
4762  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4763  " 0: ", u, r);
4764  result = 0;
4765  FLOWLOCK_UNLOCK(f);
4766  goto end;
4767  }
4768  FLOWLOCK_UNLOCK(f);
4769  }
4770 
4771  htp_state = f->alstate;
4772  if (htp_state == NULL) {
4773  printf("no http state: ");
4774  result = 0;
4775  goto end;
4776  }
4777 
4778  if (HTPStateGetTxCnt(htp_state) != 2) {
4779  printf("HTPStateGetTxCnt(htp_state) failure\n");
4780  goto end;
4781  }
4782 
4783  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4784  if (tx == NULL)
4785  goto end;
4786  if (tx->cfg != htp) {
4787  printf("wrong HTP config (%p instead of %p - default=%p): ",
4788  tx->cfg, htp, cfglist.cfg);
4789  goto end;
4790  }
4791  tx = HTPStateGetTx(htp_state, 1);
4792  if (tx == NULL)
4793  goto end;
4794  if (tx->cfg != htp) {
4795  printf("wrong HTP config (%p instead of %p - default=%p): ",
4796  tx->cfg, htp, cfglist.cfg);
4797  goto end;
4798  }
4799 
4800 end:
4801  if (alp_tctx != NULL)
4802  AppLayerParserThreadCtxFree(alp_tctx);
4803  HTPFreeConfig();
4804  ConfDeInit();
4807 
4809  UTHFreeFlow(f);
4810  return result;
4811 }
4812 
4813 /* disabled when we upgraded to libhtp 0.5.x */
4814 #if 0
4815 static int HTPParserConfigTest04(void)
4816 {
4817  int result = 0;
4818 
4819  char input[] = "\
4820 %YAML 1.1\n\
4821 ---\n\
4822 libhtp:\n\
4823 \n\
4824  default-config:\n\
4825  personality: IDS\n\
4826  path-control-char-handling: status_400\n\
4827  path-convert-utf8: yes\n\
4828  path-invalid-encoding-handling: remove_percent\n\
4829 \n\
4830  server-config:\n\
4831 \n\
4832  - apache-tomcat:\n\
4833  personality: Tomcat_6_0\n\
4834  path-invalid-utf8-handling: none\n\
4835  path-nul-encoded-handling: status_404\n\
4836  path-nul-raw-handling: status_400\n\
4837 \n\
4838  - iis7:\n\
4839  personality: IIS_7_0\n\
4840  path-replacement-char: o\n\
4841  path-unicode-mapping: status_400\n\
4842 ";
4843 
4845  ConfInit();
4847 
4848  ConfYamlLoadString(input, strlen(input));
4849 
4850  HTPConfigure();
4851 
4852  HTPCfgRec *cfg_rec = &cfglist;
4853  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4854  cfg_rec->cfg->path_convert_utf8 != 1 ||
4855  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4856  printf("failed 1\n");
4857  goto end;
4858  }
4859 
4860  cfg_rec = cfg_rec->next;
4861  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4862  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4863  printf("failed 2\n");
4864  goto end;
4865  }
4866 
4867  cfg_rec = cfg_rec->next;
4868  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4869  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4870  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4871  printf("failed 3\n");
4872  goto end;
4873  }
4874 
4875  result = 1;
4876 
4877 end:
4878  HTPFreeConfig();
4879  ConfDeInit();
4882 
4883  return result;
4884 }
4885 #endif
4886 
4887 /** \test Test %2f decoding in profile Apache_2_2
4888  *
4889  * %2f in path is left untouched
4890  * %2f in query string is normalized to %2F
4891  * %252f in query string is decoded/normalized to %2F
4892  */
4893 static int HTPParserDecodingTest01(void)
4894 {
4895  int result = 0;
4896  Flow *f = NULL;
4897  uint8_t httpbuf1[] =
4898  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4899  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4900  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4901  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4902  TcpSession ssn;
4904 
4905  HtpState *htp_state = NULL;
4906  int r = 0;
4907  char input[] = "\
4908 %YAML 1.1\n\
4909 ---\n\
4910 libhtp:\n\
4911 \n\
4912  default-config:\n\
4913  personality: Apache_2\n\
4914 ";
4915 
4917  ConfInit();
4919  ConfYamlLoadString(input, strlen(input));
4920  HTPConfigure();
4921  const char *addr = "4.3.2.1";
4922  memset(&ssn, 0, sizeof(ssn));
4923 
4924  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4925  if (f == NULL)
4926  goto end;
4927  f->protoctx = &ssn;
4928  f->proto = IPPROTO_TCP;
4929  f->alproto = ALPROTO_HTTP;
4930 
4932 
4933  uint32_t u;
4934  for (u = 0; u < httplen1; u++) {
4935  uint8_t flags = 0;
4936 
4937  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4938  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4939  else flags = STREAM_TOSERVER;
4940 
4941  FLOWLOCK_WRLOCK(f);
4942  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4943  &httpbuf1[u], 1);
4944  if (r != 0) {
4945  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4946  " 0: ", u, r);
4947  FLOWLOCK_UNLOCK(f);
4948  goto end;
4949  }
4950  FLOWLOCK_UNLOCK(f);
4951  }
4952 
4953  htp_state = f->alstate;
4954  if (htp_state == NULL) {
4955  printf("no http state: ");
4956  goto end;
4957  }
4958 
4959  uint8_t ref1[] = "/abc%2fdef";
4960  size_t reflen = sizeof(ref1) - 1;
4961 
4962  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4963  if (tx == NULL)
4964  goto end;
4965  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4966  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4967  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4968  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4969  (uintmax_t)reflen,
4970  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4971  goto end;
4972  }
4973 
4974  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4975  bstr_len(tx_ud->request_uri_normalized)) != 0)
4976  {
4977  printf("normalized uri \"");
4978  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4979  printf("\" != \"");
4980  PrintRawUriFp(stdout, ref1, reflen);
4981  printf("\": ");
4982  goto end;
4983  }
4984  }
4985 
4986  uint8_t ref2[] = "/abc/def?ghi/jkl";
4987  reflen = sizeof(ref2) - 1;
4988 
4989  tx = HTPStateGetTx(htp_state, 1);
4990  if (tx == NULL)
4991  goto end;
4992  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4993  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4994  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4995  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4996  (uintmax_t)reflen,
4997  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4998  goto end;
4999  }
5000 
5001  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5002  bstr_len(tx_ud->request_uri_normalized)) != 0)
5003  {
5004  printf("normalized uri \"");
5005  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5006  printf("\" != \"");
5007  PrintRawUriFp(stdout, ref2, reflen);
5008  printf("\": ");
5009  goto end;
5010  }
5011  }
5012 
5013  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5014  reflen = sizeof(ref3) - 1;
5015  tx = HTPStateGetTx(htp_state, 2);
5016  if (tx == NULL)
5017  goto end;
5018  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5019  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5020  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5021  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5022  (uintmax_t)reflen,
5023  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5024  goto end;
5025  }
5026 
5027  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5028  bstr_len(tx_ud->request_uri_normalized)) != 0)
5029  {
5030  printf("normalized uri \"");
5031  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5032  printf("\" != \"");
5033  PrintRawUriFp(stdout, ref3, reflen);
5034  printf("\": ");
5035  goto end;
5036  }
5037  }
5038 
5039  result = 1;
5040 
5041 end:
5042  if (alp_tctx != NULL)
5043  AppLayerParserThreadCtxFree(alp_tctx);
5044  HTPFreeConfig();
5045  ConfDeInit();
5048 
5050  UTHFreeFlow(f);
5051  return result;
5052 }
5053 
5054 /** \test Test %2f decoding in profile IDS
5055  *
5056  * %2f in path decoded to /
5057  * %2f in query string is decoded to /
5058  * %252f in query string is decoded to %2F
5059  */
5060 static int HTPParserDecodingTest02(void)
5061 {
5062  int result = 0;
5063  Flow *f = NULL;
5064  uint8_t httpbuf1[] =
5065  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5066  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5067  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5068  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5069  TcpSession ssn;
5071 
5072  HtpState *htp_state = NULL;
5073  int r = 0;
5074  char input[] = "\
5075 %YAML 1.1\n\
5076 ---\n\
5077 libhtp:\n\
5078 \n\
5079  default-config:\n\
5080  personality: IDS\n\
5081  double-decode-path: no\n\
5082  double-decode-query: no\n\
5083 ";
5084 
5086  ConfInit();
5088  ConfYamlLoadString(input, strlen(input));
5089  HTPConfigure();
5090  const char *addr = "4.3.2.1";
5091  memset(&ssn, 0, sizeof(ssn));
5092 
5093  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5094  if (f == NULL)
5095  goto end;
5096  f->protoctx = &ssn;
5097  f->proto = IPPROTO_TCP;
5098  f->alproto = ALPROTO_HTTP;
5099 
5101 
5102  uint32_t u;
5103  for (u = 0; u < httplen1; u++) {
5104  uint8_t flags = 0;
5105 
5106  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5107  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5108  else flags = STREAM_TOSERVER;
5109 
5110  FLOWLOCK_WRLOCK(f);
5111  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5112  &httpbuf1[u], 1);
5113  if (r != 0) {
5114  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5115  " 0: ", u, r);
5116  FLOWLOCK_UNLOCK(f);
5117  goto end;
5118  }
5119  FLOWLOCK_UNLOCK(f);
5120  }
5121 
5122  htp_state = f->alstate;
5123  if (htp_state == NULL) {
5124  printf("no http state: ");
5125  goto end;
5126  }
5127 
5128  uint8_t ref1[] = "/abc/def";
5129  size_t reflen = sizeof(ref1) - 1;
5130 
5131  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5132  if (tx == NULL)
5133  goto end;
5134  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5135  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5136  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5137  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5138  (uintmax_t)reflen,
5139  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5140  goto end;
5141  }
5142 
5143  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5144  bstr_len(tx_ud->request_uri_normalized)) != 0)
5145  {
5146  printf("normalized uri \"");
5147  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5148  printf("\" != \"");
5149  PrintRawUriFp(stdout, ref1, reflen);
5150  printf("\": ");
5151  goto end;
5152  }
5153  }
5154 
5155  uint8_t ref2[] = "/abc/def?ghi/jkl";
5156  reflen = sizeof(ref2) - 1;
5157 
5158  tx = HTPStateGetTx(htp_state, 1);
5159  if (tx == NULL)
5160  goto end;
5161  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5162  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5163  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5164  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5165  (uintmax_t)reflen,
5166  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5167  goto end;
5168  }
5169 
5170  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5171  bstr_len(tx_ud->request_uri_normalized)) != 0)
5172  {
5173  printf("normalized uri \"");
5174  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5175  printf("\" != \"");
5176  PrintRawUriFp(stdout, ref2, reflen);
5177  printf("\": ");
5178  goto end;
5179  }
5180  }
5181 
5182  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5183  reflen = sizeof(ref3) - 1;
5184  tx = HTPStateGetTx(htp_state, 2);
5185  if (tx == NULL)
5186  goto end;
5187  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5188  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5189  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5190  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5191  (uintmax_t)reflen,
5192  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5193  goto end;
5194  }
5195 
5196  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5197  bstr_len(tx_ud->request_uri_normalized)) != 0)
5198  {
5199  printf("normalized uri \"");
5200  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5201  printf("\" != \"");
5202  PrintRawUriFp(stdout, ref3, reflen);
5203  printf("\": ");
5204  goto end;
5205  }
5206  }
5207 
5208  result = 1;
5209 
5210 end:
5211  if (alp_tctx != NULL)
5212  AppLayerParserThreadCtxFree(alp_tctx);
5213  HTPFreeConfig();
5214  ConfDeInit();
5217 
5219  UTHFreeFlow(f);
5220  return result;
5221 }
5222 
5223 /** \test Test %2f decoding in profile IDS with double-decode-* options
5224  *
5225  * %252f in path decoded to /
5226  * %252f in query string is decoded to /
5227  */
5228 static int HTPParserDecodingTest03(void)
5229 {
5230  int result = 0;
5231  Flow *f = NULL;
5232  uint8_t httpbuf1[] =
5233  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5234  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5235  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5236  TcpSession ssn;
5238 
5239  HtpState *htp_state = NULL;
5240  int r = 0;
5241  char input[] = "\
5242 %YAML 1.1\n\
5243 ---\n\
5244 libhtp:\n\
5245 \n\
5246  default-config:\n\
5247  personality: IDS\n\
5248  double-decode-path: yes\n\
5249  double-decode-query: yes\n\
5250 ";
5251 
5253  ConfInit();
5255  ConfYamlLoadString(input, strlen(input));
5256  HTPConfigure();
5257  const char *addr = "4.3.2.1";
5258  memset(&ssn, 0, sizeof(ssn));
5259 
5260  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5261  if (f == NULL)
5262  goto end;
5263  f->protoctx = &ssn;
5264  f->proto = IPPROTO_TCP;
5265  f->alproto = ALPROTO_HTTP;
5266 
5268 
5269  uint32_t u;
5270  for (u = 0; u < httplen1; u++) {
5271  uint8_t flags = 0;
5272 
5273  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5274  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5275  else flags = STREAM_TOSERVER;
5276 
5277  FLOWLOCK_WRLOCK(f);
5278  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5279  &httpbuf1[u], 1);
5280  if (r != 0) {
5281  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5282  " 0: ", u, r);
5283  FLOWLOCK_UNLOCK(f);
5284  goto end;
5285  }
5286  FLOWLOCK_UNLOCK(f);
5287  }
5288 
5289  htp_state = f->alstate;
5290  if (htp_state == NULL) {
5291  printf("no http state: ");
5292  goto end;
5293  }
5294 
5295  uint8_t ref1[] = "/abc/def";
5296  size_t reflen = sizeof(ref1) - 1;
5297 
5298  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5299  if (tx == NULL)
5300  goto end;
5301  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5302  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5303  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5304  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5305  (uintmax_t)reflen,
5306  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5307  goto end;
5308  }
5309 
5310  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5311  bstr_len(tx_ud->request_uri_normalized)) != 0)
5312  {
5313  printf("normalized uri \"");
5314  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5315  printf("\" != \"");
5316  PrintRawUriFp(stdout, ref1, reflen);
5317  printf("\": ");
5318  goto end;
5319  }
5320  }
5321 
5322  uint8_t ref2[] = "/abc/def?ghi/jkl";
5323  reflen = sizeof(ref2) - 1;
5324 
5325  tx = HTPStateGetTx(htp_state, 1);
5326  if (tx == NULL)
5327  goto end;
5328  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5329  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5330  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5331  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5332  (uintmax_t)reflen,
5333  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5334  goto end;
5335  }
5336 
5337  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5338  bstr_len(tx_ud->request_uri_normalized)) != 0)
5339  {
5340  printf("normalized uri \"");
5341  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5342  printf("\" != \"");
5343  PrintRawUriFp(stdout, ref2, reflen);
5344  printf("\": ");
5345  goto end;
5346  }
5347  }
5348 
5349  result = 1;
5350 
5351 end:
5352  if (alp_tctx != NULL)
5353  AppLayerParserThreadCtxFree(alp_tctx);
5354  HTPFreeConfig();
5355  ConfDeInit();
5358 
5360  UTHFreeFlow(f);
5361  return result;
5362 }
5363 
5364 /** \test Test http:// in query profile IDS
5365  */
5366 static int HTPParserDecodingTest04(void)
5367 {
5368  int result = 0;
5369  Flow *f = NULL;
5370  uint8_t httpbuf1[] =
5371  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5372  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5373  TcpSession ssn;
5375 
5376  HtpState *htp_state = NULL;
5377  int r = 0;
5378  char input[] = "\
5379 %YAML 1.1\n\
5380 ---\n\
5381 libhtp:\n\
5382 \n\
5383  default-config:\n\
5384  personality: IDS\n\
5385  double-decode-path: yes\n\
5386  double-decode-query: yes\n\
5387 ";
5388 
5390  ConfInit();
5392  ConfYamlLoadString(input, strlen(input));
5393  HTPConfigure();
5394  const char *addr = "4.3.2.1";
5395  memset(&ssn, 0, sizeof(ssn));
5396 
5397  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5398  if (f == NULL)
5399  goto end;
5400  f->protoctx = &ssn;
5401  f->proto = IPPROTO_TCP;
5402  f->alproto = ALPROTO_HTTP;
5403 
5405 
5406  uint32_t u;
5407  for (u = 0; u < httplen1; u++) {
5408  uint8_t flags = 0;
5409 
5410  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5411  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5412  else flags = STREAM_TOSERVER;
5413 
5414  FLOWLOCK_WRLOCK(f);
5415  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5416  &httpbuf1[u], 1);
5417  if (r != 0) {
5418  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5419  " 0: ", u, r);
5420  FLOWLOCK_UNLOCK(f);
5421  goto end;
5422  }
5423  FLOWLOCK_UNLOCK(f);
5424  }
5425 
5426  htp_state = f->alstate;
5427  if (htp_state == NULL) {
5428  printf("no http state: ");
5429  goto end;
5430  }
5431 
5432  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5433  size_t reflen = sizeof(ref1) - 1;
5434 
5435  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5436  if (tx == NULL)
5437  goto end;
5438  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5439  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5440  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5441  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5442  (uintmax_t)reflen,
5443  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5444  goto end;
5445  }
5446 
5447  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5448  bstr_len(tx_ud->request_uri_normalized)) != 0)
5449  {
5450  printf("normalized uri \"");
5451  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5452  printf("\" != \"");
5453  PrintRawUriFp(stdout, ref1, reflen);
5454  printf("\": ");
5455  goto end;
5456  }
5457  }
5458 
5459  result = 1;
5460 
5461 end:
5462  if (alp_tctx != NULL)
5463  AppLayerParserThreadCtxFree(alp_tctx);
5464  HTPFreeConfig();
5465  ConfDeInit();
5468 
5470  UTHFreeFlow(f);
5471  return result;
5472 }
5473 
5474 /** \test Test \ char in query profile IDS. Bug 739
5475  */
5476 static int HTPParserDecodingTest05(void)
5477 {
5478  int result = 0;
5479  Flow *f = NULL;
5480  uint8_t httpbuf1[] =
5481  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5482  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5483  TcpSession ssn;
5485 
5486  HtpState *htp_state = NULL;
5487  int r = 0;
5488  char input[] = "\
5489 %YAML 1.1\n\
5490 ---\n\
5491 libhtp:\n\
5492 \n\
5493  default-config:\n\
5494  personality: IDS\n\
5495  double-decode-path: yes\n\
5496  double-decode-query: yes\n\
5497 ";
5498 
5500  ConfInit();
5502  ConfYamlLoadString(input, strlen(input));
5503  HTPConfigure();
5504  const char *addr = "4.3.2.1";
5505  memset(&ssn, 0, sizeof(ssn));
5506 
5507  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5508  if (f == NULL)
5509  goto end;
5510  f->protoctx = &ssn;
5511  f->proto = IPPROTO_TCP;
5512  f->alproto = ALPROTO_HTTP;
5513 
5515 
5516  uint32_t u;
5517  for (u = 0; u < httplen1; u++) {
5518  uint8_t flags = 0;
5519 
5520  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5521  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5522  else flags = STREAM_TOSERVER;
5523 
5524  FLOWLOCK_WRLOCK(f);
5525  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5526  &httpbuf1[u], 1);
5527  if (r != 0) {
5528  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5529  " 0: ", u, r);
5530  FLOWLOCK_UNLOCK(f);
5531  goto end;
5532  }
5533  FLOWLOCK_UNLOCK(f);
5534  }
5535 
5536  htp_state = f->alstate;
5537  if (htp_state == NULL) {
5538  printf("no http state: ");
5539  goto end;
5540  }
5541 
5542  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5543  size_t reflen = sizeof(ref1) - 1;
5544 
5545  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5546  if (tx == NULL)
5547  goto end;
5548  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5549  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5550  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5551  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5552  (uintmax_t)reflen,
5553  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5554  goto end;
5555  }
5556 
5557  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5558  bstr_len(tx_ud->request_uri_normalized)) != 0)
5559  {
5560  printf("normalized uri \"");
5561  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5562  printf("\" != \"");
5563  PrintRawUriFp(stdout, ref1, reflen);
5564  printf("\": ");
5565  goto end;
5566  }
5567  }
5568 
5569  result = 1;
5570 
5571 end:
5572  if (alp_tctx != NULL)
5573  AppLayerParserThreadCtxFree(alp_tctx);
5574  HTPFreeConfig();
5575  ConfDeInit();
5578 
5580  UTHFreeFlow(f);
5581  return result;
5582 }
5583 
5584 /** \test Test + char in query. Bug 1035
5585  */
5586 static int HTPParserDecodingTest06(void)
5587 {
5588  int result = 0;
5589  Flow *f = NULL;
5590  uint8_t httpbuf1[] =
5591  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5592  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5593  TcpSession ssn;
5595 
5596  HtpState *htp_state = NULL;
5597  int r = 0;
5598  char input[] = "\
5599 %YAML 1.1\n\
5600 ---\n\
5601 libhtp:\n\
5602 \n\
5603  default-config:\n\
5604  personality: IDS\n\
5605  double-decode-path: yes\n\
5606  double-decode-query: yes\n\
5607 ";
5608 
5610  ConfInit();
5612  ConfYamlLoadString(input, strlen(input));
5613  HTPConfigure();
5614  const char *addr = "4.3.2.1";
5615  memset(&ssn, 0, sizeof(ssn));
5616 
5617  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5618  if (f == NULL)
5619  goto end;
5620  f->protoctx = &ssn;
5621  f->proto = IPPROTO_TCP;
5622  f->alproto = ALPROTO_HTTP;
5623 
5625 
5626  uint32_t u;
5627  for (u = 0; u < httplen1; u++) {
5628  uint8_t flags = 0;
5629 
5630  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5631  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5632  else flags = STREAM_TOSERVER;
5633 
5634  FLOWLOCK_WRLOCK(f);
5635  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5636  &httpbuf1[u], 1);
5637  if (r != 0) {
5638  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5639  " 0: ", u, r);
5640  FLOWLOCK_UNLOCK(f);
5641  goto end;
5642  }
5643  FLOWLOCK_UNLOCK(f);
5644  }
5645 
5646  htp_state = f->alstate;
5647  if (htp_state == NULL) {
5648  printf("no http state: ");
5649  goto end;
5650  }
5651 
5652  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5653  size_t reflen = sizeof(ref1) - 1;
5654 
5655  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5656  if (tx == NULL)
5657  goto end;
5658  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5659  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5660  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5661  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5662  (uintmax_t)reflen,
5663  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5664  goto end;
5665  }
5666 
5667  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5668  bstr_len(tx_ud->request_uri_normalized)) != 0)
5669  {
5670  printf("normalized uri \"");
5671  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5672  printf("\" != \"");
5673  PrintRawUriFp(stdout, ref1, reflen);
5674  printf("\": ");
5675  goto end;
5676  }
5677  }
5678 
5679  result = 1;
5680 
5681 end:
5682  if (alp_tctx != NULL)
5683  AppLayerParserThreadCtxFree(alp_tctx);
5684  HTPFreeConfig();
5685  ConfDeInit();
5688 
5690  UTHFreeFlow(f);
5691  return result;
5692 }
5693 
5694 /** \test Test + char in query. Bug 1035
5695  */
5696 static int HTPParserDecodingTest07(void)
5697 {
5698  int result = 0;
5699  Flow *f = NULL;
5700  uint8_t httpbuf1[] =
5701  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5702  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5703  TcpSession ssn;
5705 
5706  HtpState *htp_state = NULL;
5707  int r = 0;
5708  char input[] = "\
5709 %YAML 1.1\n\
5710 ---\n\
5711 libhtp:\n\
5712 \n\
5713  default-config:\n\
5714  personality: IDS\n\
5715  double-decode-path: yes\n\
5716  double-decode-query: yes\n\
5717  query-plusspace-decode: yes\n\
5718 ";
5719 
5721  ConfInit();
5723  ConfYamlLoadString(input, strlen(input));
5724  HTPConfigure();
5725  const char *addr = "4.3.2.1";
5726  memset(&ssn, 0, sizeof(ssn));
5727 
5728  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5729  if (f == NULL)
5730  goto end;
5731  f->protoctx = &ssn;
5732  f->proto = IPPROTO_TCP;
5733  f->alproto = ALPROTO_HTTP;
5734 
5736 
5737  uint32_t u;
5738  for (u = 0; u < httplen1; u++) {
5739  uint8_t flags = 0;
5740 
5741  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5742  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5743  else flags = STREAM_TOSERVER;
5744 
5745  FLOWLOCK_WRLOCK(f);
5746  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5747  &httpbuf1[u], 1);
5748  if (r != 0) {
5749  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5750  " 0: ", u, r);
5751  FLOWLOCK_UNLOCK(f);
5752  goto end;
5753  }
5754  FLOWLOCK_UNLOCK(f);
5755  }
5756 
5757  htp_state = f->alstate;
5758  if (htp_state == NULL) {
5759  printf("no http state: ");
5760  goto end;
5761  }
5762 
5763  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5764  size_t reflen = sizeof(ref1) - 1;
5765 
5766  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5767  if (tx == NULL)
5768  goto end;
5769  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5770  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5771  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5772  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5773  (uintmax_t)reflen,
5774  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5775  goto end;
5776  }
5777 
5778  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5779  bstr_len(tx_ud->request_uri_normalized)) != 0)
5780  {
5781  printf("normalized uri \"");
5782  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5783  printf("\" != \"");
5784  PrintRawUriFp(stdout, ref1, reflen);
5785  printf("\": ");
5786  goto end;
5787  }
5788  }
5789 
5790  result = 1;
5791 
5792 end:
5793  if (alp_tctx != NULL)
5794  AppLayerParserThreadCtxFree(alp_tctx);
5795  HTPFreeConfig();
5796  ConfDeInit();
5799 
5801  UTHFreeFlow(f);
5802  return result;
5803 }
5804 
5805 /** \test Test 'proxy' URI normalization. Ticket 1008
5806  */
5807 static int HTPParserDecodingTest08(void)
5808 {
5809  int result = 0;
5810  Flow *f = NULL;
5811  uint8_t httpbuf1[] =
5812  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5813  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5814  TcpSession ssn;
5816 
5817  HtpState *htp_state = NULL;
5818  int r = 0;
5819  char input[] = "\
5820 %YAML 1.1\n\
5821 ---\n\
5822 libhtp:\n\
5823 \n\
5824  default-config:\n\
5825  personality: IDS\n\
5826 ";
5827 
5829  ConfInit();
5831  ConfYamlLoadString(input, strlen(input));
5832  HTPConfigure();
5833  const char *addr = "4.3.2.1";
5834  memset(&ssn, 0, sizeof(ssn));
5835 
5836  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5837  if (f == NULL)
5838  goto end;
5839  f->protoctx = &ssn;
5840  f->proto = IPPROTO_TCP;
5841  f->alproto = ALPROTO_HTTP;
5842 
5844 
5845  uint32_t u;
5846  for (u = 0; u < httplen1; u++) {
5847  uint8_t flags = 0;
5848 
5849  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5850  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5851  else flags = STREAM_TOSERVER;
5852 
5853  FLOWLOCK_WRLOCK(f);
5854  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5855  &httpbuf1[u], 1);
5856  if (r != 0) {
5857  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5858  " 0: ", u, r);
5859  FLOWLOCK_UNLOCK(f);
5860  goto end;
5861  }
5862  FLOWLOCK_UNLOCK(f);
5863  }
5864 
5865  htp_state = f->alstate;
5866  if (htp_state == NULL) {
5867  printf("no http state: ");
5868  goto end;
5869  }
5870 
5871  uint8_t ref1[] = "/blah/";
5872  size_t reflen = sizeof(ref1) - 1;
5873 
5874  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5875  if (tx == NULL)
5876  goto end;
5877  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5878  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5879  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5880  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5881  (uintmax_t)reflen,
5882  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5883  goto end;
5884  }
5885 
5886  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5887  bstr_len(tx_ud->request_uri_normalized)) != 0)
5888  {
5889  printf("normalized uri \"");
5890  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5891  printf("\" != \"");
5892  PrintRawUriFp(stdout, ref1, reflen);
5893  printf("\": ");
5894  goto end;
5895  }
5896  }
5897 
5898  result = 1;
5899 
5900 end:
5901  if (alp_tctx != NULL)
5902  AppLayerParserThreadCtxFree(alp_tctx);
5903  HTPFreeConfig();
5904  ConfDeInit();
5907 
5909  UTHFreeFlow(f);
5910  return result;
5911 }
5912 
5913 /** \test Test 'proxy' URI normalization. Ticket 1008
5914  */
5915 static int HTPParserDecodingTest09(void)
5916 {
5917  int result = 0;
5918  Flow *f = NULL;
5919  uint8_t httpbuf1[] =
5920  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5921  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5922  TcpSession ssn;
5924 
5925  HtpState *htp_state = NULL;
5926  int r = 0;
5927  char input[] = "\
5928 %YAML 1.1\n\
5929 ---\n\
5930 libhtp:\n\
5931 \n\
5932  default-config:\n\
5933  personality: IDS\n\
5934  uri-include-all: true\n\
5935 ";
5936 
5938  ConfInit();
5940  ConfYamlLoadString(input, strlen(input));
5941  HTPConfigure();
5942  const char *addr = "4.3.2.1";
5943  memset(&ssn, 0, sizeof(ssn));
5944 
5945  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5946  if (f == NULL)
5947  goto end;
5948  f->protoctx = &ssn;
5949  f->proto = IPPROTO_TCP;
5950  f->alproto = ALPROTO_HTTP;
5951