suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-htp.h"
59 #include "app-layer-htp-body.h"
60 #include "app-layer-htp-file.h"
61 #include "app-layer-htp-libhtp.h"
62 #include "app-layer-htp-xff.h"
63 
64 #include "util-spm.h"
65 #include "util-debug.h"
66 #include "util-time.h"
67 #include "util-misc.h"
68 
69 #include "util-unittest.h"
70 #include "util-unittest-helper.h"
71 #include "flow-util.h"
72 
73 #include "detect-engine.h"
74 #include "detect-engine-state.h"
75 #include "detect-parse.h"
76 
77 #include "decode-events.h"
78 
79 #include "util-memcmp.h"
80 #include "util-random.h"
81 #include "util-validate.h"
82 
83 //#define PRINT
84 
85 /** Fast lookup tree (radix) for the various HTP configurations */
86 static SCRadixTree *cfgtree;
87 /** List of HTP configurations. */
88 static HTPCfgRec cfglist;
89 
90 #ifdef DEBUG
91 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
92 static uint64_t htp_state_memuse = 0;
93 static uint64_t htp_state_memcnt = 0;
94 #endif
95 
97  { "UNKNOWN_ERROR",
99  { "GZIP_DECOMPRESSION_FAILED",
101  { "REQUEST_FIELD_MISSING_COLON",
103  { "RESPONSE_FIELD_MISSING_COLON",
105  { "INVALID_REQUEST_CHUNK_LEN",
107  { "INVALID_RESPONSE_CHUNK_LEN",
109  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
111  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
113  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
115  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
117  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
119  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
121  { "100_CONTINUE_ALREADY_SEEN",
123  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
125  { "INVALID_SERVER_PORT_IN_REQUEST",
127  { "INVALID_AUTHORITY_PORT",
129  { "REQUEST_HEADER_INVALID",
131  { "RESPONSE_HEADER_INVALID",
133  { "MISSING_HOST_HEADER",
135  { "HOST_HEADER_AMBIGUOUS",
137  { "INVALID_REQUEST_FIELD_FOLDING",
139  { "INVALID_RESPONSE_FIELD_FOLDING",
141  { "REQUEST_FIELD_TOO_LONG",
143  { "RESPONSE_FIELD_TOO_LONG",
145  { "REQUEST_LINE_INVALID",
147  { "REQUEST_BODY_UNEXPECTED",
149  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
151  { "REQUEST_URI_HOST_INVALID",
153  { "REQUEST_HEADER_HOST_INVALID",
155  { "REQUEST_AUTH_UNRECOGNIZED",
157  { "REQUEST_HEADER_REPETITION",
159  { "RESPONSE_HEADER_REPETITION",
161  { "DOUBLE_ENCODED_URI",
163  { "URI_DELIM_NON_COMPLIANT",
165  { "METHOD_DELIM_NON_COMPLIANT",
167  { "REQUEST_LINE_LEADING_WHITESPACE",
169  { "TOO_MANY_ENCODING_LAYERS",
171  { "ABNORMAL_CE_HEADER",
173  { "RESPONSE_MULTIPART_BYTERANGES",
175  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
177  { "RESPONSE_CHUNKED_OLD_PROTO",
179  { "RESPONSE_INVALID_PROTOCOL",
181  { "RESPONSE_INVALID_STATUS",
183  { "REQUEST_LINE_INCOMPLETE",
185 
186  { "LZMA_MEMLIMIT_REACHED",
188  { "COMPRESSION_BOMB",
190 
191  /* suricata warnings/errors */
192  { "MULTIPART_GENERIC_ERROR",
194  { "MULTIPART_NO_FILEDATA",
196  { "MULTIPART_INVALID_HEADER",
198 
199  { NULL, -1 },
200 };
201 
202 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
203 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
204 static uint64_t HTPStateGetTxCnt(void *alstate);
205 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction);
206 
207 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
208 {
209  uint64_t id = HTPStateGetTxCnt(s);
210  BUG_ON(id == 0);
211  return id - 1;
212 }
213 
214 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
215 {
216  return s->transaction_cnt;
217 }
218 
219 #ifdef DEBUG
220 /**
221  * \internal
222  *
223  * \brief Lookup the HTP personality string from the numeric personality.
224  *
225  * \todo This needs to be a libhtp function.
226  */
227 static const char *HTPLookupPersonalityString(int p)
228 {
229 #define CASE_HTP_PERSONALITY_STRING(p) \
230  case HTP_SERVER_ ## p: return #p
231 
232  switch (p) {
233  CASE_HTP_PERSONALITY_STRING(MINIMAL);
234  CASE_HTP_PERSONALITY_STRING(GENERIC);
235  CASE_HTP_PERSONALITY_STRING(IDS);
236  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
237  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
238  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
239  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
240  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
241  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
242  CASE_HTP_PERSONALITY_STRING(APACHE_2);
243  }
244 
245  return NULL;
246 }
247 #endif /* DEBUG */
248 
249 /**
250  * \internal
251  *
252  * \brief Lookup the numeric HTP personality from a string.
253  *
254  * \todo This needs to be a libhtp function.
255  */
256 static int HTPLookupPersonality(const char *str)
257 {
258 #define IF_HTP_PERSONALITY_NUM(p) \
259  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
260 
261  IF_HTP_PERSONALITY_NUM(MINIMAL);
262  IF_HTP_PERSONALITY_NUM(GENERIC);
264  IF_HTP_PERSONALITY_NUM(IIS_4_0);
265  IF_HTP_PERSONALITY_NUM(IIS_5_0);
266  IF_HTP_PERSONALITY_NUM(IIS_5_1);
267  IF_HTP_PERSONALITY_NUM(IIS_6_0);
268  IF_HTP_PERSONALITY_NUM(IIS_7_0);
269  IF_HTP_PERSONALITY_NUM(IIS_7_5);
270  IF_HTP_PERSONALITY_NUM(APACHE_2);
271  if (strcasecmp("TOMCAT_6_0", str) == 0) {
272  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
273  "longer supported by libhtp.", str);
274  return -1;
275  } else if ((strcasecmp("APACHE", str) == 0) ||
276  (strcasecmp("APACHE_2_2", str) == 0))
277  {
278  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
279  "longer supported by libhtp, failing back to "
280  "Apache2 personality.", str);
281  return HTP_SERVER_APACHE_2;
282  }
283 
284  return -1;
285 }
286 
287 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
288  const uint8_t dir, const uint8_t e)
289 {
290  SCLogDebug("setting event %u", e);
291 
292  if (htud) {
294  s->events++;
295  return;
296  }
297 
298  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
299  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
300 
301  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
302  if (tx == NULL && tx_id > 0)
303  tx = HTPStateGetTx(s, tx_id - 1);
304  if (tx != NULL) {
305  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
306  if (htud != NULL) {
308  s->events++;
309  return;
310  }
311  }
312  SCLogDebug("couldn't set event %u", e);
313 }
314 
315 static AppLayerDecoderEvents *HTPGetEvents(void *tx)
316 {
317  SCLogDebug("get HTTP events for TX %p", tx);
318 
319  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
320  if (htud != NULL) {
321  SCLogDebug("has htud, htud->decoder_events %p", htud->decoder_events);
322  return htud->decoder_events;
323  }
324 
325  return NULL;
326 }
327 
328 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
329  * connection parser to be used by the HTP library
330  */
331 static void *HTPStateAlloc(void)
332 {
333  SCEnter();
334 
335  HtpState *s = HTPMalloc(sizeof(HtpState));
336  if (unlikely(s == NULL)) {
337  SCReturnPtr(NULL, "void");
338  }
339 
340  memset(s, 0x00, sizeof(HtpState));
341 
342 #ifdef DEBUG
343  SCMutexLock(&htp_state_mem_lock);
344  htp_state_memcnt++;
345  htp_state_memuse += sizeof(HtpState);
346  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
347  SCMutexUnlock(&htp_state_mem_lock);
348 #endif
349 
350  SCReturnPtr((void *)s, "void");
351 }
352 
353 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
354 {
355  if (likely(htud)) {
356  HtpBodyFree(&htud->request_body);
357  HtpBodyFree(&htud->response_body);
358  bstr_free(htud->request_uri_normalized);
359  if (htud->request_headers_raw)
361  if (htud->response_headers_raw)
364  if (htud->boundary)
365  HTPFree(htud->boundary, htud->boundary_len);
366  if (htud->de_state != NULL) {
368  }
369  HTPFree(htud, sizeof(HtpTxUserData));
370  }
371 }
372 
373 /** \brief Function to frees the HTTP state memory and also frees the HTTP
374  * connection parser memory which was used by the HTP library
375  */
376 void HTPStateFree(void *state)
377 {
378  SCEnter();
379 
380  HtpState *s = (HtpState *)state;
381  if (s == NULL) {
382  SCReturn;
383  }
384 
385  /* free the connection parser memory used by HTP library */
386  if (s->connp != NULL) {
387  SCLogDebug("freeing HTP state");
388 
389  uint64_t tx_id;
390  uint64_t total_txs = HTPStateGetTxCnt(state);
391  /* free the list of body chunks */
392  if (s->conn != NULL) {
393  for (tx_id = 0; tx_id < total_txs; tx_id++) {
394  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
395  if (tx != NULL) {
396  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
397  HtpTxUserDataFree(s, htud);
398  htp_tx_set_user_data(tx, NULL);
399  }
400  }
401  }
402  htp_connp_destroy_all(s->connp);
403  }
404 
407  HTPFree(s, sizeof(HtpState));
408 
409 #ifdef DEBUG
410  SCMutexLock(&htp_state_mem_lock);
411  htp_state_memcnt--;
412  htp_state_memuse -= sizeof(HtpState);
413  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
414  SCMutexUnlock(&htp_state_mem_lock);
415 #endif
416 
417  SCReturn;
418 }
419 
420 /**
421  * \brief HTP transaction cleanup callback
422  *
423  * \warning We cannot actually free the transactions here. It seems that
424  * HTP only accepts freeing of transactions in the response callback.
425  */
426 static void HTPStateTransactionFree(void *state, uint64_t id)
427 {
428  SCEnter();
429 
430  HtpState *s = (HtpState *)state;
431 
432  SCLogDebug("state %p, id %"PRIu64, s, id);
433 
434  htp_tx_t *tx = HTPStateGetTx(s, id);
435  if (tx != NULL) {
436  /* This will remove obsolete body chunks */
437  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
438  HtpTxUserDataFree(s, htud);
439  htp_tx_set_user_data(tx, NULL);
440 
441  /* hack: even if libhtp considers the tx incomplete, we want to
442  * free it here. htp_tx_destroy however, will refuse to do this.
443  * As htp_tx_destroy_incomplete isn't available in the public API,
444  * we hack around it here. */
445  if (unlikely(!(
446  tx->request_progress == HTP_REQUEST_COMPLETE &&
447  tx->response_progress == HTP_RESPONSE_COMPLETE)))
448  {
449  tx->request_progress = HTP_REQUEST_COMPLETE;
450  tx->response_progress = HTP_RESPONSE_COMPLETE;
451  }
452  htp_tx_destroy(tx);
453  }
454 }
455 
456 /**
457  * \brief Sets a flag that informs the HTP app layer that some module in the
458  * engine needs the http request body data.
459  * \initonly
460  */
462 {
463  SCEnter();
464 
465  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
466  SCReturn;
467 }
468 
469 /**
470  * \brief Sets a flag that informs the HTP app layer that some module in the
471  * engine needs the http request body data.
472  * \initonly
473  */
475 {
476  SCEnter();
477 
478  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
479  SCReturn;
480 }
481 
482 /**
483  * \brief Sets a flag that informs the HTP app layer that some module in the
484  * engine needs the http request multi part header.
485  *
486  * \initonly
487  */
488 static void AppLayerHtpNeedMultipartHeader(void)
489 {
490  SCEnter();
492 
493  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
494  SCReturn;
495 }
496 
497 /**
498  * \brief Sets a flag that informs the HTP app layer that some module in the
499  * engine needs the http request file.
500  *
501  * \initonly
502  */
504 {
505  SCEnter();
506  AppLayerHtpNeedMultipartHeader();
509 
510  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
511  SCReturn;
512 }
513 
514 static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
515 {
516  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
517  if (tx_ud) {
518  if (flags & STREAM_TOCLIENT) {
519  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
520  } else {
521  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
522  }
523  }
524 }
525 
526 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
527 {
528  if (flags & HTP_STREAM_DEPTH_SET) {
529  uint32_t stream_depth = FileReassemblyDepth();
530  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
531  return true;
532  }
533  } else {
534  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
535  return true;
536  }
537  }
538  return false;
539 }
540 
541 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
542  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
543 {
544  uint32_t chunk_len = 0;
545  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
546  (content_len_so_far < (uint64_t)body_limit) &&
547  (content_len_so_far + (uint64_t)data_len) > body_limit)
548  {
549  chunk_len = body_limit - content_len_so_far;
550  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
551  (content_len_so_far < (uint64_t)stream_depth) &&
552  (content_len_so_far + (uint64_t)data_len) > stream_depth)
553  {
554  chunk_len = stream_depth - content_len_so_far;
555  }
556  SCLogDebug("len %u", chunk_len);
557  return (chunk_len == 0 ? data_len : chunk_len);
558 }
559 
560 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
561 struct {
562  const char *msg;
563  int de;
564 } htp_errors[] = {
565  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
566  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
567  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
568  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
569  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
570 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
571  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
572 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
573  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
574  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
575  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
576  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
577 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
578  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
579  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
580  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
581  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
582 };
583 
584 struct {
585  const char *msg;
586  int de;
587 } htp_warnings[] = {
588  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
589  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
590  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
591  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
592  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
593 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
594  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
595  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
596  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
597  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
598  * luckily, "Request server port=" is unique */
599 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
601  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
602  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
603  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
604  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
605  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
606  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
607  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
608  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
609  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
610  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
611  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
612  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
613  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
614  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
615  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
616  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
617  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
618  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
619 };
620 
621 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
622 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
623 
624 /**
625  * \internal
626  *
627  * \brief Get the warning id for the warning msg.
628  *
629  * \param msg warning message
630  *
631  * \retval id the id or 0 in case of not found
632  */
633 static int HTPHandleWarningGetId(const char *msg)
634 {
635  SCLogDebug("received warning \"%s\"", msg);
636  size_t idx;
637  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
638  if (strncmp(htp_warnings[idx].msg, msg,
639  strlen(htp_warnings[idx].msg)) == 0)
640  {
641  return htp_warnings[idx].de;
642  }
643  }
644 
645  return 0;
646 }
647 
648 /**
649  * \internal
650  *
651  * \brief Get the error id for the error msg.
652  *
653  * \param msg error message
654  *
655  * \retval id the id or 0 in case of not found
656  */
657 static int HTPHandleErrorGetId(const char *msg)
658 {
659  SCLogDebug("received error \"%s\"", msg);
660 
661  size_t idx;
662  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
663  if (strncmp(htp_errors[idx].msg, msg,
664  strlen(htp_errors[idx].msg)) == 0)
665  {
666  return htp_errors[idx].de;
667  }
668  }
669 
670  return 0;
671 }
672 
673 /**
674  * \internal
675  *
676  * \brief Check state for errors, warnings and add any as events
677  *
678  * \param s state
679  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
680  */
681 static void HTPHandleError(HtpState *s, const uint8_t dir)
682 {
683  if (s == NULL || s->conn == NULL ||
684  s->conn->messages == NULL) {
685  return;
686  }
687 
688  size_t size = htp_list_size(s->conn->messages);
689  size_t msg;
690 
691  for (msg = s->htp_messages_offset; msg < size; msg++) {
692  htp_log_t *log = htp_list_get(s->conn->messages, msg);
693  if (log == NULL)
694  continue;
695 
696  HtpTxUserData *htud = NULL;
697  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
698  if (tx != NULL)
699  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
700 
701  SCLogDebug("message %s", log->msg);
702 
703  int id = HTPHandleErrorGetId(log->msg);
704  if (id == 0) {
705  id = HTPHandleWarningGetId(log->msg);
706  if (id == 0)
708  }
709 
710  if (id > 0) {
711  HTPSetEvent(s, htud, dir, id);
712  }
713  }
714  s->htp_messages_offset = (uint16_t)msg;
715  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
716 }
717 
718 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
719 {
720 #ifdef DEBUG
721  BUG_ON(s == NULL || tx == NULL);
722 #endif
723  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
724  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
725  HTP_HOSTH_INVALID))
726  {
727  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
728  if (htud == NULL)
729  return;
730 
731  if (tx->flags & HTP_REQUEST_INVALID_T_E)
732  HTPSetEvent(s, htud, STREAM_TOSERVER,
734  if (tx->flags & HTP_REQUEST_INVALID_C_L)
735  HTPSetEvent(s, htud, STREAM_TOSERVER,
737  if (tx->flags & HTP_HOST_MISSING)
738  HTPSetEvent(s, htud, STREAM_TOSERVER,
740  if (tx->flags & HTP_HOST_AMBIGUOUS)
741  HTPSetEvent(s, htud, STREAM_TOSERVER,
743  if (tx->flags & HTP_HOSTU_INVALID)
744  HTPSetEvent(s, htud, STREAM_TOSERVER,
746  if (tx->flags & HTP_HOSTH_INVALID)
747  HTPSetEvent(s, htud, STREAM_TOSERVER,
749  }
750  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
751  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
752  if (htud == NULL)
753  return;
754  HTPSetEvent(s, htud, STREAM_TOSERVER,
756  }
757  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
758  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
759  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
760  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
761  if (htud == NULL)
762  return;
763  HTPSetEvent(s, htud, STREAM_TOSERVER,
765  }
766 }
767 
768 static int Setup(Flow *f, HtpState *hstate)
769 {
770  /* store flow ref in state so callbacks can access it */
771  hstate->f = f;
772 
773  HTPCfgRec *htp_cfg_rec = &cfglist;
774  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
775  void *user_data = NULL;
776 
777  if (FLOW_IS_IPV4(f)) {
778  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
779  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
780  }
781  else if (FLOW_IS_IPV6(f)) {
782  SCLogDebug("Looking up HTP config for ipv6");
783  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
784  }
785  else {
786  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
787  goto error;
788  }
789 
790  if (user_data != NULL) {
791  htp_cfg_rec = user_data;
792  htp = htp_cfg_rec->cfg;
793  SCLogDebug("LIBHTP using config: %p", htp);
794  } else {
795  SCLogDebug("Using default HTP config: %p", htp);
796  }
797 
798  if (NULL == htp) {
799 #ifdef DEBUG_VALIDATION
800  BUG_ON(htp == NULL);
801 #endif
802  /* should never happen if HTPConfigure is properly invoked */
803  goto error;
804  }
805 
806  hstate->connp = htp_connp_create(htp);
807  if (hstate->connp == NULL) {
808  goto error;
809  }
810 
811  hstate->conn = htp_connp_get_connection(hstate->connp);
812 
813  htp_connp_set_user_data(hstate->connp, (void *)hstate);
814  hstate->cfg = htp_cfg_rec;
815 
816  SCLogDebug("New hstate->connp %p", hstate->connp);
817 
818  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
819 
821  htp_cfg_rec->request.inspect_min_size);
823  htp_cfg_rec->response.inspect_min_size);
824  return 0;
825 error:
826  return -1;
827 }
828 
829 /**
830  * \brief Function to handle the reassembled data from client and feed it to
831  * the HTP library to process it.
832  *
833  * \param flow Pointer to the flow the data belong to
834  * \param htp_state Pointer the state in which the parsed value to be stored
835  * \param pstate Application layer parser state for this session
836  * \param input Pointer the received HTTP client data
837  * \param input_len Length in bytes of the received data
838  * \param output Pointer to the output (not used in this function)
839  *
840  * \retval On success returns 1 or on failure returns -1.
841  */
842 static int HTPHandleRequestData(Flow *f, void *htp_state,
843  AppLayerParserState *pstate,
844  const uint8_t *input, uint32_t input_len,
845  void *local_data, const uint8_t flags)
846 {
847  SCEnter();
848  int ret = 1;
849  HtpState *hstate = (HtpState *)htp_state;
850 
851  /* On the first invocation, create the connection parser structure to
852  * be used by HTP library. This is looked up via IP in the radix
853  * tree. Failing that, the default HTP config is used.
854  */
855  if (NULL == hstate->conn) {
856  if (Setup(f, hstate) != 0) {
857  goto error;
858  }
859  }
860  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
861 
862  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
863  /* pass the new data to the htp parser */
864  if (input_len > 0) {
865  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
866  switch (r) {
867  case HTP_STREAM_ERROR:
868  ret = -1;
869  break;
870  default:
871  break;
872  }
873  HTPHandleError(hstate, STREAM_TOSERVER);
874  }
875 
876  /* if the TCP connection is closed, then close the HTTP connection */
878  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
879  {
880  htp_connp_req_close(hstate->connp, &ts);
881  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
882  SCLogDebug("stream eof encountered, closing htp handle for ts");
883  }
884 
885  SCLogDebug("hstate->connp %p", hstate->connp);
886  SCReturnInt(ret);
887 
888 error:
889  SCReturnInt(-1);
890 }
891 
892 /**
893  * \brief Function to handle the reassembled data from server and feed it to
894  * the HTP library to process it.
895  *
896  * \param flow Pointer to the flow the data belong to
897  * \param htp_state Pointer the state in which the parsed value to be stored
898  * \param pstate Application layer parser state for this session
899  * \param input Pointer the received HTTP server data
900  * \param input_len Length in bytes of the received data
901  * \param output Pointer to the output (not used in this function)
902  *
903  * \retval On success returns 1 or on failure returns -1
904  */
905 static int HTPHandleResponseData(Flow *f, void *htp_state,
906  AppLayerParserState *pstate,
907  const uint8_t *input, uint32_t input_len,
908  void *local_data, const uint8_t flags)
909 {
910  SCEnter();
911  int ret = 1;
912  HtpState *hstate = (HtpState *)htp_state;
913 
914  /* On the first invocation, create the connection parser structure to
915  * be used by HTP library. This is looked up via IP in the radix
916  * tree. Failing that, the default HTP config is used.
917  */
918  if (NULL == hstate->conn) {
919  if (Setup(f, hstate) != 0) {
920  goto error;
921  }
922  }
923  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
924 
925  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
926  if (input_len > 0) {
927  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
928  switch (r) {
929  case HTP_STREAM_ERROR:
930  ret = -1;
931  break;
932  default:
933  break;
934  }
935  HTPHandleError(hstate, STREAM_TOCLIENT);
936  }
937 
938  /* if we the TCP connection is closed, then close the HTTP connection */
940  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
941  {
942  htp_connp_close(hstate->connp, &ts);
943  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
944  }
945 
946  SCLogDebug("hstate->connp %p", hstate->connp);
947  SCReturnInt(ret);
948 error:
949  SCReturnInt(-1);
950 }
951 
952 /**
953  * \param name /Lowercase/ version of the variable name
954  */
955 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
956  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
957 {
958 #ifdef PRINT
959  printf("DATA START: \n");
960  PrintRawDataFp(stdout, data, len);
961  printf("DATA END: \n");
962 #endif
963  size_t x;
964  int quote = 0;
965 
966  for (x = 0; x < len; x++) {
967  if (!(isspace(data[x])))
968  break;
969  }
970 
971  if (x >= len)
972  return 0;
973 
974  uint8_t *line = data+x;
975  size_t line_len = len-x;
976  size_t offset = 0;
977 #ifdef PRINT
978  printf("LINE START: \n");
979  PrintRawDataFp(stdout, line, line_len);
980  printf("LINE END: \n");
981 #endif
982  for (x = 0 ; x < line_len; x++) {
983  if (x > 0) {
984  if (line[x - 1] != '\\' && line[x] == '\"') {
985  quote++;
986  }
987 
988  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
989  uint8_t *token = line + offset;
990  size_t token_len = x - offset;
991 
992  if ((x + 1) == line_len) {
993  token_len++;
994  }
995 
996  offset = x + 1;
997 
998  while (offset < line_len && isspace(line[offset])) {
999  x++;
1000  offset++;
1001  }
1002 #ifdef PRINT
1003  printf("TOKEN START: \n");
1004  PrintRawDataFp(stdout, token, token_len);
1005  printf("TOKEN END: \n");
1006 #endif
1007  if (token_len > name_len) {
1008  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1009  uint8_t *value = token + name_len;
1010  size_t value_len = token_len - name_len;
1011 
1012  if (value[0] == '\"') {
1013  value++;
1014  value_len--;
1015  }
1016  if (value[value_len-1] == '\"') {
1017  value_len--;
1018  }
1019 #ifdef PRINT
1020  printf("VALUE START: \n");
1021  PrintRawDataFp(stdout, value, value_len);
1022  printf("VALUE END: \n");
1023 #endif
1024  *retptr = value;
1025  *retlen = value_len;
1026  return 1;
1027  }
1028  }
1029  }
1030  }
1031  }
1032 
1033  return 0;
1034 }
1035 
1036 /**
1037  * \param name /Lowercase/ version of the variable name
1038  */
1039 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1040  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1041 {
1042  SCEnter();
1043 #ifdef PRINT
1044  printf("DATA START: \n");
1045  PrintRawDataFp(stdout, data, len);
1046  printf("DATA END: \n");
1047 #endif
1048  size_t x;
1049  int quote = 0;
1050 
1051  for (x = 0; x < len; x++) {
1052  if (!(isspace(data[x])))
1053  break;
1054  }
1055 
1056  if (x >= len) {
1057  SCReturnInt(0);
1058  }
1059 
1060  uint8_t *line = data+x;
1061  size_t line_len = len-x;
1062  size_t offset = 0;
1063 #ifdef PRINT
1064  printf("LINE START: \n");
1065  PrintRawDataFp(stdout, line, line_len);
1066  printf("LINE END: \n");
1067 #endif
1068  for (x = 0 ; x < line_len; x++) {
1069  if (x > 0) {
1070  if (line[x - 1] != '\\' && line[x] == '\"') {
1071  quote++;
1072  }
1073 
1074  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1075  uint8_t *token = line + offset;
1076  size_t token_len = x - offset;
1077 
1078  if ((x + 1) == line_len) {
1079  token_len++;
1080  }
1081 
1082  offset = x + 1;
1083 
1084  while (offset < line_len && isspace(line[offset])) {
1085  x++;
1086  offset++;
1087  }
1088 #ifdef PRINT
1089  printf("TOKEN START: \n");
1090  PrintRawDataFp(stdout, token, token_len);
1091  printf("TOKEN END: \n");
1092 #endif
1093  if (token_len > name_len) {
1094  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1095  uint8_t *value = token + name_len;
1096  size_t value_len = token_len - name_len;
1097 
1098  if (value[0] == '\"') {
1099  value++;
1100  value_len--;
1101  }
1102  if (value[value_len-1] == '\"') {
1103  value_len--;
1104  }
1105 #ifdef PRINT
1106  printf("VALUE START: \n");
1107  PrintRawDataFp(stdout, value, value_len);
1108  printf("VALUE END: \n");
1109 #endif
1110  *retptr = value;
1111  *retlen = value_len;
1112  SCReturnInt(1);
1113  }
1114  }
1115  }
1116  }
1117  }
1118 
1119  SCReturnInt(0);
1120 }
1121 
1122 /**
1123  * \brief setup multipart parsing: extract boundary and store it
1124  *
1125  * \param d HTTP transaction
1126  * \param htud transaction userdata
1127  *
1128  * \retval 1 ok, multipart set up
1129  * \retval 0 ok, not multipart though
1130  * \retval -1 error: problem with the boundary
1131  *
1132  * If the request contains a multipart message, this function will
1133  * set the HTP_BOUNDARY_SET in the transaction.
1134  */
1135 static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud)
1136 {
1137  htp_header_t *h = (htp_header_t *)htp_table_get_c(d->tx->request_headers,
1138  "Content-Type");
1139  if (h != NULL && bstr_len(h->value) > 0) {
1140  uint8_t *boundary = NULL;
1141  size_t boundary_len = 0;
1142 
1143  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1144  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1145  &boundary, &boundary_len);
1146  if (r == 1) {
1147 #ifdef PRINT
1148  printf("BOUNDARY START: \n");
1149  PrintRawDataFp(stdout, boundary, boundary_len);
1150  printf("BOUNDARY END: \n");
1151 #endif
1152  if (boundary_len < HTP_BOUNDARY_MAX) {
1153  htud->boundary = HTPMalloc(boundary_len);
1154  if (htud->boundary == NULL) {
1155  return -1;
1156  }
1157  htud->boundary_len = (uint8_t)boundary_len;
1158  memcpy(htud->boundary, boundary, boundary_len);
1159 
1160  htud->tsflags |= HTP_BOUNDARY_SET;
1161  } else {
1162  SCLogDebug("invalid boundary");
1163  return -1;
1164  }
1165  SCReturnInt(1);
1166  }
1167  //SCReturnInt(1);
1168  }
1169  SCReturnInt(0);
1170 }
1171 
1172 #define C_D_HDR "content-disposition:"
1173 #define C_D_HDR_LEN 20
1174 #define C_T_HDR "content-type:"
1175 #define C_T_HDR_LEN 13
1176 
1177 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1178  HtpTxUserData *htud,
1179  uint8_t *header, uint32_t header_len,
1180  uint8_t **filename, uint16_t *filename_len,
1181  uint8_t **filetype, uint16_t *filetype_len)
1182 {
1183  uint8_t *fn = NULL;
1184  size_t fn_len = 0;
1185  uint8_t *ft = NULL;
1186  size_t ft_len = 0;
1187 
1188 #ifdef PRINT
1189  printf("HEADER START: \n");
1190  PrintRawDataFp(stdout, header, header_len);
1191  printf("HEADER END: \n");
1192 #endif
1193 
1194  while (header_len > 0) {
1195  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1196  uint8_t *line = header;
1197  uint32_t line_len;
1198 
1199  if (next_line == NULL) {
1200  line_len = header_len;
1201  } else {
1202  line_len = next_line - header;
1203  }
1204  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1205  if (sc == NULL) {
1206  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1208  /* if the : we found is the final char, it means we have
1209  * no value */
1210  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1211  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1213  } else {
1214 #ifdef PRINT
1215  printf("LINE START: \n");
1216  PrintRawDataFp(stdout, line, line_len);
1217  printf("LINE END: \n");
1218 #endif
1219  if (line_len >= C_D_HDR_LEN &&
1220  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1221  uint8_t *value = line + C_D_HDR_LEN;
1222  uint32_t value_len = line_len - C_D_HDR_LEN;
1223 
1224  /* parse content-disposition */
1225  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1226  value, value_len, &fn, &fn_len);
1227  } else if (line_len >= C_T_HDR_LEN &&
1228  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1229  SCLogDebug("content-type line");
1230  uint8_t *value = line + C_T_HDR_LEN;
1231  uint32_t value_len = line_len - C_T_HDR_LEN;
1232 
1233  (void)HTTPParseContentTypeHeader(NULL, 0,
1234  value, value_len, &ft, &ft_len);
1235  }
1236  }
1237 
1238  if (next_line == NULL) {
1239  SCLogDebug("no next_line");
1240  break;
1241  }
1242  header_len -= ((next_line + 2) - header);
1243  header = next_line + 2;
1244  } /* while (header_len > 0) */
1245 
1246  if (fn_len > USHRT_MAX)
1247  fn_len = USHRT_MAX;
1248  if (ft_len > USHRT_MAX)
1249  ft_len = USHRT_MAX;
1250 
1251  *filename = fn;
1252  *filename_len = fn_len;
1253  *filetype = ft;
1254  *filetype_len = ft_len;
1255 }
1256 
1257 /**
1258  * \brief Create a single buffer from the HtpBodyChunks in our list
1259  *
1260  * \param htud transaction user data
1261  * \param chunks_buffers pointer to pass back the buffer to the caller
1262  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1263  */
1264 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1265  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1266 {
1268  chunks_buffer, chunks_buffer_len,
1269  htud->request_body.body_parsed);
1270 }
1271 
1272 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1273 {
1274  SCEnter();
1275  if (tx && tx->de_state) {
1276  if (dir == STREAM_TOSERVER) {
1277  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1279  } else if (STREAM_TOCLIENT) {
1280  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1282  }
1283  }
1284 }
1285 
1286 /**
1287  * \brief Setup boundary buffers
1288  */
1289 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1290  uint8_t *boundary, uint32_t boundary_len)
1291 {
1292  memset(boundary, '-', boundary_len);
1293  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1294 }
1295 
1296 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1297  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1298 {
1299  int result = 0;
1300  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1301  uint32_t expected_boundary_len = htud->boundary_len + 2;
1302  uint32_t expected_boundary_end_len = htud->boundary_len + 4;
1303  int tx_progress = 0;
1304 
1305 #ifdef PRINT
1306  printf("CHUNK START: \n");
1307  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1308  printf("CHUNK END: \n");
1309 #endif
1310 
1311  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1312 
1313  /* search for the header start, header end and form end */
1314  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1315  boundary, expected_boundary_len);
1316  /* end of the multipart form */
1317  const uint8_t *form_end = NULL;
1318  /* end marker belonging to header_start */
1319  const uint8_t *header_end = NULL;
1320  if (header_start != NULL) {
1321  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1322  (uint8_t *)"\r\n\r\n", 4);
1323  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1324  boundary, expected_boundary_end_len);
1325  }
1326 
1327  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1328  header_end, form_end);
1329 
1330  /* we currently only handle multipart for ts. When we support it for tc,
1331  * we will need to supply right direction */
1332  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, STREAM_TOSERVER);
1333  /* if we're in the file storage process, deal with that now */
1334  if (htud->tsflags & HTP_FILENAME_SET) {
1335  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1336  SCLogDebug("reached the end of the file");
1337 
1338  const uint8_t *filedata = chunks_buffer;
1339  uint32_t filedata_len = 0;
1340  uint8_t flags = 0;
1341 
1342  if (header_start != NULL) {
1343  if (header_start == filedata + 2) {
1344  /* last chunk had all data, but not the boundary */
1345  SCLogDebug("last chunk had all data, but not the boundary");
1346  filedata_len = 0;
1347  } else if (header_start > filedata + 2) {
1348  SCLogDebug("some data from last file before the boundary");
1349  /* some data from last file before the boundary */
1350  filedata_len = header_start - filedata - 2;
1351  }
1352  }
1353  /* body parsing done, we did not get our form end. Use all data
1354  * we still have and signal to files API we have an issue. */
1355  if (tx_progress > HTP_REQUEST_BODY) {
1356  filedata_len = chunks_buffer_len;
1357  flags = FILE_TRUNCATED;
1358  }
1359 
1360  if (filedata_len > chunks_buffer_len) {
1361  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1363  goto end;
1364  }
1365 #ifdef PRINT
1366  printf("FILEDATA (final chunk) START: \n");
1367  PrintRawDataFp(stdout, filedata, filedata_len);
1368  printf("FILEDATA (final chunk) END: \n");
1369 #endif
1370  if (!(htud->tsflags & HTP_DONTSTORE)) {
1371  if (HTPFileClose(hstate, filedata, filedata_len, flags,
1372  STREAM_TOSERVER) == -1)
1373  {
1374  goto end;
1375  }
1376  }
1377 
1378  htud->tsflags &=~ HTP_FILENAME_SET;
1379 
1380  /* fall through */
1381  } else {
1382  SCLogDebug("not yet at the end of the file");
1383 
1384  if (chunks_buffer_len > expected_boundary_end_len) {
1385  const uint8_t *filedata = chunks_buffer;
1386  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1387 #ifdef PRINT
1388  printf("FILEDATA (part) START: \n");
1389  PrintRawDataFp(stdout, filedata, filedata_len);
1390  printf("FILEDATA (part) END: \n");
1391 #endif
1392 
1393  if (!(htud->tsflags & HTP_DONTSTORE)) {
1394  result = HTPFileStoreChunk(hstate, filedata,
1395  filedata_len, STREAM_TOSERVER);
1396  if (result == -1) {
1397  goto end;
1398  } else if (result == -2) {
1399  /* we know for sure we're not storing the file */
1400  htud->tsflags |= HTP_DONTSTORE;
1401  }
1402  }
1403 
1404  htud->request_body.body_parsed += filedata_len;
1405  } else {
1406  SCLogDebug("chunk too small to already process in part");
1407  }
1408 
1409  goto end;
1410  }
1411  }
1412 
1413  while (header_start != NULL && header_end != NULL &&
1414  header_end != form_end &&
1415  header_start < (chunks_buffer + chunks_buffer_len) &&
1416  header_end < (chunks_buffer + chunks_buffer_len) &&
1417  header_start < header_end)
1418  {
1419  uint8_t *filename = NULL;
1420  uint16_t filename_len = 0;
1421  uint8_t *filetype = NULL;
1422  uint16_t filetype_len = 0;
1423 
1424  uint32_t header_len = header_end - header_start;
1425  SCLogDebug("header_len %u", header_len);
1426  uint8_t *header = (uint8_t *)header_start;
1427 
1428  /* skip empty records */
1429  if (expected_boundary_len == header_len) {
1430  goto next;
1431  } else if ((expected_boundary_len + 2) <= header_len) {
1432  header_len -= (expected_boundary_len + 2);
1433  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1434  }
1435 
1436  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1437  &filename, &filename_len, &filetype, &filetype_len);
1438 
1439  if (filename != NULL) {
1440  const uint8_t *filedata = NULL;
1441  uint32_t filedata_len = 0;
1442 
1443  SCLogDebug("we have a filename");
1444 
1445  htud->tsflags |= HTP_FILENAME_SET;
1446  htud->tsflags &= ~HTP_DONTSTORE;
1447 
1448  SCLogDebug("header_end %p", header_end);
1449  SCLogDebug("form_end %p", form_end);
1450 
1451  /* everything until the final boundary is the file */
1452  if (form_end != NULL) {
1453  SCLogDebug("have form_end");
1454 
1455  filedata = header_end + 4;
1456  if (form_end == filedata) {
1457  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1459  goto end;
1460  } else if (form_end < filedata) {
1461  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1463  goto end;
1464  }
1465 
1466  filedata_len = form_end - (header_end + 4 + 2);
1467  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1468 
1469  /* or is it? */
1470  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1471  boundary, expected_boundary_len);
1472  if (header_next != NULL) {
1473  filedata_len -= (form_end - header_next);
1474  }
1475 
1476  if (filedata_len > chunks_buffer_len) {
1477  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1479  goto end;
1480  }
1481  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1482 #ifdef PRINT
1483  printf("FILEDATA START: \n");
1484  PrintRawDataFp(stdout, filedata, filedata_len);
1485  printf("FILEDATA END: \n");
1486 #endif
1487 
1488  result = HTPFileOpen(hstate, filename, filename_len,
1489  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1490  STREAM_TOSERVER);
1491  if (result == -1) {
1492  goto end;
1493  } else if (result == -2) {
1494  htud->tsflags |= HTP_DONTSTORE;
1495  } else {
1496  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1497  goto end;
1498  }
1499  }
1500  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1501 
1502  htud->request_body.body_parsed += (header_end - chunks_buffer);
1503  htud->tsflags &= ~HTP_FILENAME_SET;
1504  } else {
1505  SCLogDebug("chunk doesn't contain form end");
1506 
1507  filedata = header_end + 4;
1508  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1509  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1510 
1511  if (filedata_len > chunks_buffer_len) {
1512  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1514  goto end;
1515  }
1516 
1517 #ifdef PRINT
1518  printf("FILEDATA START: \n");
1519  PrintRawDataFp(stdout, filedata, filedata_len);
1520  printf("FILEDATA END: \n");
1521 #endif
1522  /* form doesn't end in this chunk, but the part might. Lets
1523  * see if have another coming up */
1524  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1525  boundary, expected_boundary_len);
1526  SCLogDebug("header_next %p", header_next);
1527  if (header_next == NULL) {
1528  SCLogDebug("more file data to come");
1529 
1530  uint32_t offset = (header_end + 4) - chunks_buffer;
1531  SCLogDebug("offset %u", offset);
1532  htud->request_body.body_parsed += offset;
1533 
1534  if (filedata_len >= (expected_boundary_len + 2)) {
1535  filedata_len -= (expected_boundary_len + 2 - 1);
1536  SCLogDebug("opening file with partial data");
1537  } else {
1538  filedata = NULL;
1539  filedata_len = 0;
1540  }
1541  result = HTPFileOpen(hstate, filename, filename_len,
1542  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1543  STREAM_TOSERVER);
1544  if (result == -1) {
1545  goto end;
1546  } else if (result == -2) {
1547  htud->tsflags |= HTP_DONTSTORE;
1548  }
1549  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1550  htud->request_body.body_parsed += filedata_len;
1551  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1552 
1553  } else if (header_next - filedata > 2) {
1554  filedata_len = header_next - filedata - 2;
1555  SCLogDebug("filedata_len %u", filedata_len);
1556 
1557  result = HTPFileOpen(hstate, filename, filename_len,
1558  filedata, filedata_len, HtpGetActiveRequestTxID(hstate),
1559  STREAM_TOSERVER);
1560  if (result == -1) {
1561  goto end;
1562  } else if (result == -2) {
1563  htud->tsflags |= HTP_DONTSTORE;
1564  } else {
1565  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1566  goto end;
1567  }
1568  }
1569  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1570 
1571  htud->tsflags &= ~HTP_FILENAME_SET;
1572  htud->request_body.body_parsed += (header_end - chunks_buffer);
1573  }
1574  }
1575  }
1576 next:
1577  SCLogDebug("header_start %p, header_end %p, form_end %p",
1578  header_start, header_end, form_end);
1579 
1580  /* Search next boundary entry after the start of body */
1581  uint32_t cursizeread = header_end - chunks_buffer;
1582  header_start = Bs2bmSearch(header_end + 4,
1583  chunks_buffer_len - (cursizeread + 4),
1584  boundary, expected_boundary_len);
1585  if (header_start != NULL) {
1586  header_end = Bs2bmSearch(header_end + 4,
1587  chunks_buffer_len - (cursizeread + 4),
1588  (uint8_t *) "\r\n\r\n", 4);
1589  }
1590  }
1591 
1592  /* if we're parsing the multipart and we're not currently processing a
1593  * file, we move the body pointer forward. */
1594  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1595  if (chunks_buffer_len > expected_boundary_end_len) {
1596  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1597 
1598  htud->request_body.body_parsed += move;
1599  SCLogDebug("form not ready, file not set, parsing non-file "
1600  "record: moved %u", move);
1601  }
1602  }
1603 
1604 end:
1605  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1606  return 0;
1607 }
1608 
1609 /** \brief setup things for put request
1610  * \todo really needed? */
1611 static int HtpRequestBodySetupPUT(htp_tx_data_t *d, HtpTxUserData *htud)
1612 {
1613 // if (d->tx->parsed_uri == NULL || d->tx->parsed_uri->path == NULL) {
1614 // return -1;
1615 // }
1616 
1617  /* filename is d->tx->parsed_uri->path */
1618 
1619  return 0;
1620 }
1621 
1622 /** \internal
1623  * \brief Handle POST, no multipart body data
1624  */
1625 static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
1626  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1627 {
1628  int result = 0;
1629 
1630  /* see if we need to open the file */
1631  if (!(htud->tsflags & HTP_FILENAME_SET))
1632  {
1633  uint8_t *filename = NULL;
1634  size_t filename_len = 0;
1635 
1636  /* get the name */
1637  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1638  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1639  filename_len = bstr_len(tx->parsed_uri->path);
1640  }
1641 
1642  if (filename != NULL) {
1643  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1644  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1645  if (result == -1) {
1646  goto end;
1647  } else if (result == -2) {
1648  htud->tsflags |= HTP_DONTSTORE;
1649  } else {
1650  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1651  htud->tsflags |= HTP_FILENAME_SET;
1652  htud->tsflags &= ~HTP_DONTSTORE;
1653  }
1654  }
1655  }
1656  else
1657  {
1658  /* otherwise, just store the data */
1659 
1660  if (!(htud->tsflags & HTP_DONTSTORE)) {
1661  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1662  if (result == -1) {
1663  goto end;
1664  } else if (result == -2) {
1665  /* we know for sure we're not storing the file */
1666  htud->tsflags |= HTP_DONTSTORE;
1667  }
1668  }
1669  }
1670 
1671  return 0;
1672 end:
1673  return -1;
1674 }
1675 
1676 /** \internal
1677  * \brief Handle PUT body data
1678  */
1679 static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
1680  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1681 {
1682  int result = 0;
1683 
1684  /* see if we need to open the file */
1685  if (!(htud->tsflags & HTP_FILENAME_SET))
1686  {
1687  uint8_t *filename = NULL;
1688  size_t filename_len = 0;
1689 
1690  /* get the name */
1691  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1692  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1693  filename_len = bstr_len(tx->parsed_uri->path);
1694  }
1695 
1696  if (filename != NULL) {
1697  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1698  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1699  if (result == -1) {
1700  goto end;
1701  } else if (result == -2) {
1702  htud->tsflags |= HTP_DONTSTORE;
1703  } else {
1704  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1705  htud->tsflags |= HTP_FILENAME_SET;
1706  htud->tsflags &= ~HTP_DONTSTORE;
1707  }
1708  }
1709  }
1710  else
1711  {
1712  /* otherwise, just store the data */
1713 
1714  if (!(htud->tsflags & HTP_DONTSTORE)) {
1715  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1716  if (result == -1) {
1717  goto end;
1718  } else if (result == -2) {
1719  /* we know for sure we're not storing the file */
1720  htud->tsflags |= HTP_DONTSTORE;
1721  }
1722  }
1723  }
1724 
1725  return 0;
1726 end:
1727  return -1;
1728 }
1729 
1730 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1731  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1732 {
1733  SCEnter();
1734 
1735  int result = 0;
1736 
1737  /* see if we need to open the file
1738  * we check for tx->response_line in case of junk
1739  * interpreted as body before response line
1740  */
1741  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1742  (tx->response_line != NULL || tx->is_protocol_0_9))
1743  {
1744  SCLogDebug("setting up file name");
1745 
1746  uint8_t *filename = NULL;
1747  size_t filename_len = 0;
1748 
1749  /* try Content-Disposition header first */
1750  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1751  "Content-Disposition");
1752  if (h != NULL && bstr_len(h->value) > 0) {
1753  /* parse content-disposition */
1754  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1755  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1756  }
1757 
1758  /* fall back to name from the uri */
1759  if (filename == NULL) {
1760  /* get the name */
1761  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1762  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1763  filename_len = bstr_len(tx->parsed_uri->path);
1764  }
1765  }
1766 
1767  if (filename != NULL) {
1768  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len,
1769  data, data_len, HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1770  SCLogDebug("result %d", result);
1771  if (result == -1) {
1772  goto end;
1773  } else if (result == -2) {
1774  htud->tcflags |= HTP_DONTSTORE;
1775  } else {
1776  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1777  htud->tcflags |= HTP_FILENAME_SET;
1778  htud->tcflags &= ~HTP_DONTSTORE;
1779  }
1780  //set range if present
1781  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1782  if (h_content_range != NULL) {
1783  HTPFileSetRange(hstate, h_content_range->value);
1784  }
1785  }
1786  }
1787  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1788  {
1789  /* otherwise, just store the data */
1790 
1791  if (!(htud->tcflags & HTP_DONTSTORE)) {
1792  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1793  SCLogDebug("result %d", result);
1794  if (result == -1) {
1795  goto end;
1796  } else if (result == -2) {
1797  /* we know for sure we're not storing the file */
1798  htud->tcflags |= HTP_DONTSTORE;
1799  }
1800  }
1801  }
1802 
1803  htud->response_body.body_parsed += data_len;
1804  return 0;
1805 end:
1806  return -1;
1807 }
1808 
1809 /**
1810  * \brief Function callback to append chunks for Requests
1811  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1812  * \retval int HTP_OK if all goes well
1813  */
1814 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1815 {
1816  SCEnter();
1817 
1818  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1819  SCReturnInt(HTP_OK);
1820 
1821  if (d->data == NULL || d->len == 0)
1822  SCReturnInt(HTP_OK);
1823 
1824 #ifdef PRINT
1825  printf("HTPBODY START: \n");
1826  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1827  printf("HTPBODY END: \n");
1828 #endif
1829 
1830  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1831  if (hstate == NULL) {
1832  SCReturnInt(HTP_ERROR);
1833  }
1834 
1835  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1836  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1837 
1838  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1839  if (tx_ud == NULL) {
1840  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1841  if (unlikely(tx_ud == NULL)) {
1842  SCReturnInt(HTP_OK);
1843  }
1844  memset(tx_ud, 0, sizeof(HtpTxUserData));
1845 
1846  /* Set the user data for handling body chunks on this transaction */
1847  htp_tx_set_user_data(d->tx, tx_ud);
1848  }
1849  if (!tx_ud->response_body_init) {
1850  tx_ud->response_body_init = 1;
1851 
1852  if (d->tx->request_method_number == HTP_M_POST) {
1853  SCLogDebug("POST");
1854  int r = HtpRequestBodySetupMultipart(d, tx_ud);
1855  if (r == 1) {
1857  } else if (r == 0) {
1859  SCLogDebug("not multipart");
1860  }
1861  } else if (d->tx->request_method_number == HTP_M_PUT) {
1862  if (HtpRequestBodySetupPUT(d, tx_ud) == 0) {
1864  }
1865  }
1866  }
1867 
1868  /* see if we can get rid of htp body chunks */
1869  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1870 
1871  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1872  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1873 
1874  /* within limits, add the body chunk to the state. */
1875  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1876  uint32_t stream_depth = FileReassemblyDepth();
1877  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1878  hstate->cfg->request.body_limit,
1879  stream_depth,
1880  tx_ud->tsflags,
1881  (uint32_t)d->len);
1882  BUG_ON(len > (uint32_t)d->len);
1883 
1884  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1885 
1886  const uint8_t *chunks_buffer = NULL;
1887  uint32_t chunks_buffer_len = 0;
1888 
1890  /* multi-part body handling starts here */
1891  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1892  goto end;
1893  }
1894 
1895  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1896  if (chunks_buffer == NULL) {
1897  goto end;
1898  }
1899 #ifdef PRINT
1900  printf("REASSCHUNK START: \n");
1901  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1902  printf("REASSCHUNK END: \n");
1903 #endif
1904 
1905  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1906 
1907  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
1908  HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1909  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
1910  HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1911  }
1912 
1913  } else {
1914  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1915  SCLogDebug("closing file that was being stored");
1916  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1917  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1918  }
1919  }
1920 
1921 end:
1922  if (hstate->conn != NULL) {
1923  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1925  hstate->cfg->request.inspect_min_size,
1926  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1927 
1928  /* if we reach the inspect_min_size we'll trigger inspection,
1929  * so make sure that raw stream is also inspected. Set the
1930  * data to be used to the amount of raw bytes we've seen to
1931  * get here. */
1932  if (tx_ud->request_body.body_inspected == 0 &&
1934  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1935  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1936  {
1937  uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1938 
1939  /* body still in progress, but due to min inspect size we need to inspect now */
1942  }
1943  /* after the start of the body, disable the depth logic */
1944  } else if (tx_ud->request_body.body_inspected > 0) {
1946  }
1947  }
1948  SCReturnInt(HTP_OK);
1949 }
1950 
1951 /**
1952  * \brief Function callback to append chunks for Responses
1953  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1954  * \retval int HTP_OK if all goes well
1955  */
1956 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1957 {
1958  SCEnter();
1959 
1960  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1961  SCReturnInt(HTP_OK);
1962 
1963  if (d->data == NULL || d->len == 0)
1964  SCReturnInt(HTP_OK);
1965 
1966  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1967  if (hstate == NULL) {
1968  SCReturnInt(HTP_ERROR);
1969  }
1970 
1971  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1972  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1973 
1974  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1975  if (tx_ud == NULL) {
1976  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1977  if (unlikely(tx_ud == NULL)) {
1978  SCReturnInt(HTP_OK);
1979  }
1980  memset(tx_ud, 0, sizeof(HtpTxUserData));
1981 
1982  /* Set the user data for handling body chunks on this transaction */
1983  htp_tx_set_user_data(d->tx, tx_ud);
1984  }
1985  if (!tx_ud->request_body_init) {
1986  tx_ud->request_body_init = 1;
1987  }
1988 
1989  /* see if we can get rid of htp body chunks */
1990  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1991 
1992  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1993  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1994 
1995  /* within limits, add the body chunk to the state. */
1996  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1997  uint32_t stream_depth = FileReassemblyDepth();
1998  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1999  hstate->cfg->response.body_limit,
2000  stream_depth,
2001  tx_ud->tcflags,
2002  (uint32_t)d->len);
2003  BUG_ON(len > (uint32_t)d->len);
2004 
2005  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
2006 
2007  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
2008  } else {
2009  if (tx_ud->tcflags & HTP_FILENAME_SET) {
2010  SCLogDebug("closing file that was being stored");
2011  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
2012  tx_ud->tcflags &= ~HTP_FILENAME_SET;
2013  }
2014  }
2015 
2016  if (hstate->conn != NULL) {
2017  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
2019  hstate->cfg->response.inspect_min_size,
2020  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
2021  /* if we reach the inspect_min_size we'll trigger inspection,
2022  * so make sure that raw stream is also inspected. Set the
2023  * data to be used to the amount of raw bytes we've seen to
2024  * get here. */
2025  if (tx_ud->response_body.body_inspected == 0 &&
2027  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2028  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2029  {
2030  uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
2031 
2032  /* body still in progress, but due to min inspect size we need to inspect now */
2035  }
2036  /* after the start of the body, disable the depth logic */
2037  } else if (tx_ud->response_body.body_inspected > 0) {
2039  }
2040  }
2041  SCReturnInt(HTP_OK);
2042 }
2043 
2044 /**
2045  * \brief Print the stats of the HTTP requests
2046  */
2048 {
2049 #ifdef DEBUG
2050  SCEnter();
2051  SCMutexLock(&htp_state_mem_lock);
2052  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2053  htp_state_memcnt, htp_state_memuse);
2054  SCMutexUnlock(&htp_state_mem_lock);
2055  SCReturn;
2056 #endif
2057 }
2058 
2059 /** \brief Clears the HTTP server configuration memory used by HTP library */
2060 void HTPFreeConfig(void)
2061 {
2062  SCEnter();
2063 
2064  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2065  !AppLayerParserConfParserEnabled("tcp", "http"))
2066  {
2067  SCReturn;
2068  }
2069 
2070  HTPCfgRec *nextrec = cfglist.next;
2071  SCRadixReleaseRadixTree(cfgtree);
2072  cfgtree = NULL;
2073  htp_config_destroy(cfglist.cfg);
2074  while (nextrec != NULL) {
2075  HTPCfgRec *htprec = nextrec;
2076  nextrec = nextrec->next;
2077 
2078  htp_config_destroy(htprec->cfg);
2079  SCFree(htprec);
2080  }
2081  HTPDestroyMemcap();
2082  SCReturn;
2083 }
2084 
2085 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2086 {
2087  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2088  if (htud != NULL) {
2089  htud->request_has_trailers = 1;
2090  }
2091  return HTP_OK;
2092 }
2093 
2094 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2095 {
2096  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2097  if (htud != NULL) {
2098  htud->response_has_trailers = 1;
2099  }
2100  return HTP_OK;
2101 }
2102 
2103 /**\internal
2104  * \brief called at start of request
2105  * Set min inspect size.
2106  */
2107 static int HTPCallbackRequestStart(htp_tx_t *tx)
2108 {
2109  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2110  if (hstate == NULL) {
2111  SCReturnInt(HTP_ERROR);
2112  }
2113 
2114  if (hstate->cfg)
2116  hstate->cfg->request.inspect_min_size);
2117  SCReturnInt(HTP_OK);
2118 }
2119 
2120 /**\internal
2121  * \brief called at start of response
2122  * Set min inspect size.
2123  */
2124 static int HTPCallbackResponseStart(htp_tx_t *tx)
2125 {
2126  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2127  if (hstate == NULL) {
2128  SCReturnInt(HTP_ERROR);
2129  }
2130 
2131  if (hstate->cfg)
2133  hstate->cfg->response.inspect_min_size);
2134  SCReturnInt(HTP_OK);
2135 }
2136 
2137 
2138 /**
2139  * \brief callback for request to store the recent incoming request
2140  in to the recent_in_tx for the given htp state
2141  * \param connp pointer to the current connection parser which has the htp
2142  * state in it as user data
2143  */
2144 static int HTPCallbackRequest(htp_tx_t *tx)
2145 {
2146  SCEnter();
2147 
2148  if (tx == NULL) {
2149  SCReturnInt(HTP_ERROR);
2150  }
2151 
2152  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2153  if (hstate == NULL) {
2154  SCReturnInt(HTP_ERROR);
2155  }
2156 
2157  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2158  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2159 
2160  SCLogDebug("HTTP request completed");
2161 
2162  HTPErrorCheckTxRequestFlags(hstate, tx);
2163 
2164  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2165  if (htud != NULL) {
2166  if (htud->tsflags & HTP_FILENAME_SET) {
2167  SCLogDebug("closing file that was being stored");
2168  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
2169  htud->tsflags &= ~HTP_FILENAME_SET;
2171  (uint32_t)hstate->conn->in_data_counter);
2172  }
2173  }
2174 
2175  hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
2176  /* request done, do raw reassembly now to inspect state and stream
2177  * at the same time. */
2179  SCReturnInt(HTP_OK);
2180 }
2181 
2182 /**
2183  * \brief callback for response to remove the recent received requests
2184  from the recent_in_tx for the given htp state
2185  * \param connp pointer to the current connection parser which has the htp
2186  * state in it as user data
2187  */
2188 static int HTPCallbackResponse(htp_tx_t *tx)
2189 {
2190  SCEnter();
2191 
2192  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2193  if (hstate == NULL) {
2194  SCReturnInt(HTP_ERROR);
2195  }
2196 
2197  /* we have one whole transaction now */
2198  hstate->transaction_cnt++;
2199 
2200  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2201  if (htud != NULL) {
2202  if (htud->tcflags & HTP_FILENAME_SET) {
2203  SCLogDebug("closing file that was being stored");
2204  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
2205  htud->tcflags &= ~HTP_FILENAME_SET;
2206  }
2207  }
2208 
2209  /* response done, do raw reassembly now to inspect state and stream
2210  * at the same time. */
2212 
2213  /* handle HTTP CONNECT */
2214  if (tx->request_method_number == HTP_M_CONNECT) {
2215  /* any 2XX status response implies that the connection will become
2216  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2217  if ((tx->response_status_number >= 200) &&
2218  (tx->response_status_number < 300) &&
2219  (hstate->transaction_cnt == 1)) {
2220  uint16_t dp = 0;
2221  if (tx->request_port_number != -1) {
2222  dp = (uint16_t)tx->request_port_number;
2223  }
2224  // both ALPROTO_HTTP and ALPROTO_TLS are normal options
2226  tx->request_progress = HTP_REQUEST_COMPLETE;
2227  tx->response_progress = HTP_RESPONSE_COMPLETE;
2228  }
2229  }
2230 
2231  hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
2232  SCReturnInt(HTP_OK);
2233 }
2234 
2235 static int HTPCallbackRequestLine(htp_tx_t *tx)
2236 {
2237  HtpTxUserData *tx_ud;
2238  bstr *request_uri_normalized;
2239  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2240  const HTPCfgRec *cfg = hstate->cfg;
2241 
2242  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2243  if (request_uri_normalized == NULL)
2244  return HTP_OK;
2245 
2246  tx_ud = htp_tx_get_user_data(tx);
2247  if (likely(tx_ud == NULL)) {
2248  tx_ud = HTPMalloc(sizeof(*tx_ud));
2249  if (unlikely(tx_ud == NULL)) {
2250  bstr_free(request_uri_normalized);
2251  return HTP_OK;
2252  }
2253  memset(tx_ud, 0, sizeof(*tx_ud));
2254  htp_tx_set_user_data(tx, tx_ud);
2255  }
2256  if (unlikely(tx_ud->request_uri_normalized != NULL))
2257  bstr_free(tx_ud->request_uri_normalized);
2258  tx_ud->request_uri_normalized = request_uri_normalized;
2259 
2260  if (tx->flags) {
2261  HTPErrorCheckTxRequestFlags(hstate, tx);
2262  }
2263  return HTP_OK;
2264 }
2265 
2266 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2267 {
2268  if (part == NULL)
2269  return HTP_OK;
2270 
2271  uint64_t flags = 0;
2272  size_t prevlen = bstr_len(part);
2273  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2274  // shorter string means that uri was encoded
2275  if (res == HTP_OK && prevlen > bstr_len(part)) {
2276  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2277  if (likely(htud == NULL)) {
2278  htud = HTPCalloc(1, sizeof(*htud));
2279  if (unlikely(htud == NULL))
2280  return HTP_OK;
2281  htp_tx_set_user_data(tx, htud);
2282  }
2283  HtpState *s = htp_connp_get_user_data(tx->connp);
2284  if (s == NULL)
2285  return HTP_OK;
2286  HTPSetEvent(s, htud, STREAM_TOSERVER,
2288  }
2289 
2290  return HTP_OK;
2291 }
2292 
2293 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2294 {
2295  if (tx->parsed_uri == NULL)
2296  return HTP_OK;
2297 
2298  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2299 }
2300 
2301 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2302 {
2303  if (tx->parsed_uri == NULL)
2304  return HTP_OK;
2305 
2306  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2307 }
2308 
2309 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2310 {
2311  void *ptmp;
2312  if (tx_data->len == 0 || tx_data->tx == NULL)
2313  return HTP_OK;
2314 
2315  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2316  if (tx_ud == NULL) {
2317  tx_ud = HTPMalloc(sizeof(*tx_ud));
2318  if (unlikely(tx_ud == NULL))
2319  return HTP_OK;
2320  memset(tx_ud, 0, sizeof(*tx_ud));
2321  htp_tx_set_user_data(tx_data->tx, tx_ud);
2322  }
2323  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2324  tx_ud->request_headers_raw_len,
2325  tx_ud->request_headers_raw_len + tx_data->len);
2326  if (ptmp == NULL) {
2327  /* error: we're freeing the entire user data */
2328  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2329  HtpTxUserDataFree(hstate, tx_ud);
2330  htp_tx_set_user_data(tx_data->tx, NULL);
2331  return HTP_OK;
2332  }
2333  tx_ud->request_headers_raw = ptmp;
2334 
2335  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2336  tx_data->data, tx_data->len);
2337  tx_ud->request_headers_raw_len += tx_data->len;
2338 
2339  if (tx_data->tx && tx_data->tx->flags) {
2340  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2341  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2342  }
2343  return HTP_OK;
2344 }
2345 
2346 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2347 {
2348  void *ptmp;
2349  if (tx_data->len == 0 || tx_data->tx == NULL)
2350  return HTP_OK;
2351 
2352  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2353  if (tx_ud == NULL) {
2354  tx_ud = HTPMalloc(sizeof(*tx_ud));
2355  if (unlikely(tx_ud == NULL))
2356  return HTP_OK;
2357  memset(tx_ud, 0, sizeof(*tx_ud));
2358  htp_tx_set_user_data(tx_data->tx, tx_ud);
2359  }
2360  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2361  tx_ud->response_headers_raw_len,
2362  tx_ud->response_headers_raw_len + tx_data->len);
2363  if (ptmp == NULL) {
2364  /* error: we're freeing the entire user data */
2365  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2366  HtpTxUserDataFree(hstate, tx_ud);
2367  htp_tx_set_user_data(tx_data->tx, NULL);
2368  return HTP_OK;
2369  }
2370  tx_ud->response_headers_raw = ptmp;
2371 
2372  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2373  tx_data->data, tx_data->len);
2374  tx_ud->response_headers_raw_len += tx_data->len;
2375 
2376  return HTP_OK;
2377 }
2378 
2379 /*
2380  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2381  */
2382 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2383 {
2384  cfg_prec->uri_include_all = FALSE;
2391 
2392  if (!g_disable_randomness) {
2394  } else {
2395  cfg_prec->randomize = 0;
2396  }
2398 
2399  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2400  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2401  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2402  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2403 
2404  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2405  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2406 
2407  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2408  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2409 
2410  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2411  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
2412 
2413  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2414  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
2415 
2416  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2417 
2418  /* don't convert + to space by default */
2419  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2420 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2421  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2423 #endif
2424 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2425  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2427 #endif
2428  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2429  * only the hard limit. So we set both here to the (current) htp defaults.
2430  * The reason we do this is that if the user sets the hard limit in the
2431  * config, we have to set the soft limit as well. If libhtp starts using
2432  * the soft limit in the future, we at least make sure we control what
2433  * it's value is. */
2434  htp_config_set_field_limits(cfg_prec->cfg,
2437  return;
2438 }
2439 
2440 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2441  * but we can get both <0 and >RAND_MAX values from RandomGet
2442  */
2443 static int RandomGetWrap(void)
2444 {
2445  unsigned long r;
2446 
2447  do {
2448  r = RandomGet();
2449  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2450 
2451  return r % RAND_MAX;
2452 }
2453 
2454 /*
2455  * We have this splitup so that in case double decoding has been enabled
2456  * for query and path, they would be called first on the callback queue,
2457  * before the callback set by Phase2() is called. We need this, since
2458  * the callback in Phase2() generates the normalized uri which utilizes
2459  * the query and path. */
2460 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2461 {
2462  /* randomize inspection size if needed */
2463  if (cfg_prec->randomize) {
2464  int rdrange = cfg_prec->randomize_range;
2465 
2466  long int r = RandomGetWrap();
2467  cfg_prec->request.inspect_min_size +=
2468  (int) (cfg_prec->request.inspect_min_size *
2469  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2470 
2471  r = RandomGetWrap();
2472  cfg_prec->request.inspect_window +=
2473  (int) (cfg_prec->request.inspect_window *
2474  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2475  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2476  " %d and 'request-body-inspect-window' set to %d after"
2477  " randomization.",
2478  name,
2479  cfg_prec->request.inspect_min_size,
2480  cfg_prec->request.inspect_window);
2481 
2482 
2483  r = RandomGetWrap();
2484  cfg_prec->response.inspect_min_size +=
2485  (int) (cfg_prec->response.inspect_min_size *
2486  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2487 
2488  r = RandomGetWrap();
2489  cfg_prec->response.inspect_window +=
2490  (int) (cfg_prec->response.inspect_window *
2491  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2492 
2493  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2494  " %d and 'response-body-inspect-window' set to %d after"
2495  " randomization.",
2496  name,
2497  cfg_prec->response.inspect_min_size,
2498  cfg_prec->response.inspect_window);
2499  }
2500 
2501  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2502 
2503  cfg_prec->request.sbcfg.flags = 0;
2504  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2505  cfg_prec->request.inspect_window : 256;
2506  cfg_prec->request.sbcfg.buf_slide = 0;
2507  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2508  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2509  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2510  cfg_prec->request.sbcfg.Free = HTPFree;
2511 
2512  cfg_prec->response.sbcfg.flags = 0;
2513  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2514  cfg_prec->response.inspect_window : 256;
2515  cfg_prec->response.sbcfg.buf_slide = 0;
2516  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2517  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2518  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2519  cfg_prec->response.sbcfg.Free = HTPFree;
2520  return;
2521 }
2522 
2523 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2524  SCRadixTree *tree)
2525 {
2526  if (cfg_prec == NULL || s == NULL || tree == NULL)
2527  return;
2528 
2529  ConfNode *p = NULL;
2530 
2531  /* Default Parameters */
2532  TAILQ_FOREACH(p, &s->head, next) {
2533 
2534  if (strcasecmp("address", p->name) == 0) {
2535  ConfNode *pval;
2536  /* Addresses */
2537  TAILQ_FOREACH(pval, &p->head, next) {
2538  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2539  pval->val);
2540 
2541  /* IPV6 or IPV4? */
2542  if (strchr(pval->val, ':') != NULL) {
2543  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2544  s->name, pval->val, cfg_prec->cfg);
2545  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2546  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2547  "add ipv6 server %s, ignoring", pval->val);
2548  }
2549  } else {
2550  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2551  s->name, pval->val, cfg_prec->cfg);
2552  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2553  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2554  "to add ipv4 server %s, ignoring",
2555  pval->val);
2556  }
2557  } /* else - if (strchr(pval->val, ':') != NULL) */
2558  } /* TAILQ_FOREACH(pval, &p->head, next) */
2559 
2560  } else if (strcasecmp("personality", p->name) == 0) {
2561  /* Personalities */
2562  int personality = HTPLookupPersonality(p->val);
2563  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2564  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2565 
2566  if (personality >= 0) {
2567  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2568  personality);
2569  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2570  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2571  "personality \"%s\", ignoring", p->val);
2572  } else {
2573  SCLogDebug("LIBHTP personality set to %s",
2574  HTPLookupPersonalityString(personality));
2575  }
2576 
2577  /* The IDS personality by default converts the path (and due to
2578  * our query string callback also the query string) to lowercase.
2579  * Signatures do not expect this, so override it. */
2580  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2581  } else {
2582  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2583  "\"%s\", ignoring", p->val);
2584  continue;
2585  }
2586 
2587  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2588  strcasecmp("request_body_limit", p->name) == 0) {
2589  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2590  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2591  "from conf file - %s. Killing engine", p->val);
2592  exit(EXIT_FAILURE);
2593  }
2594 
2595  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2596  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2597  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2598  "from conf file - %s. Killing engine", p->val);
2599  exit(EXIT_FAILURE);
2600  }
2601 
2602  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2603  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2604  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2605  "from conf file - %s. Killing engine", p->val);
2606  exit(EXIT_FAILURE);
2607  }
2608 
2609  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2610  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2611  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2612  "from conf file - %s. Killing engine", p->val);
2613  exit(EXIT_FAILURE);
2614  }
2615 
2616  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2617  if (ConfValIsTrue(p->val)) {
2618  htp_config_register_request_line(cfg_prec->cfg,
2619  HTPCallbackDoubleDecodeQuery);
2620  }
2621 
2622  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2623  if (ConfValIsTrue(p->val)) {
2624  htp_config_register_request_line(cfg_prec->cfg,
2625  HTPCallbackDoubleDecodePath);
2626  }
2627 
2628  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2629  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2630  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2631  "from conf file - %s. Killing engine", p->val);
2632  exit(EXIT_FAILURE);
2633  }
2634 
2635  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2636  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2637  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2638  "from conf file - %s. Killing engine", p->val);
2639  exit(EXIT_FAILURE);
2640  }
2641 
2642  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2643  uint32_t value = 2;
2644  if (ParseSizeStringU32(p->val, &value) < 0) {
2645  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2646  "from conf file - %s. Killing engine", p->val);
2647  exit(EXIT_FAILURE);
2648  }
2649 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2650  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2651 #else
2652  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2653  "to %u, libhtp version too old", value);
2654 #endif
2655  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2656  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2657  HTP_DECODER_URL_PATH,
2658  ConfValIsTrue(p->val));
2659  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2660  if (strlen(p->val) == 1) {
2661  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2662  HTP_DECODER_URL_PATH,
2663  p->val[0]);
2664  } else {
2665  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2666  "for libhtp param path-bestfit-replacement-char");
2667  }
2668  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2669  htp_config_set_convert_lowercase(cfg_prec->cfg,
2670  HTP_DECODER_URL_PATH,
2671  ConfValIsTrue(p->val));
2672  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2673  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2674  HTP_DECODER_URL_PATH,
2675  ConfValIsTrue(p->val));
2676  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2677  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2678  HTP_DECODER_URL_PATH,
2679  ConfValIsTrue(p->val));
2680  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2681  htp_config_set_path_separators_compress(cfg_prec->cfg,
2682  HTP_DECODER_URL_PATH,
2683  ConfValIsTrue(p->val));
2684  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2685  htp_config_set_path_separators_decode(cfg_prec->cfg,
2686  HTP_DECODER_URL_PATH,
2687  ConfValIsTrue(p->val));
2688  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2689  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2690  HTP_DECODER_URL_PATH,
2691  ConfValIsTrue(p->val));
2692  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2693  enum htp_url_encoding_handling_t handling;
2694  if (strcasecmp(p->val, "preserve_percent") == 0) {
2695  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2696  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2697  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2698  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2699  handling = HTP_URL_DECODE_PROCESS_INVALID;
2700  } else {
2701  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2702  "for libhtp param path-url-encoding-invalid-handling");
2703  return;
2704  }
2705  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2706  HTP_DECODER_URL_PATH,
2707  handling);
2708  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2709  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2710  HTP_DECODER_URL_PATH,
2711  ConfValIsTrue(p->val));
2712  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2713  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2714  SCLogDebug("uri-include-all %s",
2715  cfg_prec->uri_include_all ? "enabled" : "disabled");
2716  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2717  htp_config_set_plusspace_decode(cfg_prec->cfg,
2718  HTP_DECODER_URLENCODED,
2719  ConfValIsTrue(p->val));
2720  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2721  uint32_t limit = 0;
2722  if (ParseSizeStringU32(p->val, &limit) < 0) {
2723  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2724  "from conf file - %s. Killing engine", p->val);
2725  exit(EXIT_FAILURE);
2726  }
2727  if (limit == 0) {
2728  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2729  "from conf file cannot be 0. Killing engine");
2730  exit(EXIT_FAILURE);
2731  }
2732  /* set default soft-limit with our new hard limit */
2733  htp_config_set_field_limits(cfg_prec->cfg,
2735  (size_t)limit);
2736 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2737  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2738  uint32_t limit = 0;
2739  if (ParseSizeStringU32(p->val, &limit) < 0) {
2740  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'lzma-memlimit' "
2741  "from conf file - %s.", p->val);
2742  }
2743  if (limit == 0) {
2744  FatalError(SC_ERR_SIZE_PARSE, "'lzma-memlimit' "
2745  "from conf file cannot be 0.");
2746  }
2747  /* set default soft-limit with our new hard limit */
2748  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2749  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2750 #endif
2751 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2752  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2753  if (ConfValIsFalse(p->val)) {
2754  htp_config_set_lzma_memlimit(cfg_prec->cfg, 0);
2755  }
2756 #endif
2757 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2758  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2759  uint32_t limit = 0;
2760  if (ParseSizeStringU32(p->val, &limit) < 0) {
2761  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'compression-bomb-limit' "
2762  "from conf file - %s.", p->val);
2763  }
2764  if (limit == 0) {
2765  FatalError(SC_ERR_SIZE_PARSE, "'compression-bomb-limit' "
2766  "from conf file cannot be 0.");
2767  }
2768  /* set default soft-limit with our new hard limit */
2769  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2770  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2771 #endif
2772  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2773  if (!g_disable_randomness) {
2774  cfg_prec->randomize = ConfValIsTrue(p->val);
2775  }
2776  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2777  uint32_t range = atoi(p->val);
2778  if (range > 100) {
2779  SCLogError(SC_ERR_SIZE_PARSE, "Invalid value for randomize"
2780  " inspection range setting from conf file - %s."
2781  " It should be inferior to 100."
2782  " Killing engine",
2783  p->val);
2784  exit(EXIT_FAILURE);
2785  }
2786  cfg_prec->randomize_range = range;
2787  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2788  if (ConfValIsTrue(p->val)) {
2789  cfg_prec->http_body_inline = 1;
2790  } else if (ConfValIsFalse(p->val)) {
2791  cfg_prec->http_body_inline = 0;
2792  } else {
2793  if (strcmp("auto", p->val) != 0) {
2794  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2795  }
2796  if (EngineModeIsIPS()) {
2797  cfg_prec->http_body_inline = 1;
2798  } else {
2799  cfg_prec->http_body_inline = 0;
2800  }
2801  }
2802  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2803  ConfNode *pval;
2804 
2805  TAILQ_FOREACH(pval, &p->head, next) {
2806  if (strcasecmp("enabled", pval->name) == 0) {
2807  if (ConfValIsTrue(pval->val)) {
2808  cfg_prec->swf_decompression_enabled = 1;
2809  } else if (ConfValIsFalse(pval->val)) {
2810  cfg_prec->swf_decompression_enabled = 0;
2811  } else {
2812  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2813  }
2814  } else if (strcasecmp("type", pval->name) == 0) {
2815  if (strcasecmp("no", pval->val) == 0) {
2817  } else if (strcasecmp("deflate", pval->val) == 0) {
2819  } else if (strcasecmp("lzma", pval->val) == 0) {
2821  } else if (strcasecmp("both", pval->val) == 0) {
2823  } else {
2825  "Invalid entry for "
2826  "swf-decompression.type: %s - "
2827  "Killing engine", pval->val);
2828  exit(EXIT_FAILURE);
2829  }
2830  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2831  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2833  "Error parsing swf-decompression.compression-depth "
2834  "from conf file - %s. Killing engine", p->val);
2835  exit(EXIT_FAILURE);
2836  }
2837  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2838  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2840  "Error parsing swf-decompression.decompression-depth "
2841  "from conf file - %s. Killing engine", p->val);
2842  exit(EXIT_FAILURE);
2843  }
2844  } else {
2845  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2846  }
2847  }
2848  } else {
2849  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2850  "default config: %s", p->name);
2851  }
2852  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2853 
2854  return;
2855 }
2856 
2857 void HTPConfigure(void)
2858 {
2859  SCEnter();
2860 
2861  cfglist.next = NULL;
2862 
2863  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2864  if (NULL == cfgtree)
2865  exit(EXIT_FAILURE);
2866 
2867  /* Default Config */
2868  cfglist.cfg = htp_config_create();
2869  if (NULL == cfglist.cfg) {
2870  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP default config");
2871  exit(EXIT_FAILURE);
2872  }
2873  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2874  HTPConfigSetDefaultsPhase1(&cfglist);
2875  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2876  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2877  cfgtree);
2878  } else {
2879  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2880  }
2881  HTPConfigSetDefaultsPhase2("default", &cfglist);
2882 
2883  HTPParseMemcap();
2884 
2885  /* Read server config and create a parser for each IP in radix tree */
2886  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2887  if (server_config == NULL) {
2888  server_config = ConfGetNode("libhtp.server-config");
2889  if (server_config == NULL) {
2890  SCLogDebug("LIBHTP Configuring %p", server_config);
2891  SCReturn;
2892  }
2893  }
2894  SCLogDebug("LIBHTP Configuring %p", server_config);
2895 
2896  ConfNode *si;
2897  /* Server Nodes */
2898  TAILQ_FOREACH(si, &server_config->head, next) {
2899  /* Need the named node, not the index */
2900  ConfNode *s = TAILQ_FIRST(&si->head);
2901  if (NULL == s) {
2902  SCLogDebug("LIBHTP s NULL");
2903  continue;
2904  }
2905 
2906  SCLogDebug("LIBHTP server %s", s->name);
2907 
2908  HTPCfgRec *nextrec = cfglist.next;
2909  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2910  if (NULL == htprec)
2911  exit(EXIT_FAILURE);
2912  memset(htprec, 0x00, sizeof(*htprec));
2913 
2914  cfglist.next = htprec;
2915 
2916  cfglist.next->next = nextrec;
2917  cfglist.next->cfg = htp_config_create();
2918  if (NULL == cfglist.next->cfg) {
2919  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP server config");
2920  exit(EXIT_FAILURE);
2921  }
2922 
2923  HTPConfigSetDefaultsPhase1(htprec);
2924  HTPConfigParseParameters(htprec, s, cfgtree);
2925  HTPConfigSetDefaultsPhase2(s->name, htprec);
2926  }
2927 
2928  SCReturn;
2929 }
2930 
2932 {
2933 #ifdef DEBUG
2934  SCMutexLock(&htp_state_mem_lock);
2935  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2936  SCMutexUnlock(&htp_state_mem_lock);
2937 #endif
2938 }
2939 
2940 /** \internal
2941  * \brief get files callback
2942  * \param state state ptr
2943  * \param direction flow direction
2944  * \retval files files ptr
2945  */
2946 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
2947 {
2948  if (state == NULL)
2949  return NULL;
2950 
2951  HtpState *http_state = (HtpState *)state;
2952 
2953  if (direction & STREAM_TOCLIENT) {
2954  SCReturnPtr(http_state->files_tc, "FileContainer");
2955  } else {
2956  SCReturnPtr(http_state->files_ts, "FileContainer");
2957  }
2958 }
2959 
2960 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2961 {
2962  if (direction & STREAM_TOSERVER)
2963  return ((htp_tx_t *)tx)->request_progress;
2964  else
2965  return ((htp_tx_t *)tx)->response_progress;
2966 }
2967 
2968 static uint64_t HTPStateGetTxCnt(void *alstate)
2969 {
2970  HtpState *http_state = (HtpState *)alstate;
2971 
2972  if (http_state != NULL && http_state->conn != NULL) {
2973  const uint64_t size = (uint64_t)htp_list_size(http_state->conn->transactions);
2974  SCLogDebug("size %"PRIu64, size);
2975  return size;
2976  } else {
2977  return 0ULL;
2978  }
2979 }
2980 
2981 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2982 {
2983  HtpState *http_state = (HtpState *)alstate;
2984 
2985  if (http_state != NULL && http_state->conn != NULL)
2986  return htp_list_get(http_state->conn->transactions, tx_id);
2987  else
2988  return NULL;
2989 }
2990 
2991 static void HTPStateSetTxLogged(void *alstate, void *vtx, LoggerId bits)
2992 {
2993  htp_tx_t *tx = (htp_tx_t *)vtx;
2994  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2995  if (tx_ud)
2996  tx_ud->logged = bits;
2997 }
2998 
2999 static LoggerId HTPStateGetTxLogged(void *alstate, void *vtx)
3000 {
3001  htp_tx_t *tx = (htp_tx_t *)vtx;
3002  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3003  if (tx_ud != NULL)
3004  return tx_ud->logged;
3005 
3006  return 0;
3007 }
3008 
3009 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction)
3010 {
3011  return (direction & STREAM_TOSERVER) ? HTP_REQUEST_COMPLETE : HTP_RESPONSE_COMPLETE;
3012 }
3013 
3014 static int HTPStateGetEventInfo(const char *event_name,
3015  int *event_id, AppLayerEventType *event_type)
3016 {
3017  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3018  if (*event_id == -1) {
3019  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
3020  "http's enum map table.", event_name);
3021  /* this should be treated as fatal */
3022  return -1;
3023  }
3024 
3025  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3026 
3027  return 0;
3028 }
3029 
3030 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3032 {
3033  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3034  if (*event_name == NULL) {
3035  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
3036  "http's enum map table.", event_id);
3037  /* this should be treated as fatal */
3038  return -1;
3039  }
3040 
3041  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3042 
3043  return 0;
3044 }
3045 
3046 static void HTPStateTruncate(void *state, uint8_t direction)
3047 {
3048  FileContainer *fc = HTPStateGetFiles(state, direction);
3049  if (fc != NULL) {
3051  }
3052 }
3053 
3054 static DetectEngineState *HTPGetTxDetectState(void *vtx)
3055 {
3056  htp_tx_t *tx = (htp_tx_t *)vtx;
3057  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3058  return tx_ud ? tx_ud->de_state : NULL;
3059 }
3060 
3061 static int HTPSetTxDetectState(void *vtx, DetectEngineState *s)
3062 {
3063  htp_tx_t *tx = (htp_tx_t *)vtx;
3064  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3065  if (tx_ud == NULL) {
3066  tx_ud = HTPMalloc(sizeof(*tx_ud));
3067  if (unlikely(tx_ud == NULL))
3068  return -ENOMEM;
3069  memset(tx_ud, 0, sizeof(*tx_ud));
3070  htp_tx_set_user_data(tx, tx_ud);
3071  }
3072  tx_ud->de_state = s;
3073  return 0;
3074 }
3075 
3076 static uint64_t HTPGetTxDetectFlags(void *vtx, uint8_t dir)
3077 {
3078  htp_tx_t *tx = (htp_tx_t *)vtx;
3079  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3080  if (tx_ud) {
3081  if (dir & STREAM_TOSERVER) {
3082  return tx_ud->detect_flags_ts;
3083  } else {
3084  return tx_ud->detect_flags_tc;
3085  }
3086  }
3087  return 0;
3088 }
3089 
3090 static void HTPSetTxDetectFlags(void *vtx, uint8_t dir, uint64_t detect_flags)
3091 {
3092  htp_tx_t *tx = (htp_tx_t *)vtx;
3093  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3094  if (tx_ud == NULL) {
3095  tx_ud = HTPMalloc(sizeof(*tx_ud));
3096  if (unlikely(tx_ud == NULL))
3097  return;
3098  memset(tx_ud, 0, sizeof(*tx_ud));
3099  htp_tx_set_user_data(tx, tx_ud);
3100  }
3101  if (dir & STREAM_TOSERVER) {
3102  tx_ud->detect_flags_ts = detect_flags;
3103  } else {
3104  tx_ud->detect_flags_tc = detect_flags;
3105  }
3106  return;
3107 }
3108 
3109 static int HTPRegisterPatternsForProtocolDetection(void)
3110 {
3111  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3112  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3113  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3114  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3115  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3116  const char *spacings[] = { "|20|", "|09|", NULL };
3117  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3118 
3119  int methods_pos;
3120  int spacings_pos;
3121  int versions_pos;
3122  int register_result;
3123  char method_buffer[32] = "";
3124 
3125  /* Loop through all the methods ands spacings and register the patterns */
3126  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3127  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3128 
3129  /* Combine the method name and the spacing */
3130  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3131 
3132  /* Register the new method+spacing pattern
3133  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3134  * but the pattern matching should only be one char
3135  */
3136  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
3137  ALPROTO_HTTP, method_buffer, strlen(method_buffer)-3, 0, STREAM_TOSERVER);
3138  if (register_result < 0) {
3139  return -1;
3140  }
3141  }
3142  }
3143 
3144  /* Loop through all the http verions patterns that are TO_CLIENT */
3145  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3146  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
3147  ALPROTO_HTTP, versions[versions_pos], strlen(versions[versions_pos]),
3148  0, STREAM_TOCLIENT);
3149  if (register_result < 0) {
3150  return -1;
3151  }
3152  }
3153 
3154  return 0;
3155 }
3156 
3157 /**
3158  * \brief Register the HTTP protocol and state handling functions to APP layer
3159  * of the engine.
3160  */
3162 {
3163  SCEnter();
3164 
3165  const char *proto_name = "http";
3166 
3167  /** HTTP */
3168  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3170  if (HTPRegisterPatternsForProtocolDetection() < 0)
3171  return;
3172  } else {
3173  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3174  proto_name);
3175  return;
3176  }
3177 
3178  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3179  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateAlloc, HTPStateFree);
3180  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTransactionFree);
3181  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetFiles);
3182  AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetAlstateProgress);
3183  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxCnt);
3184  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTx);
3185  AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxLogged,
3186  HTPStateSetTxLogged);
3188  HTPStateGetAlstateProgressCompletionStatus);
3189  AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPGetEvents);
3190  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfo);
3191  AppLayerParserRegisterGetEventInfoById(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfoById);
3192 
3193  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTruncate);
3195  HTPGetTxDetectState, HTPSetTxDetectState);
3197  HTPGetTxDetectFlags, HTPSetTxDetectFlags);
3198 
3200  AppLayerHtpSetStreamDepthFlag);
3201 
3203  HTPHandleRequestData);
3205  HTPHandleResponseData);
3206  SC_ATOMIC_INIT(htp_config_flags);
3209  HTPConfigure();
3210  } else {
3211  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3212  "still on.", proto_name);
3213  }
3214 #ifdef UNITTESTS
3216 #endif
3217 
3218  SCReturn;
3219 }
3220 
3221 #ifdef UNITTESTS
3222 static HTPCfgRec cfglist_backup;
3223 
3225 {
3226  cfglist_backup = cfglist;
3227 
3228  return;
3229 }
3230 
3232 {
3233  cfglist = cfglist_backup;
3234 
3235  return;
3236 }
3237 
3238 /** \test Test case where chunks are sent in smaller chunks and check the
3239  * response of the parser from HTP library. */
3240 static int HTPParserTest01(void)
3241 {
3242  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3243  " Data is c0oL!";
3244  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3245 
3246  TcpSession ssn;
3247  memset(&ssn, 0, sizeof(ssn));
3248 
3250  FAIL_IF_NULL(alp_tctx);
3251 
3252  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3253  FAIL_IF_NULL(f);
3254  f->protoctx = &ssn;
3255  f->proto = IPPROTO_TCP;
3256  f->alproto = ALPROTO_HTTP;
3257 
3259 
3260  uint32_t u;
3261  for (u = 0; u < httplen1; u++) {
3262  uint8_t flags = 0;
3263 
3264  if (u == 0)
3265  flags = STREAM_TOSERVER|STREAM_START;
3266  else if (u == (httplen1 - 1))
3267  flags = STREAM_TOSERVER|STREAM_EOF;
3268  else
3269  flags = STREAM_TOSERVER;
3270 
3271  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3272  &httpbuf1[u], 1);
3273  FAIL_IF(r != 0);
3274  }
3275 
3276  HtpState *htp_state = f->alstate;
3277  FAIL_IF_NULL(htp_state);
3278 
3279  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3280  FAIL_IF_NULL(tx);
3281 
3282  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3283  FAIL_IF_NULL(h);
3284 
3285  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3286  FAIL_IF(tx->request_method_number != HTP_M_POST);
3287  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3288 
3289  AppLayerParserThreadCtxFree(alp_tctx);
3291  UTHFreeFlow(f);
3292  PASS;
3293 }
3294 
3295 /** \test Test folding in 1 read case */
3296 static int HTPParserTest01b(void)
3297 {
3298  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3299  " Data is c0oL!";
3300  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3301 
3302  TcpSession ssn;
3303  memset(&ssn, 0, sizeof(ssn));
3304 
3306  FAIL_IF_NULL(alp_tctx);
3307 
3308  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3309  FAIL_IF_NULL(f);
3310  f->protoctx = &ssn;
3311  f->proto = IPPROTO_TCP;
3312  f->alproto = ALPROTO_HTTP;
3313 
3315 
3317  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3318  httpbuf1, httplen1);
3319  FAIL_IF(r != 0);
3320 
3321  HtpState *htp_state = f->alstate;
3322  FAIL_IF_NULL(htp_state);
3323 
3324  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3325  FAIL_IF_NULL(tx);
3326 
3327  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3328  FAIL_IF_NULL(h);
3329 
3330  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3331  FAIL_IF(tx->request_method_number != HTP_M_POST);
3332  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3333 
3334  AppLayerParserThreadCtxFree(alp_tctx);
3336  UTHFreeFlow(f);
3337  PASS;
3338 }
3339 
3340 /** \test Test folding in 1byte per read case */
3341 static int HTPParserTest01c(void)
3342 {
3343  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3344  " Data is c0oL!";
3345  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3346 
3347  TcpSession ssn;
3348  memset(&ssn, 0, sizeof(ssn));
3349 
3351  FAIL_IF_NULL(alp_tctx);
3352 
3353  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3354  FAIL_IF_NULL(f);
3355  f->protoctx = &ssn;
3356  f->proto = IPPROTO_TCP;
3357  f->alproto = ALPROTO_HTTP;
3358 
3360 
3361  uint32_t u;
3362  for (u = 0; u < httplen1; u++) {
3363  uint8_t flags = 0;
3364 
3365  if (u == 0)
3366  flags = STREAM_TOSERVER|STREAM_START;
3367  else if (u == (httplen1 - 1))
3368  flags = STREAM_TOSERVER|STREAM_EOF;
3369  else
3370  flags = STREAM_TOSERVER;
3371 
3372  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3373  &httpbuf1[u], 1);
3374  FAIL_IF(r != 0);
3375  }
3376 
3377  HtpState *htp_state = f->alstate;
3378  FAIL_IF_NULL(htp_state);
3379 
3380  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3381  FAIL_IF_NULL(tx);
3382 
3383  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3384  FAIL_IF_NULL(h);
3385 
3386  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3387  FAIL_IF(tx->request_method_number != HTP_M_POST);
3388  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3389 
3390  AppLayerParserThreadCtxFree(alp_tctx);
3392  UTHFreeFlow(f);
3393  PASS;
3394 }
3395 
3396 /** \test Test case where chunks are sent in smaller chunks and check the
3397  * response of the parser from HTP library. */
3398 static int HTPParserTest01a(void)
3399 {
3400  int result = 0;
3401  Flow *f = NULL;
3402  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3403  " Data is c0oL!";
3404  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3405  TcpSession ssn;
3406  HtpState *htp_state = NULL;
3407  int r = 0;
3409 
3410  memset(&ssn, 0, sizeof(ssn));
3411 
3412  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3413  if (f == NULL)
3414  goto end;
3415  f->protoctx = &ssn;
3416  f->proto = IPPROTO_TCP;
3417  f->alproto = ALPROTO_HTTP;
3418 
3420 
3421  uint32_t u;
3422  for (u = 0; u < httplen1; u++) {
3423  uint8_t flags = 0;
3424 
3425  if (u == 0)
3426  flags = STREAM_TOSERVER|STREAM_START;
3427  else if (u == (httplen1 - 1))
3428  flags = STREAM_TOSERVER|STREAM_EOF;
3429  else
3430  flags = STREAM_TOSERVER;
3431 
3432  FLOWLOCK_WRLOCK(f);
3433  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3434  &httpbuf1[u], 1);
3435  if (r != 0) {
3436  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3437  " 0: ", u, r);
3438  FLOWLOCK_UNLOCK(f);
3439  goto end;
3440  }
3441  FLOWLOCK_UNLOCK(f);
3442  }
3443 
3444  htp_state = f->alstate;
3445  if (htp_state == NULL) {
3446  printf("no http state: ");
3447  goto end;
3448  }
3449 
3450  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3451  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3452  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3453  || tx->request_method_number != HTP_M_POST ||
3454  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3455  {
3456  printf("expected header value: Victor/1.0 and got %s: and expected"
3457  " method: POST and got %s, expected protocol number HTTP/1.0"
3458  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3459  bstr_util_strdup_to_c(tx->request_method),
3460  bstr_util_strdup_to_c(tx->request_protocol));
3461  goto end;
3462  }
3463  result = 1;
3464 end:
3465  if (alp_tctx != NULL)
3466  AppLayerParserThreadCtxFree(alp_tctx);
3468  UTHFreeFlow(f);
3469  return result;
3470 }
3471 
3472 /** \test See how it deals with an incomplete request. */
3473 static int HTPParserTest02(void)
3474 {
3475  int result = 0;
3476  Flow *f = NULL;
3477  uint8_t httpbuf1[] = "POST";
3478  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3479  TcpSession ssn;
3480  HtpState *http_state = NULL;
3482 
3483  memset(&ssn, 0, sizeof(ssn));
3484 
3485  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3486  if (f == NULL)
3487  goto end;
3488  f->protoctx = &ssn;
3489  f->proto = IPPROTO_TCP;
3490  f->alproto = ALPROTO_HTTP;
3491 
3493 
3494  FLOWLOCK_WRLOCK(f);
3495  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3497  httpbuf1,
3498  httplen1);
3499  if (r != 0) {
3500  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3501  FLOWLOCK_UNLOCK(f);
3502  goto end;
3503  }
3504  FLOWLOCK_UNLOCK(f);
3505 
3506  http_state = f->alstate;
3507  if (http_state == NULL) {
3508  printf("no http state: ");
3509  goto end;
3510  }
3511 
3512  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3513  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3514  if ((tx->request_method) != NULL || h != NULL)
3515  {
3516  printf("expected method NULL, got %s \n", bstr_util_strdup_to_c(tx->request_method));
3517  goto end;
3518  }
3519  result = 1;
3520 end:
3521  if (alp_tctx != NULL)
3522  AppLayerParserThreadCtxFree(alp_tctx);
3524  UTHFreeFlow(f);
3525  return result;
3526 }
3527 
3528 /** \test Test case where method is invalid and data is sent in smaller chunks
3529  * and check the response of the parser from HTP library. */
3530 static int HTPParserTest03(void)
3531 {
3532  int result = 0;
3533  Flow *f = NULL;
3534  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3535  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3536  TcpSession ssn;
3537  HtpState *htp_state = NULL;
3538  int r = 0;
3540 
3541  memset(&ssn, 0, sizeof(ssn));
3542 
3543  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3544  if (f == NULL)
3545  goto end;
3546  f->protoctx = &ssn;
3547  f->proto = IPPROTO_TCP;
3548  f->alproto = ALPROTO_HTTP;
3549 
3551 
3552  uint32_t u;
3553  for (u = 0; u < httplen1; u++) {
3554  uint8_t flags = 0;
3555 
3556  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3557  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3558  else flags = STREAM_TOSERVER;
3559 
3560  FLOWLOCK_WRLOCK(f);
3561  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3562  &httpbuf1[u], 1);
3563  if (r != 0) {
3564  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3565  " 0: ", u, r);
3566  FLOWLOCK_UNLOCK(f);
3567  goto end;
3568  }
3569  FLOWLOCK_UNLOCK(f);
3570  }
3571  htp_state = f->alstate;
3572  if (htp_state == NULL) {
3573  printf("no http state: ");
3574  goto end;
3575  }
3576 
3577  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3578 
3579  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3580  if (tx->request_method_number != HTP_M_UNKNOWN ||
3581  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3582  {
3583  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3584  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3585  bstr_util_strdup_to_c(tx->request_protocol));
3586  goto end;
3587  }
3588  result = 1;
3589 end:
3590  if (alp_tctx != NULL)
3591  AppLayerParserThreadCtxFree(alp_tctx);
3593  UTHFreeFlow(f);
3594  return result;
3595 }
3596 
3597 /** \test Test case where invalid data is sent and check the response of the
3598  * parser from HTP library. */
3599 static int HTPParserTest04(void)
3600 {
3601  int result = 0;
3602  Flow *f = NULL;
3603  HtpState *htp_state = NULL;
3604  uint8_t httpbuf1[] = "World!\r\n";
3605  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3606  TcpSession ssn;
3607  int r = 0;
3609 
3610  memset(&ssn, 0, sizeof(ssn));
3611 
3612  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3613  if (f == NULL)
3614  goto end;
3615  f->protoctx = &ssn;
3616  f->proto = IPPROTO_TCP;
3617  f->alproto = ALPROTO_HTTP;
3618 
3620 
3621  FLOWLOCK_WRLOCK(f);
3622  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3624  httpbuf1,
3625  httplen1);
3626  if (r != 0) {
3627  FLOWLOCK_UNLOCK(f);
3628  goto end;
3629  }
3630  FLOWLOCK_UNLOCK(f);
3631 
3632  htp_state = f->alstate;
3633  if (htp_state == NULL) {
3634  printf("no http state: ");
3635  goto end;
3636  }
3637 
3638  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3639  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3640  if (tx->request_method_number != HTP_M_UNKNOWN ||
3641  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3642  {
3643  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3644  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3645  bstr_util_strdup_to_c(tx->request_protocol));
3646  goto end;
3647  }
3648  result = 1;
3649 end:
3650  if (alp_tctx != NULL)
3651  AppLayerParserThreadCtxFree(alp_tctx);
3653  UTHFreeFlow(f);
3654  return result;
3655 }
3656 
3657 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3658  * properly parsed them and also keeps them separated. */
3659 static int HTPParserTest05(void)
3660 {
3661  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3662  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3663  uint8_t httpbuf2[] = "Post D";
3664  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3665  uint8_t httpbuf3[] = "ata is c0oL!";
3666  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3667 
3668  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3669  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3670  uint8_t httpbuf5[] = "post R";
3671  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3672  uint8_t httpbuf6[] = "esults are tha bomb!";
3673  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3674 
3675  TcpSession ssn;
3676  memset(&ssn, 0, sizeof(ssn));
3677 
3679  FAIL_IF_NULL(alp_tctx);
3680 
3681  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3682  FAIL_IF_NULL(f);
3683  f->protoctx = &ssn;
3684  f->proto = IPPROTO_TCP;
3685  f->alproto = ALPROTO_HTTP;
3686 
3688 
3689  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3690  STREAM_TOSERVER | STREAM_START, httpbuf1,
3691  httplen1);
3692  FAIL_IF(r != 0);
3693 
3694  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3695  STREAM_TOCLIENT | STREAM_START, httpbuf4,
3696  httplen4);
3697  FAIL_IF(r != 0);
3698 
3699  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOCLIENT,
3700  httpbuf5, httplen5);
3701  FAIL_IF(r != 0);
3702 
3703  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOSERVER,
3704  httpbuf2, httplen2);
3705  FAIL_IF(r != 0);
3706 
3707  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3708  STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3709  FAIL_IF(r != 0);
3710 
3711  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3712  STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3713  FAIL_IF(r != 0);
3714 
3715  HtpState *http_state = f->alstate;
3716  FAIL_IF_NULL(http_state);
3717 
3718  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3719  FAIL_IF_NULL(tx);
3720  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3721  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3722 
3723  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3724  FAIL_IF_NULL(h);
3725 
3726  FAIL_IF_NOT(tx->response_status_number == 200);
3727 
3728  AppLayerParserThreadCtxFree(alp_tctx);
3730  UTHFreeFlow(f);
3731  PASS;
3732 }
3733 
3734 /** \test Test proper chunked encoded response body
3735  */
3736 static int HTPParserTest06(void)
3737 {
3738  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3739  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3740  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3741  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3742  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3743  "GMT\r\n"
3744  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3745  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3746  "FrontPage/5.0.2.2510\r\n"
3747  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3748  "chunked\r\n"
3749  "Content-Type: text/html\r\n\r\n"
3750  "580\r\n"
3751  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3752  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3753  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3754  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3755  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3756  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3757  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3758  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3759  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3760  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3761  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3762  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3763  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3764  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3765  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3766  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3767  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3768  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3769  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3770  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3771  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3772  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3773  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3774  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3775  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3776  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3777  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3778  "aHA=\r\n0\r\n\r\n";
3779  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3780  TcpSession ssn;
3781 
3783  FAIL_IF_NULL(alp_tctx);
3784 
3785  memset(&ssn, 0, sizeof(ssn));
3786 
3787  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3788  FAIL_IF_NULL(f);
3789  f->protoctx = &ssn;
3790  f->proto = IPPROTO_TCP;
3791  f->alproto = ALPROTO_HTTP;
3792 
3794 
3795  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3796  STREAM_TOSERVER | STREAM_START, httpbuf1,
3797  httplen1);
3798  FAIL_IF(r != 0);
3799  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3800  STREAM_TOCLIENT | STREAM_START, httpbuf2,
3801  httplen2);
3802  FAIL_IF(r != 0);
3803 
3804  HtpState *http_state = f->alstate;
3805  FAIL_IF_NULL(http_state);
3806 
3807  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3808  FAIL_IF_NULL(tx);
3809 
3810  FAIL_IF(tx->request_method_number != HTP_M_GET);
3811  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3812 
3813  FAIL_IF(tx->response_status_number != 200);
3814  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3815 
3816  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3817  FAIL_IF_NULL(h);
3818 
3819  AppLayerParserThreadCtxFree(alp_tctx);
3821  UTHFreeFlow(f);
3822  PASS;
3823 }
3824 
3825 /** \test
3826  */
3827 static int HTPParserTest07(void)
3828 {
3829  int result = 0;
3830  Flow *f = NULL;
3831  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3832  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3833  TcpSession ssn;
3834  HtpState *htp_state = NULL;
3835  int r = 0;
3837 
3838  memset(&ssn, 0, sizeof(ssn));
3839 
3840  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3841  if (f == NULL)
3842  goto end;
3843  f->protoctx = &ssn;
3844  f->proto = IPPROTO_TCP;
3845  f->alproto = ALPROTO_HTTP;
3846 
3848 
3849  uint32_t u;
3850  for (u = 0; u < httplen1; u++) {
3851  uint8_t flags = 0;
3852 
3853  if (u == 0)
3854  flags = STREAM_TOSERVER|STREAM_START;
3855  else if (u == (httplen1 - 1))
3856  flags = STREAM_TOSERVER|STREAM_EOF;
3857  else
3858  flags = STREAM_TOSERVER;
3859 
3860  FLOWLOCK_WRLOCK(f);
3861  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3862  &httpbuf1[u], 1);
3863  if (r != 0) {
3864  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3865  " 0: ", u, r);
3866  FLOWLOCK_UNLOCK(f);
3867  goto end;
3868  }
3869  FLOWLOCK_UNLOCK(f);
3870  }
3871 
3872  htp_state = f->alstate;
3873  if (htp_state == NULL) {
3874  printf("no http state: ");
3875  goto end;
3876  }
3877 
3878  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3879  size_t reflen = sizeof(ref) - 1;
3880 
3881  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3882  if (tx == NULL)
3883  goto end;
3884  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3885  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3886  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3887  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3888  (uintmax_t)reflen,
3889  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3890  goto end;
3891  }
3892 
3893  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3894  bstr_len(tx_ud->request_uri_normalized)) != 0)
3895  {
3896  printf("normalized uri \"");
3897  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3898  printf("\" != \"");
3899  PrintRawUriFp(stdout, ref, reflen);
3900  printf("\": ");
3901  goto end;
3902  }
3903  }
3904 
3905  result = 1;
3906 end:
3907  if (alp_tctx != NULL)
3908  AppLayerParserThreadCtxFree(alp_tctx);
3910  UTHFreeFlow(f);
3911  return result;
3912 }
3913 
3914 #include "conf-yaml-loader.h"
3915 
3916 /** \test Abort
3917  */
3918 static int HTPParserTest08(void)
3919 {
3920  int result = 0;
3921  Flow *f = NULL;
3922  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3923  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3924  TcpSession ssn;
3926 
3927  char input[] = "\
3928 %YAML 1.1\n\
3929 ---\n\
3930 libhtp:\n\
3931 \n\
3932  default-config:\n\
3933  personality: IDS\n\
3934 ";
3935 
3937  ConfInit();
3939 
3940  ConfYamlLoadString(input, strlen(input));
3941  HTPConfigure();
3942 
3943  HtpState *htp_state = NULL;
3944  int r = 0;
3945  memset(&ssn, 0, sizeof(ssn));
3946 
3947  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3948  if (f == NULL)
3949  goto end;
3950  f->protoctx = &ssn;
3951  f->proto = IPPROTO_TCP;
3952  f->alproto = ALPROTO_HTTP;
3953 
3955 
3956  uint8_t flags = 0;
3958 
3959  FLOWLOCK_WRLOCK(f);
3960  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
3961  httplen1);
3962  if (r != 0) {
3963  printf("toserver chunk returned %" PRId32 ", expected"
3964  " 0: ", r);
3965  result = 0;
3966  FLOWLOCK_UNLOCK(f);
3967  goto end;
3968  }
3969  FLOWLOCK_UNLOCK(f);
3970 
3971  htp_state = f->alstate;
3972  if (htp_state == NULL) {
3973  printf("no http state: ");
3974  result = 0;
3975  goto end;
3976  }
3977 
3978  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3979  if (tx == NULL)
3980  goto end;
3981  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3982  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3983  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3984  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3985  bstr_len(tx_ud->request_uri_normalized));
3986  }
3987 
3988  result = 1;
3989 end:
3990  if (alp_tctx != NULL)
3991  AppLayerParserThreadCtxFree(alp_tctx);
3993  HTPFreeConfig();
3994  ConfDeInit();
3997  UTHFreeFlow(f);
3998  return result;
3999 }
4000 
4001 /** \test Abort
4002  */
4003 static int HTPParserTest09(void)
4004 {
4005  int result = 0;
4006  Flow *f = NULL;
4007  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
4008  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4009  TcpSession ssn;
4011 
4012  char input[] = "\
4013 %YAML 1.1\n\
4014 ---\n\
4015 libhtp:\n\
4016 \n\
4017  default-config:\n\
4018  personality: Apache_2_2\n\
4019 ";
4020 
4022  ConfInit();
4024 
4025  ConfYamlLoadString(input, strlen(input));
4026  HTPConfigure();
4027 
4028  HtpState *htp_state = NULL;
4029  int r = 0;
4030 
4031  memset(&ssn, 0, sizeof(ssn));
4032 
4033  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4034  if (f == NULL)
4035  goto end;
4036  f->protoctx = &ssn;
4037  f->proto = IPPROTO_TCP;
4038  f->alproto = ALPROTO_HTTP;
4039 
4041 
4042  uint8_t flags = 0;
4044 
4045  FLOWLOCK_WRLOCK(f);
4046  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
4047  httplen1);
4048  if (r != 0) {
4049  printf("toserver chunk returned %" PRId32 ", expected"
4050  " 0: ", r);
4051  FLOWLOCK_UNLOCK(f);
4052  goto end;
4053  }
4054  FLOWLOCK_UNLOCK(f);
4055 
4056  htp_state = f->alstate;
4057  if (htp_state == NULL) {
4058  printf("no http state: ");
4059  goto end;
4060  }
4061 
4062  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4063  if (tx == NULL)
4064  goto end;
4065  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4066  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4067  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4068  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4069  bstr_len(tx_ud->request_uri_normalized));
4070  }
4071 
4072  result = 1;
4073 end:
4074  if (alp_tctx != NULL)
4075  AppLayerParserThreadCtxFree(alp_tctx);
4077  HTPFreeConfig();
4078  ConfDeInit();
4081  UTHFreeFlow(f);
4082  return result;
4083 }
4084 
4085 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4086  */
4087 static int HTPParserTest10(void)
4088 {
4089  int result = 0;
4090  Flow *f = NULL;
4091  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4092  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4093  TcpSession ssn;
4094  HtpState *htp_state = NULL;
4095  int r = 0;
4097 
4098  memset(&ssn, 0, sizeof(ssn));
4099 
4100  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4101  if (f == NULL)
4102  goto end;
4103  f->protoctx = &ssn;
4104  f->proto = IPPROTO_TCP;
4105  f->alproto = ALPROTO_HTTP;
4106 
4108 
4109  uint32_t u;
4110  for (u = 0; u < httplen1; u++) {
4111  uint8_t flags = 0;
4112 
4113  if (u == 0)
4114  flags = STREAM_TOSERVER|STREAM_START;
4115  else if (u == (httplen1 - 1))
4116  flags = STREAM_TOSERVER|STREAM_EOF;
4117  else
4118  flags = STREAM_TOSERVER;
4119 
4120  FLOWLOCK_WRLOCK(f);
4121  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4122  &httpbuf1[u], 1);
4123  if (r != 0) {
4124  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4125  " 0: ", u, r);
4126  FLOWLOCK_UNLOCK(f);
4127  goto end;
4128  }
4129  FLOWLOCK_UNLOCK(f);
4130  }
4131 
4132  htp_state = f->alstate;
4133  if (htp_state == NULL) {
4134  printf("no http state: ");
4135  goto end;
4136  }
4137 
4138  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4139  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4140  if (h == NULL) {
4141  goto end;
4142  }
4143 
4144  char *name = bstr_util_strdup_to_c(h->name);
4145  if (name == NULL) {
4146  goto end;
4147  }
4148 
4149  if (strcmp(name, "Host") != 0) {
4150  printf("header name not \"Host\", instead \"%s\": ", name);
4151  free(name);
4152  goto end;
4153  }
4154  free(name);
4155 
4156  char *value = bstr_util_strdup_to_c(h->value);
4157  if (value == NULL) {
4158  goto end;
4159  }
4160 
4161  if (strcmp(value, "www.google.com") != 0) {
4162  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4163  free(value);
4164  goto end;
4165  }
4166  free(value);
4167 
4168  result = 1;
4169 end:
4170  if (alp_tctx != NULL)
4171  AppLayerParserThreadCtxFree(alp_tctx);
4173  UTHFreeFlow(f);
4174  return result;
4175 }
4176 
4177 /** \test double encoding in path
4178  */
4179 static int HTPParserTest11(void)
4180 {
4181  int result = 0;
4182  Flow *f = NULL;
4183  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4184  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4185  TcpSession ssn;
4186  HtpState *htp_state = NULL;
4187  int r = 0;
4189 
4190  memset(&ssn, 0, sizeof(ssn));
4191 
4192  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4193  if (f == NULL)
4194  goto end;
4195  f->protoctx = &ssn;
4196  f->proto = IPPROTO_TCP;
4197  f->alproto = ALPROTO_HTTP;
4198 
4200 
4201  uint32_t u;
4202  for (u = 0; u < httplen1; u++) {
4203  uint8_t flags = 0;
4204 
4205  if (u == 0)
4206  flags = STREAM_TOSERVER|STREAM_START;
4207  else if (u == (httplen1 - 1))
4208  flags = STREAM_TOSERVER|STREAM_EOF;
4209  else
4210  flags = STREAM_TOSERVER;
4211 
4212  FLOWLOCK_WRLOCK(f);
4213  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4214  &httpbuf1[u], 1);
4215  if (r != 0) {
4216  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4217  " 0: ", u, r);
4218  FLOWLOCK_UNLOCK(f);
4219  goto end;
4220  }
4221  FLOWLOCK_UNLOCK(f);
4222  }
4223 
4224  htp_state = f->alstate;
4225  if (htp_state == NULL) {
4226  printf("no http state: ");
4227  goto end;
4228  }
4229 
4230  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4231  if (tx == NULL)
4232  goto end;
4233  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4234  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4235  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4236  printf("normalized uri len should be 2, is %"PRIuMAX,
4237  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4238  goto end;
4239  }
4240 
4241  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4242  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4243  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4244  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4245  {
4246  printf("normalized uri \"");
4247  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4248  printf("\": ");
4249  goto end;
4250  }
4251  }
4252 
4253  result = 1;
4254 end:
4255  if (alp_tctx != NULL)
4256  AppLayerParserThreadCtxFree(alp_tctx);
4258  UTHFreeFlow(f);
4259  return result;
4260 }
4261 
4262 /** \test double encoding in query
4263  */
4264 static int HTPParserTest12(void)
4265 {
4266  int result = 0;
4267  Flow *f = NULL;
4268  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4269  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4270  TcpSession ssn;
4271  HtpState *htp_state = NULL;
4272  int r = 0;
4274 
4275  memset(&ssn, 0, sizeof(ssn));
4276 
4277  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4278  if (f == NULL)
4279  goto end;
4280  f->protoctx = &ssn;
4281  f->proto = IPPROTO_TCP;
4282  f->alproto = ALPROTO_HTTP;
4283 
4285 
4286  uint32_t u;
4287  for (u = 0; u < httplen1; u++) {
4288  uint8_t flags = 0;
4289 
4290  if (u == 0)
4291  flags = STREAM_TOSERVER|STREAM_START;
4292  else if (u == (httplen1 - 1))
4293  flags = STREAM_TOSERVER|STREAM_EOF;
4294  else
4295  flags = STREAM_TOSERVER;
4296 
4297  FLOWLOCK_WRLOCK(f);
4298  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4299  &httpbuf1[u], 1);
4300  if (r != 0) {
4301  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4302  " 0: ", u, r);
4303  FLOWLOCK_UNLOCK(f);
4304  goto end;
4305  }
4306  FLOWLOCK_UNLOCK(f);
4307  }
4308 
4309  htp_state = f->alstate;
4310  if (htp_state == NULL) {
4311  printf("no http state: ");
4312  goto end;
4313  }
4314 
4315  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4316  if (tx == NULL)
4317  goto end;
4318  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4319  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4320  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4321  printf("normalized uri len should be 5, is %"PRIuMAX,
4322  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4323  goto end;
4324  }
4325 
4326  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4327  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4328  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4329  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4330  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4331  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4332  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4333  {
4334  printf("normalized uri \"");
4335  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4336  printf("\": ");
4337  goto end;
4338  }
4339  }
4340 
4341  result = 1;
4342  end:
4343  if (alp_tctx != NULL)
4344  AppLayerParserThreadCtxFree(alp_tctx);
4346  UTHFreeFlow(f);
4347  return result;
4348 }
4349 
4350 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4351  */
4352 static int HTPParserTest13(void)
4353 {
4354  int result = 0;
4355  Flow *f = NULL;
4356  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4357  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4358  TcpSession ssn;
4359  HtpState *htp_state = NULL;
4360  int r = 0;
4362 
4363  memset(&ssn, 0, sizeof(ssn));
4364 
4365  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4366  if (f == NULL)
4367  goto end;
4368  f->protoctx = &ssn;
4369  f->proto = IPPROTO_TCP;
4370  f->alproto = ALPROTO_HTTP;
4371 
4373 
4374  uint32_t u;
4375  for (u = 0; u < httplen1; u++) {
4376  uint8_t flags = 0;
4377 
4378  if (u == 0)
4379  flags = STREAM_TOSERVER|STREAM_START;
4380  else if (u == (httplen1 - 1))
4381  flags = STREAM_TOSERVER|STREAM_EOF;
4382  else
4383  flags = STREAM_TOSERVER;
4384 
4385  FLOWLOCK_WRLOCK(f);
4386  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4387  &httpbuf1[u], 1);
4388  if (r != 0) {
4389  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4390  " 0: ", u, r);
4391  FLOWLOCK_UNLOCK(f);
4392  goto end;
4393  }
4394  FLOWLOCK_UNLOCK(f);
4395  }
4396 
4397  htp_state = f->alstate;
4398  if (htp_state == NULL) {
4399  printf("no http state: ");
4400  goto end;
4401  }
4402 
4403  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4404  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4405  if (h == NULL) {
4406  goto end;
4407  }
4408 
4409  char *name = bstr_util_strdup_to_c(h->name);
4410  if (name == NULL) {
4411  goto end;
4412  }
4413 
4414  if (strcmp(name, "Host") != 0) {
4415  printf("header name not \"Host\", instead \"%s\": ", name);
4416  free(name);
4417  goto end;
4418  }
4419  free(name);
4420 
4421  char *value = bstr_util_strdup_to_c(h->value);
4422  if (value == NULL) {
4423  goto end;
4424  }
4425 
4426  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4427  printf("header value not \"www.google.com\", instead \"");
4428  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4429  printf("\": ");
4430  free(value);
4431  goto end;
4432  }
4433  free(value);
4434 
4435  result = 1;
4436 end:
4437  if (alp_tctx != NULL)
4438  AppLayerParserThreadCtxFree(alp_tctx);
4440  UTHFreeFlow(f);
4441  return result;
4442 }
4443 
4444 /** \test Test basic config */
4445 static int HTPParserConfigTest01(void)
4446 {
4447  int ret = 0;
4448  char input[] = "\
4449 %YAML 1.1\n\
4450 ---\n\
4451 libhtp:\n\
4452 \n\
4453  default-config:\n\
4454  personality: IDS\n\
4455 \n\
4456  server-config:\n\
4457 \n\
4458  - apache-tomcat:\n\
4459  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4460  personality: Tomcat_6_0\n\
4461 \n\
4462  - iis7:\n\
4463  address: \n\
4464  - 192.168.0.0/24\n\
4465  - 192.168.10.0/24\n\
4466  personality: IIS_7_0\n\
4467 ";
4468 
4470  ConfInit();
4471 
4472  ConfYamlLoadString(input, strlen(input));
4473 
4474  ConfNode *outputs;
4475  outputs = ConfGetNode("libhtp.default-config.personality");
4476  if (outputs == NULL) {
4477  goto end;
4478  }
4479 
4480  outputs = ConfGetNode("libhtp.server-config");
4481  if (outputs == NULL) {
4482  goto end;
4483  }
4484 
4485  ConfNode *node = TAILQ_FIRST(&outputs->head);
4486  if (node == NULL) {
4487  goto end;
4488  }
4489  if (strcmp(node->name, "0") != 0) {
4490  goto end;
4491  }
4492  node = TAILQ_FIRST(&node->head);
4493  if (node == NULL) {
4494  goto end;
4495  }
4496  if (strcmp(node->name, "apache-tomcat") != 0) {
4497  goto end;
4498  }
4499 
4500  int i = 0;
4501  ConfNode *n;
4502 
4503  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4504  if (node2 == NULL) {
4505  goto end;
4506  }
4507  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4508  goto end;
4509  }
4510 
4511  node = ConfNodeLookupChild(node, "address");
4512  if (node == NULL) {
4513  goto end;
4514  }
4515  TAILQ_FOREACH(n, &node->head, next) {
4516  if (n == NULL) {
4517  goto end;
4518  }
4519 
4520  switch(i) {
4521  case 0:
4522  if (strcmp(n->name, "0") != 0) {
4523  goto end;
4524  }
4525  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4526  goto end;
4527  }
4528  break;
4529  case 1:
4530  if (strcmp(n->name, "1") != 0) {
4531  goto end;
4532  }
4533  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4534  goto end;
4535  }
4536  break;
4537  case 2:
4538  if (strcmp(n->name, "2") != 0) {
4539  goto end;
4540  }
4541  if (strcmp(n->val, "::1") != 0) {
4542  goto end;
4543  }
4544  break;
4545  default:
4546  goto end;
4547  }
4548  i++;
4549  }
4550 
4551  outputs = ConfGetNode("libhtp.server-config");
4552  if (outputs == NULL) {
4553  goto end;
4554  }
4555 
4556  node = TAILQ_FIRST(&outputs->head);
4557  node = TAILQ_NEXT(node, next);
4558  if (node == NULL) {
4559  goto end;
4560  }
4561  if (strcmp(node->name, "1") != 0) {
4562  goto end;
4563  }
4564  node = TAILQ_FIRST(&node->head);
4565  if (node == NULL) {
4566  goto end;
4567  }
4568  if (strcmp(node->name, "iis7") != 0) {
4569  goto end;
4570  }
4571 
4572  node2 = ConfNodeLookupChild(node, "personality");
4573  if (node2 == NULL) {
4574  goto end;
4575  }
4576  if (strcmp(node2->val, "IIS_7_0") != 0) {
4577  goto end;
4578  }
4579 
4580  node = ConfNodeLookupChild(node, "address");
4581  if (node == NULL) {
4582  goto end;
4583  }
4584 
4585  i = 0;
4586  TAILQ_FOREACH(n, &node->head, next) {
4587  if (n == NULL) {
4588  goto end;
4589  }
4590 
4591  switch(i) {
4592  case 0:
4593  if (strcmp(n->name, "0") != 0) {
4594  goto end;
4595  }
4596  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4597  goto end;
4598  }
4599  break;
4600  case 1:
4601  if (strcmp(n->name, "1") != 0) {
4602  goto end;
4603  }
4604  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4605  goto end;
4606  }
4607  break;
4608  default:
4609  goto end;
4610  }
4611  i++;
4612  }
4613 
4614  ret = 1;
4615 
4616 end:
4617  ConfDeInit();
4619 
4620  return ret;
4621 }
4622 
4623 /** \test Test config builds radix correctly */
4624 static int HTPParserConfigTest02(void)
4625 {
4626  int ret = 0;
4627  char input[] = "\
4628 %YAML 1.1\n\
4629 ---\n\
4630 libhtp:\n\
4631 \n\
4632  default-config:\n\
4633  personality: IDS\n\
4634 \n\
4635  server-config:\n\
4636 \n\
4637  - apache-tomcat:\n\
4638  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4639  personality: Tomcat_6_0\n\
4640 \n\
4641  - iis7:\n\
4642  address: \n\
4643  - 192.168.0.0/24\n\
4644  - 192.168.10.0/24\n\
4645  personality: IIS_7_0\n\
4646 ";
4647 
4649  ConfInit();
4651 
4652  ConfYamlLoadString(input, strlen(input));
4653 
4654  HTPConfigure();
4655 
4656  if (cfglist.cfg == NULL) {
4657  printf("No default config created.\n");
4658  goto end;
4659  }
4660 
4661  if (cfgtree == NULL) {
4662  printf("No config tree created.\n");
4663  goto end;
4664  }
4665 
4666  htp_cfg_t *htp = cfglist.cfg;
4667  uint8_t buf[128];
4668  const char *addr;
4669  void *user_data = NULL;
4670 
4671  addr = "192.168.10.42";
4672  if (inet_pton(AF_INET, addr, buf) == 1) {
4673  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4674  if (user_data != NULL) {
4675  HTPCfgRec *htp_cfg_rec = user_data;
4676  htp = htp_cfg_rec->cfg;
4677  SCLogDebug("LIBHTP using config: %p", htp);
4678  }
4679  if (htp == NULL) {
4680  printf("Could not get config for: %s\n", addr);
4681  goto end;
4682  }
4683  }
4684  else {
4685  printf("Failed to parse address: %s\n", addr);
4686  goto end;
4687  }
4688 
4689  user_data = NULL;
4690  addr = "::1";
4691  if (inet_pton(AF_INET6, addr, buf) == 1) {
4692  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4693  if (user_data != NULL) {
4694  HTPCfgRec *htp_cfg_rec = user_data;
4695  htp = htp_cfg_rec->cfg;
4696  SCLogDebug("LIBHTP using config: %p", htp);
4697  }
4698  if (htp == NULL) {
4699  printf("Could not get config for: %s\n", addr);
4700  goto end;
4701  }
4702  }
4703  else {
4704  printf("Failed to parse address: %s\n", addr);
4705  goto end;
4706  }
4707 
4708  ret = 1;
4709 
4710 end:
4711  HTPFreeConfig();
4712  ConfDeInit();
4715 
4716  return ret;
4717 }
4718 
4719 /** \test Test traffic is handled by the correct htp config */
4720 static int HTPParserConfigTest03(void)
4721 {
4722  int result = 1;
4723  Flow *f = NULL;
4724  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4725  " Data is c0oL!";
4726  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4727  TcpSession ssn;
4729 
4730  HtpState *htp_state = NULL;
4731  int r = 0;
4732  char input[] = "\
4733 %YAML 1.1\n\
4734 ---\n\
4735 libhtp:\n\
4736 \n\
4737  default-config:\n\
4738  personality: IDS\n\
4739 \n\
4740  server-config:\n\
4741 \n\
4742  - apache-tomcat:\n\
4743  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4744  personality: Tomcat_6_0\n\
4745 \n\
4746  - iis7:\n\
4747  address: \n\
4748  - 192.168.0.0/24\n\
4749  - 192.168.10.0/24\n\
4750  personality: IIS_7_0\n\
4751 ";
4752 
4754  ConfInit();
4756 
4757  ConfYamlLoadString(input, strlen(input));
4758 
4759  HTPConfigure();
4760 
4761  const char *addr = "192.168.10.42";
4762 
4763  memset(&ssn, 0, sizeof(ssn));
4764 
4765  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4766  if (f == NULL)
4767  goto end;
4768  f->protoctx = &ssn;
4769  f->proto = IPPROTO_TCP;
4770  f->alproto = ALPROTO_HTTP;
4771 
4772  htp_cfg_t *htp = cfglist.cfg;
4773 
4774  void *user_data = NULL;
4775  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4776  if (user_data != NULL) {
4777  HTPCfgRec *htp_cfg_rec = user_data;
4778  htp = htp_cfg_rec->cfg;
4779  SCLogDebug("LIBHTP using config: %p", htp);
4780  }
4781  if (htp == NULL) {
4782  printf("Could not get config for: %s\n", addr);
4783  goto end;
4784  }
4785 
4787 
4788  uint32_t u;
4789  for (u = 0; u < httplen1; u++) {
4790  uint8_t flags = 0;
4791 
4792  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4793  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4794  else flags = STREAM_TOSERVER;
4795 
4796  FLOWLOCK_WRLOCK(f);
4797  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4798  &httpbuf1[u], 1);
4799  if (r != 0) {
4800  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4801  " 0: ", u, r);
4802  result = 0;
4803  FLOWLOCK_UNLOCK(f);
4804  goto end;
4805  }
4806  FLOWLOCK_UNLOCK(f);
4807  }
4808 
4809  htp_state = f->alstate;
4810  if (htp_state == NULL) {
4811  printf("no http state: ");
4812  result = 0;
4813  goto end;
4814  }
4815 
4816  if (HTPStateGetTxCnt(htp_state) != 2) {
4817  printf("HTPStateGetTxCnt(htp_state) failure\n");
4818  goto end;
4819  }
4820 
4821  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4822  if (tx == NULL)
4823  goto end;
4824  if (tx->cfg != htp) {
4825  printf("wrong HTP config (%p instead of %p - default=%p): ",
4826  tx->cfg, htp, cfglist.cfg);
4827  goto end;
4828  }
4829  tx = HTPStateGetTx(htp_state, 1);
4830  if (tx == NULL)
4831  goto end;
4832  if (tx->cfg != htp) {
4833  printf("wrong HTP config (%p instead of %p - default=%p): ",
4834  tx->cfg, htp, cfglist.cfg);
4835  goto end;
4836  }
4837 
4838 end:
4839  if (alp_tctx != NULL)
4840  AppLayerParserThreadCtxFree(alp_tctx);
4841  HTPFreeConfig();
4842  ConfDeInit();
4845 
4847  UTHFreeFlow(f);
4848  return result;
4849 }
4850 
4851 /* disabled when we upgraded to libhtp 0.5.x */
4852 #if 0
4853 static int HTPParserConfigTest04(void)
4854 {
4855  int result = 0;
4856 
4857  char input[] = "\
4858 %YAML 1.1\n\
4859 ---\n\
4860 libhtp:\n\
4861 \n\
4862  default-config:\n\
4863  personality: IDS\n\
4864  path-control-char-handling: status_400\n\
4865  path-convert-utf8: yes\n\
4866  path-invalid-encoding-handling: remove_percent\n\
4867 \n\
4868  server-config:\n\
4869 \n\
4870  - apache-tomcat:\n\
4871  personality: Tomcat_6_0\n\
4872  path-invalid-utf8-handling: none\n\
4873  path-nul-encoded-handling: status_404\n\
4874  path-nul-raw-handling: status_400\n\
4875 \n\
4876  - iis7:\n\
4877  personality: IIS_7_0\n\
4878  path-replacement-char: o\n\
4879  path-unicode-mapping: status_400\n\
4880 ";
4881 
4883  ConfInit();
4885 
4886  ConfYamlLoadString(input, strlen(input));
4887 
4888  HTPConfigure();
4889 
4890  HTPCfgRec *cfg_rec = &cfglist;
4891  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4892  cfg_rec->cfg->path_convert_utf8 != 1 ||
4893  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4894  printf("failed 1\n");
4895  goto end;
4896  }
4897 
4898  cfg_rec = cfg_rec->next;
4899  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4900  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4901  printf("failed 2\n");
4902  goto end;
4903  }
4904 
4905  cfg_rec = cfg_rec->next;
4906  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4907  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4908  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4909  printf("failed 3\n");
4910  goto end;
4911  }
4912 
4913  result = 1;
4914 
4915 end:
4916  HTPFreeConfig();
4917  ConfDeInit();
4920 
4921  return result;
4922 }
4923 #endif
4924 
4925 /** \test Test %2f decoding in profile Apache_2_2
4926  *
4927  * %2f in path is left untouched
4928  * %2f in query string is normalized to %2F
4929  * %252f in query string is decoded/normalized to %2F
4930  */
4931 static int HTPParserDecodingTest01(void)
4932 {
4933  int result = 0;
4934  Flow *f = NULL;
4935  uint8_t httpbuf1[] =
4936  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4937  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4938  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4939  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4940  TcpSession ssn;
4942 
4943  HtpState *htp_state = NULL;
4944  int r = 0;
4945  char input[] = "\
4946 %YAML 1.1\n\
4947 ---\n\
4948 libhtp:\n\
4949 \n\
4950  default-config:\n\
4951  personality: Apache_2\n\
4952 ";
4953 
4955  ConfInit();
4957  ConfYamlLoadString(input, strlen(input));
4958  HTPConfigure();
4959  const char *addr = "4.3.2.1";
4960  memset(&ssn, 0, sizeof(ssn));
4961 
4962  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4963  if (f == NULL)
4964  goto end;
4965  f->protoctx = &ssn;
4966  f->proto = IPPROTO_TCP;
4967  f->alproto = ALPROTO_HTTP;
4968 
4970 
4971  uint32_t u;
4972  for (u = 0; u < httplen1; u++) {
4973  uint8_t flags = 0;
4974 
4975  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4976  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4977  else flags = STREAM_TOSERVER;
4978 
4979  FLOWLOCK_WRLOCK(f);
4980  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4981  &httpbuf1[u], 1);
4982  if (r != 0) {
4983  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4984  " 0: ", u, r);
4985  FLOWLOCK_UNLOCK(f);
4986  goto end;
4987  }
4988  FLOWLOCK_UNLOCK(f);
4989  }
4990 
4991  htp_state = f->alstate;
4992  if (htp_state == NULL) {
4993  printf("no http state: ");
4994  goto end;
4995  }
4996 
4997  uint8_t ref1[] = "/abc%2fdef";
4998  size_t reflen = sizeof(ref1) - 1;
4999 
5000  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5001  if (tx == NULL)
5002  goto end;
5003  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5004  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5005  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5006  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5007  (uintmax_t)reflen,
5008  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5009  goto end;
5010  }
5011 
5012  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5013  bstr_len(tx_ud->request_uri_normalized)) != 0)
5014  {
5015  printf("normalized uri \"");
5016  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5017  printf("\" != \"");
5018  PrintRawUriFp(stdout, ref1, reflen);
5019  printf("\": ");
5020  goto end;
5021  }
5022  }
5023 
5024  uint8_t ref2[] = "/abc/def?ghi/jkl";
5025  reflen = sizeof(ref2) - 1;
5026 
5027  tx = HTPStateGetTx(htp_state, 1);
5028  if (tx == NULL)
5029  goto end;
5030  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5031  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5032  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5033  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5034  (uintmax_t)reflen,
5035  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5036  goto end;
5037  }
5038 
5039  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5040  bstr_len(tx_ud->request_uri_normalized)) != 0)
5041  {
5042  printf("normalized uri \"");
5043  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5044  printf("\" != \"");
5045  PrintRawUriFp(stdout, ref2, reflen);
5046  printf("\": ");
5047  goto end;
5048  }
5049  }
5050 
5051  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5052  reflen = sizeof(ref3) - 1;
5053  tx = HTPStateGetTx(htp_state, 2);
5054  if (tx == NULL)
5055  goto end;
5056  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5057  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5058  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5059  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5060  (uintmax_t)reflen,
5061  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5062  goto end;
5063  }
5064 
5065  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5066  bstr_len(tx_ud->request_uri_normalized)) != 0)
5067  {
5068  printf("normalized uri \"");
5069  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5070  printf("\" != \"");
5071  PrintRawUriFp(stdout, ref3, reflen);
5072  printf("\": ");
5073  goto end;
5074  }
5075  }
5076 
5077  result = 1;
5078 
5079 end:
5080  if (alp_tctx != NULL)
5081  AppLayerParserThreadCtxFree(alp_tctx);
5082  HTPFreeConfig();
5083  ConfDeInit();
5086 
5088  UTHFreeFlow(f);
5089  return result;
5090 }
5091 
5092 /** \test Test %2f decoding in profile IDS
5093  *
5094  * %2f in path decoded to /
5095  * %2f in query string is decoded to /
5096  * %252f in query string is decoded to %2F
5097  */
5098 static int HTPParserDecodingTest02(void)
5099 {
5100  int result = 0;
5101  Flow *f = NULL;
5102  uint8_t httpbuf1[] =
5103  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5104  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5105  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5106  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5107  TcpSession ssn;
5109 
5110  HtpState *htp_state = NULL;
5111  int r = 0;
5112  char input[] = "\
5113 %YAML 1.1\n\
5114 ---\n\
5115 libhtp:\n\
5116 \n\
5117  default-config:\n\
5118  personality: IDS\n\
5119  double-decode-path: no\n\
5120  double-decode-query: no\n\
5121 ";
5122 
5124  ConfInit();
5126  ConfYamlLoadString(input, strlen(input));
5127  HTPConfigure();
5128  const char *addr = "4.3.2.1";
5129  memset(&ssn, 0, sizeof(ssn));
5130 
5131  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5132  if (f == NULL)
5133  goto end;
5134  f->protoctx = &ssn;
5135  f->proto = IPPROTO_TCP;
5136  f->alproto = ALPROTO_HTTP;
5137 
5139 
5140  uint32_t u;
5141  for (u = 0; u < httplen1; u++) {
5142  uint8_t flags = 0;
5143 
5144  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5145  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5146  else flags = STREAM_TOSERVER;
5147 
5148  FLOWLOCK_WRLOCK(f);
5149  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5150  &httpbuf1[u], 1);
5151  if (r != 0) {
5152  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5153  " 0: ", u, r);
5154  FLOWLOCK_UNLOCK(f);
5155  goto end;
5156  }
5157  FLOWLOCK_UNLOCK(f);
5158  }
5159 
5160  htp_state = f->alstate;
5161  if (htp_state == NULL) {
5162  printf("no http state: ");
5163  goto end;
5164  }
5165 
5166  uint8_t ref1[] = "/abc/def";
5167  size_t reflen = sizeof(ref1) - 1;
5168 
5169  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5170  if (tx == NULL)
5171  goto end;
5172  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5173  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5174  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5175  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5176  (uintmax_t)reflen,
5177  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5178  goto end;
5179  }
5180 
5181  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5182  bstr_len(tx_ud->request_uri_normalized)) != 0)
5183  {
5184  printf("normalized uri \"");
5185  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5186  printf("\" != \"");
5187  PrintRawUriFp(stdout, ref1, reflen);
5188  printf("\": ");
5189  goto end;
5190  }
5191  }
5192 
5193  uint8_t ref2[] = "/abc/def?ghi/jkl";
5194  reflen = sizeof(ref2) - 1;
5195 
5196  tx = HTPStateGetTx(htp_state, 1);
5197  if (tx == NULL)
5198  goto end;
5199  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5200  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5201  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5202  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5203  (uintmax_t)reflen,
5204  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5205  goto end;
5206  }
5207 
5208  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5209  bstr_len(tx_ud->request_uri_normalized)) != 0)
5210  {
5211  printf("normalized uri \"");
5212  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5213  printf("\" != \"");
5214  PrintRawUriFp(stdout, ref2, reflen);
5215  printf("\": ");
5216  goto end;
5217  }
5218  }
5219 
5220  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5221  reflen = sizeof(ref3) - 1;
5222  tx = HTPStateGetTx(htp_state, 2);
5223  if (tx == NULL)
5224  goto end;
5225  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5226  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5227  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5228  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5229  (uintmax_t)reflen,
5230  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5231  goto end;
5232  }
5233 
5234  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5235  bstr_len(tx_ud->request_uri_normalized)) != 0)
5236  {
5237  printf("normalized uri \"");
5238  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5239  printf("\" != \"");
5240  PrintRawUriFp(stdout, ref3, reflen);
5241  printf("\": ");
5242  goto end;
5243  }
5244  }
5245 
5246  result = 1;
5247 
5248 end:
5249  if (alp_tctx != NULL)
5250  AppLayerParserThreadCtxFree(alp_tctx);
5251  HTPFreeConfig();
5252  ConfDeInit();
5255 
5257  UTHFreeFlow(f);
5258  return result;
5259 }
5260 
5261 /** \test Test %2f decoding in profile IDS with double-decode-* options
5262  *
5263  * %252f in path decoded to /
5264  * %252f in query string is decoded to /
5265  */
5266 static int HTPParserDecodingTest03(void)
5267 {
5268  int result = 0;
5269  Flow *f = NULL;
5270  uint8_t httpbuf1[] =
5271  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5272  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5273  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5274  TcpSession ssn;
5276 
5277  HtpState *htp_state = NULL;
5278  int r = 0;
5279  char input[] = "\
5280 %YAML 1.1\n\
5281 ---\n\
5282 libhtp:\n\
5283 \n\
5284  default-config:\n\
5285  personality: IDS\n\
5286  double-decode-path: yes\n\
5287  double-decode-query: yes\n\
5288 ";
5289 
5291  ConfInit();
5293  ConfYamlLoadString(input, strlen(input));
5294  HTPConfigure();
5295  const char *addr = "4.3.2.1";
5296  memset(&ssn, 0, sizeof(ssn));
5297 
5298  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5299  if (f == NULL)
5300  goto end;
5301  f->protoctx = &ssn;
5302  f->proto = IPPROTO_TCP;
5303  f->alproto = ALPROTO_HTTP;
5304 
5306 
5307  uint32_t u;
5308  for (u = 0; u < httplen1; u++) {
5309  uint8_t flags = 0;
5310 
5311  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5312  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5313  else flags = STREAM_TOSERVER;
5314 
5315  FLOWLOCK_WRLOCK(f);
5316  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5317  &httpbuf1[u], 1);
5318  if (r != 0) {
5319  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5320  " 0: ", u, r);
5321  FLOWLOCK_UNLOCK(f);
5322  goto end;
5323  }
5324  FLOWLOCK_UNLOCK(f);
5325  }
5326 
5327  htp_state = f->alstate;
5328  if (htp_state == NULL) {
5329  printf("no http state: ");
5330  goto end;
5331  }
5332 
5333  uint8_t ref1[] = "/abc/def";
5334  size_t reflen = sizeof(ref1) - 1;
5335 
5336  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5337  if (tx == NULL)
5338  goto end;
5339  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5340  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5341  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5342  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5343  (uintmax_t)reflen,
5344  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5345  goto end;
5346  }
5347 
5348  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5349  bstr_len(tx_ud->request_uri_normalized)) != 0)
5350  {
5351  printf("normalized uri \"");
5352  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5353  printf("\" != \"");
5354  PrintRawUriFp(stdout, ref1, reflen);
5355  printf("\": ");
5356  goto end;
5357  }
5358  }
5359 
5360  uint8_t ref2[] = "/abc/def?ghi/jkl";
5361  reflen = sizeof(ref2) - 1;
5362 
5363  tx = HTPStateGetTx(htp_state, 1);
5364  if (tx == NULL)
5365  goto end;
5366  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5367  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5368  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5369  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5370  (uintmax_t)reflen,
5371  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5372  goto end;
5373  }
5374 
5375  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5376  bstr_len(tx_ud->request_uri_normalized)) != 0)
5377  {
5378  printf("normalized uri \"");
5379  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5380  printf("\" != \"");
5381  PrintRawUriFp(stdout, ref2, reflen);
5382  printf("\": ");
5383  goto end;
5384  }
5385  }
5386 
5387  result = 1;
5388 
5389 end:
5390  if (alp_tctx != NULL)
5391  AppLayerParserThreadCtxFree(alp_tctx);
5392  HTPFreeConfig();
5393  ConfDeInit();
5396 
5398  UTHFreeFlow(f);
5399  return result;
5400 }
5401 
5402 /** \test Test http:// in query profile IDS
5403  */
5404 static int HTPParserDecodingTest04(void)
5405 {
5406  int result = 0;
5407  Flow *f = NULL;
5408  uint8_t httpbuf1[] =
5409  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5410  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5411  TcpSession ssn;
5413 
5414  HtpState *htp_state = NULL;
5415  int r = 0;
5416  char input[] = "\
5417 %YAML 1.1\n\
5418 ---\n\
5419 libhtp:\n\
5420 \n\
5421  default-config:\n\
5422  personality: IDS\n\
5423  double-decode-path: yes\n\
5424  double-decode-query: yes\n\
5425 ";
5426 
5428  ConfInit();
5430  ConfYamlLoadString(input, strlen(input));
5431  HTPConfigure();
5432  const char *addr = "4.3.2.1";
5433  memset(&ssn, 0, sizeof(ssn));
5434 
5435  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5436  if (f == NULL)
5437  goto end;
5438  f->protoctx = &ssn;
5439  f->proto = IPPROTO_TCP;
5440  f->alproto = ALPROTO_HTTP;
5441 
5443 
5444  uint32_t u;
5445  for (u = 0; u < httplen1; u++) {
5446  uint8_t flags = 0;
5447 
5448  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5449  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5450  else flags = STREAM_TOSERVER;
5451 
5452  FLOWLOCK_WRLOCK(f);
5453  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5454  &httpbuf1[u], 1);
5455  if (r != 0) {
5456  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5457  " 0: ", u, r);
5458  FLOWLOCK_UNLOCK(f);
5459  goto end;
5460  }
5461  FLOWLOCK_UNLOCK(f);
5462  }
5463 
5464  htp_state = f->alstate;
5465  if (htp_state == NULL) {
5466  printf("no http state: ");
5467  goto end;
5468  }
5469 
5470  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5471  size_t reflen = sizeof(ref1) - 1;
5472 
5473  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5474  if (tx == NULL)
5475  goto end;
5476  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5477  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5478  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5479  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5480  (uintmax_t)reflen,
5481  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5482  goto end;
5483  }
5484 
5485  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5486  bstr_len(tx_ud->request_uri_normalized)) != 0)
5487  {
5488  printf("normalized uri \"");
5489  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5490  printf("\" != \"");
5491  PrintRawUriFp(stdout, ref1, reflen);
5492  printf("\": ");
5493  goto end;
5494  }
5495  }
5496 
5497  result = 1;
5498 
5499 end:
5500  if (alp_tctx != NULL)
5501  AppLayerParserThreadCtxFree(alp_tctx);
5502  HTPFreeConfig();
5503  ConfDeInit();
5506 
5508  UTHFreeFlow(f);
5509  return result;
5510 }
5511 
5512 /** \test Test \ char in query profile IDS. Bug 739
5513  */
5514 static int HTPParserDecodingTest05(void)
5515 {
5516  int result = 0;
5517  Flow *f = NULL;
5518  uint8_t httpbuf1[] =
5519  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5520  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5521  TcpSession ssn;
5523 
5524  HtpState *htp_state = NULL;
5525  int r = 0;
5526  char input[] = "\
5527 %YAML 1.1\n\
5528 ---\n\
5529 libhtp:\n\
5530 \n\
5531  default-config:\n\
5532  personality: IDS\n\
5533  double-decode-path: yes\n\
5534  double-decode-query: yes\n\
5535 ";
5536 
5538  ConfInit();
5540  ConfYamlLoadString(input, strlen(input));
5541  HTPConfigure();
5542  const char *addr = "4.3.2.1";
5543  memset(&ssn, 0, sizeof(ssn));
5544 
5545  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5546  if (f == NULL)
5547  goto end;
5548  f->protoctx = &ssn;
5549  f->proto = IPPROTO_TCP;
5550  f->alproto = ALPROTO_HTTP;
5551 
5553 
5554  uint32_t u;
5555  for (u = 0; u < httplen1; u++) {
5556  uint8_t flags = 0;
5557 
5558  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5559  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5560  else flags = STREAM_TOSERVER;
5561 
5562  FLOWLOCK_WRLOCK(f);
5563  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5564  &httpbuf1[u], 1);
5565  if (r != 0) {
5566  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5567  " 0: ", u, r);
5568  FLOWLOCK_UNLOCK(f);
5569  goto end;
5570  }
5571  FLOWLOCK_UNLOCK(f);
5572  }
5573 
5574  htp_state = f->alstate;
5575  if (htp_state == NULL) {
5576  printf("no http state: ");
5577  goto end;
5578  }
5579 
5580  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5581  size_t reflen = sizeof(ref1) - 1;
5582 
5583  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5584  if (tx == NULL)
5585  goto end;
5586  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5587  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5588  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5589  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5590  (uintmax_t)reflen,
5591  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5592  goto end;
5593  }
5594 
5595  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5596  bstr_len(tx_ud->request_uri_normalized)) != 0)
5597  {
5598  printf("normalized uri \"");
5599  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5600  printf("\" != \"");
5601  PrintRawUriFp(stdout, ref1, reflen);
5602  printf("\": ");
5603  goto end;
5604  }
5605  }
5606 
5607  result = 1;
5608 
5609 end:
5610  if (alp_tctx != NULL)
5611  AppLayerParserThreadCtxFree(alp_tctx);
5612  HTPFreeConfig();
5613  ConfDeInit();
5616 
5618  UTHFreeFlow(f);
5619  return result;
5620 }
5621 
5622 /** \test Test + char in query. Bug 1035
5623  */
5624 static int HTPParserDecodingTest06(void)
5625 {
5626  int result = 0;
5627  Flow *f = NULL;
5628  uint8_t httpbuf1[] =
5629  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5630  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5631  TcpSession ssn;
5633 
5634  HtpState *htp_state = NULL;
5635  int r = 0;
5636  char input[] = "\
5637 %YAML 1.1\n\
5638 ---\n\
5639 libhtp:\n\
5640 \n\
5641  default-config:\n\
5642  personality: IDS\n\
5643  double-decode-path: yes\n\
5644  double-decode-query: yes\n\
5645 ";
5646 
5648  ConfInit();
5650  ConfYamlLoadString(input, strlen(input));
5651  HTPConfigure();
5652  const char *addr = "4.3.2.1";
5653  memset(&ssn, 0, sizeof(ssn));
5654 
5655  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5656  if (f == NULL)
5657  goto end;
5658  f->protoctx = &ssn;
5659  f->proto = IPPROTO_TCP;
5660  f->alproto = ALPROTO_HTTP;
5661 
5663 
5664  uint32_t u;
5665  for (u = 0; u < httplen1; u++) {
5666  uint8_t flags = 0;
5667 
5668  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5669  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5670  else flags = STREAM_TOSERVER;
5671 
5672  FLOWLOCK_WRLOCK(f);
5673  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5674  &httpbuf1[u], 1);
5675  if (r != 0) {
5676  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5677  " 0: ", u, r);
5678  FLOWLOCK_UNLOCK(f);
5679  goto end;
5680  }
5681  FLOWLOCK_UNLOCK(f);
5682  }
5683 
5684  htp_state = f->alstate;
5685  if (htp_state == NULL) {
5686  printf("no http state: ");
5687  goto end;
5688  }
5689 
5690  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5691  size_t reflen = sizeof(ref1) - 1;
5692 
5693  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5694  if (tx == NULL)
5695  goto end;
5696  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5697  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5698  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5699  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5700  (uintmax_t)reflen,
5701  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5702  goto end;
5703  }
5704 
5705  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5706  bstr_len(tx_ud->request_uri_normalized)) != 0)
5707  {
5708  printf("normalized uri \"");
5709  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5710  printf("\" != \"");
5711  PrintRawUriFp(stdout, ref1, reflen);
5712  printf("\": ");
5713  goto end;
5714  }
5715  }
5716 
5717  result = 1;
5718 
5719 end:
5720  if (alp_tctx != NULL)
5721  AppLayerParserThreadCtxFree(alp_tctx);
5722  HTPFreeConfig();
5723  ConfDeInit();
5726 
5728  UTHFreeFlow(f);
5729  return result;
5730 }
5731 
5732 /** \test Test + char in query. Bug 1035
5733  */
5734 static int HTPParserDecodingTest07(void)
5735 {
5736  int result = 0;
5737  Flow *f = NULL;
5738  uint8_t httpbuf1[] =
5739  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5740  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5741  TcpSession ssn;
5743 
5744  HtpState *htp_state = NULL;
5745  int r = 0;
5746  char input[] = "\
5747 %YAML 1.1\n\
5748 ---\n\
5749 libhtp:\n\
5750 \n\
5751  default-config:\n\
5752  personality: IDS\n\
5753  double-decode-path: yes\n\
5754  double-decode-query: yes\n\
5755  query-plusspace-decode: yes\n\
5756 ";
5757 
5759  ConfInit();
5761  ConfYamlLoadString(input, strlen(input));
5762  HTPConfigure();
5763  const char *addr = "4.3.2.1";
5764  memset(&ssn, 0, sizeof(ssn));
5765 
5766  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5767  if (f == NULL)
5768  goto end;
5769  f->protoctx = &ssn;
5770  f->proto = IPPROTO_TCP;
5771  f->alproto = ALPROTO_HTTP;
5772 
5774 
5775  uint32_t u;
5776  for (u = 0; u < httplen1; u++) {
5777  uint8_t flags = 0;
5778 
5779  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5780  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5781  else flags = STREAM_TOSERVER;
5782 
5783  FLOWLOCK_WRLOCK(f);
5784  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5785  &httpbuf1[u], 1);
5786  if (r != 0) {
5787  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5788  " 0: ", u, r);
5789  FLOWLOCK_UNLOCK(f);
5790  goto end;
5791  }
5792  FLOWLOCK_UNLOCK(f);
5793  }
5794 
5795  htp_state = f->alstate;
5796  if (htp_state == NULL) {
5797  printf("no http state: ");
5798  goto end;
5799  }
5800 
5801  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5802  size_t reflen = sizeof(ref1) - 1;
5803 
5804  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5805  if (tx == NULL)
5806  goto end;
5807  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5808  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5809  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5810  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5811  (uintmax_t)reflen,
5812  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5813  goto end;
5814  }
5815 
5816  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5817  bstr_len(tx_ud->request_uri_normalized)) != 0)
5818  {
5819  printf("normalized uri \"");
5820  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5821  printf("\" != \"");
5822  PrintRawUriFp(stdout, ref1, reflen);
5823  printf("\": ");
5824  goto end;
5825  }
5826  }
5827 
5828  result = 1;
5829 
5830 end:
5831  if (alp_tctx != NULL)
5832  AppLayerParserThreadCtxFree(alp_tctx);
5833  HTPFreeConfig();
5834  ConfDeInit();
5837 
5839  UTHFreeFlow(f);
5840  return result;
5841 }
5842 
5843 /** \test Test 'proxy' URI normalization. Ticket 1008
5844  */
5845 static int HTPParserDecodingTest08(void)
5846 {
5847  int result = 0;
5848  Flow *f = NULL;
5849  uint8_t httpbuf1[] =
5850  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5851  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5852  TcpSession ssn;
5854 
5855  HtpState *htp_state = NULL;
5856  int r = 0;
5857  char input[] = "\
5858 %YAML 1.1\n\
5859 ---\n\
5860 libhtp:\n\
5861 \n\
5862  default-config:\n\
5863  personality: IDS\n\
5864 ";
5865 
5867  ConfInit();
5869  ConfYamlLoadString(input, strlen(input));
5870  HTPConfigure();
5871  const char *addr = "4.3.2.1";
5872  memset(&ssn, 0, sizeof(ssn));
5873 
5874  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5875  if (f == NULL)
5876  goto end;
5877  f->protoctx = &ssn;
5878  f->proto = IPPROTO_TCP;
5879  f->alproto = ALPROTO_HTTP;
5880 
5882 
5883  uint32_t u;
5884  for (u = 0; u < httplen1; u++) {
5885  uint8_t flags = 0;
5886 
5887  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5888  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5889  else flags = STREAM_TOSERVER;
5890 
5891  FLOWLOCK_WRLOCK(f);
5892  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5893  &httpbuf1[u], 1);
5894  if (r != 0) {
5895  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5896  " 0: ", u, r);
5897  FLOWLOCK_UNLOCK(f);
5898  goto end;
5899  }
5900  FLOWLOCK_UNLOCK(f);
5901  }
5902 
5903  htp_state = f->alstate;
5904  if (htp_state == NULL) {
5905  printf("no http state: ");
5906  goto end;
5907  }
5908 
5909  uint8_t ref1[] = "/blah/";
5910  size_t reflen = sizeof(ref1) - 1;
5911 
5912  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5913  if (tx == NULL)
5914  goto end;
5915  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5916  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5917  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5918  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5919  (uintmax_t)reflen,
5920  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5921  goto end;
5922  }
5923 
5924  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5925  bstr_len(tx_ud->request_uri_normalized)) != 0)
5926  {
5927  printf("normalized uri \"");
5928  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5929  printf("\" != \"");
5930  PrintRawUriFp(stdout, ref1, reflen);
5931  printf("\": ");
5932  goto end;
5933  }
5934  }
5935 
5936  result = 1;
5937 
5938 end:
5939  if (alp_tctx != NULL)
5940  AppLayerParserThreadCtxFree(alp_tctx);
5941  HTPFreeConfig();
5942  ConfDeInit();
5944