suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-htp.h"
59 #include "app-layer-htp-body.h"
60 #include "app-layer-htp-file.h"
61 #include "app-layer-htp-libhtp.h"
62 #include "app-layer-htp-xff.h"
63 
64 #include "util-spm.h"
65 #include "util-debug.h"
66 #include "util-time.h"
67 #include "util-misc.h"
68 
69 #include "util-unittest.h"
70 #include "util-unittest-helper.h"
71 #include "flow-util.h"
72 
73 #include "detect-engine.h"
74 #include "detect-engine-state.h"
75 #include "detect-parse.h"
76 
77 #include "decode-events.h"
78 
79 #include "util-memcmp.h"
80 #include "util-random.h"
81 #include "util-validate.h"
82 
83 //#define PRINT
84 
85 /** Fast lookup tree (radix) for the various HTP configurations */
86 static SCRadixTree *cfgtree;
87 /** List of HTP configurations. */
88 static HTPCfgRec cfglist;
89 
90 #ifdef DEBUG
91 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
92 static uint64_t htp_state_memuse = 0;
93 static uint64_t htp_state_memcnt = 0;
94 #endif
95 
97  { "UNKNOWN_ERROR",
99  { "GZIP_DECOMPRESSION_FAILED",
101  { "REQUEST_FIELD_MISSING_COLON",
103  { "RESPONSE_FIELD_MISSING_COLON",
105  { "INVALID_REQUEST_CHUNK_LEN",
107  { "INVALID_RESPONSE_CHUNK_LEN",
109  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
111  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
113  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
115  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
117  { "100_CONTINUE_ALREADY_SEEN",
119  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
121  { "INVALID_SERVER_PORT_IN_REQUEST",
123  { "INVALID_AUTHORITY_PORT",
125  { "REQUEST_HEADER_INVALID",
127  { "RESPONSE_HEADER_INVALID",
129  { "MISSING_HOST_HEADER",
131  { "HOST_HEADER_AMBIGUOUS",
133  { "INVALID_REQUEST_FIELD_FOLDING",
135  { "INVALID_RESPONSE_FIELD_FOLDING",
137  { "REQUEST_FIELD_TOO_LONG",
139  { "RESPONSE_FIELD_TOO_LONG",
141  { "REQUEST_LINE_INVALID",
143  { "REQUEST_BODY_UNEXPECTED",
145  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
147  { "REQUEST_URI_HOST_INVALID",
149  { "REQUEST_HEADER_HOST_INVALID",
151  { "REQUEST_AUTH_UNRECOGNIZED",
153  { "REQUEST_HEADER_REPETITION",
155  { "RESPONSE_HEADER_REPETITION",
157  { "DOUBLE_ENCODED_URI",
159  { "URI_DELIM_NON_COMPLIANT",
161  { "METHOD_DELIM_NON_COMPLIANT",
163  { "REQUEST_LINE_LEADING_WHITESPACE",
165  { "TOO_MANY_ENCODING_LAYERS",
167  { "ABNORMAL_CE_HEADER",
169  { "RESPONSE_MULTIPART_BYTERANGES",
171  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
173  { "RESPONSE_CHUNKED_OLD_PROTO",
175  { "RESPONSE_INVALID_PROTOCOL",
177  { "RESPONSE_INVALID_STATUS",
179  { "REQUEST_LINE_INCOMPLETE",
181 
182  /* suricata warnings/errors */
183  { "MULTIPART_GENERIC_ERROR",
185  { "MULTIPART_NO_FILEDATA",
187  { "MULTIPART_INVALID_HEADER",
189 
190  { NULL, -1 },
191 };
192 
193 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
194 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
195 static uint64_t HTPStateGetTxCnt(void *alstate);
196 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction);
197 
198 #ifdef DEBUG
199 /**
200  * \internal
201  *
202  * \brief Lookup the HTP personality string from the numeric personality.
203  *
204  * \todo This needs to be a libhtp function.
205  */
206 static const char *HTPLookupPersonalityString(int p)
207 {
208 #define CASE_HTP_PERSONALITY_STRING(p) \
209  case HTP_SERVER_ ## p: return #p
210 
211  switch (p) {
212  CASE_HTP_PERSONALITY_STRING(MINIMAL);
213  CASE_HTP_PERSONALITY_STRING(GENERIC);
214  CASE_HTP_PERSONALITY_STRING(IDS);
215  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
216  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
217  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
218  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
219  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
220  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
221  CASE_HTP_PERSONALITY_STRING(APACHE_2);
222  }
223 
224  return NULL;
225 }
226 #endif /* DEBUG */
227 
228 /**
229  * \internal
230  *
231  * \brief Lookup the numeric HTP personality from a string.
232  *
233  * \todo This needs to be a libhtp function.
234  */
235 static int HTPLookupPersonality(const char *str)
236 {
237 #define IF_HTP_PERSONALITY_NUM(p) \
238  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
239 
240  IF_HTP_PERSONALITY_NUM(MINIMAL);
241  IF_HTP_PERSONALITY_NUM(GENERIC);
243  IF_HTP_PERSONALITY_NUM(IIS_4_0);
244  IF_HTP_PERSONALITY_NUM(IIS_5_0);
245  IF_HTP_PERSONALITY_NUM(IIS_5_1);
246  IF_HTP_PERSONALITY_NUM(IIS_6_0);
247  IF_HTP_PERSONALITY_NUM(IIS_7_0);
248  IF_HTP_PERSONALITY_NUM(IIS_7_5);
249  IF_HTP_PERSONALITY_NUM(APACHE_2);
250  if (strcasecmp("TOMCAT_6_0", str) == 0) {
251  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
252  "longer supported by libhtp.", str);
253  return -1;
254  } else if ((strcasecmp("APACHE", str) == 0) ||
255  (strcasecmp("APACHE_2_2", str) == 0))
256  {
257  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
258  "longer supported by libhtp, failing back to "
259  "Apache2 personality.", str);
260  return HTP_SERVER_APACHE_2;
261  }
262 
263  return -1;
264 }
265 
266 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud, uint8_t e)
267 {
268  SCLogDebug("setting event %u", e);
269 
270  if (htud) {
272  s->events++;
273  return;
274  }
275 
276  htp_tx_t *tx = HTPStateGetTx(s, s->transaction_cnt);
277  if (tx == NULL && s->transaction_cnt > 0)
278  tx = HTPStateGetTx(s, s->transaction_cnt - 1);
279  if (tx != NULL) {
280  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
281  if (htud != NULL) {
283  s->events++;
284  return;
285  }
286  }
287  SCLogDebug("couldn't set event %u", e);
288 }
289 
290 static AppLayerDecoderEvents *HTPGetEvents(void *tx)
291 {
292  SCLogDebug("get HTTP events for TX %p", tx);
293 
294  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
295  if (htud != NULL) {
296  SCLogDebug("has htud, htud->decoder_events %p", htud->decoder_events);
297  return htud->decoder_events;
298  }
299 
300  return NULL;
301 }
302 
303 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
304  * connection parser to be used by the HTP library
305  */
306 static void *HTPStateAlloc(void)
307 {
308  SCEnter();
309 
310  HtpState *s = HTPMalloc(sizeof(HtpState));
311  if (unlikely(s == NULL)) {
312  SCReturnPtr(NULL, "void");
313  }
314 
315  memset(s, 0x00, sizeof(HtpState));
316 
317 #ifdef DEBUG
318  SCMutexLock(&htp_state_mem_lock);
319  htp_state_memcnt++;
320  htp_state_memuse += sizeof(HtpState);
321  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
322  SCMutexUnlock(&htp_state_mem_lock);
323 #endif
324 
325  SCReturnPtr((void *)s, "void");
326 }
327 
328 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
329 {
330  if (likely(htud)) {
331  HtpBodyFree(&htud->request_body);
332  HtpBodyFree(&htud->response_body);
333  bstr_free(htud->request_uri_normalized);
334  if (htud->request_headers_raw)
336  if (htud->response_headers_raw)
339  if (htud->boundary)
340  HTPFree(htud->boundary, htud->boundary_len);
341  if (htud->de_state != NULL) {
343  }
344  HTPFree(htud, sizeof(HtpTxUserData));
345  }
346 }
347 
348 /** \brief Function to frees the HTTP state memory and also frees the HTTP
349  * connection parser memory which was used by the HTP library
350  */
351 void HTPStateFree(void *state)
352 {
353  SCEnter();
354 
355  HtpState *s = (HtpState *)state;
356  if (s == NULL) {
357  SCReturn;
358  }
359 
360  /* free the connection parser memory used by HTP library */
361  if (s->connp != NULL) {
362  SCLogDebug("freeing HTP state");
363 
364  uint64_t tx_id;
365  uint64_t total_txs = HTPStateGetTxCnt(state);
366  /* free the list of body chunks */
367  if (s->conn != NULL) {
368  for (tx_id = 0; tx_id < total_txs; tx_id++) {
369  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
370  if (tx != NULL) {
371  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
372  HtpTxUserDataFree(s, htud);
373  htp_tx_set_user_data(tx, NULL);
374  }
375  }
376  }
377  htp_connp_destroy_all(s->connp);
378  }
379 
382  HTPFree(s, sizeof(HtpState));
383 
384 #ifdef DEBUG
385  SCMutexLock(&htp_state_mem_lock);
386  htp_state_memcnt--;
387  htp_state_memuse -= sizeof(HtpState);
388  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
389  SCMutexUnlock(&htp_state_mem_lock);
390 #endif
391 
392  SCReturn;
393 }
394 
395 /**
396  * \brief HTP transaction cleanup callback
397  *
398  * \warning We cannot actually free the transactions here. It seems that
399  * HTP only accepts freeing of transactions in the response callback.
400  */
401 static void HTPStateTransactionFree(void *state, uint64_t id)
402 {
403  SCEnter();
404 
405  HtpState *s = (HtpState *)state;
406 
407  SCLogDebug("state %p, id %"PRIu64, s, id);
408 
409  htp_tx_t *tx = HTPStateGetTx(s, id);
410  if (tx != NULL) {
411  /* This will remove obsolete body chunks */
412  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
413  HtpTxUserDataFree(s, htud);
414  htp_tx_set_user_data(tx, NULL);
415 
416  /* hack: even if libhtp considers the tx incomplete, we want to
417  * free it here. htp_tx_destroy however, will refuse to do this.
418  * As htp_tx_destroy_incomplete isn't available in the public API,
419  * we hack around it here. */
420  if (unlikely(!(
421  tx->request_progress == HTP_REQUEST_COMPLETE &&
422  tx->response_progress == HTP_RESPONSE_COMPLETE)))
423  {
424  tx->request_progress = HTP_REQUEST_COMPLETE;
425  tx->response_progress = HTP_RESPONSE_COMPLETE;
426  }
427  htp_tx_destroy(tx);
428  }
429 }
430 
431 /**
432  * \brief Sets a flag that informs the HTP app layer that some module in the
433  * engine needs the http request body data.
434  * \initonly
435  */
437 {
438  SCEnter();
439 
440  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
441  SCReturn;
442 }
443 
444 /**
445  * \brief Sets a flag that informs the HTP app layer that some module in the
446  * engine needs the http request body data.
447  * \initonly
448  */
450 {
451  SCEnter();
452 
453  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
454  SCReturn;
455 }
456 
457 /**
458  * \brief Sets a flag that informs the HTP app layer that some module in the
459  * engine needs the http request multi part header.
460  *
461  * \initonly
462  */
463 static void AppLayerHtpNeedMultipartHeader(void)
464 {
465  SCEnter();
467 
468  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
469  SCReturn;
470 }
471 
472 /**
473  * \brief Sets a flag that informs the HTP app layer that some module in the
474  * engine needs the http request file.
475  *
476  * \initonly
477  */
479 {
480  SCEnter();
481  AppLayerHtpNeedMultipartHeader();
484 
485  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
486  SCReturn;
487 }
488 
489 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
490 struct {
491  const char *msg;
492  int de;
493 } htp_errors[] = {
494  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
495  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
496  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
497  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
498  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
499 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
500  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
501 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
502  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
503  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
504  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
505  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
506 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
507  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
508  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
509  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
510 };
511 
512 struct {
513  const char *msg;
514  int de;
515 } htp_warnings[] = {
516  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
517  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
518  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
519  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
520  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
521 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
522  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
523  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
524  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
525  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
526  * luckily, "Request server port=" is unique */
527 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
529  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
530  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
531  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
532  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
533  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
534  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
535  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
536  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
537  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
538  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
539  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
540  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
541  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
542  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
543  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
544 };
545 
546 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
547 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
548 
549 /**
550  * \internal
551  *
552  * \brief Get the warning id for the warning msg.
553  *
554  * \param msg warning message
555  *
556  * \retval id the id or 0 in case of not found
557  */
558 static int HTPHandleWarningGetId(const char *msg)
559 {
560  SCLogDebug("received warning \"%s\"", msg);
561  size_t idx;
562  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
563  if (strncmp(htp_warnings[idx].msg, msg,
564  strlen(htp_warnings[idx].msg)) == 0)
565  {
566  return htp_warnings[idx].de;
567  }
568  }
569 
570  return 0;
571 }
572 
573 /**
574  * \internal
575  *
576  * \brief Get the error id for the error msg.
577  *
578  * \param msg error message
579  *
580  * \retval id the id or 0 in case of not found
581  */
582 static int HTPHandleErrorGetId(const char *msg)
583 {
584  SCLogDebug("received error \"%s\"", msg);
585 
586  size_t idx;
587  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
588  if (strncmp(htp_errors[idx].msg, msg,
589  strlen(htp_errors[idx].msg)) == 0)
590  {
591  return htp_errors[idx].de;
592  }
593  }
594 
595  return 0;
596 }
597 
598 /**
599  * \internal
600  *
601  * \brief Check state for errors, warnings and add any as events
602  *
603  * \param s state
604  */
605 static void HTPHandleError(HtpState *s)
606 {
607  if (s == NULL || s->conn == NULL ||
608  s->conn->messages == NULL) {
609  return;
610  }
611 
612  size_t size = htp_list_size(s->conn->messages);
613  size_t msg;
614 
615  for (msg = s->htp_messages_offset; msg < size; msg++) {
616  htp_log_t *log = htp_list_get(s->conn->messages, msg);
617  if (log == NULL)
618  continue;
619 
620  HtpTxUserData *htud = NULL;
621  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
622  if (tx != NULL)
623  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
624 
625  SCLogDebug("message %s", log->msg);
626 
627  int id = HTPHandleErrorGetId(log->msg);
628  if (id == 0) {
629  id = HTPHandleWarningGetId(log->msg);
630  if (id == 0)
632  }
633 
634  if (id > 0) {
635  HTPSetEvent(s, htud, id);
636  }
637  }
638  s->htp_messages_offset = (uint16_t)msg;
639  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
640 }
641 
642 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
643 {
644 #ifdef DEBUG
645  BUG_ON(s == NULL || tx == NULL);
646 #endif
647  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
648  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
649  HTP_HOSTH_INVALID))
650  {
651  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
652  if (htud == NULL)
653  return;
654 
655  if (tx->flags & HTP_REQUEST_INVALID_T_E)
656  HTPSetEvent(s, htud,
658  if (tx->flags & HTP_REQUEST_INVALID_C_L)
659  HTPSetEvent(s, htud,
661  if (tx->flags & HTP_HOST_MISSING)
662  HTPSetEvent(s, htud,
664  if (tx->flags & HTP_HOST_AMBIGUOUS)
665  HTPSetEvent(s, htud,
667  if (tx->flags & HTP_HOSTU_INVALID)
668  HTPSetEvent(s, htud,
670  if (tx->flags & HTP_HOSTH_INVALID)
671  HTPSetEvent(s, htud,
673  }
674  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
675  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
676  if (htud == NULL)
677  return;
678  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED);
679  }
680  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
681  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
682  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
683  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
684  if (htud == NULL)
685  return;
686  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_REQUEST_LINE_INVALID);
687  }
688 }
689 
690 static int Setup(Flow *f, HtpState *hstate)
691 {
692  /* store flow ref in state so callbacks can access it */
693  hstate->f = f;
694 
695  HTPCfgRec *htp_cfg_rec = &cfglist;
696  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
697  void *user_data = NULL;
698 
699  if (FLOW_IS_IPV4(f)) {
700  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
701  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
702  }
703  else if (FLOW_IS_IPV6(f)) {
704  SCLogDebug("Looking up HTP config for ipv6");
705  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
706  }
707  else {
708  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
709  goto error;
710  }
711 
712  if (user_data != NULL) {
713  htp_cfg_rec = user_data;
714  htp = htp_cfg_rec->cfg;
715  SCLogDebug("LIBHTP using config: %p", htp);
716  } else {
717  SCLogDebug("Using default HTP config: %p", htp);
718  }
719 
720  if (NULL == htp) {
721 #ifdef DEBUG_VALIDATION
722  BUG_ON(htp == NULL);
723 #endif
724  /* should never happen if HTPConfigure is properly invoked */
725  goto error;
726  }
727 
728  hstate->connp = htp_connp_create(htp);
729  if (hstate->connp == NULL) {
730  goto error;
731  }
732 
733  hstate->conn = htp_connp_get_connection(hstate->connp);
734 
735  htp_connp_set_user_data(hstate->connp, (void *)hstate);
736  hstate->cfg = htp_cfg_rec;
737 
738  SCLogDebug("New hstate->connp %p", hstate->connp);
739 
740  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
741 
743  htp_cfg_rec->request.inspect_min_size);
745  htp_cfg_rec->response.inspect_min_size);
746  return 0;
747 error:
748  return -1;
749 }
750 
751 /**
752  * \brief Function to handle the reassembled data from client and feed it to
753  * the HTP library to process it.
754  *
755  * \param flow Pointer to the flow the data belong to
756  * \param htp_state Pointer the state in which the parsed value to be stored
757  * \param pstate Application layer parser state for this session
758  * \param input Pointer the received HTTP client data
759  * \param input_len Length in bytes of the received data
760  * \param output Pointer to the output (not used in this function)
761  *
762  * \retval On success returns 1 or on failure returns -1.
763  */
764 static int HTPHandleRequestData(Flow *f, void *htp_state,
765  AppLayerParserState *pstate,
766  uint8_t *input, uint32_t input_len,
767  void *local_data, const uint8_t flags)
768 {
769  SCEnter();
770  int ret = 1;
771  HtpState *hstate = (HtpState *)htp_state;
772 
773  /* On the first invocation, create the connection parser structure to
774  * be used by HTP library. This is looked up via IP in the radix
775  * tree. Failing that, the default HTP config is used.
776  */
777  if (NULL == hstate->conn) {
778  if (Setup(f, hstate) != 0) {
779  goto error;
780  }
781  }
782  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
783 
784  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
785  /* pass the new data to the htp parser */
786  if (input_len > 0) {
787  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
788  switch (r) {
789  case HTP_STREAM_ERROR:
790  ret = -1;
791  break;
792  default:
793  break;
794  }
795  HTPHandleError(hstate);
796  }
797 
798  /* if the TCP connection is closed, then close the HTTP connection */
800  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
801  {
802  htp_connp_close(hstate->connp, &ts);
803  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
804  SCLogDebug("stream eof encountered, closing htp handle for ts");
805  }
806 
807  SCLogDebug("hstate->connp %p", hstate->connp);
808  SCReturnInt(ret);
809 
810 error:
811  SCReturnInt(-1);
812 }
813 
814 /**
815  * \brief Function to handle the reassembled data from server and feed it to
816  * the HTP library to process it.
817  *
818  * \param flow Pointer to the flow the data belong to
819  * \param htp_state Pointer the state in which the parsed value to be stored
820  * \param pstate Application layer parser state for this session
821  * \param input Pointer the received HTTP server data
822  * \param input_len Length in bytes of the received data
823  * \param output Pointer to the output (not used in this function)
824  *
825  * \retval On success returns 1 or on failure returns -1
826  */
827 static int HTPHandleResponseData(Flow *f, void *htp_state,
828  AppLayerParserState *pstate,
829  uint8_t *input, uint32_t input_len,
830  void *local_data, const uint8_t flags)
831 {
832  SCEnter();
833  int ret = 1;
834  HtpState *hstate = (HtpState *)htp_state;
835 
836  /* On the first invocation, create the connection parser structure to
837  * be used by HTP library. This is looked up via IP in the radix
838  * tree. Failing that, the default HTP config is used.
839  */
840  if (NULL == hstate->conn) {
841  if (Setup(f, hstate) != 0) {
842  goto error;
843  }
844  }
845  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
846 
847  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
848  if (input_len > 0) {
849  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
850  switch (r) {
851  case HTP_STREAM_ERROR:
852  ret = -1;
853  break;
854  default:
855  break;
856  }
857  HTPHandleError(hstate);
858  }
859 
860  /* if we the TCP connection is closed, then close the HTTP connection */
862  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
863  {
864  htp_connp_close(hstate->connp, &ts);
865  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
866  }
867 
868  SCLogDebug("hstate->connp %p", hstate->connp);
869  SCReturnInt(ret);
870 error:
871  SCReturnInt(-1);
872 }
873 
874 /**
875  * \param name /Lowercase/ version of the variable name
876  */
877 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
878  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
879 {
880 #ifdef PRINT
881  printf("DATA START: \n");
882  PrintRawDataFp(stdout, data, len);
883  printf("DATA END: \n");
884 #endif
885  size_t x;
886  int quote = 0;
887 
888  for (x = 0; x < len; x++) {
889  if (!(isspace(data[x])))
890  break;
891  }
892 
893  if (x >= len)
894  return 0;
895 
896  uint8_t *line = data+x;
897  size_t line_len = len-x;
898  size_t offset = 0;
899 #ifdef PRINT
900  printf("LINE START: \n");
901  PrintRawDataFp(stdout, line, line_len);
902  printf("LINE END: \n");
903 #endif
904  for (x = 0 ; x < line_len; x++) {
905  if (x > 0) {
906  if (line[x - 1] != '\\' && line[x] == '\"') {
907  quote++;
908  }
909 
910  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
911  uint8_t *token = line + offset;
912  size_t token_len = x - offset;
913 
914  if ((x + 1) == line_len) {
915  token_len++;
916  }
917 
918  offset = x + 1;
919 
920  while (offset < line_len && isspace(line[offset])) {
921  x++;
922  offset++;
923  }
924 #ifdef PRINT
925  printf("TOKEN START: \n");
926  PrintRawDataFp(stdout, token, token_len);
927  printf("TOKEN END: \n");
928 #endif
929  if (token_len > name_len) {
930  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
931  uint8_t *value = token + name_len;
932  size_t value_len = token_len - name_len;
933 
934  if (value[0] == '\"') {
935  value++;
936  value_len--;
937  }
938  if (value[value_len-1] == '\"') {
939  value_len--;
940  }
941 #ifdef PRINT
942  printf("VALUE START: \n");
943  PrintRawDataFp(stdout, value, value_len);
944  printf("VALUE END: \n");
945 #endif
946  *retptr = value;
947  *retlen = value_len;
948  return 1;
949  }
950  }
951  }
952  }
953  }
954 
955  return 0;
956 }
957 
958 /**
959  * \param name /Lowercase/ version of the variable name
960  */
961 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
962  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
963 {
964  SCEnter();
965 #ifdef PRINT
966  printf("DATA START: \n");
967  PrintRawDataFp(stdout, data, len);
968  printf("DATA END: \n");
969 #endif
970  size_t x;
971  int quote = 0;
972 
973  for (x = 0; x < len; x++) {
974  if (!(isspace(data[x])))
975  break;
976  }
977 
978  if (x >= len) {
979  SCReturnInt(0);
980  }
981 
982  uint8_t *line = data+x;
983  size_t line_len = len-x;
984  size_t offset = 0;
985 #ifdef PRINT
986  printf("LINE START: \n");
987  PrintRawDataFp(stdout, line, line_len);
988  printf("LINE END: \n");
989 #endif
990  for (x = 0 ; x < line_len; x++) {
991  if (x > 0) {
992  if (line[x - 1] != '\\' && line[x] == '\"') {
993  quote++;
994  }
995 
996  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
997  uint8_t *token = line + offset;
998  size_t token_len = x - offset;
999 
1000  if ((x + 1) == line_len) {
1001  token_len++;
1002  }
1003 
1004  offset = x + 1;
1005 
1006  while (offset < line_len && isspace(line[offset])) {
1007  x++;
1008  offset++;
1009  }
1010 #ifdef PRINT
1011  printf("TOKEN START: \n");
1012  PrintRawDataFp(stdout, token, token_len);
1013  printf("TOKEN END: \n");
1014 #endif
1015  if (token_len > name_len) {
1016  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1017  uint8_t *value = token + name_len;
1018  size_t value_len = token_len - name_len;
1019 
1020  if (value[0] == '\"') {
1021  value++;
1022  value_len--;
1023  }
1024  if (value[value_len-1] == '\"') {
1025  value_len--;
1026  }
1027 #ifdef PRINT
1028  printf("VALUE START: \n");
1029  PrintRawDataFp(stdout, value, value_len);
1030  printf("VALUE END: \n");
1031 #endif
1032  *retptr = value;
1033  *retlen = value_len;
1034  SCReturnInt(1);
1035  }
1036  }
1037  }
1038  }
1039  }
1040 
1041  SCReturnInt(0);
1042 }
1043 
1044 /**
1045  * \brief setup multipart parsing: extract boundary and store it
1046  *
1047  * \param d HTTP transaction
1048  * \param htud transaction userdata
1049  *
1050  * \retval 1 ok, multipart set up
1051  * \retval 0 ok, not multipart though
1052  * \retval -1 error: problem with the boundary
1053  *
1054  * If the request contains a multipart message, this function will
1055  * set the HTP_BOUNDARY_SET in the transaction.
1056  */
1057 static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud)
1058 {
1059  htp_header_t *h = (htp_header_t *)htp_table_get_c(d->tx->request_headers,
1060  "Content-Type");
1061  if (h != NULL && bstr_len(h->value) > 0) {
1062  uint8_t *boundary = NULL;
1063  size_t boundary_len = 0;
1064 
1065  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1066  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1067  &boundary, &boundary_len);
1068  if (r == 1) {
1069 #ifdef PRINT
1070  printf("BOUNDARY START: \n");
1071  PrintRawDataFp(stdout, boundary, boundary_len);
1072  printf("BOUNDARY END: \n");
1073 #endif
1074  if (boundary_len < HTP_BOUNDARY_MAX) {
1075  htud->boundary = HTPMalloc(boundary_len);
1076  if (htud->boundary == NULL) {
1077  return -1;
1078  }
1079  htud->boundary_len = (uint8_t)boundary_len;
1080  memcpy(htud->boundary, boundary, boundary_len);
1081 
1082  htud->tsflags |= HTP_BOUNDARY_SET;
1083  } else {
1084  SCLogDebug("invalid boundary");
1085  return -1;
1086  }
1087  SCReturnInt(1);
1088  }
1089  //SCReturnInt(1);
1090  }
1091  SCReturnInt(0);
1092 }
1093 
1094 #define C_D_HDR "content-disposition:"
1095 #define C_D_HDR_LEN 20
1096 #define C_T_HDR "content-type:"
1097 #define C_T_HDR_LEN 13
1098 
1099 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1100  HtpTxUserData *htud,
1101  uint8_t *header, uint32_t header_len,
1102  uint8_t **filename, uint16_t *filename_len,
1103  uint8_t **filetype, uint16_t *filetype_len)
1104 {
1105  uint8_t *fn = NULL;
1106  size_t fn_len = 0;
1107  uint8_t *ft = NULL;
1108  size_t ft_len = 0;
1109 
1110 #ifdef PRINT
1111  printf("HEADER START: \n");
1112  PrintRawDataFp(stdout, header, header_len);
1113  printf("HEADER END: \n");
1114 #endif
1115 
1116  while (header_len > 0) {
1117  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1118  uint8_t *line = header;
1119  uint32_t line_len;
1120 
1121  if (next_line == NULL) {
1122  line_len = header_len;
1123  } else {
1124  line_len = next_line - header;
1125  }
1126  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1127  if (sc == NULL) {
1128  HTPSetEvent(hstate, htud,
1130  /* if the : we found is the final char, it means we have
1131  * no value */
1132  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1133  HTPSetEvent(hstate, htud,
1135  } else {
1136 #ifdef PRINT
1137  printf("LINE START: \n");
1138  PrintRawDataFp(stdout, line, line_len);
1139  printf("LINE END: \n");
1140 #endif
1141  if (line_len >= C_D_HDR_LEN &&
1142  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1143  uint8_t *value = line + C_D_HDR_LEN;
1144  uint32_t value_len = line_len - C_D_HDR_LEN;
1145 
1146  /* parse content-disposition */
1147  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1148  value, value_len, &fn, &fn_len);
1149  } else if (line_len >= C_T_HDR_LEN &&
1150  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1151  SCLogDebug("content-type line");
1152  uint8_t *value = line + C_T_HDR_LEN;
1153  uint32_t value_len = line_len - C_T_HDR_LEN;
1154 
1155  (void)HTTPParseContentTypeHeader(NULL, 0,
1156  value, value_len, &ft, &ft_len);
1157  }
1158  }
1159 
1160  if (next_line == NULL) {
1161  SCLogDebug("no next_line");
1162  break;
1163  }
1164  header_len -= ((next_line + 2) - header);
1165  header = next_line + 2;
1166  } /* while (header_len > 0) */
1167 
1168  if (fn_len > USHRT_MAX)
1169  fn_len = USHRT_MAX;
1170  if (ft_len > USHRT_MAX)
1171  ft_len = USHRT_MAX;
1172 
1173  *filename = fn;
1174  *filename_len = fn_len;
1175  *filetype = ft;
1176  *filetype_len = ft_len;
1177 }
1178 
1179 /**
1180  * \brief Create a single buffer from the HtpBodyChunks in our list
1181  *
1182  * \param htud transaction user data
1183  * \param chunks_buffers pointer to pass back the buffer to the caller
1184  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1185  */
1186 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1187  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1188 {
1190  chunks_buffer, chunks_buffer_len,
1191  htud->request_body.body_parsed);
1192 }
1193 
1194 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1195 {
1196  SCEnter();
1197  if (tx && tx->de_state) {
1198  if (dir == STREAM_TOSERVER) {
1199  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1201  } else if (STREAM_TOCLIENT) {
1202  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1204  }
1205  }
1206 }
1207 
1208 /**
1209  * \brief Setup boundary buffers
1210  */
1211 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1212  uint8_t *boundary, uint32_t boundary_len)
1213 {
1214  memset(boundary, '-', boundary_len);
1215  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1216 }
1217 
1218 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1219  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1220 {
1221  int result = 0;
1222  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1223  uint8_t expected_boundary_len = htud->boundary_len + 2;
1224  uint8_t expected_boundary_end_len = htud->boundary_len + 4;
1225  int tx_progress = 0;
1226 
1227 #ifdef PRINT
1228  printf("CHUNK START: \n");
1229  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1230  printf("CHUNK END: \n");
1231 #endif
1232 
1233  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1234 
1235  /* search for the header start, header end and form end */
1236  uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1237  boundary, expected_boundary_len);
1238  uint8_t *header_end = NULL;
1239  if (header_start != NULL) {
1240  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1241  (uint8_t *)"\r\n\r\n", 4);
1242  }
1243  uint8_t *form_end = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1244  boundary, expected_boundary_end_len);
1245 
1246  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1247  header_end, form_end);
1248 
1249  /* we currently only handle multipart for ts. When we support it for tc,
1250  * we will need to supply right direction */
1251  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, STREAM_TOSERVER);
1252  /* if we're in the file storage process, deal with that now */
1253  if (htud->tsflags & HTP_FILENAME_SET) {
1254  if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1255  SCLogDebug("reached the end of the file");
1256 
1257  const uint8_t *filedata = chunks_buffer;
1258  uint32_t filedata_len = 0;
1259  uint8_t flags = 0;
1260 
1261  if (header_start < form_end || (header_start != NULL && form_end == NULL)) {
1262  filedata_len = header_start - filedata - 2; /* 0d 0a */
1263  } else if (form_end != NULL && form_end < header_start) {
1264  filedata_len = form_end - filedata;
1265  } else if (form_end != NULL && form_end == header_start) {
1266  filedata_len = form_end - filedata - 2; /* 0d 0a */
1267  } else if (tx_progress > HTP_RESPONSE_BODY) {
1268  filedata_len = chunks_buffer_len;
1269  flags = FILE_TRUNCATED;
1270  }
1271 
1272  if (filedata_len > chunks_buffer_len) {
1273  HTPSetEvent(hstate, htud,
1275  goto end;
1276  }
1277 #ifdef PRINT
1278  printf("FILEDATA (final chunk) START: \n");
1279  PrintRawDataFp(stdout, filedata, filedata_len);
1280  printf("FILEDATA (final chunk) END: \n");
1281 #endif
1282  if (!(htud->tsflags & HTP_DONTSTORE)) {
1283  if (HTPFileClose(hstate, filedata, filedata_len, flags,
1284  STREAM_TOSERVER) == -1)
1285  {
1286  goto end;
1287  }
1288  }
1289 
1290  htud->tsflags &=~ HTP_FILENAME_SET;
1291 
1292  /* fall through */
1293  } else {
1294  SCLogDebug("not yet at the end of the file");
1295 
1296  if (chunks_buffer_len > expected_boundary_end_len) {
1297  const uint8_t *filedata = chunks_buffer;
1298  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1299 #ifdef PRINT
1300  printf("FILEDATA (part) START: \n");
1301  PrintRawDataFp(stdout, filedata, filedata_len);
1302  printf("FILEDATA (part) END: \n");
1303 #endif
1304 
1305  if (!(htud->tsflags & HTP_DONTSTORE)) {
1306  result = HTPFileStoreChunk(hstate, filedata,
1307  filedata_len, STREAM_TOSERVER);
1308  if (result == -1) {
1309  goto end;
1310  } else if (result == -2) {
1311  /* we know for sure we're not storing the file */
1312  htud->tsflags |= HTP_DONTSTORE;
1313  }
1314  }
1315 
1316  htud->request_body.body_parsed += filedata_len;
1317  } else {
1318  SCLogDebug("chunk too small to already process in part");
1319  }
1320 
1321  goto end;
1322  }
1323  }
1324 
1325  while (header_start != NULL && header_end != NULL &&
1326  header_end != form_end &&
1327  header_start < (chunks_buffer + chunks_buffer_len) &&
1328  header_end < (chunks_buffer + chunks_buffer_len) &&
1329  header_start < header_end)
1330  {
1331  uint8_t *filename = NULL;
1332  uint16_t filename_len = 0;
1333  uint8_t *filetype = NULL;
1334  uint16_t filetype_len = 0;
1335 
1336  uint32_t header_len = header_end - header_start;
1337  SCLogDebug("header_len %u", header_len);
1338  uint8_t *header = header_start;
1339 
1340  /* skip empty records */
1341  if (expected_boundary_len == header_len) {
1342  goto next;
1343  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1344  header_len -= (expected_boundary_len + 2);
1345  header = header_start + (expected_boundary_len + 2); // + for 0d 0a
1346  }
1347 
1348  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1349  &filename, &filename_len, &filetype, &filetype_len);
1350 
1351  if (filename != NULL) {
1352  uint8_t *filedata = NULL;
1353  uint32_t filedata_len = 0;
1354 
1355  SCLogDebug("we have a filename");
1356 
1357  htud->tsflags |= HTP_FILENAME_SET;
1358  htud->tsflags &= ~HTP_DONTSTORE;
1359 
1360  SCLogDebug("header_end %p", header_end);
1361  SCLogDebug("form_end %p", form_end);
1362 
1363  /* everything until the final boundary is the file */
1364  if (form_end != NULL) {
1365  filedata = header_end + 4;
1366  if (form_end == filedata) {
1367  HTPSetEvent(hstate, htud,
1369  goto end;
1370  } else if (form_end < filedata) {
1371  HTPSetEvent(hstate, htud,
1373  goto end;
1374  }
1375 
1376  filedata_len = form_end - (header_end + 4 + 2);
1377  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1378 
1379  /* or is it? */
1380  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1381  boundary, expected_boundary_len);
1382  if (header_next != NULL) {
1383  filedata_len -= (form_end - header_next);
1384  }
1385 
1386  if (filedata_len > chunks_buffer_len) {
1387  HTPSetEvent(hstate, htud,
1389  goto end;
1390  }
1391  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1392 #ifdef PRINT
1393  printf("FILEDATA START: \n");
1394  PrintRawDataFp(stdout, filedata, filedata_len);
1395  printf("FILEDATA END: \n");
1396 #endif
1397 
1398  result = HTPFileOpen(hstate, filename, filename_len,
1399  filedata, filedata_len, hstate->transaction_cnt,
1400  STREAM_TOSERVER);
1401  if (result == -1) {
1402  goto end;
1403  } else if (result == -2) {
1404  htud->tsflags |= HTP_DONTSTORE;
1405  } else {
1406  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1407  goto end;
1408  }
1409  }
1410  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1411 
1412  htud->request_body.body_parsed += (header_end - chunks_buffer);
1413  htud->tsflags &= ~HTP_FILENAME_SET;
1414  } else {
1415  SCLogDebug("chunk doesn't contain form end");
1416 
1417  filedata = header_end + 4;
1418  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1419  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1420 
1421  if (filedata_len > chunks_buffer_len) {
1422  HTPSetEvent(hstate, htud,
1424  goto end;
1425  }
1426 
1427 #ifdef PRINT
1428  printf("FILEDATA START: \n");
1429  PrintRawDataFp(stdout, filedata, filedata_len);
1430  printf("FILEDATA END: \n");
1431 #endif
1432  /* form doesn't end in this chunk, but part might. Lets
1433  * see if have another coming up */
1434  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1435  boundary, expected_boundary_len);
1436  SCLogDebug("header_next %p", header_next);
1437  if (header_next == NULL) {
1438  /* no, but we'll handle the file data when we see the
1439  * form_end */
1440 
1441  SCLogDebug("more file data to come");
1442 
1443  uint32_t offset = (header_end + 4) - chunks_buffer;
1444  SCLogDebug("offset %u", offset);
1445  htud->request_body.body_parsed += offset;
1446 
1447  result = HTPFileOpen(hstate, filename, filename_len,
1448  NULL, 0, hstate->transaction_cnt,
1449  STREAM_TOSERVER);
1450  if (result == -1) {
1451  goto end;
1452  } else if (result == -2) {
1453  htud->tsflags |= HTP_DONTSTORE;
1454  }
1455  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1456 
1457  } else if (header_next - filedata > 2) {
1458  filedata_len = header_next - filedata - 2;
1459  SCLogDebug("filedata_len %u", filedata_len);
1460 
1461  result = HTPFileOpen(hstate, filename, filename_len,
1462  filedata, filedata_len, hstate->transaction_cnt,
1463  STREAM_TOSERVER);
1464  if (result == -1) {
1465  goto end;
1466  } else if (result == -2) {
1467  htud->tsflags |= HTP_DONTSTORE;
1468  } else {
1469  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1470  goto end;
1471  }
1472  }
1473  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1474 
1475  htud->tsflags &= ~HTP_FILENAME_SET;
1476  htud->request_body.body_parsed += (header_end - chunks_buffer);
1477  }
1478  }
1479  }
1480 next:
1481  SCLogDebug("header_start %p, header_end %p, form_end %p",
1482  header_start, header_end, form_end);
1483 
1484  /* Search next boundary entry after the start of body */
1485  uint32_t cursizeread = header_end - chunks_buffer;
1486  header_start = Bs2bmSearch(header_end + 4,
1487  chunks_buffer_len - (cursizeread + 4),
1488  boundary, expected_boundary_len);
1489  if (header_start != NULL) {
1490  header_end = Bs2bmSearch(header_end + 4,
1491  chunks_buffer_len - (cursizeread + 4),
1492  (uint8_t *) "\r\n\r\n", 4);
1493  }
1494  }
1495 
1496  /* if we're parsing the multipart and we're not currently processing a
1497  * file, we move the body pointer forward. */
1498  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1499  if (chunks_buffer_len > expected_boundary_end_len) {
1500  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1501 
1502  htud->request_body.body_parsed += move;
1503  SCLogDebug("form not ready, file not set, parsing non-file "
1504  "record: moved %u", move);
1505  }
1506  }
1507 
1508 end:
1509  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1510  return 0;
1511 }
1512 
1513 /** \brief setup things for put request
1514  * \todo really needed? */
1515 static int HtpRequestBodySetupPUT(htp_tx_data_t *d, HtpTxUserData *htud)
1516 {
1517 // if (d->tx->parsed_uri == NULL || d->tx->parsed_uri->path == NULL) {
1518 // return -1;
1519 // }
1520 
1521  /* filename is d->tx->parsed_uri->path */
1522 
1523  return 0;
1524 }
1525 
1526 /** \internal
1527  * \brief Handle POST, no multipart body data
1528  */
1529 static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
1530  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1531 {
1532  int result = 0;
1533 
1534  /* see if we need to open the file */
1535  if (!(htud->tsflags & HTP_FILENAME_SET))
1536  {
1537  uint8_t *filename = NULL;
1538  size_t filename_len = 0;
1539 
1540  /* get the name */
1541  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1542  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1543  filename_len = bstr_len(tx->parsed_uri->path);
1544  }
1545 
1546  if (filename != NULL) {
1547  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1548  hstate->transaction_cnt, STREAM_TOSERVER);
1549  if (result == -1) {
1550  goto end;
1551  } else if (result == -2) {
1552  htud->tsflags |= HTP_DONTSTORE;
1553  } else {
1554  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1555  htud->tsflags |= HTP_FILENAME_SET;
1556  htud->tsflags &= ~HTP_DONTSTORE;
1557  }
1558  }
1559  }
1560  else
1561  {
1562  /* otherwise, just store the data */
1563 
1564  if (!(htud->tsflags & HTP_DONTSTORE)) {
1565  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1566  if (result == -1) {
1567  goto end;
1568  } else if (result == -2) {
1569  /* we know for sure we're not storing the file */
1570  htud->tsflags |= HTP_DONTSTORE;
1571  }
1572  }
1573  }
1574 
1575  return 0;
1576 end:
1577  return -1;
1578 }
1579 
1580 /** \internal
1581  * \brief Handle PUT body data
1582  */
1583 static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
1584  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1585 {
1586  int result = 0;
1587 
1588  /* see if we need to open the file */
1589  if (!(htud->tsflags & HTP_FILENAME_SET))
1590  {
1591  uint8_t *filename = NULL;
1592  size_t filename_len = 0;
1593 
1594  /* get the name */
1595  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1596  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1597  filename_len = bstr_len(tx->parsed_uri->path);
1598  }
1599 
1600  if (filename != NULL) {
1601  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1602  hstate->transaction_cnt, STREAM_TOSERVER);
1603  if (result == -1) {
1604  goto end;
1605  } else if (result == -2) {
1606  htud->tsflags |= HTP_DONTSTORE;
1607  } else {
1608  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1609  htud->tsflags |= HTP_FILENAME_SET;
1610  htud->tsflags &= ~HTP_DONTSTORE;
1611  }
1612  }
1613  }
1614  else
1615  {
1616  /* otherwise, just store the data */
1617 
1618  if (!(htud->tsflags & HTP_DONTSTORE)) {
1619  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1620  if (result == -1) {
1621  goto end;
1622  } else if (result == -2) {
1623  /* we know for sure we're not storing the file */
1624  htud->tsflags |= HTP_DONTSTORE;
1625  }
1626  }
1627  }
1628 
1629  return 0;
1630 end:
1631  return -1;
1632 }
1633 
1634 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1635  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1636 {
1637  SCEnter();
1638 
1639  int result = 0;
1640 
1641  /* see if we need to open the file */
1642  if (!(htud->tcflags & HTP_FILENAME_SET))
1643  {
1644  SCLogDebug("setting up file name");
1645 
1646  uint8_t *filename = NULL;
1647  size_t filename_len = 0;
1648 
1649  /* try Content-Disposition header first */
1650  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1651  "Content-Disposition");
1652  if (h != NULL && bstr_len(h->value) > 0) {
1653  /* parse content-disposition */
1654  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1655  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1656  }
1657 
1658  /* fall back to name from the uri */
1659  if (filename == NULL) {
1660  /* get the name */
1661  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1662  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1663  filename_len = bstr_len(tx->parsed_uri->path);
1664  }
1665  }
1666 
1667  if (filename != NULL) {
1668  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len,
1669  data, data_len, hstate->transaction_cnt, STREAM_TOCLIENT);
1670  SCLogDebug("result %d", result);
1671  if (result == -1) {
1672  goto end;
1673  } else if (result == -2) {
1674  htud->tcflags |= HTP_DONTSTORE;
1675  } else {
1676  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1677  htud->tcflags |= HTP_FILENAME_SET;
1678  htud->tcflags &= ~HTP_DONTSTORE;
1679  }
1680  //set range if present
1681  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1682  if (h_content_range != NULL) {
1683  HTPFileSetRange(hstate, h_content_range->value);
1684  }
1685  }
1686  }
1687  else
1688  {
1689  /* otherwise, just store the data */
1690 
1691  if (!(htud->tcflags & HTP_DONTSTORE)) {
1692  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1693  SCLogDebug("result %d", result);
1694  if (result == -1) {
1695  goto end;
1696  } else if (result == -2) {
1697  /* we know for sure we're not storing the file */
1698  htud->tcflags |= HTP_DONTSTORE;
1699  }
1700  }
1701  }
1702 
1703  htud->response_body.body_parsed += data_len;
1704  return 0;
1705 end:
1706  return -1;
1707 }
1708 
1709 /**
1710  * \brief Function callback to append chunks for Requests
1711  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1712  * \retval int HTP_OK if all goes well
1713  */
1714 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1715 {
1716  SCEnter();
1717 
1718  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1719  SCReturnInt(HTP_OK);
1720 
1721  if (d->data == NULL || d->len == 0)
1722  SCReturnInt(HTP_OK);
1723 
1724 #ifdef PRINT
1725  printf("HTPBODY START: \n");
1726  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1727  printf("HTPBODY END: \n");
1728 #endif
1729 
1730  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1731  if (hstate == NULL) {
1732  SCReturnInt(HTP_ERROR);
1733  }
1734 
1735  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1736  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1737 
1738  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1739  if (tx_ud == NULL) {
1740  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1741  if (unlikely(tx_ud == NULL)) {
1742  SCReturnInt(HTP_OK);
1743  }
1744  memset(tx_ud, 0, sizeof(HtpTxUserData));
1745 
1746  /* Set the user data for handling body chunks on this transaction */
1747  htp_tx_set_user_data(d->tx, tx_ud);
1748  }
1749  if (!tx_ud->response_body_init) {
1750  tx_ud->response_body_init = 1;
1751 
1752  if (d->tx->request_method_number == HTP_M_POST) {
1753  SCLogDebug("POST");
1754  int r = HtpRequestBodySetupMultipart(d, tx_ud);
1755  if (r == 1) {
1757  } else if (r == 0) {
1759  SCLogDebug("not multipart");
1760  }
1761  } else if (d->tx->request_method_number == HTP_M_PUT) {
1762  if (HtpRequestBodySetupPUT(d, tx_ud) == 0) {
1764  }
1765  }
1766  }
1767 
1768  /* see if we can get rid of htp body chunks */
1769  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1770 
1771  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1772  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1773 
1774  /* within limits, add the body chunk to the state. */
1775  if (hstate->cfg->request.body_limit == 0 || tx_ud->request_body.content_len_so_far < hstate->cfg->request.body_limit)
1776  {
1777  uint32_t len = (uint32_t)d->len;
1778 
1779  if (hstate->cfg->request.body_limit > 0 &&
1780  (tx_ud->request_body.content_len_so_far + len) > hstate->cfg->request.body_limit)
1781  {
1782  len = hstate->cfg->request.body_limit - tx_ud->request_body.content_len_so_far;
1783  BUG_ON(len > (uint32_t)d->len);
1784  }
1785  SCLogDebug("len %u", len);
1786 
1787  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1788 
1789  const uint8_t *chunks_buffer = NULL;
1790  uint32_t chunks_buffer_len = 0;
1791 
1793  /* multi-part body handling starts here */
1794  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1795  goto end;
1796  }
1797 
1798  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1799  if (chunks_buffer == NULL) {
1800  goto end;
1801  }
1802 #ifdef PRINT
1803  printf("REASSCHUNK START: \n");
1804  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1805  printf("REASSCHUNK END: \n");
1806 #endif
1807 
1808  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1809 
1810  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
1811  HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1812  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
1813  HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1814  }
1815 
1816  } else {
1817  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1818  SCLogDebug("closing file that was being stored");
1819  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1820  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1821  }
1822  }
1823 
1824 end:
1825  if (hstate->conn != NULL) {
1826  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1828  hstate->cfg->request.inspect_min_size,
1829  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1830 
1831  /* if we reach the inspect_min_size we'll trigger inspection,
1832  * so make sure that raw stream is also inspected. Set the
1833  * data to be used to the amount of raw bytes we've seen to
1834  * get here. */
1835  if (tx_ud->request_body.body_inspected == 0 &&
1837  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1838  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1839  {
1840  uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1841 
1842  /* body still in progress, but due to min inspect size we need to inspect now */
1845  }
1846  /* after the start of the body, disable the depth logic */
1847  } else if (tx_ud->request_body.body_inspected > 0) {
1849  }
1850  }
1851  SCReturnInt(HTP_OK);
1852 }
1853 
1854 /**
1855  * \brief Function callback to append chunks for Responses
1856  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1857  * \retval int HTP_OK if all goes well
1858  */
1859 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1860 {
1861  SCEnter();
1862 
1863  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1864  SCReturnInt(HTP_OK);
1865 
1866  if (d->data == NULL || d->len == 0)
1867  SCReturnInt(HTP_OK);
1868 
1869  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1870  if (hstate == NULL) {
1871  SCReturnInt(HTP_ERROR);
1872  }
1873 
1874  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1875  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1876 
1877  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1878  if (tx_ud == NULL) {
1879  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1880  if (unlikely(tx_ud == NULL)) {
1881  SCReturnInt(HTP_OK);
1882  }
1883  memset(tx_ud, 0, sizeof(HtpTxUserData));
1884 
1885  /* Set the user data for handling body chunks on this transaction */
1886  htp_tx_set_user_data(d->tx, tx_ud);
1887  }
1888  if (!tx_ud->request_body_init) {
1889  tx_ud->request_body_init = 1;
1890  }
1891 
1892  /* see if we can get rid of htp body chunks */
1893  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1894 
1895  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1896  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1897 
1898  /* within limits, add the body chunk to the state. */
1899  if (hstate->cfg->response.body_limit == 0 || tx_ud->response_body.content_len_so_far < hstate->cfg->response.body_limit)
1900  {
1901  uint32_t len = (uint32_t)d->len;
1902 
1903  if (hstate->cfg->response.body_limit > 0 &&
1904  (tx_ud->response_body.content_len_so_far + len) > hstate->cfg->response.body_limit)
1905  {
1906  len = hstate->cfg->response.body_limit - tx_ud->response_body.content_len_so_far;
1907  BUG_ON(len > (uint32_t)d->len);
1908  }
1909  SCLogDebug("len %u", len);
1910 
1911  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1912 
1913  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1914  } else {
1915  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1916  SCLogDebug("closing file that was being stored");
1917  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1918  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1919  }
1920  }
1921 
1922  if (hstate->conn != NULL) {
1923  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1925  hstate->cfg->response.inspect_min_size,
1926  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
1927  /* if we reach the inspect_min_size we'll trigger inspection,
1928  * so make sure that raw stream is also inspected. Set the
1929  * data to be used to the amount of raw bytes we've seen to
1930  * get here. */
1931  if (tx_ud->response_body.body_inspected == 0 &&
1933  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
1934  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
1935  {
1936  uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
1937 
1938  /* body still in progress, but due to min inspect size we need to inspect now */
1941  }
1942  /* after the start of the body, disable the depth logic */
1943  } else if (tx_ud->response_body.body_inspected > 0) {
1945  }
1946  }
1947  SCReturnInt(HTP_OK);
1948 }
1949 
1950 /**
1951  * \brief Print the stats of the HTTP requests
1952  */
1954 {
1955 #ifdef DEBUG
1956  SCEnter();
1957  SCMutexLock(&htp_state_mem_lock);
1958  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
1959  htp_state_memcnt, htp_state_memuse);
1960  SCMutexUnlock(&htp_state_mem_lock);
1961  SCReturn;
1962 #endif
1963 }
1964 
1965 /** \brief Clears the HTTP server configuration memory used by HTP library */
1966 void HTPFreeConfig(void)
1967 {
1968  SCEnter();
1969 
1970  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
1971  !AppLayerParserConfParserEnabled("tcp", "http"))
1972  {
1973  SCReturn;
1974  }
1975 
1976  HTPCfgRec *nextrec = cfglist.next;
1977  SCRadixReleaseRadixTree(cfgtree);
1978  cfgtree = NULL;
1979  htp_config_destroy(cfglist.cfg);
1980  while (nextrec != NULL) {
1981  HTPCfgRec *htprec = nextrec;
1982  nextrec = nextrec->next;
1983 
1984  htp_config_destroy(htprec->cfg);
1985  SCFree(htprec);
1986  }
1987  HTPDestroyMemcap();
1988  SCReturn;
1989 }
1990 
1991 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
1992 {
1993  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
1994  if (htud != NULL) {
1995  htud->request_has_trailers = 1;
1996  }
1997  return HTP_OK;
1998 }
1999 
2000 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2001 {
2002  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2003  if (htud != NULL) {
2004  htud->response_has_trailers = 1;
2005  }
2006  return HTP_OK;
2007 }
2008 
2009 /**\internal
2010  * \brief called at start of request
2011  * Set min inspect size.
2012  */
2013 static int HTPCallbackRequestStart(htp_tx_t *tx)
2014 {
2015  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2016  if (hstate == NULL) {
2017  SCReturnInt(HTP_ERROR);
2018  }
2019 
2020  if (hstate->cfg)
2022  hstate->cfg->request.inspect_min_size);
2023  SCReturnInt(HTP_OK);
2024 }
2025 
2026 /**\internal
2027  * \brief called at start of response
2028  * Set min inspect size.
2029  */
2030 static int HTPCallbackResponseStart(htp_tx_t *tx)
2031 {
2032  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2033  if (hstate == NULL) {
2034  SCReturnInt(HTP_ERROR);
2035  }
2036 
2037  if (hstate->cfg)
2039  hstate->cfg->response.inspect_min_size);
2040  SCReturnInt(HTP_OK);
2041 }
2042 
2043 
2044 /**
2045  * \brief callback for request to store the recent incoming request
2046  in to the recent_in_tx for the given htp state
2047  * \param connp pointer to the current connection parser which has the htp
2048  * state in it as user data
2049  */
2050 static int HTPCallbackRequest(htp_tx_t *tx)
2051 {
2052  SCEnter();
2053 
2054  if (tx == NULL) {
2055  SCReturnInt(HTP_ERROR);
2056  }
2057 
2058  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2059  if (hstate == NULL) {
2060  SCReturnInt(HTP_ERROR);
2061  }
2062 
2063  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2064  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2065 
2066  SCLogDebug("HTTP request completed");
2067 
2068  HTPErrorCheckTxRequestFlags(hstate, tx);
2069 
2070  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2071  if (htud != NULL) {
2072  if (htud->tsflags & HTP_FILENAME_SET) {
2073  SCLogDebug("closing file that was being stored");
2074  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
2075  htud->tsflags &= ~HTP_FILENAME_SET;
2077  (uint32_t)hstate->conn->in_data_counter);
2078  }
2079  }
2080 
2081  hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
2082  /* request done, do raw reassembly now to inspect state and stream
2083  * at the same time. */
2085  SCReturnInt(HTP_OK);
2086 }
2087 
2088 /**
2089  * \brief callback for response to remove the recent received requests
2090  from the recent_in_tx for the given htp state
2091  * \param connp pointer to the current connection parser which has the htp
2092  * state in it as user data
2093  */
2094 static int HTPCallbackResponse(htp_tx_t *tx)
2095 {
2096  SCEnter();
2097 
2098  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2099  if (hstate == NULL) {
2100  SCReturnInt(HTP_ERROR);
2101  }
2102 
2103  /* we have one whole transaction now */
2104  hstate->transaction_cnt++;
2105 
2106  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2107  if (htud != NULL) {
2108  if (htud->tcflags & HTP_FILENAME_SET) {
2109  SCLogDebug("closing file that was being stored");
2110  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
2111  htud->tcflags &= ~HTP_FILENAME_SET;
2112  }
2113  }
2114 
2115  /* response done, do raw reassembly now to inspect state and stream
2116  * at the same time. */
2118 
2119  /* handle HTTP CONNECT */
2120  if (tx->request_method_number == HTP_M_CONNECT) {
2121  /* any 2XX status response implies that the connection will become
2122  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2123  if ((tx->response_status_number >= 200) &&
2124  (tx->response_status_number < 300) &&
2125  (hstate->transaction_cnt == 1)) {
2126  uint16_t dp = 0;
2127  if (tx->request_port_number != -1) {
2128  dp = (uint16_t)tx->request_port_number;
2129  }
2130  // both ALPROTO_HTTP and ALPROTO_TLS are normal options
2132  tx->request_progress = HTP_REQUEST_COMPLETE;
2133  tx->response_progress = HTP_RESPONSE_COMPLETE;
2134  }
2135  }
2136 
2137  hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
2138  SCReturnInt(HTP_OK);
2139 }
2140 
2141 static int HTPCallbackRequestLine(htp_tx_t *tx)
2142 {
2143  HtpTxUserData *tx_ud;
2144  bstr *request_uri_normalized;
2145  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2146  const HTPCfgRec *cfg = hstate->cfg;
2147 
2148  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2149  if (request_uri_normalized == NULL)
2150  return HTP_OK;
2151 
2152  tx_ud = htp_tx_get_user_data(tx);
2153  if (likely(tx_ud == NULL)) {
2154  tx_ud = HTPMalloc(sizeof(*tx_ud));
2155  if (unlikely(tx_ud == NULL)) {
2156  bstr_free(request_uri_normalized);
2157  return HTP_OK;
2158  }
2159  memset(tx_ud, 0, sizeof(*tx_ud));
2160  htp_tx_set_user_data(tx, tx_ud);
2161  }
2162  if (unlikely(tx_ud->request_uri_normalized != NULL))
2163  bstr_free(tx_ud->request_uri_normalized);
2164  tx_ud->request_uri_normalized = request_uri_normalized;
2165 
2166  if (tx->flags) {
2167  HTPErrorCheckTxRequestFlags(hstate, tx);
2168  }
2169  return HTP_OK;
2170 }
2171 
2172 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2173 {
2174  if (part == NULL)
2175  return HTP_OK;
2176 
2177  uint64_t flags = 0;
2178  size_t prevlen = bstr_len(part);
2179  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2180  // shorter string means that uri was encoded
2181  if (res == HTP_OK && prevlen > bstr_len(part)) {
2182  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2183  if (likely(htud == NULL)) {
2184  htud = HTPCalloc(1, sizeof(*htud));
2185  if (unlikely(htud == NULL))
2186  return HTP_OK;
2187  htp_tx_set_user_data(tx, htud);
2188  }
2189  HtpState *s = htp_connp_get_user_data(tx->connp);
2190  if (s == NULL)
2191  return HTP_OK;
2192  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI);
2193  }
2194 
2195  return HTP_OK;
2196 }
2197 
2198 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2199 {
2200  if (tx->parsed_uri == NULL)
2201  return HTP_OK;
2202 
2203  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2204 }
2205 
2206 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2207 {
2208  if (tx->parsed_uri == NULL)
2209  return HTP_OK;
2210 
2211  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2212 }
2213 
2214 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2215 {
2216  void *ptmp;
2217  if (tx_data->len == 0 || tx_data->tx == NULL)
2218  return HTP_OK;
2219 
2220  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2221  if (tx_ud == NULL) {
2222  tx_ud = HTPMalloc(sizeof(*tx_ud));
2223  if (unlikely(tx_ud == NULL))
2224  return HTP_OK;
2225  memset(tx_ud, 0, sizeof(*tx_ud));
2226  htp_tx_set_user_data(tx_data->tx, tx_ud);
2227  }
2228  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2229  tx_ud->request_headers_raw_len,
2230  tx_ud->request_headers_raw_len + tx_data->len);
2231  if (ptmp == NULL) {
2232  /* error: we're freeing the entire user data */
2233  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2234  HtpTxUserDataFree(hstate, tx_ud);
2235  htp_tx_set_user_data(tx_data->tx, NULL);
2236  return HTP_OK;
2237  }
2238  tx_ud->request_headers_raw = ptmp;
2239 
2240  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2241  tx_data->data, tx_data->len);
2242  tx_ud->request_headers_raw_len += tx_data->len;
2243 
2244  if (tx_data->tx && tx_data->tx->flags) {
2245  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2246  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2247  }
2248  return HTP_OK;
2249 }
2250 
2251 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2252 {
2253  void *ptmp;
2254  if (tx_data->len == 0 || tx_data->tx == NULL)
2255  return HTP_OK;
2256 
2257  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2258  if (tx_ud == NULL) {
2259  tx_ud = HTPMalloc(sizeof(*tx_ud));
2260  if (unlikely(tx_ud == NULL))
2261  return HTP_OK;
2262  memset(tx_ud, 0, sizeof(*tx_ud));
2263  htp_tx_set_user_data(tx_data->tx, tx_ud);
2264  }
2265  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2266  tx_ud->response_headers_raw_len,
2267  tx_ud->response_headers_raw_len + tx_data->len);
2268  if (ptmp == NULL) {
2269  /* error: we're freeing the entire user data */
2270  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2271  HtpTxUserDataFree(hstate, tx_ud);
2272  htp_tx_set_user_data(tx_data->tx, NULL);
2273  return HTP_OK;
2274  }
2275  tx_ud->response_headers_raw = ptmp;
2276 
2277  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2278  tx_data->data, tx_data->len);
2279  tx_ud->response_headers_raw_len += tx_data->len;
2280 
2281  return HTP_OK;
2282 }
2283 
2284 /*
2285  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2286  */
2287 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2288 {
2289  cfg_prec->uri_include_all = FALSE;
2296 
2297  if (!g_disable_randomness) {
2299  } else {
2300  cfg_prec->randomize = 0;
2301  }
2303 
2304  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2305  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2306  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2307  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2308 
2309  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2310  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2311 
2312  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2313  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2314 
2315  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2316  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
2317 
2318  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2319  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
2320 
2321  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2322 
2323  /* don't convert + to space by default */
2324  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2325 
2326  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2327  * only the hard limit. So we set both here to the (current) htp defaults.
2328  * The reason we do this is that if the user sets the hard limit in the
2329  * config, we have to set the soft limit as well. If libhtp starts using
2330  * the soft limit in the future, we at least make sure we control what
2331  * it's value is. */
2332  htp_config_set_field_limits(cfg_prec->cfg,
2335  return;
2336 }
2337 
2338 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2339  * but we can get both <0 and >RAND_MAX values from RandomGet
2340  */
2341 static int RandomGetWrap(void)
2342 {
2343  unsigned long r;
2344 
2345  do {
2346  r = RandomGet();
2347  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2348 
2349  return r % RAND_MAX;
2350 }
2351 
2352 /*
2353  * We have this splitup so that in case double decoding has been enabled
2354  * for query and path, they would be called first on the callback queue,
2355  * before the callback set by Phase2() is called. We need this, since
2356  * the callback in Phase2() generates the normalized uri which utilizes
2357  * the query and path. */
2358 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2359 {
2360  /* randomize inspection size if needed */
2361  if (cfg_prec->randomize) {
2362  int rdrange = cfg_prec->randomize_range;
2363 
2364  long int r = RandomGetWrap();
2365  cfg_prec->request.inspect_min_size +=
2366  (int) (cfg_prec->request.inspect_min_size *
2367  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2368 
2369  r = RandomGetWrap();
2370  cfg_prec->request.inspect_window +=
2371  (int) (cfg_prec->request.inspect_window *
2372  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2373  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2374  " %d and 'request-body-inspect-window' set to %d after"
2375  " randomization.",
2376  name,
2377  cfg_prec->request.inspect_min_size,
2378  cfg_prec->request.inspect_window);
2379 
2380 
2381  r = RandomGetWrap();
2382  cfg_prec->response.inspect_min_size +=
2383  (int) (cfg_prec->response.inspect_min_size *
2384  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2385 
2386  r = RandomGetWrap();
2387  cfg_prec->response.inspect_window +=
2388  (int) (cfg_prec->response.inspect_window *
2389  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2390 
2391  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2392  " %d and 'response-body-inspect-window' set to %d after"
2393  " randomization.",
2394  name,
2395  cfg_prec->response.inspect_min_size,
2396  cfg_prec->response.inspect_window);
2397  }
2398 
2399  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2400 
2401  cfg_prec->request.sbcfg.flags = 0;
2402  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2403  cfg_prec->request.inspect_window : 256;
2404  cfg_prec->request.sbcfg.buf_slide = 0;
2405  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2406  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2407  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2408  cfg_prec->request.sbcfg.Free = HTPFree;
2409 
2410  cfg_prec->response.sbcfg.flags = 0;
2411  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2412  cfg_prec->response.inspect_window : 256;
2413  cfg_prec->response.sbcfg.buf_slide = 0;
2414  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2415  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2416  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2417  cfg_prec->response.sbcfg.Free = HTPFree;
2418  return;
2419 }
2420 
2421 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2422  SCRadixTree *tree)
2423 {
2424  if (cfg_prec == NULL || s == NULL || tree == NULL)
2425  return;
2426 
2427  ConfNode *p = NULL;
2428 
2429  /* Default Parameters */
2430  TAILQ_FOREACH(p, &s->head, next) {
2431 
2432  if (strcasecmp("address", p->name) == 0) {
2433  ConfNode *pval;
2434  /* Addresses */
2435  TAILQ_FOREACH(pval, &p->head, next) {
2436  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2437  pval->val);
2438 
2439  /* IPV6 or IPV4? */
2440  if (strchr(pval->val, ':') != NULL) {
2441  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2442  s->name, pval->val, cfg_prec->cfg);
2443  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2444  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2445  "add ipv6 server %s, ignoring", pval->val);
2446  }
2447  } else {
2448  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2449  s->name, pval->val, cfg_prec->cfg);
2450  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2451  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2452  "to add ipv4 server %s, ignoring",
2453  pval->val);
2454  }
2455  } /* else - if (strchr(pval->val, ':') != NULL) */
2456  } /* TAILQ_FOREACH(pval, &p->head, next) */
2457 
2458  } else if (strcasecmp("personality", p->name) == 0) {
2459  /* Personalities */
2460  int personality = HTPLookupPersonality(p->val);
2461  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2462  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2463 
2464  if (personality >= 0) {
2465  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2466  personality);
2467  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2468  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2469  "personality \"%s\", ignoring", p->val);
2470  } else {
2471  SCLogDebug("LIBHTP personality set to %s",
2472  HTPLookupPersonalityString(personality));
2473  }
2474 
2475  /* The IDS personality by default converts the path (and due to
2476  * our query string callback also the query string) to lowercase.
2477  * Signatures do not expect this, so override it. */
2478  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2479  } else {
2480  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2481  "\"%s\", ignoring", p->val);
2482  continue;
2483  }
2484 
2485  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2486  strcasecmp("request_body_limit", p->name) == 0) {
2487  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2488  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2489  "from conf file - %s. Killing engine", p->val);
2490  exit(EXIT_FAILURE);
2491  }
2492 
2493  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2494  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2495  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2496  "from conf file - %s. Killing engine", p->val);
2497  exit(EXIT_FAILURE);
2498  }
2499 
2500  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2501  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2502  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2503  "from conf file - %s. Killing engine", p->val);
2504  exit(EXIT_FAILURE);
2505  }
2506 
2507  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2508  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2509  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2510  "from conf file - %s. Killing engine", p->val);
2511  exit(EXIT_FAILURE);
2512  }
2513 
2514  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2515  if (ConfValIsTrue(p->val)) {
2516  htp_config_register_request_line(cfg_prec->cfg,
2517  HTPCallbackDoubleDecodeQuery);
2518  }
2519 
2520  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2521  if (ConfValIsTrue(p->val)) {
2522  htp_config_register_request_line(cfg_prec->cfg,
2523  HTPCallbackDoubleDecodePath);
2524  }
2525 
2526  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2527  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2528  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2529  "from conf file - %s. Killing engine", p->val);
2530  exit(EXIT_FAILURE);
2531  }
2532 
2533  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2534  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2535  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2536  "from conf file - %s. Killing engine", p->val);
2537  exit(EXIT_FAILURE);
2538  }
2539 
2540  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2541  uint32_t value = 2;
2542  if (ParseSizeStringU32(p->val, &value) < 0) {
2543  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2544  "from conf file - %s. Killing engine", p->val);
2545  exit(EXIT_FAILURE);
2546  }
2547 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2548  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2549 #else
2550  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2551  "to %u, libhtp version too old", value);
2552 #endif
2553  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2554  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2555  HTP_DECODER_URL_PATH,
2556  ConfValIsTrue(p->val));
2557  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2558  if (strlen(p->val) == 1) {
2559  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2560  HTP_DECODER_URL_PATH,
2561  p->val[0]);
2562  } else {
2563  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2564  "for libhtp param path-bestfit-replacement-char");
2565  }
2566  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2567  htp_config_set_convert_lowercase(cfg_prec->cfg,
2568  HTP_DECODER_URL_PATH,
2569  ConfValIsTrue(p->val));
2570  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2571  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2572  HTP_DECODER_URL_PATH,
2573  ConfValIsTrue(p->val));
2574  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2575  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2576  HTP_DECODER_URL_PATH,
2577  ConfValIsTrue(p->val));
2578  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2579  htp_config_set_path_separators_compress(cfg_prec->cfg,
2580  HTP_DECODER_URL_PATH,
2581  ConfValIsTrue(p->val));
2582  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2583  htp_config_set_path_separators_decode(cfg_prec->cfg,
2584  HTP_DECODER_URL_PATH,
2585  ConfValIsTrue(p->val));
2586  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2587  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2588  HTP_DECODER_URL_PATH,
2589  ConfValIsTrue(p->val));
2590  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2591  enum htp_url_encoding_handling_t handling;
2592  if (strcasecmp(p->val, "preserve_percent") == 0) {
2593  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2594  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2595  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2596  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2597  handling = HTP_URL_DECODE_PROCESS_INVALID;
2598  } else {
2599  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2600  "for libhtp param path-url-encoding-invalid-handling");
2601  return;
2602  }
2603  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2604  HTP_DECODER_URL_PATH,
2605  handling);
2606  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2607  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2608  HTP_DECODER_URL_PATH,
2609  ConfValIsTrue(p->val));
2610  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2611  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2612  SCLogDebug("uri-include-all %s",
2613  cfg_prec->uri_include_all ? "enabled" : "disabled");
2614  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2615  htp_config_set_plusspace_decode(cfg_prec->cfg,
2616  HTP_DECODER_URLENCODED,
2617  ConfValIsTrue(p->val));
2618  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2619  uint32_t limit = 0;
2620  if (ParseSizeStringU32(p->val, &limit) < 0) {
2621  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2622  "from conf file - %s. Killing engine", p->val);
2623  exit(EXIT_FAILURE);
2624  }
2625  if (limit == 0) {
2626  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2627  "from conf file cannot be 0. Killing engine");
2628  exit(EXIT_FAILURE);
2629  }
2630  /* set default soft-limit with our new hard limit */
2631  htp_config_set_field_limits(cfg_prec->cfg,
2633  (size_t)limit);
2634  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2635  if (!g_disable_randomness) {
2636  cfg_prec->randomize = ConfValIsTrue(p->val);
2637  }
2638  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2639  uint32_t range = atoi(p->val);
2640  if (range > 100) {
2641  SCLogError(SC_ERR_SIZE_PARSE, "Invalid value for randomize"
2642  " inspection range setting from conf file - %s."
2643  " It should be inferior to 100."
2644  " Killing engine",
2645  p->val);
2646  exit(EXIT_FAILURE);
2647  }
2648  cfg_prec->randomize_range = range;
2649  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2650  if (ConfValIsTrue(p->val)) {
2651  cfg_prec->http_body_inline = 1;
2652  } else if (ConfValIsFalse(p->val)) {
2653  cfg_prec->http_body_inline = 0;
2654  } else {
2655  if (strcmp("auto", p->val) != 0) {
2656  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2657  }
2658  if (EngineModeIsIPS()) {
2659  cfg_prec->http_body_inline = 1;
2660  } else {
2661  cfg_prec->http_body_inline = 0;
2662  }
2663  }
2664  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2665  ConfNode *pval;
2666 
2667  TAILQ_FOREACH(pval, &p->head, next) {
2668  if (strcasecmp("enabled", pval->name) == 0) {
2669  if (ConfValIsTrue(pval->val)) {
2670  cfg_prec->swf_decompression_enabled = 1;
2671  } else if (ConfValIsFalse(pval->val)) {
2672  cfg_prec->swf_decompression_enabled = 0;
2673  } else {
2674  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2675  }
2676  } else if (strcasecmp("type", pval->name) == 0) {
2677  if (strcasecmp("no", pval->val) == 0) {
2679  } else if (strcasecmp("deflate", pval->val) == 0) {
2681  } else if (strcasecmp("lzma", pval->val) == 0) {
2683  } else if (strcasecmp("both", pval->val) == 0) {
2685  } else {
2687  "Invalid entry for "
2688  "swf-decompression.type: %s - "
2689  "Killing engine", pval->val);
2690  exit(EXIT_FAILURE);
2691  }
2692  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2693  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2695  "Error parsing swf-decompression.compression-depth "
2696  "from conf file - %s. Killing engine", p->val);
2697  exit(EXIT_FAILURE);
2698  }
2699  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2700  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2702  "Error parsing swf-decompression.decompression-depth "
2703  "from conf file - %s. Killing engine", p->val);
2704  exit(EXIT_FAILURE);
2705  }
2706  } else {
2707  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2708  }
2709  }
2710  } else {
2711  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2712  "default config: %s", p->name);
2713  }
2714  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2715 
2716  return;
2717 }
2718 
2719 void HTPConfigure(void)
2720 {
2721  SCEnter();
2722 
2723  cfglist.next = NULL;
2724 
2725  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2726  if (NULL == cfgtree)
2727  exit(EXIT_FAILURE);
2728 
2729  /* Default Config */
2730  cfglist.cfg = htp_config_create();
2731  if (NULL == cfglist.cfg) {
2732  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP default config");
2733  exit(EXIT_FAILURE);
2734  }
2735  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2736  HTPConfigSetDefaultsPhase1(&cfglist);
2737  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2738  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2739  cfgtree);
2740  } else {
2741  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2742  }
2743  HTPConfigSetDefaultsPhase2("default", &cfglist);
2744 
2745  HTPParseMemcap();
2746 
2747  /* Read server config and create a parser for each IP in radix tree */
2748  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2749  if (server_config == NULL) {
2750  server_config = ConfGetNode("libhtp.server-config");
2751  if (server_config == NULL) {
2752  SCLogDebug("LIBHTP Configuring %p", server_config);
2753  SCReturn;
2754  }
2755  }
2756  SCLogDebug("LIBHTP Configuring %p", server_config);
2757 
2758  ConfNode *si;
2759  /* Server Nodes */
2760  TAILQ_FOREACH(si, &server_config->head, next) {
2761  /* Need the named node, not the index */
2762  ConfNode *s = TAILQ_FIRST(&si->head);
2763  if (NULL == s) {
2764  SCLogDebug("LIBHTP s NULL");
2765  continue;
2766  }
2767 
2768  SCLogDebug("LIBHTP server %s", s->name);
2769 
2770  HTPCfgRec *nextrec = cfglist.next;
2771  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2772  if (NULL == htprec)
2773  exit(EXIT_FAILURE);
2774  memset(htprec, 0x00, sizeof(*htprec));
2775 
2776  cfglist.next = htprec;
2777 
2778  cfglist.next->next = nextrec;
2779  cfglist.next->cfg = htp_config_create();
2780  if (NULL == cfglist.next->cfg) {
2781  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP server config");
2782  exit(EXIT_FAILURE);
2783  }
2784 
2785  HTPConfigSetDefaultsPhase1(htprec);
2786  HTPConfigParseParameters(htprec, s, cfgtree);
2787  HTPConfigSetDefaultsPhase2(s->name, htprec);
2788  }
2789 
2790  SCReturn;
2791 }
2792 
2794 {
2795 #ifdef DEBUG
2796  SCMutexLock(&htp_state_mem_lock);
2797  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2798  SCMutexUnlock(&htp_state_mem_lock);
2799 #endif
2800 }
2801 
2802 /** \internal
2803  * \brief get files callback
2804  * \param state state ptr
2805  * \param direction flow direction
2806  * \retval files files ptr
2807  */
2808 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
2809 {
2810  if (state == NULL)
2811  return NULL;
2812 
2813  HtpState *http_state = (HtpState *)state;
2814 
2815  if (direction & STREAM_TOCLIENT) {
2816  SCReturnPtr(http_state->files_tc, "FileContainer");
2817  } else {
2818  SCReturnPtr(http_state->files_ts, "FileContainer");
2819  }
2820 }
2821 
2822 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2823 {
2824  if (direction & STREAM_TOSERVER)
2825  return ((htp_tx_t *)tx)->request_progress;
2826  else
2827  return ((htp_tx_t *)tx)->response_progress;
2828 }
2829 
2830 static uint64_t HTPStateGetTxCnt(void *alstate)
2831 {
2832  HtpState *http_state = (HtpState *)alstate;
2833 
2834  if (http_state != NULL && http_state->conn != NULL) {
2835  const uint64_t size = (uint64_t)htp_list_size(http_state->conn->transactions);
2836  SCLogDebug("size %"PRIu64, size);
2837  return size;
2838  } else {
2839  return 0ULL;
2840  }
2841 }
2842 
2843 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2844 {
2845  HtpState *http_state = (HtpState *)alstate;
2846 
2847  if (http_state != NULL && http_state->conn != NULL)
2848  return htp_list_get(http_state->conn->transactions, tx_id);
2849  else
2850  return NULL;
2851 }
2852 
2853 static void HTPStateSetTxLogged(void *alstate, void *vtx, LoggerId bits)
2854 {
2855  htp_tx_t *tx = (htp_tx_t *)vtx;
2856  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2857  if (tx_ud)
2858  tx_ud->logged = bits;
2859 }
2860 
2861 static LoggerId HTPStateGetTxLogged(void *alstate, void *vtx)
2862 {
2863  htp_tx_t *tx = (htp_tx_t *)vtx;
2864  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2865  if (tx_ud != NULL)
2866  return tx_ud->logged;
2867 
2868  return 0;
2869 }
2870 
2871 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction)
2872 {
2873  return (direction & STREAM_TOSERVER) ? HTP_REQUEST_COMPLETE : HTP_RESPONSE_COMPLETE;
2874 }
2875 
2876 static int HTPStateGetEventInfo(const char *event_name,
2877  int *event_id, AppLayerEventType *event_type)
2878 {
2879  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
2880  if (*event_id == -1) {
2881  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
2882  "http's enum map table.", event_name);
2883  /* this should be treated as fatal */
2884  return -1;
2885  }
2886 
2887  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2888 
2889  return 0;
2890 }
2891 
2892 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
2894 {
2895  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
2896  if (*event_name == NULL) {
2897  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
2898  "http's enum map table.", event_id);
2899  /* this should be treated as fatal */
2900  return -1;
2901  }
2902 
2903  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2904 
2905  return 0;
2906 }
2907 
2908 static void HTPStateTruncate(void *state, uint8_t direction)
2909 {
2910  FileContainer *fc = HTPStateGetFiles(state, direction);
2911  if (fc != NULL) {
2913  }
2914 }
2915 
2916 static DetectEngineState *HTPGetTxDetectState(void *vtx)
2917 {
2918  htp_tx_t *tx = (htp_tx_t *)vtx;
2919  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2920  return tx_ud ? tx_ud->de_state : NULL;
2921 }
2922 
2923 static int HTPSetTxDetectState(void *vtx, DetectEngineState *s)
2924 {
2925  htp_tx_t *tx = (htp_tx_t *)vtx;
2926  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2927  if (tx_ud == NULL) {
2928  tx_ud = HTPMalloc(sizeof(*tx_ud));
2929  if (unlikely(tx_ud == NULL))
2930  return -ENOMEM;
2931  memset(tx_ud, 0, sizeof(*tx_ud));
2932  htp_tx_set_user_data(tx, tx_ud);
2933  }
2934  tx_ud->de_state = s;
2935  return 0;
2936 }
2937 
2938 static uint64_t HTPGetTxDetectFlags(void *vtx, uint8_t dir)
2939 {
2940  htp_tx_t *tx = (htp_tx_t *)vtx;
2941  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2942  if (tx_ud) {
2943  if (dir & STREAM_TOSERVER) {
2944  return tx_ud->detect_flags_ts;
2945  } else {
2946  return tx_ud->detect_flags_tc;
2947  }
2948  }
2949  return 0;
2950 }
2951 
2952 static void HTPSetTxDetectFlags(void *vtx, uint8_t dir, uint64_t detect_flags)
2953 {
2954  htp_tx_t *tx = (htp_tx_t *)vtx;
2955  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2956  if (tx_ud == NULL) {
2957  tx_ud = HTPMalloc(sizeof(*tx_ud));
2958  if (unlikely(tx_ud == NULL))
2959  return;
2960  memset(tx_ud, 0, sizeof(*tx_ud));
2961  htp_tx_set_user_data(tx, tx_ud);
2962  }
2963  if (dir & STREAM_TOSERVER) {
2964  tx_ud->detect_flags_ts = detect_flags;
2965  } else {
2966  tx_ud->detect_flags_tc = detect_flags;
2967  }
2968  return;
2969 }
2970 
2971 static int HTPRegisterPatternsForProtocolDetection(void)
2972 {
2973  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
2974  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
2975  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
2976  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
2977  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
2978  const char *spacings[] = { "|20|", "|09|", NULL };
2979  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
2980 
2981  int methods_pos;
2982  int spacings_pos;
2983  int versions_pos;
2984  int register_result;
2985  char method_buffer[32] = "";
2986 
2987  /* Loop through all the methods ands spacings and register the patterns */
2988  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
2989  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
2990 
2991  /* Combine the method name and the spacing */
2992  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
2993 
2994  /* Register the new method+spacing pattern
2995  * 3 is subtracted from the length since the spacing is hex typed as |xx|
2996  * but the pattern matching should only be one char
2997  */
2998  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
2999  ALPROTO_HTTP, method_buffer, strlen(method_buffer)-3, 0, STREAM_TOSERVER);
3000  if (register_result < 0) {
3001  return -1;
3002  }
3003  }
3004  }
3005 
3006  /* Loop through all the http verions patterns that are TO_CLIENT */
3007  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3008  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
3009  ALPROTO_HTTP, versions[versions_pos], strlen(versions[versions_pos]),
3010  0, STREAM_TOCLIENT);
3011  if (register_result < 0) {
3012  return -1;
3013  }
3014  }
3015 
3016  return 0;
3017 }
3018 
3019 /**
3020  * \brief Register the HTTP protocol and state handling functions to APP layer
3021  * of the engine.
3022  */
3024 {
3025  SCEnter();
3026 
3027  const char *proto_name = "http";
3028 
3029  /** HTTP */
3030  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3032  if (HTPRegisterPatternsForProtocolDetection() < 0)
3033  return;
3034  } else {
3035  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3036  proto_name);
3037  return;
3038  }
3039 
3040  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3041  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateAlloc, HTPStateFree);
3042  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTransactionFree);
3043  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetFiles);
3044  AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetAlstateProgress);
3045  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxCnt);
3046  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTx);
3047  AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxLogged,
3048  HTPStateSetTxLogged);
3050  HTPStateGetAlstateProgressCompletionStatus);
3051  AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPGetEvents);
3052  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfo);
3053  AppLayerParserRegisterGetEventInfoById(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfoById);
3054 
3055  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTruncate);
3057  HTPGetTxDetectState, HTPSetTxDetectState);
3059  HTPGetTxDetectFlags, HTPSetTxDetectFlags);
3060 
3062  HTPHandleRequestData);
3064  HTPHandleResponseData);
3065  SC_ATOMIC_INIT(htp_config_flags);
3068  HTPConfigure();
3069  } else {
3070  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3071  "still on.", proto_name);
3072  }
3073 #ifdef UNITTESTS
3075 #endif
3076 
3077  SCReturn;
3078 }
3079 
3080 #ifdef UNITTESTS
3081 static HTPCfgRec cfglist_backup;
3082 
3084 {
3085  cfglist_backup = cfglist;
3086 
3087  return;
3088 }
3089 
3091 {
3092  cfglist = cfglist_backup;
3093 
3094  return;
3095 }
3096 
3097 /** \test Test case where chunks are sent in smaller chunks and check the
3098  * response of the parser from HTP library. */
3099 static int HTPParserTest01(void)
3100 {
3101  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3102  " Data is c0oL!";
3103  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3104 
3105  TcpSession ssn;
3106  memset(&ssn, 0, sizeof(ssn));
3107 
3109  FAIL_IF_NULL(alp_tctx);
3110 
3111  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3112  FAIL_IF_NULL(f);
3113  f->protoctx = &ssn;
3114  f->proto = IPPROTO_TCP;
3115  f->alproto = ALPROTO_HTTP;
3116 
3118 
3119  uint32_t u;
3120  for (u = 0; u < httplen1; u++) {
3121  uint8_t flags = 0;
3122 
3123  if (u == 0)
3124  flags = STREAM_TOSERVER|STREAM_START;
3125  else if (u == (httplen1 - 1))
3126  flags = STREAM_TOSERVER|STREAM_EOF;
3127  else
3128  flags = STREAM_TOSERVER;
3129 
3130  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3131  &httpbuf1[u], 1);
3132  FAIL_IF(r != 0);
3133  }
3134 
3135  HtpState *htp_state = f->alstate;
3136  FAIL_IF_NULL(htp_state);
3137 
3138  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3139  FAIL_IF_NULL(tx);
3140 
3141  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3142  FAIL_IF_NULL(h);
3143 
3144  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3145  FAIL_IF(tx->request_method_number != HTP_M_POST);
3146  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3147 
3148  AppLayerParserThreadCtxFree(alp_tctx);
3150  UTHFreeFlow(f);
3151  PASS;
3152 }
3153 
3154 /** \test Test folding in 1 read case */
3155 static int HTPParserTest01b(void)
3156 {
3157  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3158  " Data is c0oL!";
3159  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3160 
3161  TcpSession ssn;
3162  memset(&ssn, 0, sizeof(ssn));
3163 
3165  FAIL_IF_NULL(alp_tctx);
3166 
3167  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3168  FAIL_IF_NULL(f);
3169  f->protoctx = &ssn;
3170  f->proto = IPPROTO_TCP;
3171  f->alproto = ALPROTO_HTTP;
3172 
3174 
3176  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3177  httpbuf1, httplen1);
3178  FAIL_IF(r != 0);
3179 
3180  HtpState *htp_state = f->alstate;
3181  FAIL_IF_NULL(htp_state);
3182 
3183  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3184  FAIL_IF_NULL(tx);
3185 
3186  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3187  FAIL_IF_NULL(h);
3188 
3189  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3190  FAIL_IF(tx->request_method_number != HTP_M_POST);
3191  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3192 
3193  AppLayerParserThreadCtxFree(alp_tctx);
3195  UTHFreeFlow(f);
3196  PASS;
3197 }
3198 
3199 /** \test Test folding in 1byte per read case */
3200 static int HTPParserTest01c(void)
3201 {
3202  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3203  " Data is c0oL!";
3204  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3205 
3206  TcpSession ssn;
3207  memset(&ssn, 0, sizeof(ssn));
3208 
3210  FAIL_IF_NULL(alp_tctx);
3211 
3212  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3213  FAIL_IF_NULL(f);
3214  f->protoctx = &ssn;
3215  f->proto = IPPROTO_TCP;
3216  f->alproto = ALPROTO_HTTP;
3217 
3219 
3220  uint32_t u;
3221  for (u = 0; u < httplen1; u++) {
3222  uint8_t flags = 0;
3223 
3224  if (u == 0)
3225  flags = STREAM_TOSERVER|STREAM_START;
3226  else if (u == (httplen1 - 1))
3227  flags = STREAM_TOSERVER|STREAM_EOF;
3228  else
3229  flags = STREAM_TOSERVER;
3230 
3231  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3232  &httpbuf1[u], 1);
3233  FAIL_IF(r != 0);
3234  }
3235 
3236  HtpState *htp_state = f->alstate;
3237  FAIL_IF_NULL(htp_state);
3238 
3239  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3240  FAIL_IF_NULL(tx);
3241 
3242  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3243  FAIL_IF_NULL(h);
3244 
3245  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3246  FAIL_IF(tx->request_method_number != HTP_M_POST);
3247  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3248 
3249  AppLayerParserThreadCtxFree(alp_tctx);
3251  UTHFreeFlow(f);
3252  PASS;
3253 }
3254 
3255 /** \test Test case where chunks are sent in smaller chunks and check the
3256  * response of the parser from HTP library. */
3257 static int HTPParserTest01a(void)
3258 {
3259  int result = 0;
3260  Flow *f = NULL;
3261  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3262  " Data is c0oL!";
3263  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3264  TcpSession ssn;
3265  HtpState *htp_state = NULL;
3266  int r = 0;
3268 
3269  memset(&ssn, 0, sizeof(ssn));
3270 
3271  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3272  if (f == NULL)
3273  goto end;
3274  f->protoctx = &ssn;
3275  f->proto = IPPROTO_TCP;
3276  f->alproto = ALPROTO_HTTP;
3277 
3279 
3280  uint32_t u;
3281  for (u = 0; u < httplen1; u++) {
3282  uint8_t flags = 0;
3283 
3284  if (u == 0)
3285  flags = STREAM_TOSERVER|STREAM_START;
3286  else if (u == (httplen1 - 1))
3287  flags = STREAM_TOSERVER|STREAM_EOF;
3288  else
3289  flags = STREAM_TOSERVER;
3290 
3291  FLOWLOCK_WRLOCK(f);
3292  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3293  &httpbuf1[u], 1);
3294  if (r != 0) {
3295  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3296  " 0: ", u, r);
3297  FLOWLOCK_UNLOCK(f);
3298  goto end;
3299  }
3300  FLOWLOCK_UNLOCK(f);
3301  }
3302 
3303  htp_state = f->alstate;
3304  if (htp_state == NULL) {
3305  printf("no http state: ");
3306  goto end;
3307  }
3308 
3309  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3310  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3311  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3312  || tx->request_method_number != HTP_M_POST ||
3313  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3314  {
3315  printf("expected header value: Victor/1.0 and got %s: and expected"
3316  " method: POST and got %s, expected protocol number HTTP/1.0"
3317  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3318  bstr_util_strdup_to_c(tx->request_method),
3319  bstr_util_strdup_to_c(tx->request_protocol));
3320  goto end;
3321  }
3322  result = 1;
3323 end:
3324  if (alp_tctx != NULL)
3325  AppLayerParserThreadCtxFree(alp_tctx);
3327  UTHFreeFlow(f);
3328  return result;
3329 }
3330 
3331 /** \test See how it deals with an incomplete request. */
3332 static int HTPParserTest02(void)
3333 {
3334  int result = 0;
3335  Flow *f = NULL;
3336  uint8_t httpbuf1[] = "POST";
3337  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3338  TcpSession ssn;
3339  HtpState *http_state = NULL;
3341 
3342  memset(&ssn, 0, sizeof(ssn));
3343 
3344  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3345  if (f == NULL)
3346  goto end;
3347  f->protoctx = &ssn;
3348  f->proto = IPPROTO_TCP;
3349  f->alproto = ALPROTO_HTTP;
3350 
3352 
3353  FLOWLOCK_WRLOCK(f);
3354  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3356  httpbuf1,
3357  httplen1);
3358  if (r != 0) {
3359  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3360  FLOWLOCK_UNLOCK(f);
3361  goto end;
3362  }
3363  FLOWLOCK_UNLOCK(f);
3364 
3365  http_state = f->alstate;
3366  if (http_state == NULL) {
3367  printf("no http state: ");
3368  goto end;
3369  }
3370 
3371  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3372  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3373  if ((tx->request_method) != NULL || h != NULL)
3374  {
3375  printf("expected method NULL, got %s \n", bstr_util_strdup_to_c(tx->request_method));
3376  goto end;
3377  }
3378  result = 1;
3379 end:
3380  if (alp_tctx != NULL)
3381  AppLayerParserThreadCtxFree(alp_tctx);
3383  UTHFreeFlow(f);
3384  return result;
3385 }
3386 
3387 /** \test Test case where method is invalid and data is sent in smaller chunks
3388  * and check the response of the parser from HTP library. */
3389 static int HTPParserTest03(void)
3390 {
3391  int result = 0;
3392  Flow *f = NULL;
3393  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3394  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3395  TcpSession ssn;
3396  HtpState *htp_state = NULL;
3397  int r = 0;
3399 
3400  memset(&ssn, 0, sizeof(ssn));
3401 
3402  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3403  if (f == NULL)
3404  goto end;
3405  f->protoctx = &ssn;
3406  f->proto = IPPROTO_TCP;
3407  f->alproto = ALPROTO_HTTP;
3408 
3410 
3411  uint32_t u;
3412  for (u = 0; u < httplen1; u++) {
3413  uint8_t flags = 0;
3414 
3415  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3416  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3417  else flags = STREAM_TOSERVER;
3418 
3419  FLOWLOCK_WRLOCK(f);
3420  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3421  &httpbuf1[u], 1);
3422  if (r != 0) {
3423  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3424  " 0: ", u, r);
3425  FLOWLOCK_UNLOCK(f);
3426  goto end;
3427  }
3428  FLOWLOCK_UNLOCK(f);
3429  }
3430  htp_state = f->alstate;
3431  if (htp_state == NULL) {
3432  printf("no http state: ");
3433  goto end;
3434  }
3435 
3436  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3437 
3438  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3439  if (tx->request_method_number != HTP_M_UNKNOWN ||
3440  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3441  {
3442  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3443  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3444  bstr_util_strdup_to_c(tx->request_protocol));
3445  goto end;
3446  }
3447  result = 1;
3448 end:
3449  if (alp_tctx != NULL)
3450  AppLayerParserThreadCtxFree(alp_tctx);
3452  UTHFreeFlow(f);
3453  return result;
3454 }
3455 
3456 /** \test Test case where invalid data is sent and check the response of the
3457  * parser from HTP library. */
3458 static int HTPParserTest04(void)
3459 {
3460  int result = 0;
3461  Flow *f = NULL;
3462  HtpState *htp_state = NULL;
3463  uint8_t httpbuf1[] = "World!\r\n";
3464  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3465  TcpSession ssn;
3466  int r = 0;
3468 
3469  memset(&ssn, 0, sizeof(ssn));
3470 
3471  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3472  if (f == NULL)
3473  goto end;
3474  f->protoctx = &ssn;
3475  f->proto = IPPROTO_TCP;
3476  f->alproto = ALPROTO_HTTP;
3477 
3479 
3480  FLOWLOCK_WRLOCK(f);
3481  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3483  httpbuf1,
3484  httplen1);
3485  if (r != 0) {
3486  FLOWLOCK_UNLOCK(f);
3487  goto end;
3488  }
3489  FLOWLOCK_UNLOCK(f);
3490 
3491  htp_state = f->alstate;
3492  if (htp_state == NULL) {
3493  printf("no http state: ");
3494  goto end;
3495  }
3496 
3497  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3498  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3499  if (tx->request_method_number != HTP_M_UNKNOWN ||
3500  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3501  {
3502  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3503  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3504  bstr_util_strdup_to_c(tx->request_protocol));
3505  goto end;
3506  }
3507  result = 1;
3508 end:
3509  if (alp_tctx != NULL)
3510  AppLayerParserThreadCtxFree(alp_tctx);
3512  UTHFreeFlow(f);
3513  return result;
3514 }
3515 
3516 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3517  * properly parsed them and also keeps them separated. */
3518 static int HTPParserTest05(void)
3519 {
3520  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3521  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3522  uint8_t httpbuf2[] = "Post D";
3523  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3524  uint8_t httpbuf3[] = "ata is c0oL!";
3525  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3526 
3527  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3528  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3529  uint8_t httpbuf5[] = "post R";
3530  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3531  uint8_t httpbuf6[] = "esults are tha bomb!";
3532  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3533 
3534  TcpSession ssn;
3535  memset(&ssn, 0, sizeof(ssn));
3536 
3538  FAIL_IF_NULL(alp_tctx);
3539 
3540  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3541  FAIL_IF_NULL(f);
3542  f->protoctx = &ssn;
3543  f->proto = IPPROTO_TCP;
3544  f->alproto = ALPROTO_HTTP;
3545 
3547 
3548  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3549  STREAM_TOSERVER | STREAM_START, httpbuf1,
3550  httplen1);
3551  FAIL_IF(r != 0);
3552 
3553  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3554  STREAM_TOCLIENT | STREAM_START, httpbuf4,
3555  httplen4);
3556  FAIL_IF(r != 0);
3557 
3558  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOCLIENT,
3559  httpbuf5, httplen5);
3560  FAIL_IF(r != 0);
3561 
3562  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOSERVER,
3563  httpbuf2, httplen2);
3564  FAIL_IF(r != 0);
3565 
3566  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3567  STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3568  FAIL_IF(r != 0);
3569 
3570  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3571  STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3572  FAIL_IF(r != 0);
3573 
3574  HtpState *http_state = f->alstate;
3575  FAIL_IF_NULL(http_state);
3576 
3577  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3578  FAIL_IF_NULL(tx);
3579  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3580  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3581 
3582  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3583  FAIL_IF_NULL(h);
3584 
3585  FAIL_IF_NOT(tx->response_status_number == 200);
3586 
3587  AppLayerParserThreadCtxFree(alp_tctx);
3589  UTHFreeFlow(f);
3590  PASS;
3591 }
3592 
3593 /** \test Test proper chunked encoded response body
3594  */
3595 static int HTPParserTest06(void)
3596 {
3597  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3598  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3599  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3600  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3601  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3602  "GMT\r\n"
3603  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3604  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3605  "FrontPage/5.0.2.2510\r\n"
3606  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3607  "chunked\r\n"
3608  "Content-Type: text/html\r\n\r\n"
3609  "580\r\n"
3610  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3611  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3612  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3613  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3614  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3615  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3616  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3617  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3618  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3619  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3620  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3621  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3622  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3623  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3624  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3625  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3626  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3627  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3628  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3629  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3630  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3631  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3632  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3633  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3634  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3635  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3636  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3637  "aHA=\r\n0\r\n\r\n";
3638  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3639  TcpSession ssn;
3640 
3642  FAIL_IF_NULL(alp_tctx);
3643 
3644  memset(&ssn, 0, sizeof(ssn));
3645 
3646  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3647  FAIL_IF_NULL(f);
3648  f->protoctx = &ssn;
3649  f->proto = IPPROTO_TCP;
3650  f->alproto = ALPROTO_HTTP;
3651 
3653 
3654  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3655  STREAM_TOSERVER | STREAM_START, httpbuf1,
3656  httplen1);
3657  FAIL_IF(r != 0);
3658  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3659  STREAM_TOCLIENT | STREAM_START, httpbuf2,
3660  httplen2);
3661  FAIL_IF(r != 0);
3662 
3663  HtpState *http_state = f->alstate;
3664  FAIL_IF_NULL(http_state);
3665 
3666  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3667  FAIL_IF_NULL(tx);
3668 
3669  FAIL_IF(tx->request_method_number != HTP_M_GET);
3670  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3671 
3672  FAIL_IF(tx->response_status_number != 200);
3673  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3674 
3675  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3676  FAIL_IF_NULL(h);
3677 
3678  AppLayerParserThreadCtxFree(alp_tctx);
3680  UTHFreeFlow(f);
3681  PASS;
3682 }
3683 
3684 /** \test
3685  */
3686 static int HTPParserTest07(void)
3687 {
3688  int result = 0;
3689  Flow *f = NULL;
3690  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3691  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3692  TcpSession ssn;
3693  HtpState *htp_state = NULL;
3694  int r = 0;
3696 
3697  memset(&ssn, 0, sizeof(ssn));
3698 
3699  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3700  if (f == NULL)
3701  goto end;
3702  f->protoctx = &ssn;
3703  f->proto = IPPROTO_TCP;
3704  f->alproto = ALPROTO_HTTP;
3705 
3707 
3708  uint32_t u;
3709  for (u = 0; u < httplen1; u++) {
3710  uint8_t flags = 0;
3711 
3712  if (u == 0)
3713  flags = STREAM_TOSERVER|STREAM_START;
3714  else if (u == (httplen1 - 1))
3715  flags = STREAM_TOSERVER|STREAM_EOF;
3716  else
3717  flags = STREAM_TOSERVER;
3718 
3719  FLOWLOCK_WRLOCK(f);
3720  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3721  &httpbuf1[u], 1);
3722  if (r != 0) {
3723  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3724  " 0: ", u, r);
3725  FLOWLOCK_UNLOCK(f);
3726  goto end;
3727  }
3728  FLOWLOCK_UNLOCK(f);
3729  }
3730 
3731  htp_state = f->alstate;
3732  if (htp_state == NULL) {
3733  printf("no http state: ");
3734  goto end;
3735  }
3736 
3737  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3738  size_t reflen = sizeof(ref) - 1;
3739 
3740  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3741  if (tx == NULL)
3742  goto end;
3743  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3744  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3745  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3746  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3747  (uintmax_t)reflen,
3748  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3749  goto end;
3750  }
3751 
3752  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3753  bstr_len(tx_ud->request_uri_normalized)) != 0)
3754  {
3755  printf("normalized uri \"");
3756  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3757  printf("\" != \"");
3758  PrintRawUriFp(stdout, ref, reflen);
3759  printf("\": ");
3760  goto end;
3761  }
3762  }
3763 
3764  result = 1;
3765 end:
3766  if (alp_tctx != NULL)
3767  AppLayerParserThreadCtxFree(alp_tctx);
3769  UTHFreeFlow(f);
3770  return result;
3771 }
3772 
3773 #include "conf-yaml-loader.h"
3774 
3775 /** \test Abort
3776  */
3777 static int HTPParserTest08(void)
3778 {
3779  int result = 0;
3780  Flow *f = NULL;
3781  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3782  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3783  TcpSession ssn;
3785 
3786  char input[] = "\
3787 %YAML 1.1\n\
3788 ---\n\
3789 libhtp:\n\
3790 \n\
3791  default-config:\n\
3792  personality: IDS\n\
3793 ";
3794 
3796  ConfInit();
3798 
3799  ConfYamlLoadString(input, strlen(input));
3800  HTPConfigure();
3801 
3802  HtpState *htp_state = NULL;
3803  int r = 0;
3804  memset(&ssn, 0, sizeof(ssn));
3805 
3806  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3807  if (f == NULL)
3808  goto end;
3809  f->protoctx = &ssn;
3810  f->proto = IPPROTO_TCP;
3811  f->alproto = ALPROTO_HTTP;
3812 
3814 
3815  uint8_t flags = 0;
3817 
3818  FLOWLOCK_WRLOCK(f);
3819  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
3820  httplen1);
3821  if (r != 0) {
3822  printf("toserver chunk returned %" PRId32 ", expected"
3823  " 0: ", r);
3824  result = 0;
3825  FLOWLOCK_UNLOCK(f);
3826  goto end;
3827  }
3828  FLOWLOCK_UNLOCK(f);
3829 
3830  htp_state = f->alstate;
3831  if (htp_state == NULL) {
3832  printf("no http state: ");
3833  result = 0;
3834  goto end;
3835  }
3836 
3837  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3838  if (tx == NULL)
3839  goto end;
3840  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3841  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3842  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3843  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3844  bstr_len(tx_ud->request_uri_normalized));
3845  }
3846 
3847  result = 1;
3848 end:
3849  if (alp_tctx != NULL)
3850  AppLayerParserThreadCtxFree(alp_tctx);
3852  HTPFreeConfig();
3853  ConfDeInit();
3856  UTHFreeFlow(f);
3857  return result;
3858 }
3859 
3860 /** \test Abort
3861  */
3862 static int HTPParserTest09(void)
3863 {
3864  int result = 0;
3865  Flow *f = NULL;
3866  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3867  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3868  TcpSession ssn;
3870 
3871  char input[] = "\
3872 %YAML 1.1\n\
3873 ---\n\
3874 libhtp:\n\
3875 \n\
3876  default-config:\n\
3877  personality: Apache_2_2\n\
3878 ";
3879 
3881  ConfInit();
3883 
3884  ConfYamlLoadString(input, strlen(input));
3885  HTPConfigure();
3886 
3887  HtpState *htp_state = NULL;
3888  int r = 0;
3889 
3890  memset(&ssn, 0, sizeof(ssn));
3891 
3892  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3893  if (f == NULL)
3894  goto end;
3895  f->protoctx = &ssn;
3896  f->proto = IPPROTO_TCP;
3897  f->alproto = ALPROTO_HTTP;
3898 
3900 
3901  uint8_t flags = 0;
3903 
3904  FLOWLOCK_WRLOCK(f);
3905  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
3906  httplen1);
3907  if (r != 0) {
3908  printf("toserver chunk returned %" PRId32 ", expected"
3909  " 0: ", r);
3910  FLOWLOCK_UNLOCK(f);
3911  goto end;
3912  }
3913  FLOWLOCK_UNLOCK(f);
3914 
3915  htp_state = f->alstate;
3916  if (htp_state == NULL) {
3917  printf("no http state: ");
3918  goto end;
3919  }
3920 
3921  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3922  if (tx == NULL)
3923  goto end;
3924  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3925  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3926  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3927  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3928  bstr_len(tx_ud->request_uri_normalized));
3929  }
3930 
3931  result = 1;
3932 end:
3933  if (alp_tctx != NULL)
3934  AppLayerParserThreadCtxFree(alp_tctx);
3936  HTPFreeConfig();
3937  ConfDeInit();
3940  UTHFreeFlow(f);
3941  return result;
3942 }
3943 
3944 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
3945  */
3946 static int HTPParserTest10(void)
3947 {
3948  int result = 0;
3949  Flow *f = NULL;
3950  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
3951  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3952  TcpSession ssn;
3953  HtpState *htp_state = NULL;
3954  int r = 0;
3956 
3957  memset(&ssn, 0, sizeof(ssn));
3958 
3959  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3960  if (f == NULL)
3961  goto end;
3962  f->protoctx = &ssn;
3963  f->proto = IPPROTO_TCP;
3964  f->alproto = ALPROTO_HTTP;
3965 
3967 
3968  uint32_t u;
3969  for (u = 0; u < httplen1; u++) {
3970  uint8_t flags = 0;
3971 
3972  if (u == 0)
3973  flags = STREAM_TOSERVER|STREAM_START;
3974  else if (u == (httplen1 - 1))
3975  flags = STREAM_TOSERVER|STREAM_EOF;
3976  else
3977  flags = STREAM_TOSERVER;
3978 
3979  FLOWLOCK_WRLOCK(f);
3980  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3981  &httpbuf1[u], 1);
3982  if (r != 0) {
3983  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3984  " 0: ", u, r);
3985  FLOWLOCK_UNLOCK(f);
3986  goto end;
3987  }
3988  FLOWLOCK_UNLOCK(f);
3989  }
3990 
3991  htp_state = f->alstate;
3992  if (htp_state == NULL) {
3993  printf("no http state: ");
3994  goto end;
3995  }
3996 
3997  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3998  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3999  if (h == NULL) {
4000  goto end;
4001  }
4002 
4003  char *name = bstr_util_strdup_to_c(h->name);
4004  if (name == NULL) {
4005  goto end;
4006  }
4007 
4008  if (strcmp(name, "Host") != 0) {
4009  printf("header name not \"Host\", instead \"%s\": ", name);
4010  free(name);
4011  goto end;
4012  }
4013  free(name);
4014 
4015  char *value = bstr_util_strdup_to_c(h->value);
4016  if (value == NULL) {
4017  goto end;
4018  }
4019 
4020  if (strcmp(value, "www.google.com") != 0) {
4021  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4022  free(value);
4023  goto end;
4024  }
4025  free(value);
4026 
4027  result = 1;
4028 end:
4029  if (alp_tctx != NULL)
4030  AppLayerParserThreadCtxFree(alp_tctx);
4032  UTHFreeFlow(f);
4033  return result;
4034 }
4035 
4036 /** \test double encoding in path
4037  */
4038 static int HTPParserTest11(void)
4039 {
4040  int result = 0;
4041  Flow *f = NULL;
4042  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4043  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4044  TcpSession ssn;
4045  HtpState *htp_state = NULL;
4046  int r = 0;
4048 
4049  memset(&ssn, 0, sizeof(ssn));
4050 
4051  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4052  if (f == NULL)
4053  goto end;
4054  f->protoctx = &ssn;
4055  f->proto = IPPROTO_TCP;
4056  f->alproto = ALPROTO_HTTP;
4057 
4059 
4060  uint32_t u;
4061  for (u = 0; u < httplen1; u++) {
4062  uint8_t flags = 0;
4063 
4064  if (u == 0)
4065  flags = STREAM_TOSERVER|STREAM_START;
4066  else if (u == (httplen1 - 1))
4067  flags = STREAM_TOSERVER|STREAM_EOF;
4068  else
4069  flags = STREAM_TOSERVER;
4070 
4071  FLOWLOCK_WRLOCK(f);
4072  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4073  &httpbuf1[u], 1);
4074  if (r != 0) {
4075  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4076  " 0: ", u, r);
4077  FLOWLOCK_UNLOCK(f);
4078  goto end;
4079  }
4080  FLOWLOCK_UNLOCK(f);
4081  }
4082 
4083  htp_state = f->alstate;
4084  if (htp_state == NULL) {
4085  printf("no http state: ");
4086  goto end;
4087  }
4088 
4089  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4090  if (tx == NULL)
4091  goto end;
4092  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4093  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4094  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4095  printf("normalized uri len should be 2, is %"PRIuMAX,
4096  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4097  goto end;
4098  }
4099 
4100  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4101  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4102  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4103  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4104  {
4105  printf("normalized uri \"");
4106  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4107  printf("\": ");
4108  goto end;
4109  }
4110  }
4111 
4112  result = 1;
4113 end:
4114  if (alp_tctx != NULL)
4115  AppLayerParserThreadCtxFree(alp_tctx);
4117  UTHFreeFlow(f);
4118  return result;
4119 }
4120 
4121 /** \test double encoding in query
4122  */
4123 static int HTPParserTest12(void)
4124 {
4125  int result = 0;
4126  Flow *f = NULL;
4127  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4128  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4129  TcpSession ssn;
4130  HtpState *htp_state = NULL;
4131  int r = 0;
4133 
4134  memset(&ssn, 0, sizeof(ssn));
4135 
4136  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4137  if (f == NULL)
4138  goto end;
4139  f->protoctx = &ssn;
4140  f->proto = IPPROTO_TCP;
4141  f->alproto = ALPROTO_HTTP;
4142 
4144 
4145  uint32_t u;
4146  for (u = 0; u < httplen1; u++) {
4147  uint8_t flags = 0;
4148 
4149  if (u == 0)
4150  flags = STREAM_TOSERVER|STREAM_START;
4151  else if (u == (httplen1 - 1))
4152  flags = STREAM_TOSERVER|STREAM_EOF;
4153  else
4154  flags = STREAM_TOSERVER;
4155 
4156  FLOWLOCK_WRLOCK(f);
4157  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4158  &httpbuf1[u], 1);
4159  if (r != 0) {
4160  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4161  " 0: ", u, r);
4162  FLOWLOCK_UNLOCK(f);
4163  goto end;
4164  }
4165  FLOWLOCK_UNLOCK(f);
4166  }
4167 
4168  htp_state = f->alstate;
4169  if (htp_state == NULL) {
4170  printf("no http state: ");
4171  goto end;
4172  }
4173 
4174  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4175  if (tx == NULL)
4176  goto end;
4177  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4178  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4179  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4180  printf("normalized uri len should be 5, is %"PRIuMAX,
4181  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4182  goto end;
4183  }
4184 
4185  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4186  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4187  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4188  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4189  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4190  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4191  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4192  {
4193  printf("normalized uri \"");
4194  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4195  printf("\": ");
4196  goto end;
4197  }
4198  }
4199 
4200  result = 1;
4201  end:
4202  if (alp_tctx != NULL)
4203  AppLayerParserThreadCtxFree(alp_tctx);
4205  UTHFreeFlow(f);
4206  return result;
4207 }
4208 
4209 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4210  */
4211 static int HTPParserTest13(void)
4212 {
4213  int result = 0;
4214  Flow *f = NULL;
4215  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4216  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4217  TcpSession ssn;
4218  HtpState *htp_state = NULL;
4219  int r = 0;
4221 
4222  memset(&ssn, 0, sizeof(ssn));
4223 
4224  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4225  if (f == NULL)
4226  goto end;
4227  f->protoctx = &ssn;
4228  f->proto = IPPROTO_TCP;
4229  f->alproto = ALPROTO_HTTP;
4230 
4232 
4233  uint32_t u;
4234  for (u = 0; u < httplen1; u++) {
4235  uint8_t flags = 0;
4236 
4237  if (u == 0)
4238  flags = STREAM_TOSERVER|STREAM_START;
4239  else if (u == (httplen1 - 1))
4240  flags = STREAM_TOSERVER|STREAM_EOF;
4241  else
4242  flags = STREAM_TOSERVER;
4243 
4244  FLOWLOCK_WRLOCK(f);
4245  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4246  &httpbuf1[u], 1);
4247  if (r != 0) {
4248  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4249  " 0: ", u, r);
4250  FLOWLOCK_UNLOCK(f);
4251  goto end;
4252  }
4253  FLOWLOCK_UNLOCK(f);
4254  }
4255 
4256  htp_state = f->alstate;
4257  if (htp_state == NULL) {
4258  printf("no http state: ");
4259  goto end;
4260  }
4261 
4262  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4263  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4264  if (h == NULL) {
4265  goto end;
4266  }
4267 
4268  char *name = bstr_util_strdup_to_c(h->name);
4269  if (name == NULL) {
4270  goto end;
4271  }
4272 
4273  if (strcmp(name, "Host") != 0) {
4274  printf("header name not \"Host\", instead \"%s\": ", name);
4275  free(name);
4276  goto end;
4277  }
4278  free(name);
4279 
4280  char *value = bstr_util_strdup_to_c(h->value);
4281  if (value == NULL) {
4282  goto end;
4283  }
4284 
4285  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4286  printf("header value not \"www.google.com\", instead \"");
4287  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4288  printf("\": ");
4289  free(value);
4290  goto end;
4291  }
4292  free(value);
4293 
4294  result = 1;
4295 end:
4296  if (alp_tctx != NULL)
4297  AppLayerParserThreadCtxFree(alp_tctx);
4299  UTHFreeFlow(f);
4300  return result;
4301 }
4302 
4303 /** \test Test basic config */
4304 static int HTPParserConfigTest01(void)
4305 {
4306  int ret = 0;
4307  char input[] = "\
4308 %YAML 1.1\n\
4309 ---\n\
4310 libhtp:\n\
4311 \n\
4312  default-config:\n\
4313  personality: IDS\n\
4314 \n\
4315  server-config:\n\
4316 \n\
4317  - apache-tomcat:\n\
4318  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4319  personality: Tomcat_6_0\n\
4320 \n\
4321  - iis7:\n\
4322  address: \n\
4323  - 192.168.0.0/24\n\
4324  - 192.168.10.0/24\n\
4325  personality: IIS_7_0\n\
4326 ";
4327 
4329  ConfInit();
4330 
4331  ConfYamlLoadString(input, strlen(input));
4332 
4333  ConfNode *outputs;
4334  outputs = ConfGetNode("libhtp.default-config.personality");
4335  if (outputs == NULL) {
4336  goto end;
4337  }
4338 
4339  outputs = ConfGetNode("libhtp.server-config");
4340  if (outputs == NULL) {
4341  goto end;
4342  }
4343 
4344  ConfNode *node = TAILQ_FIRST(&outputs->head);
4345  if (node == NULL) {
4346  goto end;
4347  }
4348  if (strcmp(node->name, "0") != 0) {
4349  goto end;
4350  }
4351  node = TAILQ_FIRST(&node->head);
4352  if (node == NULL) {
4353  goto end;
4354  }
4355  if (strcmp(node->name, "apache-tomcat") != 0) {
4356  goto end;
4357  }
4358 
4359  int i = 0;
4360  ConfNode *n;
4361 
4362  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4363  if (node2 == NULL) {
4364  goto end;
4365  }
4366  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4367  goto end;
4368  }
4369 
4370  node = ConfNodeLookupChild(node, "address");
4371  if (node == NULL) {
4372  goto end;
4373  }
4374  TAILQ_FOREACH(n, &node->head, next) {
4375  if (n == NULL) {
4376  goto end;
4377  }
4378 
4379  switch(i) {
4380  case 0:
4381  if (strcmp(n->name, "0") != 0) {
4382  goto end;
4383  }
4384  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4385  goto end;
4386  }
4387  break;
4388  case 1:
4389  if (strcmp(n->name, "1") != 0) {
4390  goto end;
4391  }
4392  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4393  goto end;
4394  }
4395  break;
4396  case 2:
4397  if (strcmp(n->name, "2") != 0) {
4398  goto end;
4399  }
4400  if (strcmp(n->val, "::1") != 0) {
4401  goto end;
4402  }
4403  break;
4404  default:
4405  goto end;
4406  }
4407  i++;
4408  }
4409 
4410  outputs = ConfGetNode("libhtp.server-config");
4411  if (outputs == NULL) {
4412  goto end;
4413  }
4414 
4415  node = TAILQ_FIRST(&outputs->head);
4416  node = TAILQ_NEXT(node, next);
4417  if (node == NULL) {
4418  goto end;
4419  }
4420  if (strcmp(node->name, "1") != 0) {
4421  goto end;
4422  }
4423  node = TAILQ_FIRST(&node->head);
4424  if (node == NULL) {
4425  goto end;
4426  }
4427  if (strcmp(node->name, "iis7") != 0) {
4428  goto end;
4429  }
4430 
4431  node2 = ConfNodeLookupChild(node, "personality");
4432  if (node2 == NULL) {
4433  goto end;
4434  }
4435  if (strcmp(node2->val, "IIS_7_0") != 0) {
4436  goto end;
4437  }
4438 
4439  node = ConfNodeLookupChild(node, "address");
4440  if (node == NULL) {
4441  goto end;
4442  }
4443 
4444  i = 0;
4445  TAILQ_FOREACH(n, &node->head, next) {
4446  if (n == NULL) {
4447  goto end;
4448  }
4449 
4450  switch(i) {
4451  case 0:
4452  if (strcmp(n->name, "0") != 0) {
4453  goto end;
4454  }
4455  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4456  goto end;
4457  }
4458  break;
4459  case 1:
4460  if (strcmp(n->name, "1") != 0) {
4461  goto end;
4462  }
4463  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4464  goto end;
4465  }
4466  break;
4467  default:
4468  goto end;
4469  }
4470  i++;
4471  }
4472 
4473  ret = 1;
4474 
4475 end:
4476  ConfDeInit();
4478 
4479  return ret;
4480 }
4481 
4482 /** \test Test config builds radix correctly */
4483 static int HTPParserConfigTest02(void)
4484 {
4485  int ret = 0;
4486  char input[] = "\
4487 %YAML 1.1\n\
4488 ---\n\
4489 libhtp:\n\
4490 \n\
4491  default-config:\n\
4492  personality: IDS\n\
4493 \n\
4494  server-config:\n\
4495 \n\
4496  - apache-tomcat:\n\
4497  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4498  personality: Tomcat_6_0\n\
4499 \n\
4500  - iis7:\n\
4501  address: \n\
4502  - 192.168.0.0/24\n\
4503  - 192.168.10.0/24\n\
4504  personality: IIS_7_0\n\
4505 ";
4506 
4508  ConfInit();
4510 
4511  ConfYamlLoadString(input, strlen(input));
4512 
4513  HTPConfigure();
4514 
4515  if (cfglist.cfg == NULL) {
4516  printf("No default config created.\n");
4517  goto end;
4518  }
4519 
4520  if (cfgtree == NULL) {
4521  printf("No config tree created.\n");
4522  goto end;
4523  }
4524 
4525  htp_cfg_t *htp = cfglist.cfg;
4526  uint8_t buf[128];
4527  const char *addr;
4528  void *user_data = NULL;
4529 
4530  addr = "192.168.10.42";
4531  if (inet_pton(AF_INET, addr, buf) == 1) {
4532  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4533  if (user_data != NULL) {
4534  HTPCfgRec *htp_cfg_rec = user_data;
4535  htp = htp_cfg_rec->cfg;
4536  SCLogDebug("LIBHTP using config: %p", htp);
4537  }
4538  if (htp == NULL) {
4539  printf("Could not get config for: %s\n", addr);
4540  goto end;
4541  }
4542  }
4543  else {
4544  printf("Failed to parse address: %s\n", addr);
4545  goto end;
4546  }
4547 
4548  user_data = NULL;
4549  addr = "::1";
4550  if (inet_pton(AF_INET6, addr, buf) == 1) {
4551  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4552  if (user_data != NULL) {
4553  HTPCfgRec *htp_cfg_rec = user_data;
4554  htp = htp_cfg_rec->cfg;
4555  SCLogDebug("LIBHTP using config: %p", htp);
4556  }
4557  if (htp == NULL) {
4558  printf("Could not get config for: %s\n", addr);
4559  goto end;
4560  }
4561  }
4562  else {
4563  printf("Failed to parse address: %s\n", addr);
4564  goto end;
4565  }
4566 
4567  ret = 1;
4568 
4569 end:
4570  HTPFreeConfig();
4571  ConfDeInit();
4574 
4575  return ret;
4576 }
4577 
4578 /** \test Test traffic is handled by the correct htp config */
4579 static int HTPParserConfigTest03(void)
4580 {
4581  int result = 1;
4582  Flow *f = NULL;
4583  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4584  " Data is c0oL!";
4585  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4586  TcpSession ssn;
4588 
4589  HtpState *htp_state = NULL;
4590  int r = 0;
4591  char input[] = "\
4592 %YAML 1.1\n\
4593 ---\n\
4594 libhtp:\n\
4595 \n\
4596  default-config:\n\
4597  personality: IDS\n\
4598 \n\
4599  server-config:\n\
4600 \n\
4601  - apache-tomcat:\n\
4602  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4603  personality: Tomcat_6_0\n\
4604 \n\
4605  - iis7:\n\
4606  address: \n\
4607  - 192.168.0.0/24\n\
4608  - 192.168.10.0/24\n\
4609  personality: IIS_7_0\n\
4610 ";
4611 
4613  ConfInit();
4615 
4616  ConfYamlLoadString(input, strlen(input));
4617 
4618  HTPConfigure();
4619 
4620  const char *addr = "192.168.10.42";
4621 
4622  memset(&ssn, 0, sizeof(ssn));
4623 
4624  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4625  if (f == NULL)
4626  goto end;
4627  f->protoctx = &ssn;
4628  f->proto = IPPROTO_TCP;
4629  f->alproto = ALPROTO_HTTP;
4630 
4631  htp_cfg_t *htp = cfglist.cfg;
4632 
4633  void *user_data = NULL;
4634  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4635  if (user_data != NULL) {
4636  HTPCfgRec *htp_cfg_rec = user_data;
4637  htp = htp_cfg_rec->cfg;
4638  SCLogDebug("LIBHTP using config: %p", htp);
4639  }
4640  if (htp == NULL) {
4641  printf("Could not get config for: %s\n", addr);
4642  goto end;
4643  }
4644 
4646 
4647  uint32_t u;
4648  for (u = 0; u < httplen1; u++) {
4649  uint8_t flags = 0;
4650 
4651  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4652  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4653  else flags = STREAM_TOSERVER;
4654 
4655  FLOWLOCK_WRLOCK(f);
4656  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4657  &httpbuf1[u], 1);
4658  if (r != 0) {
4659  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4660  " 0: ", u, r);
4661  result = 0;
4662  FLOWLOCK_UNLOCK(f);
4663  goto end;
4664  }
4665  FLOWLOCK_UNLOCK(f);
4666  }
4667 
4668  htp_state = f->alstate;
4669  if (htp_state == NULL) {
4670  printf("no http state: ");
4671  result = 0;
4672  goto end;
4673  }
4674 
4675  if (HTPStateGetTxCnt(htp_state) != 2) {
4676  printf("HTPStateGetTxCnt(htp_state) failure\n");
4677  goto end;
4678  }
4679 
4680  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4681  if (tx == NULL)
4682  goto end;
4683  if (tx->cfg != htp) {
4684  printf("wrong HTP config (%p instead of %p - default=%p): ",
4685  tx->cfg, htp, cfglist.cfg);
4686  goto end;
4687  }
4688  tx = HTPStateGetTx(htp_state, 1);
4689  if (tx == NULL)
4690  goto end;
4691  if (tx->cfg != htp) {
4692  printf("wrong HTP config (%p instead of %p - default=%p): ",
4693  tx->cfg, htp, cfglist.cfg);
4694  goto end;
4695  }
4696 
4697 end:
4698  if (alp_tctx != NULL)
4699  AppLayerParserThreadCtxFree(alp_tctx);
4700  HTPFreeConfig();
4701  ConfDeInit();
4704 
4706  UTHFreeFlow(f);
4707  return result;
4708 }
4709 
4710 /* disabled when we upgraded to libhtp 0.5.x */
4711 #if 0
4712 static int HTPParserConfigTest04(void)
4713 {
4714  int result = 0;
4715 
4716  char input[] = "\
4717 %YAML 1.1\n\
4718 ---\n\
4719 libhtp:\n\
4720 \n\
4721  default-config:\n\
4722  personality: IDS\n\
4723  path-control-char-handling: status_400\n\
4724  path-convert-utf8: yes\n\
4725  path-invalid-encoding-handling: remove_percent\n\
4726 \n\
4727  server-config:\n\
4728 \n\
4729  - apache-tomcat:\n\
4730  personality: Tomcat_6_0\n\
4731  path-invalid-utf8-handling: none\n\
4732  path-nul-encoded-handling: status_404\n\
4733  path-nul-raw-handling: status_400\n\
4734 \n\
4735  - iis7:\n\
4736  personality: IIS_7_0\n\
4737  path-replacement-char: o\n\
4738  path-unicode-mapping: status_400\n\
4739 ";
4740 
4742  ConfInit();
4744 
4745  ConfYamlLoadString(input, strlen(input));
4746 
4747  HTPConfigure();
4748 
4749  HTPCfgRec *cfg_rec = &cfglist;
4750  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4751  cfg_rec->cfg->path_convert_utf8 != 1 ||
4752  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4753  printf("failed 1\n");
4754  goto end;
4755  }
4756 
4757  cfg_rec = cfg_rec->next;
4758  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4759  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4760  printf("failed 2\n");
4761  goto end;
4762  }
4763 
4764  cfg_rec = cfg_rec->next;
4765  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4766  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4767  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4768  printf("failed 3\n");
4769  goto end;
4770  }
4771 
4772  result = 1;
4773 
4774 end:
4775  HTPFreeConfig();
4776  ConfDeInit();
4779 
4780  return result;
4781 }
4782 #endif
4783 
4784 /** \test Test %2f decoding in profile Apache_2_2
4785  *
4786  * %2f in path is left untouched
4787  * %2f in query string is normalized to %2F
4788  * %252f in query string is decoded/normalized to %2F
4789  */
4790 static int HTPParserDecodingTest01(void)
4791 {
4792  int result = 0;
4793  Flow *f = NULL;
4794  uint8_t httpbuf1[] =
4795  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4796  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4797  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4798  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4799  TcpSession ssn;
4801 
4802  HtpState *htp_state = NULL;
4803  int r = 0;
4804  char input[] = "\
4805 %YAML 1.1\n\
4806 ---\n\
4807 libhtp:\n\
4808 \n\
4809  default-config:\n\
4810  personality: Apache_2\n\
4811 ";
4812 
4814  ConfInit();
4816  ConfYamlLoadString(input, strlen(input));
4817  HTPConfigure();
4818  const char *addr = "4.3.2.1";
4819  memset(&ssn, 0, sizeof(ssn));
4820 
4821  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4822  if (f == NULL)
4823  goto end;
4824  f->protoctx = &ssn;
4825  f->proto = IPPROTO_TCP;
4826  f->alproto = ALPROTO_HTTP;
4827 
4829 
4830  uint32_t u;
4831  for (u = 0; u < httplen1; u++) {
4832  uint8_t flags = 0;
4833 
4834  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4835  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4836  else flags = STREAM_TOSERVER;
4837 
4838  FLOWLOCK_WRLOCK(f);
4839  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4840  &httpbuf1[u], 1);
4841  if (r != 0) {
4842  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4843  " 0: ", u, r);
4844  FLOWLOCK_UNLOCK(f);
4845  goto end;
4846  }
4847  FLOWLOCK_UNLOCK(f);
4848  }
4849 
4850  htp_state = f->alstate;
4851  if (htp_state == NULL) {
4852  printf("no http state: ");
4853  goto end;
4854  }
4855 
4856  uint8_t ref1[] = "/abc%2fdef";
4857  size_t reflen = sizeof(ref1) - 1;
4858 
4859  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4860  if (tx == NULL)
4861  goto end;
4862  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4863  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4864  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4865  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4866  (uintmax_t)reflen,
4867  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4868  goto end;
4869  }
4870 
4871  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4872  bstr_len(tx_ud->request_uri_normalized)) != 0)
4873  {
4874  printf("normalized uri \"");
4875  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4876  printf("\" != \"");
4877  PrintRawUriFp(stdout, ref1, reflen);
4878  printf("\": ");
4879  goto end;
4880  }
4881  }
4882 
4883  uint8_t ref2[] = "/abc/def?ghi/jkl";
4884  reflen = sizeof(ref2) - 1;
4885 
4886  tx = HTPStateGetTx(htp_state, 1);
4887  if (tx == NULL)
4888  goto end;
4889  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4890  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4891  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4892  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4893  (uintmax_t)reflen,
4894  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4895  goto end;
4896  }
4897 
4898  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4899  bstr_len(tx_ud->request_uri_normalized)) != 0)
4900  {
4901  printf("normalized uri \"");
4902  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4903  printf("\" != \"");
4904  PrintRawUriFp(stdout, ref2, reflen);
4905  printf("\": ");
4906  goto end;
4907  }
4908  }
4909 
4910  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4911  reflen = sizeof(ref3) - 1;
4912  tx = HTPStateGetTx(htp_state, 2);
4913  if (tx == NULL)
4914  goto end;
4915  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4916  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4917  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4918  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4919  (uintmax_t)reflen,
4920  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4921  goto end;
4922  }
4923 
4924  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4925  bstr_len(tx_ud->request_uri_normalized)) != 0)
4926  {
4927  printf("normalized uri \"");
4928  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4929  printf("\" != \"");
4930  PrintRawUriFp(stdout, ref3, reflen);
4931  printf("\": ");
4932  goto end;
4933  }
4934  }
4935 
4936  result = 1;
4937 
4938 end:
4939  if (alp_tctx != NULL)
4940  AppLayerParserThreadCtxFree(alp_tctx);
4941  HTPFreeConfig();
4942  ConfDeInit();
4945 
4947  UTHFreeFlow(f);
4948  return result;
4949 }
4950 
4951 /** \test Test %2f decoding in profile IDS
4952  *
4953  * %2f in path decoded to /
4954  * %2f in query string is decoded to /
4955  * %252f in query string is decoded to %2F
4956  */
4957 static int HTPParserDecodingTest02(void)
4958 {
4959  int result = 0;
4960  Flow *f = NULL;
4961  uint8_t httpbuf1[] =
4962  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4963  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4964  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4965  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4966  TcpSession ssn;
4968 
4969  HtpState *htp_state = NULL;
4970  int r = 0;
4971  char input[] = "\
4972 %YAML 1.1\n\
4973 ---\n\
4974 libhtp:\n\
4975 \n\
4976  default-config:\n\
4977  personality: IDS\n\
4978  double-decode-path: no\n\
4979  double-decode-query: no\n\
4980 ";
4981 
4983  ConfInit();
4985  ConfYamlLoadString(input, strlen(input));
4986  HTPConfigure();
4987  const char *addr = "4.3.2.1";
4988  memset(&ssn, 0, sizeof(ssn));
4989 
4990  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4991  if (f == NULL)
4992  goto end;
4993  f->protoctx = &ssn;
4994  f->proto = IPPROTO_TCP;
4995  f->alproto = ALPROTO_HTTP;
4996 
4998 
4999  uint32_t u;
5000  for (u = 0; u < httplen1; u++) {
5001  uint8_t flags = 0;
5002 
5003  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5004  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5005  else flags = STREAM_TOSERVER;
5006 
5007  FLOWLOCK_WRLOCK(f);
5008  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5009  &httpbuf1[u], 1);
5010  if (r != 0) {
5011  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5012  " 0: ", u, r);
5013  FLOWLOCK_UNLOCK(f);
5014  goto end;
5015  }
5016  FLOWLOCK_UNLOCK(f);
5017  }
5018 
5019  htp_state = f->alstate;
5020  if (htp_state == NULL) {
5021  printf("no http state: ");
5022  goto end;
5023  }
5024 
5025  uint8_t ref1[] = "/abc/def";
5026  size_t reflen = sizeof(ref1) - 1;
5027 
5028  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5029  if (tx == NULL)
5030  goto end;
5031  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5032  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5033  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5034  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5035  (uintmax_t)reflen,
5036  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5037  goto end;
5038  }
5039 
5040  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5041  bstr_len(tx_ud->request_uri_normalized)) != 0)
5042  {
5043  printf("normalized uri \"");
5044  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5045  printf("\" != \"");
5046  PrintRawUriFp(stdout, ref1, reflen);
5047  printf("\": ");
5048  goto end;
5049  }
5050  }
5051 
5052  uint8_t ref2[] = "/abc/def?ghi/jkl";
5053  reflen = sizeof(ref2) - 1;
5054 
5055  tx = HTPStateGetTx(htp_state, 1);
5056  if (tx == NULL)
5057  goto end;
5058  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5059  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5060  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5061  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5062  (uintmax_t)reflen,
5063  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5064  goto end;
5065  }
5066 
5067  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5068  bstr_len(tx_ud->request_uri_normalized)) != 0)
5069  {
5070  printf("normalized uri \"");
5071  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5072  printf("\" != \"");
5073  PrintRawUriFp(stdout, ref2, reflen);
5074  printf("\": ");
5075  goto end;
5076  }
5077  }
5078 
5079  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5080  reflen = sizeof(ref3) - 1;
5081  tx = HTPStateGetTx(htp_state, 2);
5082  if (tx == NULL)
5083  goto end;
5084  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5085  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5086  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5087  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5088  (uintmax_t)reflen,
5089  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5090  goto end;
5091  }
5092 
5093  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5094  bstr_len(tx_ud->request_uri_normalized)) != 0)
5095  {
5096  printf("normalized uri \"");
5097  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5098  printf("\" != \"");
5099  PrintRawUriFp(stdout, ref3, reflen);
5100  printf("\": ");
5101  goto end;
5102  }
5103  }
5104 
5105  result = 1;
5106 
5107 end:
5108  if (alp_tctx != NULL)
5109  AppLayerParserThreadCtxFree(alp_tctx);
5110  HTPFreeConfig();
5111  ConfDeInit();
5114 
5116  UTHFreeFlow(f);
5117  return result;
5118 }
5119 
5120 /** \test Test %2f decoding in profile IDS with double-decode-* options
5121  *
5122  * %252f in path decoded to /
5123  * %252f in query string is decoded to /
5124  */
5125 static int HTPParserDecodingTest03(void)
5126 {
5127  int result = 0;
5128  Flow *f = NULL;
5129  uint8_t httpbuf1[] =
5130  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5131  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5132  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5133  TcpSession ssn;
5135 
5136  HtpState *htp_state = NULL;
5137  int r = 0;
5138  char input[] = "\
5139 %YAML 1.1\n\
5140 ---\n\
5141 libhtp:\n\
5142 \n\
5143  default-config:\n\
5144  personality: IDS\n\
5145  double-decode-path: yes\n\
5146  double-decode-query: yes\n\
5147 ";
5148 
5150  ConfInit();
5152  ConfYamlLoadString(input, strlen(input));
5153  HTPConfigure();
5154  const char *addr = "4.3.2.1";
5155  memset(&ssn, 0, sizeof(ssn));
5156 
5157  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5158  if (f == NULL)
5159  goto end;
5160  f->protoctx = &ssn;
5161  f->proto = IPPROTO_TCP;
5162  f->alproto = ALPROTO_HTTP;
5163 
5165 
5166  uint32_t u;
5167  for (u = 0; u < httplen1; u++) {
5168  uint8_t flags = 0;
5169 
5170  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5171  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5172  else flags = STREAM_TOSERVER;
5173 
5174  FLOWLOCK_WRLOCK(f);
5175  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5176  &httpbuf1[u], 1);
5177  if (r != 0) {
5178  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5179  " 0: ", u, r);
5180  FLOWLOCK_UNLOCK(f);
5181  goto end;
5182  }
5183  FLOWLOCK_UNLOCK(f);
5184  }
5185 
5186  htp_state = f->alstate;
5187  if (htp_state == NULL) {
5188  printf("no http state: ");
5189  goto end;
5190  }
5191 
5192  uint8_t ref1[] = "/abc/def";
5193  size_t reflen = sizeof(ref1) - 1;
5194 
5195  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5196  if (tx == NULL)
5197  goto end;
5198  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5199  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5200  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5201  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5202  (uintmax_t)reflen,
5203  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5204  goto end;
5205  }
5206 
5207  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5208  bstr_len(tx_ud->request_uri_normalized)) != 0)
5209  {
5210  printf("normalized uri \"");
5211  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5212  printf("\" != \"");
5213  PrintRawUriFp(stdout, ref1, reflen);
5214  printf("\": ");
5215  goto end;
5216  }
5217  }
5218 
5219  uint8_t ref2[] = "/abc/def?ghi/jkl";
5220  reflen = sizeof(ref2) - 1;
5221 
5222  tx = HTPStateGetTx(htp_state, 1);
5223  if (tx == NULL)
5224  goto end;
5225  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5226  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5227  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5228  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5229  (uintmax_t)reflen,
5230  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5231  goto end;
5232  }
5233 
5234  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5235  bstr_len(tx_ud->request_uri_normalized)) != 0)
5236  {
5237  printf("normalized uri \"");
5238  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5239  printf("\" != \"");
5240  PrintRawUriFp(stdout, ref2, reflen);
5241  printf("\": ");
5242  goto end;
5243  }
5244  }
5245 
5246  result = 1;
5247 
5248 end:
5249  if (alp_tctx != NULL)
5250  AppLayerParserThreadCtxFree(alp_tctx);
5251  HTPFreeConfig();
5252  ConfDeInit();
5255 
5257  UTHFreeFlow(f);
5258  return result;
5259 }
5260 
5261 /** \test Test http:// in query profile IDS
5262  */
5263 static int HTPParserDecodingTest04(void)
5264 {
5265  int result = 0;
5266  Flow *f = NULL;
5267  uint8_t httpbuf1[] =
5268  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5269  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5270  TcpSession ssn;
5272 
5273  HtpState *htp_state = NULL;
5274  int r = 0;
5275  char input[] = "\
5276 %YAML 1.1\n\
5277 ---\n\
5278 libhtp:\n\
5279 \n\
5280  default-config:\n\
5281  personality: IDS\n\
5282  double-decode-path: yes\n\
5283  double-decode-query: yes\n\
5284 ";
5285 
5287  ConfInit();
5289  ConfYamlLoadString(input, strlen(input));
5290  HTPConfigure();
5291  const char *addr = "4.3.2.1";
5292  memset(&ssn, 0, sizeof(ssn));
5293 
5294  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5295  if (f == NULL)
5296  goto end;
5297  f->protoctx = &ssn;
5298  f->proto = IPPROTO_TCP;
5299  f->alproto = ALPROTO_HTTP;
5300 
5302 
5303  uint32_t u;
5304  for (u = 0; u < httplen1; u++) {
5305  uint8_t flags = 0;
5306 
5307  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5308  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5309  else flags = STREAM_TOSERVER;
5310 
5311  FLOWLOCK_WRLOCK(f);
5312  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5313  &httpbuf1[u], 1);
5314  if (r != 0) {
5315  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5316  " 0: ", u, r);
5317  FLOWLOCK_UNLOCK(f);
5318  goto end;
5319  }
5320  FLOWLOCK_UNLOCK(f);
5321  }
5322 
5323  htp_state = f->alstate;
5324  if (htp_state == NULL) {
5325  printf("no http state: ");
5326  goto end;
5327  }
5328 
5329  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5330  size_t reflen = sizeof(ref1) - 1;
5331 
5332  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5333  if (tx == NULL)
5334  goto end;
5335  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5336  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5337  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5338  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5339  (uintmax_t)reflen,
5340  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5341  goto end;
5342  }
5343 
5344  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5345  bstr_len(tx_ud->request_uri_normalized)) != 0)
5346  {
5347  printf("normalized uri \"");
5348  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5349  printf("\" != \"");
5350  PrintRawUriFp(stdout, ref1, reflen);
5351  printf("\": ");
5352  goto end;
5353  }
5354  }
5355 
5356  result = 1;
5357 
5358 end:
5359  if (alp_tctx != NULL)
5360  AppLayerParserThreadCtxFree(alp_tctx);
5361  HTPFreeConfig();
5362  ConfDeInit();
5365 
5367  UTHFreeFlow(f);
5368  return result;
5369 }
5370 
5371 /** \test Test \ char in query profile IDS. Bug 739
5372  */
5373 static int HTPParserDecodingTest05(void)
5374 {
5375  int result = 0;
5376  Flow *f = NULL;
5377  uint8_t httpbuf1[] =
5378  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5379  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5380  TcpSession ssn;
5382 
5383  HtpState *htp_state = NULL;
5384  int r = 0;
5385  char input[] = "\
5386 %YAML 1.1\n\
5387 ---\n\
5388 libhtp:\n\
5389 \n\
5390  default-config:\n\
5391  personality: IDS\n\
5392  double-decode-path: yes\n\
5393  double-decode-query: yes\n\
5394 ";
5395 
5397  ConfInit();
5399  ConfYamlLoadString(input, strlen(input));
5400  HTPConfigure();
5401  const char *addr = "4.3.2.1";
5402  memset(&ssn, 0, sizeof(ssn));
5403 
5404  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5405  if (f == NULL)
5406  goto end;
5407  f->protoctx = &ssn;
5408  f->proto = IPPROTO_TCP;
5409  f->alproto = ALPROTO_HTTP;
5410 
5412 
5413  uint32_t u;
5414  for (u = 0; u < httplen1; u++) {
5415  uint8_t flags = 0;
5416 
5417  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5418  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5419  else flags = STREAM_TOSERVER;
5420 
5421  FLOWLOCK_WRLOCK(f);
5422  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5423  &httpbuf1[u], 1);
5424  if (r != 0) {
5425  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5426  " 0: ", u, r);
5427  FLOWLOCK_UNLOCK(f);
5428  goto end;
5429  }
5430  FLOWLOCK_UNLOCK(f);
5431  }
5432 
5433  htp_state = f->alstate;
5434  if (htp_state == NULL) {
5435  printf("no http state: ");
5436  goto end;
5437  }
5438 
5439  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5440  size_t reflen = sizeof(ref1) - 1;
5441 
5442  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5443  if (tx == NULL)
5444  goto end;
5445  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5446  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5447  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5448  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5449  (uintmax_t)reflen,
5450  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5451  goto end;
5452  }
5453 
5454  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5455  bstr_len(tx_ud->request_uri_normalized)) != 0)
5456  {
5457  printf("normalized uri \"");
5458  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5459  printf("\" != \"");
5460  PrintRawUriFp(stdout, ref1, reflen);
5461  printf("\": ");
5462  goto end;
5463  }
5464  }
5465 
5466  result = 1;
5467 
5468 end:
5469  if (alp_tctx != NULL)
5470  AppLayerParserThreadCtxFree(alp_tctx);
5471  HTPFreeConfig();
5472  ConfDeInit();
5475 
5477  UTHFreeFlow(f);
5478  return result;
5479 }
5480 
5481 /** \test Test + char in query. Bug 1035
5482  */
5483 static int HTPParserDecodingTest06(void)
5484 {
5485  int result = 0;
5486  Flow *f = NULL;
5487  uint8_t httpbuf1[] =
5488  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5489  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5490  TcpSession ssn;
5492 
5493  HtpState *htp_state = NULL;
5494  int r = 0;
5495  char input[] = "\
5496 %YAML 1.1\n\
5497 ---\n\
5498 libhtp:\n\
5499 \n\
5500  default-config:\n\
5501  personality: IDS\n\
5502  double-decode-path: yes\n\
5503  double-decode-query: yes\n\
5504 ";
5505 
5507  ConfInit();
5509  ConfYamlLoadString(input, strlen(input));
5510  HTPConfigure();
5511  const char *addr = "4.3.2.1";
5512  memset(&ssn, 0, sizeof(ssn));
5513 
5514  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5515  if (f == NULL)
5516  goto end;
5517  f->protoctx = &ssn;
5518  f->proto = IPPROTO_TCP;
5519  f->alproto = ALPROTO_HTTP;
5520 
5522 
5523  uint32_t u;
5524  for (u = 0; u < httplen1; u++) {
5525  uint8_t flags = 0;
5526 
5527  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5528  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5529  else flags = STREAM_TOSERVER;
5530 
5531  FLOWLOCK_WRLOCK(f);
5532  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5533  &httpbuf1[u], 1);
5534  if (r != 0) {
5535  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5536  " 0: ", u, r);
5537  FLOWLOCK_UNLOCK(f);
5538  goto end;
5539  }
5540  FLOWLOCK_UNLOCK(f);
5541  }
5542 
5543  htp_state = f->alstate;
5544  if (htp_state == NULL) {
5545  printf("no http state: ");
5546  goto end;
5547  }
5548 
5549  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5550  size_t reflen = sizeof(ref1) - 1;
5551 
5552  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5553  if (tx == NULL)
5554  goto end;
5555  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5556  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5557  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5558  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5559  (uintmax_t)reflen,
5560  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5561  goto end;
5562  }
5563 
5564  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5565  bstr_len(tx_ud->request_uri_normalized)) != 0)
5566  {
5567  printf("normalized uri \"");
5568  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5569  printf("\" != \"");
5570  PrintRawUriFp(stdout, ref1, reflen);
5571  printf("\": ");
5572  goto end;
5573  }
5574  }
5575 
5576  result = 1;
5577 
5578 end:
5579  if (alp_tctx != NULL)
5580  AppLayerParserThreadCtxFree(alp_tctx);
5581  HTPFreeConfig();
5582  ConfDeInit();
5585 
5587  UTHFreeFlow(f);
5588  return result;
5589 }
5590 
5591 /** \test Test + char in query. Bug 1035
5592  */
5593 static int HTPParserDecodingTest07(void)
5594 {
5595  int result = 0;
5596  Flow *f = NULL;
5597  uint8_t httpbuf1[] =
5598  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5599  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5600  TcpSession ssn;
5602 
5603  HtpState *htp_state = NULL;
5604  int r = 0;
5605  char input[] = "\
5606 %YAML 1.1\n\
5607 ---\n\
5608 libhtp:\n\
5609 \n\
5610  default-config:\n\
5611  personality: IDS\n\
5612  double-decode-path: yes\n\
5613  double-decode-query: yes\n\
5614  query-plusspace-decode: yes\n\
5615 ";
5616 
5618  ConfInit();
5620  ConfYamlLoadString(input, strlen(input));
5621  HTPConfigure();
5622  const char *addr = "4.3.2.1";
5623  memset(&ssn, 0, sizeof(ssn));
5624 
5625  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5626  if (f == NULL)
5627  goto end;
5628  f->protoctx = &ssn;
5629  f->proto = IPPROTO_TCP;
5630  f->alproto = ALPROTO_HTTP;
5631 
5633 
5634  uint32_t u;
5635  for (u = 0; u < httplen1; u++) {
5636  uint8_t flags = 0;
5637 
5638  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5639  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5640  else flags = STREAM_TOSERVER;
5641 
5642  FLOWLOCK_WRLOCK(f);
5643  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5644  &httpbuf1[u], 1);
5645  if (r != 0) {
5646  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5647  " 0: ", u, r);
5648  FLOWLOCK_UNLOCK(f);
5649  goto end;
5650  }
5651  FLOWLOCK_UNLOCK(f);
5652  }
5653 
5654  htp_state = f->alstate;
5655  if (htp_state == NULL) {
5656  printf("no http state: ");
5657  goto end;
5658  }
5659 
5660  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5661  size_t reflen = sizeof(ref1) - 1;
5662 
5663  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5664  if (tx == NULL)
5665  goto end;
5666  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5667  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5668  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5669  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5670  (uintmax_t)reflen,
5671  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5672  goto end;
5673  }
5674 
5675  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5676  bstr_len(tx_ud->request_uri_normalized)) != 0)
5677  {
5678  printf("normalized uri \"");
5679  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5680  printf("\" != \"");
5681  PrintRawUriFp(stdout, ref1, reflen);
5682  printf("\": ");
5683  goto end;
5684  }
5685  }
5686 
5687  result = 1;
5688 
5689 end:
5690  if (alp_tctx != NULL)
5691  AppLayerParserThreadCtxFree(alp_tctx);
5692  HTPFreeConfig();
5693  ConfDeInit();
5696 
5698  UTHFreeFlow(f);
5699  return result;
5700 }
5701 
5702 /** \test Test 'proxy' URI normalization. Ticket 1008
5703  */
5704 static int HTPParserDecodingTest08(void)
5705 {
5706  int result = 0;
5707  Flow *f = NULL;
5708  uint8_t httpbuf1[] =
5709  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5710  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5711  TcpSession ssn;
5713 
5714  HtpState *htp_state = NULL;
5715  int r = 0;
5716  char input[] = "\
5717 %YAML 1.1\n\
5718 ---\n\
5719 libhtp:\n\
5720 \n\
5721  default-config:\n\
5722  personality: IDS\n\
5723 ";
5724 
5726  ConfInit();
5728  ConfYamlLoadString(input, strlen(input));
5729  HTPConfigure();
5730  const char *addr = "4.3.2.1";
5731  memset(&ssn, 0, sizeof(ssn));
5732 
5733  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5734  if (f == NULL)
5735  goto end;
5736  f->protoctx = &ssn;
5737  f->proto = IPPROTO_TCP;
5738  f->alproto = ALPROTO_HTTP;
5739 
5741 
5742  uint32_t u;
5743  for (u = 0; u < httplen1; u++) {
5744  uint8_t flags = 0;
5745 
5746  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5747  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5748  else flags = STREAM_TOSERVER;
5749 
5750  FLOWLOCK_WRLOCK(f);
5751  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5752  &httpbuf1[u], 1);
5753  if (r != 0) {
5754  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5755  " 0: ", u, r);
5756  FLOWLOCK_UNLOCK(f);
5757  goto end;
5758  }
5759  FLOWLOCK_UNLOCK(f);
5760  }
5761 
5762  htp_state = f->alstate;
5763  if (htp_state == NULL) {
5764  printf("no http state: ");
5765  goto end;
5766  }
5767 
5768  uint8_t ref1[] = "/blah/";
5769  size_t reflen = sizeof(ref1) - 1;
5770 
5771  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5772  if (tx == NULL)
5773  goto end;
5774  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5775  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5776  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5777  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5778  (uintmax_t)reflen,
5779  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5780  goto end;
5781  }
5782 
5783  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5784  bstr_len(tx_ud->request_uri_normalized)) != 0)
5785  {
5786  printf("normalized uri \"");
5787  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5788  printf("\" != \"");
5789  PrintRawUriFp(stdout, ref1, reflen);
5790  printf("\": ");
5791  goto end;
5792  }
5793  }
5794 
5795  result = 1;
5796 
5797 end:
5798  if (alp_tctx != NULL)
5799  AppLayerParserThreadCtxFree(alp_tctx);
5800  HTPFreeConfig();
5801  ConfDeInit();
5804 
5806  UTHFreeFlow(f);
5807  return result;
5808 }
5809 
5810 /** \test Test 'proxy' URI normalization. Ticket 1008
5811  */
5812 static int HTPParserDecodingTest09(void)
5813 {
5814  int result = 0;
5815  Flow *f = NULL;
5816  uint8_t httpbuf1[] =
5817  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5818  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5819  TcpSession ssn;
5821 
5822  HtpState *htp_state = NULL;
5823  int r = 0;
5824  char input[] = "\
5825 %YAML 1.1\n\
5826 ---\n\
5827 libhtp:\n\
5828 \n\
5829  default-config:\n\
5830  personality: IDS\n\
5831  uri-include-all: true\n\
5832 ";
5833 
5835  ConfInit();
5837  ConfYamlLoadString(input, strlen(input));
5838  HTPConfigure();
5839  const char *addr = "4.3.2.1";
5840  memset(&ssn, 0, sizeof(ssn));
5841 
5842  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5843  if (f == NULL)
5844  goto end;
5845  f->protoctx = &ssn;
5846  f->proto = IPPROTO_TCP;
5847  f->alproto = ALPROTO_HTTP;
5848 
5850 
5851  uint32_t u;
5852  for (u = 0; u < httplen1; u++) {
5853  uint8_t flags = 0;
5854 
5855  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5856  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5857  else flags = STREAM_TOSERVER;
5858 
5859  FLOWLOCK_WRLOCK(f);
5860  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5861  &httpbuf1[u], 1);
5862  if (r != 0) {
5863  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5864  " 0: ", u, r);
5865  FLOWLOCK_UNLOCK(f);
5866  goto end;
5867  }
5868  FLOWLOCK_UNLOCK(f);
5869  }
5870 
5871  htp_state = f->alstate;
5872  if (htp_state == NULL) {
5873  printf("no http state: ");
5874  goto end;
5875  }
5876 
5877  uint8_t ref1[] = "http://suricata-ids.org/blah/";
5878  size_t reflen = sizeof(ref1) - 1;
5879 
5880  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5881  if (tx == NULL)
5882  goto end;
5883  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5884  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5885  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5886  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5887  (uintmax_t)reflen,
5888  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5889  goto end;
5890  }
5891 
5892  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5893  bstr_len(tx_ud->request_uri_normalized)) != 0)
5894  {
5895  printf("normalized uri \"");
5896  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5897  printf("\" != \"");
5898  PrintRawUriFp(stdout, ref1, reflen);
5899  printf("\": ");
5900  goto end;
5901  }
5902  }
5903 
5904  result = 1;
5905 
5906 end:
5907  if (alp_tctx != NULL)
5908  AppLayerParserThreadCtxFree(alp_tctx);
5909  HTPFreeConfig();
5910  ConfDeInit();
5913 
5915  UTHFreeFlow(f);
5916  return result;
5917 }
5918 
5919 /** \test BG box crash -- chunks are messed up. Observed for real. */
5920 static int HTPBodyReassemblyTest01(void)
5921 {
5922  int result = 0;
5923  HtpTxUserData htud;
5924  memset(&htud, 0x00, sizeof(htud));
5925  HtpState hstate;
5926  memset(&hstate, 0x00, sizeof(hstate));
5927  Flow flow;
5928  memset(&flow, 0x00, sizeof(flow));
5930  htp_tx_t tx;
5931  memset(&tx, 0, sizeof(tx));
5932 
5933  hstate.f = &flow;
5934  flow.alparser = parser;
5935 
5936  uint8_t chunk1[] = "--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5937  uint8_t chunk2[] = "POST /uri HTTP/1.1\r\nHost: hostname.com\r\nKeep-Alive: 115\r\nAccept-Charset: utf-8\r\nUser-Agent: Mozilla/5.0 (X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nConnection: keep-alive\r\nContent-length: 68102\r\nReferer: http://otherhost.com\r\nAccept-Encoding: gzip\r\nContent-Type: multipart/form-data; boundary=e5a320f21416a02493a0a6f561b1c494\r\nCookie: blah\r\nAccept-Language: us\r\n\r\n--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5938 
5939  int r = HtpBodyAppendChunk(NULL, &htud.request_body, chunk1, sizeof(chunk1)-1);
5940  BUG_ON(r != 0);
5941  r = HtpBodyAppendChunk(NULL, &htud.request_body, chunk2, sizeof(chunk2)-1);
5942  BUG_ON(r != 0);
5943 
5944  const uint8_t *chunks_buffer = NULL;
5945  uint32_t chunks_buffer_len = 0;
5946 
5947  HtpRequestBodyReassemble(&htud, &chunks_buffer, &chunks_buffer_len);
5948  if (chunks_buffer == NULL) {
5949  goto end;
5950  }
5951 #ifdef PRINT
5952  printf("REASSCHUNK START: \n");
5953  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
5954  printf("REASSCHUNK END: \n");
5955 #endif
5956 
5957  HtpRequestBodyHandleMultipart(&hstate, &htud, &tx, chunks_buffer, chunks_buffer_len);
5958 
5959  if (htud.request_body.content_len_so_far != 669) {
5960  printf("htud.request_body.content_len_so_far %"PRIu64": ", htud.request_body.content_len_so_far);
5961  goto end;
5962  }
5963 
5964  if (hstate.fil