suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 #include "util-byte.h"
49 
50 #include "stream-tcp-private.h"
51 #include "stream-tcp-reassemble.h"
52 #include "stream-tcp.h"
53 #include "stream.h"
54 
55 #include "app-layer-protos.h"
56 #include "app-layer-parser.h"
57 
58 #include "app-layer.h"
59 #include "app-layer-frames.h"
60 #include "app-layer-htp.h"
61 #include "app-layer-htp-body.h"
62 #include "app-layer-htp-file.h"
63 #include "app-layer-htp-libhtp.h"
64 #include "app-layer-htp-xff.h"
65 
66 #include "util-spm.h"
67 #include "util-debug.h"
68 #include "util-time.h"
69 #include "util-misc.h"
70 
71 #include "util-unittest.h"
72 #include "util-unittest-helper.h"
73 #include "flow-util.h"
74 
75 #include "detect-engine.h"
76 #include "detect-engine-state.h"
77 #include "detect-parse.h"
78 
79 #include "decode-events.h"
80 
81 #include "util-memcmp.h"
82 #include "util-random.h"
83 #include "util-validate.h"
84 
85 //#define PRINT
86 
87 /** Fast lookup tree (radix) for the various HTP configurations */
88 static SCRadixTree *cfgtree;
89 /** List of HTP configurations. */
90 static HTPCfgRec cfglist;
91 
92 /** Limit to the number of libhtp messages that can be handled */
93 #define HTP_MAX_MESSAGES 512
94 
95 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
96 
97 #ifdef DEBUG
98 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
99 static uint64_t htp_state_memuse = 0;
100 static uint64_t htp_state_memcnt = 0;
101 #endif
102 
104  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
105  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
106  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
107  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
108  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
109  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
110  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
112  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
114  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
116  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
118  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
120  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
122  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
123  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
125  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
126  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
127  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
128  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
129  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
130  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
131  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
132  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
133  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
134  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
135  { "FILE_NAME_TOO_LONG", HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG },
136  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
137  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
138  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
140  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
141  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
142  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
143  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
144  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
145  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
146  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
147  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
148  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
149  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
150  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
151  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
152  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
154  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
155  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
156  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
157  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
158 
159  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
160  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
161 
162  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
163 
164  /* suricata warnings/errors */
165  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
166  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
167  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
168 
169  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
170 
171  { NULL, -1 },
172 };
173 
174 /* app-layer-frame-documentation tag start: HttpFrameTypes */
178 };
179 
181  {
182  "request",
184  },
185  {
186  "response",
188  },
189  { NULL, -1 },
190 };
191 /* app-layer-frame-documentation tag end: HttpFrameTypes */
192 
193 static int HTTPGetFrameIdByName(const char *frame_name)
194 {
195  int id = SCMapEnumNameToValue(frame_name, http_frame_table);
196  if (id < 0) {
197  return -1;
198  }
199  return id;
200 }
201 
202 static const char *HTTPGetFrameNameById(const uint8_t frame_id)
203 {
204  const char *name = SCMapEnumValueToName(frame_id, http_frame_table);
205  return name;
206 }
207 
208 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
209 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
210 static uint64_t HTPStateGetTxCnt(void *alstate);
211 #ifdef UNITTESTS
212 static void HTPParserRegisterTests(void);
213 #endif
214 
215 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
216 {
217  uint64_t id = HTPStateGetTxCnt(s);
218  BUG_ON(id == 0);
219  return id - 1;
220 }
221 
222 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
223 {
224  return s->transaction_cnt;
225 }
226 
227 #ifdef DEBUG
228 /**
229  * \internal
230  *
231  * \brief Lookup the HTP personality string from the numeric personality.
232  *
233  * \todo This needs to be a libhtp function.
234  */
235 static const char *HTPLookupPersonalityString(int p)
236 {
237 #define CASE_HTP_PERSONALITY_STRING(p) \
238  case HTP_SERVER_ ## p: return #p
239 
240  switch (p) {
241  CASE_HTP_PERSONALITY_STRING(MINIMAL);
242  CASE_HTP_PERSONALITY_STRING(GENERIC);
243  CASE_HTP_PERSONALITY_STRING(IDS);
244  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
245  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
246  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
247  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
248  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
249  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
250  CASE_HTP_PERSONALITY_STRING(APACHE_2);
251  }
252 
253  return NULL;
254 }
255 #endif /* DEBUG */
256 
257 /**
258  * \internal
259  *
260  * \brief Lookup the numeric HTP personality from a string.
261  *
262  * \todo This needs to be a libhtp function.
263  */
264 static int HTPLookupPersonality(const char *str)
265 {
266 #define IF_HTP_PERSONALITY_NUM(p) \
267  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
268 
269  IF_HTP_PERSONALITY_NUM(MINIMAL);
270  IF_HTP_PERSONALITY_NUM(GENERIC);
272  IF_HTP_PERSONALITY_NUM(IIS_4_0);
273  IF_HTP_PERSONALITY_NUM(IIS_5_0);
274  IF_HTP_PERSONALITY_NUM(IIS_5_1);
275  IF_HTP_PERSONALITY_NUM(IIS_6_0);
276  IF_HTP_PERSONALITY_NUM(IIS_7_0);
277  IF_HTP_PERSONALITY_NUM(IIS_7_5);
278  IF_HTP_PERSONALITY_NUM(APACHE_2);
279  if (strcasecmp("TOMCAT_6_0", str) == 0) {
280  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
281  "longer supported by libhtp.", str);
282  return -1;
283  } else if ((strcasecmp("APACHE", str) == 0) ||
284  (strcasecmp("APACHE_2_2", str) == 0))
285  {
286  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
287  "longer supported by libhtp, failing back to "
288  "Apache2 personality.", str);
289  return HTP_SERVER_APACHE_2;
290  }
291 
292  return -1;
293 }
294 
295 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
296  const uint8_t dir, const uint8_t e)
297 {
298  SCLogDebug("setting event %u", e);
299 
300  if (htud) {
301  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
302  s->events++;
303  return;
304  }
305 
306  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
307  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
308 
309  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
310  if (tx == NULL && tx_id > 0)
311  tx = HTPStateGetTx(s, tx_id - 1);
312  if (tx != NULL) {
313  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
314  if (htud != NULL) {
315  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
316  s->events++;
317  return;
318  }
319  }
320  SCLogDebug("couldn't set event %u", e);
321 }
322 
323 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
324  * connection parser to be used by the HTP library
325  */
326 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
327 {
328  SCEnter();
329 
330  HtpState *s = HTPMalloc(sizeof(HtpState));
331  if (unlikely(s == NULL)) {
332  SCReturnPtr(NULL, "void");
333  }
334 
335  memset(s, 0x00, sizeof(HtpState));
336 
337 #ifdef DEBUG
338  SCMutexLock(&htp_state_mem_lock);
339  htp_state_memcnt++;
340  htp_state_memuse += sizeof(HtpState);
341  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
342  SCMutexUnlock(&htp_state_mem_lock);
343 #endif
344 
345  SCReturnPtr((void *)s, "void");
346 }
347 
348 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
349 {
350  if (likely(htud)) {
351  HtpBodyFree(&htud->request_body);
352  HtpBodyFree(&htud->response_body);
353  bstr_free(htud->request_uri_normalized);
354  if (htud->request_headers_raw)
356  if (htud->response_headers_raw)
359  if (htud->boundary)
360  HTPFree(htud->boundary, htud->boundary_len);
361  if (htud->tx_data.de_state != NULL) {
362  DetectEngineStateFree(htud->tx_data.de_state);
363  }
364  HTPFree(htud, sizeof(HtpTxUserData));
365  }
366 }
367 
368 /** \brief Function to frees the HTTP state memory and also frees the HTTP
369  * connection parser memory which was used by the HTP library
370  */
371 void HTPStateFree(void *state)
372 {
373  SCEnter();
374 
375  HtpState *s = (HtpState *)state;
376  if (s == NULL) {
377  SCReturn;
378  }
379 
380  /* free the connection parser memory used by HTP library */
381  if (s->connp != NULL) {
382  SCLogDebug("freeing HTP state");
383 
384  uint64_t tx_id;
385  uint64_t total_txs = HTPStateGetTxCnt(state);
386  /* free the list of body chunks */
387  if (s->conn != NULL) {
388  for (tx_id = 0; tx_id < total_txs; tx_id++) {
389  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
390  if (tx != NULL) {
391  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
392  HtpTxUserDataFree(s, htud);
393  htp_tx_set_user_data(tx, NULL);
394  }
395  }
396  }
397  htp_connp_destroy_all(s->connp);
398  }
399 
400  if (s->file_range) {
401  HTPFileCloseHandleRange(s->files_tc, 0, s->file_range, NULL, 0);
403  }
404 
407  HTPFree(s, sizeof(HtpState));
408 
409 #ifdef DEBUG
410  SCMutexLock(&htp_state_mem_lock);
411  htp_state_memcnt--;
412  htp_state_memuse -= sizeof(HtpState);
413  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
414  SCMutexUnlock(&htp_state_mem_lock);
415 #endif
416 
417  SCReturn;
418 }
419 
420 /**
421  * \brief HTP transaction cleanup callback
422  *
423  * \warning We cannot actually free the transactions here. It seems that
424  * HTP only accepts freeing of transactions in the response callback.
425  */
426 static void HTPStateTransactionFree(void *state, uint64_t id)
427 {
428  SCEnter();
429 
430  HtpState *s = (HtpState *)state;
431 
432  SCLogDebug("state %p, id %"PRIu64, s, id);
433 
434  htp_tx_t *tx = HTPStateGetTx(s, id);
435  if (tx != NULL) {
436  /* This will remove obsolete body chunks */
437  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
438  HtpTxUserDataFree(s, htud);
439  htp_tx_set_user_data(tx, NULL);
440 
441  /* hack: even if libhtp considers the tx incomplete, we want to
442  * free it here. htp_tx_destroy however, will refuse to do this.
443  * As htp_tx_destroy_incomplete isn't available in the public API,
444  * we hack around it here. */
445  if (unlikely(!(
446  tx->request_progress == HTP_REQUEST_COMPLETE &&
447  tx->response_progress == HTP_RESPONSE_COMPLETE)))
448  {
449  tx->request_progress = HTP_REQUEST_COMPLETE;
450  tx->response_progress = HTP_RESPONSE_COMPLETE;
451  }
452  htp_tx_destroy(tx);
453  }
454 }
455 
456 /**
457  * \brief Sets a flag that informs the HTP app layer that some module in the
458  * engine needs the http request body data.
459  * \initonly
460  */
462 {
463  SCEnter();
464 
465  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
466  SCReturn;
467 }
468 
469 /**
470  * \brief Sets a flag that informs the HTP app layer that some module in the
471  * engine needs the http request body data.
472  * \initonly
473  */
475 {
476  SCEnter();
477 
478  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
479  SCReturn;
480 }
481 
482 /**
483  * \brief Sets a flag that informs the HTP app layer that some module in the
484  * engine needs the http request multi part header.
485  *
486  * \initonly
487  */
488 static void AppLayerHtpNeedMultipartHeader(void)
489 {
490  SCEnter();
492 
493  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
494  SCReturn;
495 }
496 
497 /**
498  * \brief Sets a flag that informs the HTP app layer that some module in the
499  * engine needs the http request file.
500  *
501  * \initonly
502  */
504 {
505  SCEnter();
506  AppLayerHtpNeedMultipartHeader();
509 
510  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
511  SCReturn;
512 }
513 
514 static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
515 {
516  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
517  if (tx_ud) {
518  if (flags & STREAM_TOCLIENT) {
519  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
520  } else {
521  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
522  }
523  }
524 }
525 
526 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
527 {
528  if (flags & HTP_STREAM_DEPTH_SET) {
529  uint32_t stream_depth = FileReassemblyDepth();
530  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
531  return true;
532  }
533  } else {
534  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
535  return true;
536  }
537  }
538  return false;
539 }
540 
541 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
542  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
543 {
544  uint32_t chunk_len = 0;
545  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
546  (content_len_so_far < (uint64_t)body_limit) &&
547  (content_len_so_far + (uint64_t)data_len) > body_limit)
548  {
549  chunk_len = body_limit - content_len_so_far;
550  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
551  (content_len_so_far < (uint64_t)stream_depth) &&
552  (content_len_so_far + (uint64_t)data_len) > stream_depth)
553  {
554  chunk_len = stream_depth - content_len_so_far;
555  }
556  SCLogDebug("len %u", chunk_len);
557  return (chunk_len == 0 ? data_len : chunk_len);
558 }
559 
560 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
561 struct {
562  const char *msg;
563  uint8_t de;
564 } htp_errors[] = {
565  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
566  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
567  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
568  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
569  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
570 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
571  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
572 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
573  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
574  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
575  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
576  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
577 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
578  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
579  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
580  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
581  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
582 };
583 
584 struct {
585  const char *msg;
586  uint8_t de;
587 } htp_warnings[] = {
588  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
589  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
590  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
591  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
592  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
593 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
594  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
595  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
596  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
597  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
598  * luckily, "Request server port=" is unique */
599 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
601  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
602  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
603  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
604  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
605  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
606  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
607  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
608  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
609  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
610  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
611  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
612  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
613  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
614  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
615  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
616  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
617  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
618  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
619 };
620 
621 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
622 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
623 
624 /**
625  * \internal
626  *
627  * \brief Get the warning id for the warning msg.
628  *
629  * \param msg warning message
630  *
631  * \retval id the id or 0 in case of not found
632  */
633 static uint8_t HTPHandleWarningGetId(const char *msg)
634 {
635  SCLogDebug("received warning \"%s\"", msg);
636  size_t idx;
637  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
638  if (strncmp(htp_warnings[idx].msg, msg,
639  strlen(htp_warnings[idx].msg)) == 0)
640  {
641  return htp_warnings[idx].de;
642  }
643  }
644 
645  return 0;
646 }
647 
648 /**
649  * \internal
650  *
651  * \brief Get the error id for the error msg.
652  *
653  * \param msg error message
654  *
655  * \retval id the id or 0 in case of not found
656  */
657 static uint8_t HTPHandleErrorGetId(const char *msg)
658 {
659  SCLogDebug("received error \"%s\"", msg);
660 
661  size_t idx;
662  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
663  if (strncmp(htp_errors[idx].msg, msg,
664  strlen(htp_errors[idx].msg)) == 0)
665  {
666  return htp_errors[idx].de;
667  }
668  }
669 
670  return 0;
671 }
672 
673 /**
674  * \internal
675  *
676  * \brief Check state for errors, warnings and add any as events
677  *
678  * \param s state
679  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
680  */
681 static void HTPHandleError(HtpState *s, const uint8_t dir)
682 {
683  if (s == NULL || s->conn == NULL ||
684  s->conn->messages == NULL) {
685  return;
686  }
687 
688  size_t size = htp_list_size(s->conn->messages);
689  size_t msg;
690  if(size >= HTP_MAX_MESSAGES) {
692  //only once per HtpState
693  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
695  //too noisy in fuzzing
696  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
697  }
698  // ignore further messages
699  return;
700  }
701 
702  for (msg = s->htp_messages_offset; msg < size; msg++) {
703  htp_log_t *log = htp_list_get(s->conn->messages, msg);
704  if (log == NULL)
705  continue;
706 
707  HtpTxUserData *htud = NULL;
708  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
709  if (tx != NULL)
710  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
711 
712  SCLogDebug("message %s", log->msg);
713 
714  uint8_t id = HTPHandleErrorGetId(log->msg);
715  if (id == 0) {
716  id = HTPHandleWarningGetId(log->msg);
717  if (id == 0)
719  }
720 
721  if (id > 0) {
722  HTPSetEvent(s, htud, dir, id);
723  }
724  }
725  s->htp_messages_offset = (uint16_t)msg;
726  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
727 }
728 
729 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
730 {
731 #ifdef DEBUG
732  BUG_ON(s == NULL || tx == NULL);
733 #endif
734  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
735  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
736  HTP_HOSTH_INVALID))
737  {
738  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
739  if (htud == NULL)
740  return;
741 
742  if (tx->flags & HTP_REQUEST_INVALID_T_E)
743  HTPSetEvent(s, htud, STREAM_TOSERVER,
745  if (tx->flags & HTP_REQUEST_INVALID_C_L)
746  HTPSetEvent(s, htud, STREAM_TOSERVER,
748  if (tx->flags & HTP_HOST_MISSING)
749  HTPSetEvent(s, htud, STREAM_TOSERVER,
751  if (tx->flags & HTP_HOST_AMBIGUOUS)
752  HTPSetEvent(s, htud, STREAM_TOSERVER,
754  if (tx->flags & HTP_HOSTU_INVALID)
755  HTPSetEvent(s, htud, STREAM_TOSERVER,
757  if (tx->flags & HTP_HOSTH_INVALID)
758  HTPSetEvent(s, htud, STREAM_TOSERVER,
760  }
761  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
762  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
763  if (htud == NULL)
764  return;
765  HTPSetEvent(s, htud, STREAM_TOSERVER,
767  }
768  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
769  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
770  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
771  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
772  if (htud == NULL)
773  return;
774  HTPSetEvent(s, htud, STREAM_TOSERVER,
776  }
777 }
778 
779 static int Setup(Flow *f, HtpState *hstate)
780 {
781  /* store flow ref in state so callbacks can access it */
782  hstate->f = f;
783 
784  HTPCfgRec *htp_cfg_rec = &cfglist;
785  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
786  void *user_data = NULL;
787 
788  if (FLOW_IS_IPV4(f)) {
789  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
790  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
791  }
792  else if (FLOW_IS_IPV6(f)) {
793  SCLogDebug("Looking up HTP config for ipv6");
794  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
795  }
796  else {
797  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
798  goto error;
799  }
800 
801  if (user_data != NULL) {
802  htp_cfg_rec = user_data;
803  htp = htp_cfg_rec->cfg;
804  SCLogDebug("LIBHTP using config: %p", htp);
805  } else {
806  SCLogDebug("Using default HTP config: %p", htp);
807  }
808 
809  if (NULL == htp) {
810 #ifdef DEBUG_VALIDATION
811  BUG_ON(htp == NULL);
812 #endif
813  /* should never happen if HTPConfigure is properly invoked */
814  goto error;
815  }
816 
817  hstate->connp = htp_connp_create(htp);
818  if (hstate->connp == NULL) {
819  goto error;
820  }
821 
822  hstate->conn = htp_connp_get_connection(hstate->connp);
823 
824  htp_connp_set_user_data(hstate->connp, (void *)hstate);
825  hstate->cfg = htp_cfg_rec;
826 
827  SCLogDebug("New hstate->connp %p", hstate->connp);
828 
829  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
830 
832  htp_cfg_rec->request.inspect_min_size);
834  htp_cfg_rec->response.inspect_min_size);
835  return 0;
836 error:
837  return -1;
838 }
839 
840 /**
841  * \brief Function to handle the reassembled data from client and feed it to
842  * the HTP library to process it.
843  *
844  * \param flow Pointer to the flow the data belong to
845  * \param htp_state Pointer the state in which the parsed value to be stored
846  * \param pstate Application layer parser state for this session
847  *
848  * \retval On success returns 1 or on failure returns -1.
849  */
850 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state, AppLayerParserState *pstate,
851  StreamSlice stream_slice, void *local_data)
852 {
853  SCEnter();
854  int ret = 0;
855  HtpState *hstate = (HtpState *)htp_state;
856 
857  /* On the first invocation, create the connection parser structure to
858  * be used by HTP library. This is looked up via IP in the radix
859  * tree. Failing that, the default HTP config is used.
860  */
861  if (NULL == hstate->conn) {
862  if (Setup(f, hstate) != 0) {
864  }
865  }
866  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
867  hstate->slice = &stream_slice;
868 
869  const uint8_t *input = StreamSliceGetData(&stream_slice);
870  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
871 
872  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
873  /* pass the new data to the htp parser */
874  if (input_len > 0) {
875  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
876  switch (r) {
877  case HTP_STREAM_ERROR:
878  ret = -1;
879  break;
880  default:
881  break;
882  }
883  HTPHandleError(hstate, STREAM_TOSERVER);
884  }
885 
886  /* if the TCP connection is closed, then close the HTTP connection */
888  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
889  {
890  htp_connp_req_close(hstate->connp, &ts);
891  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
892  SCLogDebug("stream eof encountered, closing htp handle for ts");
893  }
894 
895  SCLogDebug("hstate->connp %p", hstate->connp);
896  hstate->slice = NULL;
897 
898  if (ret < 0) {
900  }
902 }
903 
904 /**
905  * \brief Function to handle the reassembled data from server and feed it to
906  * the HTP library to process it.
907  *
908  * \param flow Pointer to the flow the data belong to
909  * \param htp_state Pointer the state in which the parsed value to be stored
910  * \param pstate Application layer parser state for this session
911  * \param input Pointer the received HTTP server data
912  * \param input_len Length in bytes of the received data
913  * \param output Pointer to the output (not used in this function)
914  *
915  * \retval On success returns 1 or on failure returns -1
916  */
917 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state, AppLayerParserState *pstate,
918  StreamSlice stream_slice, void *local_data)
919 {
920  SCEnter();
921  int ret = 0;
922  HtpState *hstate = (HtpState *)htp_state;
923 
924  const uint8_t *input = StreamSliceGetData(&stream_slice);
925  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
926 
927  /* On the first invocation, create the connection parser structure to
928  * be used by HTP library. This is looked up via IP in the radix
929  * tree. Failing that, the default HTP config is used.
930  */
931  if (NULL == hstate->conn) {
932  if (Setup(f, hstate) != 0) {
934  }
935  }
936  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
937  hstate->slice = &stream_slice;
938 
939  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
940  htp_tx_t *tx = NULL;
941  size_t consumed = 0;
942  if (input_len > 0) {
943  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
944  switch (r) {
945  case HTP_STREAM_ERROR:
946  ret = -1;
947  break;
948  case HTP_STREAM_TUNNEL:
949  tx = htp_connp_get_out_tx(hstate->connp);
950  if (tx != NULL && tx->response_status_number == 101) {
951  htp_header_t *h =
952  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
953  if (h == NULL || bstr_cmp_c(h->value, "h2c") != 0) {
954  break;
955  }
957  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
958  break;
959  }
960  uint16_t dp = 0;
961  if (tx->request_port_number != -1) {
962  dp = (uint16_t)tx->request_port_number;
963  }
964  consumed = htp_connp_res_data_consumed(hstate->connp);
965  hstate->slice = NULL;
967  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
968  // and we need to parser the remaining part with HTTP2
969  if (consumed > 0 && consumed < input_len) {
970  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
971  }
973  }
974  break;
975  default:
976  break;
977  }
978  HTPHandleError(hstate, STREAM_TOCLIENT);
979  }
980 
981  /* if we the TCP connection is closed, then close the HTTP connection */
983  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
984  {
985  htp_connp_close(hstate->connp, &ts);
986  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
987  }
988 
989  SCLogDebug("hstate->connp %p", hstate->connp);
990  hstate->slice = NULL;
991 
992  if (ret < 0) {
994  }
996 }
997 
998 /**
999  * \param name /Lowercase/ version of the variable name
1000  */
1001 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1002  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1003 {
1004 #ifdef PRINT
1005  printf("DATA START: \n");
1006  PrintRawDataFp(stdout, data, len);
1007  printf("DATA END: \n");
1008 #endif
1009  size_t x;
1010  int quote = 0;
1011 
1012  for (x = 0; x < len; x++) {
1013  if (!(isspace(data[x])))
1014  break;
1015  }
1016 
1017  if (x >= len)
1018  return 0;
1019 
1020  uint8_t *line = data+x;
1021  size_t line_len = len-x;
1022  size_t offset = 0;
1023 #ifdef PRINT
1024  printf("LINE START: \n");
1025  PrintRawDataFp(stdout, line, line_len);
1026  printf("LINE END: \n");
1027 #endif
1028  for (x = 0 ; x < line_len; x++) {
1029  if (x > 0) {
1030  if (line[x - 1] != '\\' && line[x] == '\"') {
1031  quote++;
1032  }
1033 
1034  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1035  uint8_t *token = line + offset;
1036  size_t token_len = x - offset;
1037 
1038  if ((x + 1) == line_len) {
1039  token_len++;
1040  }
1041 
1042  offset = x + 1;
1043 
1044  while (offset < line_len && isspace(line[offset])) {
1045  x++;
1046  offset++;
1047  }
1048 #ifdef PRINT
1049  printf("TOKEN START: \n");
1050  PrintRawDataFp(stdout, token, token_len);
1051  printf("TOKEN END: \n");
1052 #endif
1053  if (token_len > name_len) {
1054  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1055  uint8_t *value = token + name_len;
1056  size_t value_len = token_len - name_len;
1057 
1058  if (value[0] == '\"') {
1059  value++;
1060  value_len--;
1061  }
1062  if (value[value_len-1] == '\"') {
1063  value_len--;
1064  }
1065 #ifdef PRINT
1066  printf("VALUE START: \n");
1067  PrintRawDataFp(stdout, value, value_len);
1068  printf("VALUE END: \n");
1069 #endif
1070  *retptr = value;
1071  *retlen = value_len;
1072  return 1;
1073  }
1074  }
1075  }
1076  }
1077  }
1078 
1079  return 0;
1080 }
1081 
1082 /**
1083  * \param name /Lowercase/ version of the variable name
1084  */
1085 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1086  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1087 {
1088  SCEnter();
1089 #ifdef PRINT
1090  printf("DATA START: \n");
1091  PrintRawDataFp(stdout, data, len);
1092  printf("DATA END: \n");
1093 #endif
1094  size_t x;
1095  int quote = 0;
1096 
1097  for (x = 0; x < len; x++) {
1098  if (!(isspace(data[x])))
1099  break;
1100  }
1101 
1102  if (x >= len) {
1103  SCReturnInt(0);
1104  }
1105 
1106  uint8_t *line = data+x;
1107  size_t line_len = len-x;
1108  size_t offset = 0;
1109 #ifdef PRINT
1110  printf("LINE START: \n");
1111  PrintRawDataFp(stdout, line, line_len);
1112  printf("LINE END: \n");
1113 #endif
1114  for (x = 0 ; x < line_len; x++) {
1115  if (x > 0) {
1116  if (line[x - 1] != '\\' && line[x] == '\"') {
1117  quote++;
1118  }
1119 
1120  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1121  uint8_t *token = line + offset;
1122  size_t token_len = x - offset;
1123 
1124  if ((x + 1) == line_len) {
1125  token_len++;
1126  }
1127 
1128  offset = x + 1;
1129 
1130  while (offset < line_len && isspace(line[offset])) {
1131  x++;
1132  offset++;
1133  }
1134 #ifdef PRINT
1135  printf("TOKEN START: \n");
1136  PrintRawDataFp(stdout, token, token_len);
1137  printf("TOKEN END: \n");
1138 #endif
1139  if (token_len > name_len) {
1140  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1141  uint8_t *value = token + name_len;
1142  size_t value_len = token_len - name_len;
1143 
1144  if (value[0] == '\"') {
1145  value++;
1146  value_len--;
1147  }
1148  if (value[value_len-1] == '\"') {
1149  value_len--;
1150  }
1151 #ifdef PRINT
1152  printf("VALUE START: \n");
1153  PrintRawDataFp(stdout, value, value_len);
1154  printf("VALUE END: \n");
1155 #endif
1156  *retptr = value;
1157  *retlen = value_len;
1158  SCReturnInt(1);
1159  }
1160  }
1161  }
1162  }
1163  }
1164 
1165  SCReturnInt(0);
1166 }
1167 
1168 /**
1169  * \brief setup multipart parsing: extract boundary and store it
1170  *
1171  * \param d HTTP transaction
1172  * \param htud transaction userdata
1173  *
1174  * \retval 1 ok, multipart set up
1175  * \retval 0 ok, not multipart though
1176  * \retval -1 error: problem with the boundary
1177  *
1178  * If the request contains a multipart message, this function will
1179  * set the HTP_BOUNDARY_SET in the transaction.
1180  */
1181 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1182 {
1183  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1184  "Content-Type");
1185  if (h != NULL && bstr_len(h->value) > 0) {
1186  uint8_t *boundary = NULL;
1187  size_t boundary_len = 0;
1188 
1189  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1190  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1191  &boundary, &boundary_len);
1192  if (r == 1) {
1193 #ifdef PRINT
1194  printf("BOUNDARY START: \n");
1195  PrintRawDataFp(stdout, boundary, boundary_len);
1196  printf("BOUNDARY END: \n");
1197 #endif
1198  if (boundary_len < HTP_BOUNDARY_MAX) {
1199  htud->boundary = HTPMalloc(boundary_len);
1200  if (htud->boundary == NULL) {
1201  return -1;
1202  }
1203  htud->boundary_len = (uint8_t)boundary_len;
1204  memcpy(htud->boundary, boundary, boundary_len);
1205 
1206  htud->tsflags |= HTP_BOUNDARY_SET;
1207  } else {
1208  SCLogDebug("invalid boundary");
1209  return -1;
1210  }
1211  SCReturnInt(1);
1212  }
1213  //SCReturnInt(1);
1214  }
1215  SCReturnInt(0);
1216 }
1217 
1218 #define C_D_HDR "content-disposition:"
1219 #define C_D_HDR_LEN 20
1220 #define C_T_HDR "content-type:"
1221 #define C_T_HDR_LEN 13
1223 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1224  HtpTxUserData *htud,
1225  uint8_t *header, uint32_t header_len,
1226  uint8_t **filename, uint16_t *filename_len,
1227  uint8_t **filetype, uint16_t *filetype_len)
1228 {
1229  uint8_t *fn = NULL;
1230  size_t fn_len = 0;
1231  uint8_t *ft = NULL;
1232  size_t ft_len = 0;
1233 
1234 #ifdef PRINT
1235  printf("HEADER START: \n");
1236  PrintRawDataFp(stdout, header, header_len);
1237  printf("HEADER END: \n");
1238 #endif
1239 
1240  while (header_len > 0) {
1241  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1242  uint8_t *line = header;
1243  uint32_t line_len;
1244 
1245  if (next_line == NULL) {
1246  line_len = header_len;
1247  } else {
1248  line_len = next_line - header;
1249  }
1250  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1251  if (sc == NULL) {
1252  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1254  /* if the : we found is the final char, it means we have
1255  * no value */
1256  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1257  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1259  } else {
1260 #ifdef PRINT
1261  printf("LINE START: \n");
1262  PrintRawDataFp(stdout, line, line_len);
1263  printf("LINE END: \n");
1264 #endif
1265  if (line_len >= C_D_HDR_LEN &&
1266  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1267  uint8_t *value = line + C_D_HDR_LEN;
1268  uint32_t value_len = line_len - C_D_HDR_LEN;
1269 
1270  /* parse content-disposition */
1271  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1272  value, value_len, &fn, &fn_len);
1273  } else if (line_len >= C_T_HDR_LEN &&
1274  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1275  SCLogDebug("content-type line");
1276  uint8_t *value = line + C_T_HDR_LEN;
1277  uint32_t value_len = line_len - C_T_HDR_LEN;
1278 
1279  (void)HTTPParseContentTypeHeader(NULL, 0,
1280  value, value_len, &ft, &ft_len);
1281  }
1282  }
1283 
1284  if (next_line == NULL) {
1285  SCLogDebug("no next_line");
1286  break;
1287  }
1288  header_len -= ((next_line + 2) - header);
1289  header = next_line + 2;
1290  } /* while (header_len > 0) */
1291 
1292  if (fn_len > USHRT_MAX)
1293  fn_len = USHRT_MAX;
1294  if (ft_len > USHRT_MAX)
1295  ft_len = USHRT_MAX;
1296 
1297  *filename = fn;
1298  *filename_len = (uint16_t)fn_len;
1299  *filetype = ft;
1300  *filetype_len = (uint16_t)ft_len;
1301 }
1302 
1303 /**
1304  * \brief Create a single buffer from the HtpBodyChunks in our list
1305  *
1306  * \param htud transaction user data
1307  * \param chunks_buffers pointer to pass back the buffer to the caller
1308  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1309  */
1310 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1311  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1312 {
1314  chunks_buffer, chunks_buffer_len,
1315  htud->request_body.body_parsed);
1316 }
1317 
1318 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1319 {
1320  SCEnter();
1321  if (tx && tx->tx_data.de_state) {
1322  if (dir == STREAM_TOSERVER) {
1323  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1324  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1325  } else if (STREAM_TOCLIENT) {
1326  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1327  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1328  }
1329  }
1330 }
1331 
1332 /**
1333  * \brief Setup boundary buffers
1334  */
1335 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1336  uint8_t *boundary, uint32_t boundary_len)
1337 {
1338  memset(boundary, '-', boundary_len);
1339  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1340 }
1341 
1342 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1343  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1344 {
1345  int result = 0;
1346  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1347  uint16_t expected_boundary_len = htud->boundary_len + 2;
1348  uint16_t expected_boundary_end_len = htud->boundary_len + 4;
1349  int tx_progress = 0;
1350 
1351 #ifdef PRINT
1352  printf("CHUNK START: \n");
1353  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1354  printf("CHUNK END: \n");
1355 #endif
1356 
1357  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1358 
1359  /* search for the header start, header end and form end */
1360  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1361  boundary, expected_boundary_len);
1362  /* end of the multipart form */
1363  const uint8_t *form_end = NULL;
1364  /* end marker belonging to header_start */
1365  const uint8_t *header_end = NULL;
1366  if (header_start != NULL) {
1367  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1368  (uint8_t *)"\r\n\r\n", 4);
1369  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1370  boundary, expected_boundary_end_len);
1371  }
1372 
1373  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1374  header_end, form_end);
1375 
1376  /* we currently only handle multipart for ts. When we support it for tc,
1377  * we will need to supply right direction */
1378  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1379  /* if we're in the file storage process, deal with that now */
1380  if (htud->tsflags & HTP_FILENAME_SET) {
1381  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1382  SCLogDebug("reached the end of the file");
1383 
1384  const uint8_t *filedata = chunks_buffer;
1385  uint32_t filedata_len = 0;
1386  uint8_t flags = 0;
1387 
1388  if (header_start != NULL) {
1389  if (header_start == filedata + 2) {
1390  /* last chunk had all data, but not the boundary */
1391  SCLogDebug("last chunk had all data, but not the boundary");
1392  filedata_len = 0;
1393  } else if (header_start > filedata + 2) {
1394  SCLogDebug("some data from last file before the boundary");
1395  /* some data from last file before the boundary */
1396  filedata_len = header_start - filedata - 2;
1397  }
1398  }
1399  /* body parsing done, we did not get our form end. Use all data
1400  * we still have and signal to files API we have an issue. */
1401  if (tx_progress > HTP_REQUEST_BODY) {
1402  filedata_len = chunks_buffer_len;
1404  }
1405 
1406  if (filedata_len > chunks_buffer_len) {
1407  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1409  goto end;
1410  }
1411 #ifdef PRINT
1412  printf("FILEDATA (final chunk) START: \n");
1413  PrintRawDataFp(stdout, filedata, filedata_len);
1414  printf("FILEDATA (final chunk) END: \n");
1415 #endif
1416  if (!(htud->tsflags & HTP_DONTSTORE)) {
1417  if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
1418  -1) {
1419  goto end;
1420  }
1421  }
1422 
1423  htud->tsflags &=~ HTP_FILENAME_SET;
1424 
1425  /* fall through */
1426  } else {
1427  SCLogDebug("not yet at the end of the file");
1428 
1429  if (chunks_buffer_len > expected_boundary_end_len) {
1430  const uint8_t *filedata = chunks_buffer;
1431  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1432 #ifdef PRINT
1433  printf("FILEDATA (part) START: \n");
1434  PrintRawDataFp(stdout, filedata, filedata_len);
1435  printf("FILEDATA (part) END: \n");
1436 #endif
1437 
1438  if (!(htud->tsflags & HTP_DONTSTORE)) {
1439  result = HTPFileStoreChunk(hstate, filedata,
1440  filedata_len, STREAM_TOSERVER);
1441  if (result == -1) {
1442  goto end;
1443  } else if (result == -2) {
1444  /* we know for sure we're not storing the file */
1445  htud->tsflags |= HTP_DONTSTORE;
1446  }
1447  }
1448 
1449  htud->request_body.body_parsed += filedata_len;
1450  } else {
1451  SCLogDebug("chunk too small to already process in part");
1452  }
1453 
1454  goto end;
1455  }
1456  }
1457 
1458  while (header_start != NULL && header_end != NULL &&
1459  header_end != form_end &&
1460  header_start < (chunks_buffer + chunks_buffer_len) &&
1461  header_end < (chunks_buffer + chunks_buffer_len) &&
1462  header_start < header_end)
1463  {
1464  uint8_t *filename = NULL;
1465  uint16_t filename_len = 0;
1466  uint8_t *filetype = NULL;
1467  uint16_t filetype_len = 0;
1468 
1469  uint32_t header_len = header_end - header_start;
1470  SCLogDebug("header_len %u", header_len);
1471  uint8_t *header = (uint8_t *)header_start;
1472 
1473  /* skip empty records */
1474  if (expected_boundary_len == header_len) {
1475  goto next;
1476  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1477  header_len -= (expected_boundary_len + 2);
1478  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1479  }
1480 
1481  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1482  &filename, &filename_len, &filetype, &filetype_len);
1483 
1484  if (filename != NULL) {
1485  const uint8_t *filedata = NULL;
1486  uint32_t filedata_len = 0;
1487 
1488  SCLogDebug("we have a filename");
1489 
1490  htud->tsflags |= HTP_FILENAME_SET;
1491  htud->tsflags &= ~HTP_DONTSTORE;
1492 
1493  SCLogDebug("header_end %p", header_end);
1494  SCLogDebug("form_end %p", form_end);
1495 
1496  /* everything until the final boundary is the file */
1497  if (form_end != NULL) {
1498  SCLogDebug("have form_end");
1499 
1500  filedata = header_end + 4;
1501  if (form_end == filedata) {
1502  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1504  goto end;
1505  } else if (form_end < filedata) {
1506  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1508  goto end;
1509  }
1510 
1511  filedata_len = form_end - (header_end + 4 + 2);
1512  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1513 
1514  /* or is it? */
1515  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1516  boundary, expected_boundary_len);
1517  if (header_next != NULL) {
1518  filedata_len -= (form_end - header_next);
1519  }
1520 
1521  if (filedata_len > chunks_buffer_len) {
1522  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1524  goto end;
1525  }
1526  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1527 #ifdef PRINT
1528  printf("FILEDATA START: \n");
1529  PrintRawDataFp(stdout, filedata, filedata_len);
1530  printf("FILEDATA END: \n");
1531 #endif
1532 
1533  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata, filedata_len,
1534  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1535  if (result == -1) {
1536  goto end;
1537  } else if (result == -2) {
1538  htud->tsflags |= HTP_DONTSTORE;
1539  } else {
1540  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1541  goto end;
1542  }
1543  }
1544  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1545 
1546  htud->request_body.body_parsed += (header_end - chunks_buffer);
1547  htud->tsflags &= ~HTP_FILENAME_SET;
1548  } else {
1549  SCLogDebug("chunk doesn't contain form end");
1550 
1551  filedata = header_end + 4;
1552  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1553  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1554 
1555  if (filedata_len > chunks_buffer_len) {
1556  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1558  goto end;
1559  }
1560 
1561 #ifdef PRINT
1562  printf("FILEDATA START: \n");
1563  PrintRawDataFp(stdout, filedata, filedata_len);
1564  printf("FILEDATA END: \n");
1565 #endif
1566  /* form doesn't end in this chunk, but the part might. Lets
1567  * see if have another coming up */
1568  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1569  boundary, expected_boundary_len);
1570  SCLogDebug("header_next %p", header_next);
1571  if (header_next == NULL) {
1572  SCLogDebug("more file data to come");
1573 
1574  uint32_t offset = (header_end + 4) - chunks_buffer;
1575  SCLogDebug("offset %u", offset);
1576  htud->request_body.body_parsed += offset;
1577 
1578  if (filedata_len >= (uint32_t)(expected_boundary_len + 2)) {
1579  filedata_len -= (expected_boundary_len + 2 - 1);
1580  SCLogDebug("opening file with partial data");
1581  } else {
1582  filedata = NULL;
1583  filedata_len = 0;
1584  }
1585  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1586  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1587  if (result == -1) {
1588  goto end;
1589  } else if (result == -2) {
1590  htud->tsflags |= HTP_DONTSTORE;
1591  }
1592  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1593  htud->request_body.body_parsed += filedata_len;
1594  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1595 
1596  } else if (header_next - filedata > 2) {
1597  filedata_len = header_next - filedata - 2;
1598  SCLogDebug("filedata_len %u", filedata_len);
1599 
1600  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1601  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1602  if (result == -1) {
1603  goto end;
1604  } else if (result == -2) {
1605  htud->tsflags |= HTP_DONTSTORE;
1606  } else {
1607  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1608  goto end;
1609  }
1610  }
1611  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1612 
1613  htud->tsflags &= ~HTP_FILENAME_SET;
1614  htud->request_body.body_parsed += (header_end - chunks_buffer);
1615  }
1616  }
1617  }
1618 next:
1619  SCLogDebug("header_start %p, header_end %p, form_end %p",
1620  header_start, header_end, form_end);
1621 
1622  /* Search next boundary entry after the start of body */
1623  uint32_t cursizeread = header_end - chunks_buffer;
1624  header_start = Bs2bmSearch(header_end + 4,
1625  chunks_buffer_len - (cursizeread + 4),
1626  boundary, expected_boundary_len);
1627  if (header_start != NULL) {
1628  header_end = Bs2bmSearch(header_end + 4,
1629  chunks_buffer_len - (cursizeread + 4),
1630  (uint8_t *) "\r\n\r\n", 4);
1631  }
1632  }
1633 
1634  /* if we're parsing the multipart and we're not currently processing a
1635  * file, we move the body pointer forward. */
1636  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1637  if (chunks_buffer_len > expected_boundary_end_len) {
1638  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1639 
1640  htud->request_body.body_parsed += move;
1641  SCLogDebug("form not ready, file not set, parsing non-file "
1642  "record: moved %u", move);
1643  }
1644  }
1645 
1646 end:
1647  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1648  return 0;
1649 }
1650 
1651 /** \internal
1652  * \brief Handle POST or PUT, no multipart body data
1653  */
1654 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1655  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1656 {
1657  int result = 0;
1658 
1659  /* see if we need to open the file */
1660  if (!(htud->tsflags & HTP_FILENAME_SET))
1661  {
1662  uint8_t *filename = NULL;
1663  size_t filename_len = 0;
1664 
1665  /* get the name */
1666  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1667  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1668  filename_len = bstr_len(tx->parsed_uri->path);
1669  }
1670 
1671  if (filename != NULL) {
1672  if (filename_len > SC_FILENAME_MAX) {
1673  // explicitly truncate the file name if too long
1674  filename_len = SC_FILENAME_MAX;
1675  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1676  }
1677  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1678  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1679  if (result == -1) {
1680  goto end;
1681  } else if (result == -2) {
1682  htud->tsflags |= HTP_DONTSTORE;
1683  } else {
1684  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1685  htud->tsflags |= HTP_FILENAME_SET;
1686  htud->tsflags &= ~HTP_DONTSTORE;
1687  }
1688  }
1689  }
1690  else
1691  {
1692  /* otherwise, just store the data */
1693 
1694  if (!(htud->tsflags & HTP_DONTSTORE)) {
1695  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1696  if (result == -1) {
1697  goto end;
1698  } else if (result == -2) {
1699  /* we know for sure we're not storing the file */
1700  htud->tsflags |= HTP_DONTSTORE;
1701  }
1702  }
1703  }
1704 
1705  return 0;
1706 end:
1707  return -1;
1708 }
1709 
1710 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1711  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1712 {
1713  SCEnter();
1714 
1715  int result = 0;
1716 
1717  /* see if we need to open the file
1718  * we check for tx->response_line in case of junk
1719  * interpreted as body before response line
1720  */
1721  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1722  (tx->response_line != NULL || tx->is_protocol_0_9))
1723  {
1724  SCLogDebug("setting up file name");
1725 
1726  uint8_t *filename = NULL;
1727  size_t filename_len = 0;
1728 
1729  /* try Content-Disposition header first */
1730  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1731  "Content-Disposition");
1732  if (h != NULL && bstr_len(h->value) > 0) {
1733  /* parse content-disposition */
1734  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1735  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1736  }
1737 
1738  /* fall back to name from the uri */
1739  if (filename == NULL) {
1740  /* get the name */
1741  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1742  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1743  filename_len = bstr_len(tx->parsed_uri->path);
1744  }
1745  }
1746 
1747  if (filename != NULL) {
1748  // set range if present
1749  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1750  if (filename_len > SC_FILENAME_MAX) {
1751  // explicitly truncate the file name if too long
1752  filename_len = SC_FILENAME_MAX;
1753  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1754  }
1755  if (h_content_range != NULL) {
1756  result = HTPFileOpenWithRange(hstate, htud, filename, (uint16_t)filename_len, data,
1757  data_len, HtpGetActiveResponseTxID(hstate), h_content_range->value, htud);
1758  } else {
1759  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1760  HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1761  }
1762  SCLogDebug("result %d", result);
1763  if (result == -1) {
1764  goto end;
1765  } else if (result == -2) {
1766  htud->tcflags |= HTP_DONTSTORE;
1767  } else {
1768  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1769  htud->tcflags |= HTP_FILENAME_SET;
1770  htud->tcflags &= ~HTP_DONTSTORE;
1771  }
1772  }
1773  }
1774  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1775  {
1776  /* otherwise, just store the data */
1777 
1778  if (!(htud->tcflags & HTP_DONTSTORE)) {
1779  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1780  SCLogDebug("result %d", result);
1781  if (result == -1) {
1782  goto end;
1783  } else if (result == -2) {
1784  /* we know for sure we're not storing the file */
1785  htud->tcflags |= HTP_DONTSTORE;
1786  }
1787  }
1788  }
1789 
1790  htud->response_body.body_parsed += data_len;
1791  return 0;
1792 end:
1793  return -1;
1794 }
1795 
1796 /**
1797  * \brief Function callback to append chunks for Requests
1798  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1799  * \retval int HTP_OK if all goes well
1800  */
1801 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1802 {
1803  SCEnter();
1804 
1805  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1806  SCReturnInt(HTP_OK);
1807 
1808  if (d->len == 0)
1809  SCReturnInt(HTP_OK);
1810 
1811 #ifdef PRINT
1812  printf("HTPBODY START: \n");
1813  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1814  printf("HTPBODY END: \n");
1815 #endif
1816 
1817  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1818  if (hstate == NULL) {
1819  SCReturnInt(HTP_ERROR);
1820  }
1821 
1822  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1823  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1824 
1825  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1826  if (tx_ud == NULL) {
1827  SCReturnInt(HTP_OK);
1828  }
1829  if (!tx_ud->response_body_init) {
1830  tx_ud->response_body_init = 1;
1831 
1832  if (d->tx->request_method_number == HTP_M_POST) {
1833  SCLogDebug("POST");
1834  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1835  if (r == 1) {
1837  } else if (r == 0) {
1839  SCLogDebug("not multipart");
1840  }
1841  } else if (d->tx->request_method_number == HTP_M_PUT) {
1843  }
1844  }
1845 
1846  /* see if we can get rid of htp body chunks */
1847  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1848 
1849  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1850  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1851 
1852  /* within limits, add the body chunk to the state. */
1853  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1854  uint32_t stream_depth = FileReassemblyDepth();
1855  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1856  hstate->cfg->request.body_limit,
1857  stream_depth,
1858  tx_ud->tsflags,
1859  (uint32_t)d->len);
1860  BUG_ON(len > (uint32_t)d->len);
1861 
1862  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1863 
1864  const uint8_t *chunks_buffer = NULL;
1865  uint32_t chunks_buffer_len = 0;
1866 
1868  /* multi-part body handling starts here */
1869  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1870  goto end;
1871  }
1872 
1873  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1874  if (chunks_buffer == NULL) {
1875  goto end;
1876  }
1877 #ifdef PRINT
1878  printf("REASSCHUNK START: \n");
1879  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1880  printf("REASSCHUNK END: \n");
1881 #endif
1882 
1883  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1884 
1885  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1887  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1888  }
1889 
1890  } else {
1891  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1892  SCLogDebug("closing file that was being stored");
1893  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1894  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1895  }
1896  }
1897 
1898 end:
1899  if (hstate->conn != NULL) {
1900  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1902  hstate->cfg->request.inspect_min_size,
1903  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1904 
1905  /* if we reach the inspect_min_size we'll trigger inspection,
1906  * so make sure that raw stream is also inspected. Set the
1907  * data to be used to the amount of raw bytes we've seen to
1908  * get here. */
1909  if (tx_ud->request_body.body_inspected == 0 &&
1911  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1912  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1913  {
1914  const uint32_t data_size = (uint32_t)(
1915  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1916  const uint32_t depth = MIN(data_size, hstate->cfg->request.inspect_min_size);
1917 
1918  /* body still in progress, but due to min inspect size we need to inspect now */
1919  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, depth);
1920  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1921  }
1922  /* after the start of the body, disable the depth logic */
1923  } else if (tx_ud->request_body.body_inspected > 0) {
1924  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1925  }
1926  }
1927  SCReturnInt(HTP_OK);
1928 }
1929 
1930 /**
1931  * \brief Function callback to append chunks for Responses
1932  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1933  * \retval int HTP_OK if all goes well
1934  */
1935 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1936 {
1937  SCEnter();
1938 
1939  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1940  SCReturnInt(HTP_OK);
1941 
1942  if (d->len == 0)
1943  SCReturnInt(HTP_OK);
1944 
1945  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1946  if (hstate == NULL) {
1947  SCReturnInt(HTP_ERROR);
1948  }
1949 
1950  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1951  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1952 
1953  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1954  if (tx_ud == NULL) {
1955  SCReturnInt(HTP_OK);
1956  }
1957  if (!tx_ud->request_body_init) {
1958  tx_ud->request_body_init = 1;
1959  }
1960 
1961  /* see if we can get rid of htp body chunks */
1962  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1963 
1964  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1965  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1966 
1967  /* within limits, add the body chunk to the state. */
1968  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1969  uint32_t stream_depth = FileReassemblyDepth();
1970  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1971  hstate->cfg->response.body_limit,
1972  stream_depth,
1973  tx_ud->tcflags,
1974  (uint32_t)d->len);
1975  BUG_ON(len > (uint32_t)d->len);
1976 
1977  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1978 
1979  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1980  } else {
1981  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1982  SCLogDebug("closing file that was being stored");
1983  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1984  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1985  }
1986  }
1987 
1988  if (hstate->conn != NULL) {
1989  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1991  hstate->cfg->response.inspect_min_size,
1992  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
1993  /* if we reach the inspect_min_size we'll trigger inspection,
1994  * so make sure that raw stream is also inspected. Set the
1995  * data to be used to the amount of raw bytes we've seen to
1996  * get here. */
1997  if (tx_ud->response_body.body_inspected == 0 &&
1999  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2000  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2001  {
2002  const uint32_t data_size = (uint32_t)((uint64_t)hstate->conn->out_data_counter -
2003  hstate->last_response_data_stamp);
2004  const uint32_t depth = MIN(data_size, hstate->cfg->response.inspect_min_size);
2005 
2006  /* body still in progress, but due to min inspect size we need to inspect now */
2007  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, depth);
2008  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2009  }
2010  /* after the start of the body, disable the depth logic */
2011  } else if (tx_ud->response_body.body_inspected > 0) {
2012  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
2013  }
2014  }
2015  SCReturnInt(HTP_OK);
2016 }
2017 
2018 /**
2019  * \brief Print the stats of the HTTP requests
2020  */
2022 {
2023 #ifdef DEBUG
2024  SCEnter();
2025  SCMutexLock(&htp_state_mem_lock);
2026  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2027  htp_state_memcnt, htp_state_memuse);
2028  SCMutexUnlock(&htp_state_mem_lock);
2029  SCReturn;
2030 #endif
2031 }
2032 
2033 /** \brief Clears the HTTP server configuration memory used by HTP library */
2034 void HTPFreeConfig(void)
2035 {
2036  SCEnter();
2037 
2038  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2039  !AppLayerParserConfParserEnabled("tcp", "http"))
2040  {
2041  SCReturn;
2042  }
2043 
2044  HTPCfgRec *nextrec = cfglist.next;
2045  SCRadixReleaseRadixTree(cfgtree);
2046  cfgtree = NULL;
2047  htp_config_destroy(cfglist.cfg);
2048  while (nextrec != NULL) {
2049  HTPCfgRec *htprec = nextrec;
2050  nextrec = nextrec->next;
2051 
2052  htp_config_destroy(htprec->cfg);
2053  SCFree(htprec);
2054  }
2055  SCReturn;
2056 }
2057 
2058 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2059 {
2060  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2061  if (htud != NULL) {
2062  htud->request_has_trailers = 1;
2063  }
2064  return HTP_OK;
2065 }
2066 
2067 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2068 {
2069  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2070  if (htud != NULL) {
2071  htud->response_has_trailers = 1;
2072  }
2073  return HTP_OK;
2074 }
2075 
2076 /**\internal
2077  * \brief called at start of request
2078  * Set min inspect size.
2079  */
2080 static int HTPCallbackRequestStart(htp_tx_t *tx)
2081 {
2082  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2083  if (hstate == NULL) {
2084  SCReturnInt(HTP_ERROR);
2085  }
2086 
2087  uint64_t consumed = hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2088  SCLogDebug("HTTP request start: data offset %" PRIu64 ", in_data_counter %" PRIu64, consumed,
2089  (uint64_t)hstate->conn->in_data_counter);
2090 
2091  /* app-layer-frame-documentation tag start: frame registration http request */
2093  hstate->f, hstate->slice, consumed, -1, 0, HTTP_FRAME_REQUEST);
2094  if (frame) {
2095  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2096  hstate->request_frame_id = frame->id;
2097  AppLayerFrameSetTxId(frame, HtpGetActiveRequestTxID(hstate));
2098  }
2099  /* app-layer-frame-documentation tag end: frame registration http request */
2100 
2101  if (hstate->cfg)
2102  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2103  hstate->cfg->request.inspect_min_size);
2104 
2105  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2106  if (tx_ud == NULL) {
2107  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2108  if (unlikely(tx_ud == NULL)) {
2109  SCReturnInt(HTP_OK);
2110  }
2111  htp_tx_set_user_data(tx, tx_ud);
2112  }
2113  SCReturnInt(HTP_OK);
2114 }
2115 
2116 /**\internal
2117  * \brief called at start of response
2118  * Set min inspect size.
2119  */
2120 static int HTPCallbackResponseStart(htp_tx_t *tx)
2121 {
2122  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2123  if (hstate == NULL) {
2124  SCReturnInt(HTP_ERROR);
2125  }
2126 
2127  uint64_t consumed = hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2128  SCLogDebug("HTTP response start: data offset %" PRIu64 ", out_data_counter %" PRIu64, consumed,
2129  (uint64_t)hstate->conn->out_data_counter);
2130 
2132  hstate->f, hstate->slice, consumed, -1, 1, HTTP_FRAME_RESPONSE);
2133  if (frame) {
2134  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2135  hstate->response_frame_id = frame->id;
2136  AppLayerFrameSetTxId(frame, HtpGetActiveResponseTxID(hstate));
2137  }
2138 
2139  if (hstate->cfg)
2140  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
2141  hstate->cfg->response.inspect_min_size);
2142 
2143  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2144  if (tx_ud == NULL) {
2145  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2146  if (unlikely(tx_ud == NULL)) {
2147  SCReturnInt(HTP_OK);
2148  }
2149  htp_tx_set_user_data(tx, tx_ud);
2150  }
2151  SCReturnInt(HTP_OK);
2152 }
2153 
2154 
2155 /**
2156  * \brief callback for request to store the recent incoming request
2157  in to the recent_in_tx for the given htp state
2158  * \param connp pointer to the current connection parser which has the htp
2159  * state in it as user data
2160  */
2161 static int HTPCallbackRequestComplete(htp_tx_t *tx)
2162 {
2163  SCEnter();
2164 
2165  if (tx == NULL) {
2166  SCReturnInt(HTP_ERROR);
2167  }
2168 
2169  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2170  if (hstate == NULL) {
2171  SCReturnInt(HTP_ERROR);
2172  }
2173 
2174  const uint64_t abs_right_edge =
2175  hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2176 
2177  /* app-layer-frame-documentation tag start: updating frame->len */
2178  if (hstate->request_frame_id > 0) {
2179  Frame *frame = AppLayerFrameGetById(hstate->f, 0, hstate->request_frame_id);
2180  if (frame) {
2181  const uint64_t request_size = abs_right_edge - hstate->last_request_data_stamp;
2182 
2183  SCLogDebug("HTTP request complete: data offset %" PRIu64 ", request_size %" PRIu64,
2184  hstate->last_request_data_stamp, request_size);
2185  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2186  request_size);
2187  frame->len = (int64_t)request_size;
2188  /* app-layer-frame-documentation tag end: updating frame->len */
2189  }
2190  hstate->request_frame_id = 0;
2191  }
2192 
2193  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2194  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2195 
2196  SCLogDebug("HTTP request completed");
2197 
2198  HTPErrorCheckTxRequestFlags(hstate, tx);
2199 
2200  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2201  if (htud != NULL) {
2202  if (htud->tsflags & HTP_FILENAME_SET) {
2203  SCLogDebug("closing file that was being stored");
2204  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
2205  htud->tsflags &= ~HTP_FILENAME_SET;
2206  if (abs_right_edge < (uint64_t)UINT32_MAX) {
2208  hstate->f->protoctx, STREAM_TOSERVER, (uint32_t)abs_right_edge);
2209  }
2210  }
2211  }
2212 
2213  hstate->last_request_data_stamp = abs_right_edge;
2214  /* request done, do raw reassembly now to inspect state and stream
2215  * at the same time. */
2216  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
2217  SCReturnInt(HTP_OK);
2218 }
2219 
2220 /**
2221  * \brief callback for response to remove the recent received requests
2222  from the recent_in_tx for the given htp state
2223  * \param connp pointer to the current connection parser which has the htp
2224  * state in it as user data
2225  */
2226 static int HTPCallbackResponseComplete(htp_tx_t *tx)
2227 {
2228  SCEnter();
2229 
2230  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2231  if (hstate == NULL) {
2232  SCReturnInt(HTP_ERROR);
2233  }
2234 
2235  /* we have one whole transaction now */
2236  hstate->transaction_cnt++;
2237 
2238  const uint64_t abs_right_edge =
2239  hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2240 
2241  if (hstate->response_frame_id > 0) {
2242  Frame *frame = AppLayerFrameGetById(hstate->f, 1, hstate->response_frame_id);
2243  if (frame) {
2244  const uint64_t response_size = abs_right_edge - hstate->last_response_data_stamp;
2245 
2246  SCLogDebug("HTTP response complete: data offset %" PRIu64 ", response_size %" PRIu64,
2247  hstate->last_response_data_stamp, response_size);
2248  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2249  response_size);
2250  frame->len = (int64_t)response_size;
2251  }
2252  hstate->response_frame_id = 0;
2253  }
2254 
2255  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2256  if (htud != NULL) {
2257  if (htud->tcflags & HTP_FILENAME_SET) {
2258  SCLogDebug("closing file that was being stored");
2259  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
2260  htud->tcflags &= ~HTP_FILENAME_SET;
2261  }
2262  }
2263 
2264  /* response done, do raw reassembly now to inspect state and stream
2265  * at the same time. */
2266  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2267 
2268  /* handle HTTP CONNECT */
2269  if (tx->request_method_number == HTP_M_CONNECT) {
2270  /* any 2XX status response implies that the connection will become
2271  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2272  if ((tx->response_status_number >= 200) &&
2273  (tx->response_status_number < 300) &&
2274  (hstate->transaction_cnt == 1)) {
2275  uint16_t dp = 0;
2276  if (tx->request_port_number != -1) {
2277  dp = (uint16_t)tx->request_port_number;
2278  }
2279  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2281  tx->request_progress = HTP_REQUEST_COMPLETE;
2282  tx->response_progress = HTP_RESPONSE_COMPLETE;
2283  }
2284  }
2285 
2286  hstate->last_response_data_stamp = abs_right_edge;
2287  SCReturnInt(HTP_OK);
2288 }
2289 
2290 static int HTPCallbackRequestLine(htp_tx_t *tx)
2291 {
2292  HtpTxUserData *tx_ud;
2293  bstr *request_uri_normalized;
2294  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2295  const HTPCfgRec *cfg = hstate->cfg;
2296 
2297  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2298  if (request_uri_normalized == NULL)
2299  return HTP_OK;
2300 
2301  tx_ud = htp_tx_get_user_data(tx);
2302  if (likely(tx_ud == NULL)) {
2303  tx_ud = HTPMalloc(sizeof(*tx_ud));
2304  if (unlikely(tx_ud == NULL)) {
2305  bstr_free(request_uri_normalized);
2306  return HTP_OK;
2307  }
2308  memset(tx_ud, 0, sizeof(*tx_ud));
2309  htp_tx_set_user_data(tx, tx_ud);
2310  }
2311  if (unlikely(tx_ud->request_uri_normalized != NULL))
2312  bstr_free(tx_ud->request_uri_normalized);
2313  tx_ud->request_uri_normalized = request_uri_normalized;
2314 
2315  if (tx->flags) {
2316  HTPErrorCheckTxRequestFlags(hstate, tx);
2317  }
2318  return HTP_OK;
2319 }
2320 
2321 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2322 {
2323  if (part == NULL)
2324  return HTP_OK;
2325 
2326  uint64_t flags = 0;
2327  size_t prevlen = bstr_len(part);
2328  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2329  // shorter string means that uri was encoded
2330  if (res == HTP_OK && prevlen > bstr_len(part)) {
2331  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2332  if (htud == NULL)
2333  return HTP_OK;
2334  HtpState *s = htp_connp_get_user_data(tx->connp);
2335  if (s == NULL)
2336  return HTP_OK;
2337  HTPSetEvent(s, htud, STREAM_TOSERVER,
2339  }
2340 
2341  return HTP_OK;
2342 }
2343 
2344 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2345 {
2346  if (tx->parsed_uri == NULL)
2347  return HTP_OK;
2348 
2349  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2350 }
2351 
2352 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2353 {
2354  if (tx->parsed_uri == NULL)
2355  return HTP_OK;
2356 
2357  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2358 }
2359 
2360 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2361 {
2362  void *ptmp;
2363  if (tx_data->len == 0 || tx_data->tx == NULL)
2364  return HTP_OK;
2365 
2366  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2367  if (tx_ud == NULL) {
2368  return HTP_OK;
2369  }
2370  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2371  tx_ud->request_headers_raw_len,
2372  tx_ud->request_headers_raw_len + tx_data->len);
2373  if (ptmp == NULL) {
2374  return HTP_OK;
2375  }
2376  tx_ud->request_headers_raw = ptmp;
2377 
2378  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2379  tx_data->data, tx_data->len);
2380  tx_ud->request_headers_raw_len += tx_data->len;
2381 
2382  if (tx_data->tx && tx_data->tx->flags) {
2383  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2384  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2385  }
2386  return HTP_OK;
2387 }
2388 
2389 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2390 {
2391  void *ptmp;
2392  if (tx_data->len == 0 || tx_data->tx == NULL)
2393  return HTP_OK;
2394 
2395  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2396  if (tx_ud == NULL) {
2397  return HTP_OK;
2398  }
2399  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2400  tx_ud->response_headers_raw_len,
2401  tx_ud->response_headers_raw_len + tx_data->len);
2402  if (ptmp == NULL) {
2403  return HTP_OK;
2404  }
2405  tx_ud->response_headers_raw = ptmp;
2406 
2407  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2408  tx_data->data, tx_data->len);
2409  tx_ud->response_headers_raw_len += tx_data->len;
2410 
2411  return HTP_OK;
2412 }
2413 
2414 /*
2415  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2416  */
2417 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2418 {
2419  cfg_prec->uri_include_all = FALSE;
2426 
2427  if (!g_disable_randomness) {
2429  } else {
2430  cfg_prec->randomize = 0;
2431  }
2433 
2434  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2435  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2436  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2437  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2438 
2439  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2440  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2441 
2442  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2443  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2444 
2445  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2446  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequestComplete);
2447 
2448  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2449  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponseComplete);
2450 
2451  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2452 
2453  /* don't convert + to space by default */
2454  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2455  // enables request decompression
2456  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2457 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2458  // disable by default
2459  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2460 #endif
2461 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2462  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2464 #endif
2465 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2466  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2468 #endif
2469 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2470  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2471 #endif
2472  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2473  * only the hard limit. So we set both here to the (current) htp defaults.
2474  * The reason we do this is that if the user sets the hard limit in the
2475  * config, we have to set the soft limit as well. If libhtp starts using
2476  * the soft limit in the future, we at least make sure we control what
2477  * it's value is. */
2478  htp_config_set_field_limits(cfg_prec->cfg,
2481  return;
2482 }
2483 
2484 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2485  * but we can get both <0 and >RAND_MAX values from RandomGet
2486  */
2487 static int RandomGetWrap(void)
2488 {
2489  unsigned long r;
2490 
2491  do {
2492  r = RandomGet();
2493  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2494 
2495  return r % RAND_MAX;
2496 }
2497 
2498 /*
2499  * We have this splitup so that in case double decoding has been enabled
2500  * for query and path, they would be called first on the callback queue,
2501  * before the callback set by Phase2() is called. We need this, since
2502  * the callback in Phase2() generates the normalized uri which utilizes
2503  * the query and path. */
2504 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2505 {
2506  /* randomize inspection size if needed */
2507  if (cfg_prec->randomize) {
2508  int rdrange = cfg_prec->randomize_range;
2509 
2510  long int r = RandomGetWrap();
2511  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2512  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2513 
2514  r = RandomGetWrap();
2515  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2516  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2517  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2518  " %u and 'request-body-inspect-window' set to %u after"
2519  " randomization.",
2520  name, cfg_prec->request.inspect_min_size, cfg_prec->request.inspect_window);
2521 
2522  r = RandomGetWrap();
2523  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2524  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2525 
2526  r = RandomGetWrap();
2527  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2528  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2529 
2530  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2531  " %u and 'response-body-inspect-window' set to %u after"
2532  " randomization.",
2533  name, cfg_prec->response.inspect_min_size, cfg_prec->response.inspect_window);
2534  }
2535 
2536  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2537 
2538  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2539  cfg_prec->request.inspect_window : 256;
2540  cfg_prec->request.sbcfg.buf_slide = 0;
2541  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2542  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2543  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2544  cfg_prec->request.sbcfg.Free = HTPFree;
2545 
2546  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2547  cfg_prec->response.inspect_window : 256;
2548  cfg_prec->response.sbcfg.buf_slide = 0;
2549  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2550  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2551  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2552  cfg_prec->response.sbcfg.Free = HTPFree;
2553  return;
2554 }
2555 
2556 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2557  SCRadixTree *tree)
2558 {
2559  if (cfg_prec == NULL || s == NULL || tree == NULL)
2560  return;
2561 
2562  ConfNode *p = NULL;
2563 
2564  /* Default Parameters */
2565  TAILQ_FOREACH(p, &s->head, next) {
2566 
2567  if (strcasecmp("address", p->name) == 0) {
2568  ConfNode *pval;
2569  /* Addresses */
2570  TAILQ_FOREACH(pval, &p->head, next) {
2571  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2572  pval->val);
2573 
2574  /* IPV6 or IPV4? */
2575  if (strchr(pval->val, ':') != NULL) {
2576  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2577  s->name, pval->val, cfg_prec->cfg);
2578  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2579  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2580  "add ipv6 server %s, ignoring", pval->val);
2581  }
2582  } else {
2583  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2584  s->name, pval->val, cfg_prec->cfg);
2585  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2586  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2587  "to add ipv4 server %s, ignoring",
2588  pval->val);
2589  }
2590  } /* else - if (strchr(pval->val, ':') != NULL) */
2591  } /* TAILQ_FOREACH(pval, &p->head, next) */
2592 
2593  } else if (strcasecmp("personality", p->name) == 0) {
2594  /* Personalities */
2595  int personality = HTPLookupPersonality(p->val);
2596  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2597  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2598 
2599  if (personality >= 0) {
2600  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2601  personality);
2602  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2603  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2604  "personality \"%s\", ignoring", p->val);
2605  } else {
2606  SCLogDebug("LIBHTP personality set to %s",
2607  HTPLookupPersonalityString(personality));
2608  }
2609 
2610  /* The IDS personality by default converts the path (and due to
2611  * our query string callback also the query string) to lowercase.
2612  * Signatures do not expect this, so override it. */
2613  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2614  } else {
2615  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2616  "\"%s\", ignoring", p->val);
2617  continue;
2618  }
2619 
2620  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2621  strcasecmp("request_body_limit", p->name) == 0) {
2622  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2623  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2624  "from conf file - %s. Killing engine", p->val);
2625  exit(EXIT_FAILURE);
2626  }
2627 
2628  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2629  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2630  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2631  "from conf file - %s. Killing engine", p->val);
2632  exit(EXIT_FAILURE);
2633  }
2634 
2635  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2636  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2637  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2638  "from conf file - %s. Killing engine", p->val);
2639  exit(EXIT_FAILURE);
2640  }
2641 
2642  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2643  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2644  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2645  "from conf file - %s. Killing engine", p->val);
2646  exit(EXIT_FAILURE);
2647  }
2648 
2649  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2650  if (ConfValIsTrue(p->val)) {
2651  htp_config_register_request_line(cfg_prec->cfg,
2652  HTPCallbackDoubleDecodeQuery);
2653  }
2654 
2655  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2656  if (ConfValIsTrue(p->val)) {
2657  htp_config_register_request_line(cfg_prec->cfg,
2658  HTPCallbackDoubleDecodePath);
2659  }
2660 
2661  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2662  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2663  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2664  "from conf file - %s. Killing engine", p->val);
2665  exit(EXIT_FAILURE);
2666  }
2667 
2668  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2669  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2670  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2671  "from conf file - %s. Killing engine", p->val);
2672  exit(EXIT_FAILURE);
2673  }
2674 
2675  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2676  uint32_t value = 2;
2677  if (ParseSizeStringU32(p->val, &value) < 0) {
2678  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2679  "from conf file - %s. Killing engine", p->val);
2680  exit(EXIT_FAILURE);
2681  }
2682 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2683  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2684 #else
2685  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2686  "to %u, libhtp version too old", value);
2687 #endif
2688  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2689  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2690  HTP_DECODER_URL_PATH,
2691  ConfValIsTrue(p->val));
2692  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2693  if (strlen(p->val) == 1) {
2694  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2695  HTP_DECODER_URL_PATH,
2696  p->val[0]);
2697  } else {
2698  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2699  "for libhtp param path-bestfit-replacement-char");
2700  }
2701  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2702  htp_config_set_convert_lowercase(cfg_prec->cfg,
2703  HTP_DECODER_URL_PATH,
2704  ConfValIsTrue(p->val));
2705  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2706  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2707  HTP_DECODER_URL_PATH,
2708  ConfValIsTrue(p->val));
2709  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2710  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2711  HTP_DECODER_URL_PATH,
2712  ConfValIsTrue(p->val));
2713  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2714  htp_config_set_path_separators_compress(cfg_prec->cfg,
2715  HTP_DECODER_URL_PATH,
2716  ConfValIsTrue(p->val));
2717  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2718  htp_config_set_path_separators_decode(cfg_prec->cfg,
2719  HTP_DECODER_URL_PATH,
2720  ConfValIsTrue(p->val));
2721  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2722  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2723  HTP_DECODER_URL_PATH,
2724  ConfValIsTrue(p->val));
2725  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2726  enum htp_url_encoding_handling_t handling;
2727  if (strcasecmp(p->val, "preserve_percent") == 0) {
2728  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2729  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2730  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2731  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2732  handling = HTP_URL_DECODE_PROCESS_INVALID;
2733  } else {
2734  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2735  "for libhtp param path-url-encoding-invalid-handling");
2736  return;
2737  }
2738  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2739  HTP_DECODER_URL_PATH,
2740  handling);
2741  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2742  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2743  HTP_DECODER_URL_PATH,
2744  ConfValIsTrue(p->val));
2745  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2746  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2747  SCLogDebug("uri-include-all %s",
2748  cfg_prec->uri_include_all ? "enabled" : "disabled");
2749  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2750  htp_config_set_plusspace_decode(cfg_prec->cfg,
2751  HTP_DECODER_URLENCODED,
2752  ConfValIsTrue(p->val));
2753  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2754  uint32_t limit = 0;
2755  if (ParseSizeStringU32(p->val, &limit) < 0) {
2756  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2757  "from conf file - %s. Killing engine", p->val);
2758  exit(EXIT_FAILURE);
2759  }
2760  if (limit == 0) {
2761  FatalError(SC_ERR_FATAL, "Error meta-field-limit "
2762  "from conf file cannot be 0. Killing engine");
2763  }
2764  /* set default soft-limit with our new hard limit */
2765  htp_config_set_field_limits(cfg_prec->cfg,
2767  (size_t)limit);
2768 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2769  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2770  uint32_t limit = 0;
2771  if (ParseSizeStringU32(p->val, &limit) < 0) {
2772  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'lzma-memlimit' "
2773  "from conf file - %s.", p->val);
2774  }
2775  if (limit == 0) {
2776  FatalError(SC_ERR_SIZE_PARSE, "'lzma-memlimit' "
2777  "from conf file cannot be 0.");
2778  }
2779  /* set default soft-limit with our new hard limit */
2780  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2781  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2782 #endif
2783 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2784  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2785  if (ConfValIsTrue(p->val)) {
2786  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2787  } else if (!ConfValIsFalse(p->val)) {
2788  int8_t limit;
2789  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2791  "failed to parse 'lzma-enabled' "
2792  "from conf file - %s.",
2793  p->val);
2794  }
2795  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2796  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2797  }
2798 #endif
2799 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2800  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2801  uint32_t limit = 0;
2802  if (ParseSizeStringU32(p->val, &limit) < 0) {
2803  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'compression-bomb-limit' "
2804  "from conf file - %s.", p->val);
2805  }
2806  if (limit == 0) {
2807  FatalError(SC_ERR_SIZE_PARSE, "'compression-bomb-limit' "
2808  "from conf file cannot be 0.");
2809  }
2810  /* set default soft-limit with our new hard limit */
2811  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2812  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2813 #endif
2814 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2815  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2816  uint32_t limit = 0;
2817  // between 1 usec and 1 second
2818  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2820  "failed to parse 'decompression-time-limit' "
2821  "from conf file - %s.",
2822  p->val);
2823  }
2824  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2825  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2826 #endif
2827  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2828  if (!g_disable_randomness) {
2829  cfg_prec->randomize = ConfValIsTrue(p->val);
2830  }
2831  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2832  uint32_t range;
2833  if (StringParseU32RangeCheck(&range, 10, 0,
2834  (const char *)p->val, 0, 100) < 0) {
2835  SCLogError(SC_ERR_INVALID_VALUE, "Invalid value for randomize"
2836  "-inspection-range setting from conf file - \"%s\"."
2837  " It should be a valid integer less than or equal to 100."
2838  " Killing engine",
2839  p->val);
2840  exit(EXIT_FAILURE);
2841  }
2842  cfg_prec->randomize_range = range;
2843  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2844  if (ConfValIsTrue(p->val)) {
2845  cfg_prec->http_body_inline = 1;
2846  } else if (ConfValIsFalse(p->val)) {
2847  cfg_prec->http_body_inline = 0;
2848  } else {
2849  if (strcmp("auto", p->val) != 0) {
2850  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2851  }
2852  if (EngineModeIsIPS()) {
2853  cfg_prec->http_body_inline = 1;
2854  } else {
2855  cfg_prec->http_body_inline = 0;
2856  }
2857  }
2858  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2859  ConfNode *pval;
2860 
2861  TAILQ_FOREACH(pval, &p->head, next) {
2862  if (strcasecmp("enabled", pval->name) == 0) {
2863  if (ConfValIsTrue(pval->val)) {
2864  cfg_prec->swf_decompression_enabled = 1;
2865  } else if (ConfValIsFalse(pval->val)) {
2866  cfg_prec->swf_decompression_enabled = 0;
2867  } else {
2868  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2869  }
2870  } else if (strcasecmp("type", pval->name) == 0) {
2871  if (strcasecmp("no", pval->val) == 0) {
2873  } else if (strcasecmp("deflate", pval->val) == 0) {
2875  } else if (strcasecmp("lzma", pval->val) == 0) {
2877  } else if (strcasecmp("both", pval->val) == 0) {
2879  } else {
2881  "Invalid entry for "
2882  "swf-decompression.type: %s - "
2883  "Killing engine", pval->val);
2884  exit(EXIT_FAILURE);
2885  }
2886  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2887  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2889  "Error parsing swf-decompression.compression-depth "
2890  "from conf file - %s. Killing engine", p->val);
2891  exit(EXIT_FAILURE);
2892  }
2893  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2894  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2896  "Error parsing swf-decompression.decompression-depth "
2897  "from conf file - %s. Killing engine", p->val);
2898  exit(EXIT_FAILURE);
2899  }
2900  } else {
2901  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2902  }
2903  }
2904  } else {
2905  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2906  "default config: %s", p->name);
2907  }
2908  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2909 
2910  return;
2911 }
2912 
2913 void HTPConfigure(void)
2914 {
2915  SCEnter();
2916 
2917  cfglist.next = NULL;
2918 
2919  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2920  if (NULL == cfgtree)
2921  exit(EXIT_FAILURE);
2922 
2923  /* Default Config */
2924  cfglist.cfg = htp_config_create();
2925  if (NULL == cfglist.cfg) {
2926  FatalError(SC_ERR_FATAL, "Failed to create HTP default config");
2927  }
2928  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2929  HTPConfigSetDefaultsPhase1(&cfglist);
2930  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2931  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2932  cfgtree);
2933  } else {
2934  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2935  }
2936  HTPConfigSetDefaultsPhase2("default", &cfglist);
2937 
2938  HTPParseMemcap();
2939 
2940  /* Read server config and create a parser for each IP in radix tree */
2941  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2942  if (server_config == NULL) {
2943  server_config = ConfGetNode("libhtp.server-config");
2944  if (server_config == NULL) {
2945  SCLogDebug("LIBHTP Configuring %p", server_config);
2946  SCReturn;
2947  }
2948  }
2949  SCLogDebug("LIBHTP Configuring %p", server_config);
2950 
2951  ConfNode *si;
2952  /* Server Nodes */
2953  TAILQ_FOREACH(si, &server_config->head, next) {
2954  /* Need the named node, not the index */
2955  ConfNode *s = TAILQ_FIRST(&si->head);
2956  if (NULL == s) {
2957  SCLogDebug("LIBHTP s NULL");
2958  continue;
2959  }
2960 
2961  SCLogDebug("LIBHTP server %s", s->name);
2962 
2963  HTPCfgRec *nextrec = cfglist.next;
2964  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2965  if (NULL == htprec)
2966  exit(EXIT_FAILURE);
2967  memset(htprec, 0x00, sizeof(*htprec));
2968 
2969  cfglist.next = htprec;
2970 
2971  cfglist.next->next = nextrec;
2972  cfglist.next->cfg = htp_config_create();
2973  if (NULL == cfglist.next->cfg) {
2974  FatalError(SC_ERR_FATAL, "Failed to create HTP server config");
2975  }
2976 
2977  HTPConfigSetDefaultsPhase1(htprec);
2978  HTPConfigParseParameters(htprec, s, cfgtree);
2979  HTPConfigSetDefaultsPhase2(s->name, htprec);
2980  }
2981 
2982  SCReturn;
2983 }
2984 
2986 {
2987 #ifdef DEBUG
2988  SCMutexLock(&htp_state_mem_lock);
2989  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2990  SCMutexUnlock(&htp_state_mem_lock);
2991 #endif
2992 }
2993 
2994 /** \internal
2995  * \brief get files callback
2996  * \param state state ptr
2997  * \param direction flow direction
2998  * \retval files files ptr
2999  */
3000 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
3001 {
3002  if (state == NULL)
3003  return NULL;
3004 
3005  HtpState *http_state = (HtpState *)state;
3006 
3007  if (direction & STREAM_TOCLIENT) {
3008  SCReturnPtr(http_state->files_tc, "FileContainer");
3009  } else {
3010  SCReturnPtr(http_state->files_ts, "FileContainer");
3011  }
3012 }
3013 
3014 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
3015 {
3016  if (direction & STREAM_TOSERVER)
3017  return ((htp_tx_t *)tx)->request_progress;
3018  else
3019  return ((htp_tx_t *)tx)->response_progress;
3020 }
3021 
3022 static uint64_t HTPStateGetTxCnt(void *alstate)
3023 {
3024  HtpState *http_state = (HtpState *)alstate;
3025 
3026  if (http_state != NULL && http_state->conn != NULL) {
3027  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
3028  if (size < 0)
3029  return 0ULL;
3030  SCLogDebug("size %"PRIu64, size);
3031  return (uint64_t)size;
3032  } else {
3033  return 0ULL;
3034  }
3035 }
3036 
3037 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
3038 {
3039  HtpState *http_state = (HtpState *)alstate;
3040 
3041  if (http_state != NULL && http_state->conn != NULL)
3042  return htp_list_get(http_state->conn->transactions, tx_id);
3043  else
3044  return NULL;
3045 }
3046 
3047 void *HtpGetTxForH2(void *alstate)
3048 {
3049  // gets last transaction
3050  HtpState *http_state = (HtpState *)alstate;
3051  if (http_state != NULL && http_state->conn != NULL) {
3052  size_t txid = htp_list_array_size(http_state->conn->transactions);
3053  if (txid > 0) {
3054  return htp_list_get(http_state->conn->transactions, txid - 1);
3055  }
3056  }
3057  return NULL;
3058 }
3059 
3060 static int HTPStateGetEventInfo(const char *event_name,
3061  int *event_id, AppLayerEventType *event_type)
3062 {
3063  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3064  if (*event_id == -1) {
3065  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
3066  "http's enum map table.", event_name);
3067  /* this should be treated as fatal */
3068  return -1;
3069  }
3070 
3071  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3072 
3073  return 0;
3074 }
3075 
3076 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3077  AppLayerEventType *event_type)
3078 {
3079  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3080  if (*event_name == NULL) {
3081  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
3082  "http's enum map table.", event_id);
3083  /* this should be treated as fatal */
3084  return -1;
3085  }
3086 
3087  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3088 
3089  return 0;
3090 }
3091 
3092 static void HTPStateTruncate(void *state, uint8_t direction)
3093 {
3094  FileContainer *fc = HTPStateGetFiles(state, direction);
3095  if (fc != NULL) {
3097  }
3098 }
3099 
3100 static AppLayerTxData *HTPGetTxData(void *vtx)
3101 {
3102  htp_tx_t *tx = (htp_tx_t *)vtx;
3103  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3104  if (tx_ud) {
3105  return &tx_ud->tx_data;
3106  }
3107  return NULL;
3108 }
3109 
3110 static int HTPRegisterPatternsForProtocolDetection(void)
3111 {
3112  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3113  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3114  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3115  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3116  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3117  const char *spacings[] = { "|20|", "|09|", NULL };
3118  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3119 
3120  int methods_pos;
3121  int spacings_pos;
3122  int versions_pos;
3123  int register_result;
3124  char method_buffer[32] = "";
3125 
3126  /* Loop through all the methods ands spacings and register the patterns */
3127  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3128  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3129 
3130  /* Combine the method name and the spacing */
3131  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3132 
3133  /* Register the new method+spacing pattern
3134  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3135  * but the pattern matching should only be one char
3136  */
3137  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3138  method_buffer, (uint16_t)strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3139  if (register_result < 0) {
3140  return -1;
3141  }
3142  }
3143  }
3144 
3145  /* Loop through all the http verions patterns that are TO_CLIENT */
3146  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3147  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3148  versions[versions_pos], (uint16_t)strlen(versions[versions_pos]), 0,
3149  STREAM_TOCLIENT);
3150  if (register_result < 0) {
3151  return -1;
3152  }
3153  }
3154 
3155  return 0;
3156 }
3157 
3158 /**
3159  * \brief Register the HTTP protocol and state handling functions to APP layer
3160  * of the engine.
3161  */
3163 {
3164  SCEnter();
3165 
3166  const char *proto_name = "http";
3167 
3168  /** HTTP */
3169  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3171  if (HTPRegisterPatternsForProtocolDetection() < 0)
3172  return;
3173  } else {
3174  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3175  proto_name);
3176  return;
3177  }
3178 
3179  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3180  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3181  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3182  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetFiles);
3184  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3185  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3186  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3187 
3189  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3190  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3192  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3193 
3194  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTruncate);
3195  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3196 
3198  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3199 
3201  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3203  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3204  SC_ATOMIC_INIT(htp_config_flags);
3205  /* This parser accepts gaps. */
3209  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_TOCLIENT);
3210  /* app-layer-frame-documentation tag start: registering relevant callbacks */
3212  IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById);
3213  /* app-layer-frame-documentation tag end: registering relevant callbacks */
3214  HTPConfigure();
3215  } else {
3216  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3217  "still on.", proto_name);
3218  }
3219 #ifdef UNITTESTS
3220  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3221 #endif
3222 
3223  SCReturn;
3224 }
3225 
3226 #ifdef UNITTESTS
3227 static HTPCfgRec cfglist_backup;
3228 
3230 {
3231  cfglist_backup = cfglist;
3232 
3233  return;
3234 }
3235 
3237 {
3238  cfglist = cfglist_backup;
3239 
3240  return;
3241 }
3242 
3243 /** \test Test case where chunks are sent in smaller chunks and check the
3244  * response of the parser from HTP library. */
3245 static int HTPParserTest01(void)
3246 {
3247  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3248  " Data is c0oL!";
3249  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3250 
3251  TcpSession ssn;
3252  memset(&ssn, 0, sizeof(ssn));
3253 
3256 
3257  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3258  FAIL_IF_NULL(f);
3259  f->protoctx = &ssn;
3260  f->proto = IPPROTO_TCP;
3261  f->alproto = ALPROTO_HTTP1;
3262 
3263  StreamTcpInitConfig(true);
3264 
3265  uint32_t u;
3266  for (u = 0; u < httplen1; u++) {
3267  uint8_t flags = 0;
3268 
3269  if (u == 0)
3270  flags = STREAM_TOSERVER|STREAM_START;
3271  else if (u == (httplen1 - 1))
3272  flags = STREAM_TOSERVER|STREAM_EOF;
3273  else
3274  flags = STREAM_TOSERVER;
3275 
3276  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3277  FAIL_IF(r != 0);
3278  }
3279 
3280  HtpState *htp_state = f->alstate;
3281  FAIL_IF_NULL(htp_state);
3282 
3283  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3284  FAIL_IF_NULL(tx);
3285 
3286  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3287  FAIL_IF_NULL(h);
3288 
3289  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3290  FAIL_IF(tx->request_method_number != HTP_M_POST);
3291  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3292 
3294  StreamTcpFreeConfig(true);
3295  UTHFreeFlow(f);
3296  PASS;
3297 }
3298 
3299 /** \test Test folding in 1 read case */
3300 static int HTPParserTest01b(void)
3301 {
3302  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3303  " Data is c0oL!";
3304  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3305 
3306  TcpSession ssn;
3307  memset(&ssn, 0, sizeof(ssn));
3308 
3311 
3312  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3313  FAIL_IF_NULL(f);
3314  f->protoctx = &ssn;
3315  f->proto = IPPROTO_TCP;
3316  f->alproto = ALPROTO_HTTP1;
3317 
3318  StreamTcpInitConfig(true);
3319 
3320  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3321  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3322  FAIL_IF(r != 0);
3323 
3324  HtpState *htp_state = f->alstate;
3325  FAIL_IF_NULL(htp_state);
3326 
3327  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3328  FAIL_IF_NULL(tx);
3329 
3330  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3331  FAIL_IF_NULL(h);
3332 
3333  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3334  FAIL_IF(tx->request_method_number != HTP_M_POST);
3335  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3336 
3338  StreamTcpFreeConfig(true);
3339  UTHFreeFlow(f);
3340  PASS;
3341 }
3342 
3343 /** \test Test folding in 1byte per read case */
3344 static int HTPParserTest01c(void)
3345 {
3346  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3347  " Data is c0oL!";
3348  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3349 
3350  TcpSession ssn;
3351  memset(&ssn, 0, sizeof(ssn));
3352 
3355 
3356  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3357  FAIL_IF_NULL(f);
3358  f->protoctx = &ssn;
3359  f->proto = IPPROTO_TCP;
3360  f->alproto = ALPROTO_HTTP1;
3361 
3362  StreamTcpInitConfig(true);
3363 
3364  uint32_t u;
3365  for (u = 0; u < httplen1; u++) {
3366  uint8_t flags = 0;
3367 
3368  if (u == 0)
3369  flags = STREAM_TOSERVER|STREAM_START;
3370  else if (u == (httplen1 - 1))
3371  flags = STREAM_TOSERVER|STREAM_EOF;
3372  else
3373  flags = STREAM_TOSERVER;
3374 
3375  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3376  FAIL_IF(r != 0);
3377  }
3378 
3379  HtpState *htp_state = f->alstate;
3380  FAIL_IF_NULL(htp_state);
3381 
3382  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3383  FAIL_IF_NULL(tx);
3384 
3385  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3386  FAIL_IF_NULL(h);
3387 
3388  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3389  FAIL_IF(tx->request_method_number != HTP_M_POST);
3390  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3391 
3393  StreamTcpFreeConfig(true);
3394  UTHFreeFlow(f);
3395  PASS;
3396 }
3397 
3398 /** \test Test case where chunks are sent in smaller chunks and check the
3399  * response of the parser from HTP library. */
3400 static int HTPParserTest01a(void)
3401 {
3402  int result = 0;
3403  Flow *f = NULL;
3404  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3405  " Data is c0oL!";
3406  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3407  TcpSession ssn;
3408  HtpState *htp_state = NULL;
3409  int r = 0;
3411 
3412  memset(&ssn, 0, sizeof(ssn));
3413 
3414  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3415  if (f == NULL)
3416  goto end;
3417  f->protoctx = &ssn;
3418  f->proto = IPPROTO_TCP;
3419  f->alproto = ALPROTO_HTTP1;
3420 
3421  StreamTcpInitConfig(true);
3422 
3423  uint32_t u;
3424  for (u = 0; u < httplen1; u++) {
3425  uint8_t flags = 0;
3426 
3427  if (u == 0)
3428  flags = STREAM_TOSERVER|STREAM_START;
3429  else if (u == (httplen1 - 1))
3430  flags = STREAM_TOSERVER|STREAM_EOF;
3431  else
3432  flags = STREAM_TOSERVER;
3433 
3434  FLOWLOCK_WRLOCK(f);
3435  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3436  if (r != 0) {
3437  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3438  " 0: ", u, r);
3439  FLOWLOCK_UNLOCK(f);
3440  goto end;
3441  }
3442  FLOWLOCK_UNLOCK(f);
3443  }
3444 
3445  htp_state = f->alstate;
3446  if (htp_state == NULL) {
3447  printf("no http state: ");
3448  goto end;
3449  }
3450 
3451  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3452  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3453  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3454  || tx->request_method_number != HTP_M_POST ||
3455  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3456  {
3457  printf("expected header value: Victor/1.0 and got %s: and expected"
3458  " method: POST and got %s, expected protocol number HTTP/1.0"
3459  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3460  bstr_util_strdup_to_c(tx->request_method),
3461  bstr_util_strdup_to_c(tx->request_protocol));
3462  goto end;
3463  }
3464  result = 1;
3465 end:
3466  if (alp_tctx != NULL)
3468  StreamTcpFreeConfig(true);
3469  UTHFreeFlow(f);
3470  return result;
3471 }
3472 
3473 /** \test See how it deals with an incomplete request. */
3474 static int HTPParserTest02(void)
3475 {
3476  int result = 0;
3477  Flow *f = NULL;
3478  uint8_t httpbuf1[] = "POST";
3479  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3480  TcpSession ssn;
3481  HtpState *http_state = NULL;
3483 
3484  memset(&ssn, 0, sizeof(ssn));
3485 
3486  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3487  if (f == NULL)
3488  goto end;
3489  f->protoctx = &ssn;
3490  f->proto = IPPROTO_TCP;
3491  f->alproto = ALPROTO_HTTP1;
3492 
3493  StreamTcpInitConfig(true);
3494 
3495  FLOWLOCK_WRLOCK(f);
3496  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3497  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3498  if (r != 0) {
3499  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3500  FLOWLOCK_UNLOCK(f);
3501  goto end;
3502  }
3503  FLOWLOCK_UNLOCK(f);
3504 
3505  http_state = f->alstate;
3506  if (http_state == NULL) {
3507  printf("no http state: ");
3508  goto end;
3509  }
3510 
3511  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3512  FAIL_IF_NULL(tx);
3513  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3514  FAIL_IF_NOT_NULL(h);
3515 
3516  FAIL_IF_NULL(tx->request_method);
3517  char *method = bstr_util_strdup_to_c(tx->request_method);
3518  FAIL_IF_NULL(method);
3519 
3520  FAIL_IF(strcmp(method, "POST") != 0);
3521  SCFree(method);
3522 
3523  result = 1;
3524 end:
3525  if (alp_tctx != NULL)
3527  StreamTcpFreeConfig(true);
3528  UTHFreeFlow(f);
3529  return result;
3530 }
3531 
3532 /** \test Test case where method is invalid and data is sent in smaller chunks
3533  * and check the response of the parser from HTP library. */
3534 static int HTPParserTest03(void)
3535 {
3536  int result = 0;
3537  Flow *f = NULL;
3538  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3539  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3540  TcpSession ssn;
3541  HtpState *htp_state = NULL;
3542  int r = 0;
3544 
3545  memset(&ssn, 0, sizeof(ssn));
3546 
3547  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3548  if (f == NULL)
3549  goto end;
3550  f->protoctx = &ssn;
3551  f->proto = IPPROTO_TCP;
3552  f->alproto = ALPROTO_HTTP1;
3553 
3554  StreamTcpInitConfig(true);
3555 
3556  uint32_t u;
3557  for (u = 0; u < httplen1; u++) {
3558  uint8_t flags = 0;
3559 
3560  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3561  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3562  else flags = STREAM_TOSERVER;
3563 
3564  FLOWLOCK_WRLOCK(f);
3565  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3566  if (r != 0) {
3567  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3568  " 0: ", u, r);
3569  FLOWLOCK_UNLOCK(f);
3570  goto end;
3571  }
3572  FLOWLOCK_UNLOCK(f);
3573  }
3574  htp_state = f->alstate;
3575  if (htp_state == NULL) {
3576  printf("no http state: ");
3577  goto end;
3578  }
3579 
3580  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3581 
3582  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3583  if (tx->request_method_number != HTP_M_UNKNOWN ||
3584  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3585  {
3586  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3587  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3588  bstr_util_strdup_to_c(tx->request_protocol));
3589  goto end;
3590  }
3591  result = 1;
3592 end:
3593  if (alp_tctx != NULL)
3595  StreamTcpFreeConfig(true);
3596  UTHFreeFlow(f);
3597  return result;
3598 }
3599 
3600 /** \test Test case where invalid data is sent and check the response of the
3601  * parser from HTP library. */
3602 static int HTPParserTest04(void)
3603 {
3604  int result = 0;
3605  Flow *f = NULL;
3606  HtpState *htp_state = NULL;
3607  uint8_t httpbuf1[] = "World!\r\n";
3608  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3609  TcpSession ssn;
3610  int r = 0;
3612 
3613  memset(&ssn, 0, sizeof(ssn));
3614 
3615  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3616  if (f == NULL)
3617  goto end;
3618  f->protoctx = &ssn;
3619  f->proto = IPPROTO_TCP;
3620  f->alproto = ALPROTO_HTTP1;
3621 
3622  StreamTcpInitConfig(true);
3623 
3624  FLOWLOCK_WRLOCK(f);
3626  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3627  if (r != 0) {
3628  FLOWLOCK_UNLOCK(f);
3629  goto end;
3630  }
3631  FLOWLOCK_UNLOCK(f);
3632 
3633  htp_state = f->alstate;
3634  if (htp_state == NULL) {
3635  printf("no http state: ");
3636  goto end;
3637  }
3638 
3639  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3640  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3641  if (tx->request_method_number != HTP_M_UNKNOWN ||
3642  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3643  {
3644  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3645  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3646  bstr_util_strdup_to_c(tx->request_protocol));
3647  goto end;
3648  }
3649  result = 1;
3650 end:
3651  if (alp_tctx != NULL)
3653  StreamTcpFreeConfig(true);
3654  UTHFreeFlow(f);
3655  return result;
3656 }
3657 
3658 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3659  * properly parsed them and also keeps them separated. */
3660 static int HTPParserTest05(void)
3661 {
3662  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3663  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3664  uint8_t httpbuf2[] = "Post D";
3665  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3666  uint8_t httpbuf3[] = "ata is c0oL!";
3667  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3668 
3669  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3670  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3671  uint8_t httpbuf5[] = "post R";
3672  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3673  uint8_t httpbuf6[] = "esults are tha bomb!";
3674  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3675 
3676  TcpSession ssn;
3677  memset(&ssn, 0, sizeof(ssn));
3678 
3681 
3682  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3683  FAIL_IF_NULL(f);
3684  f->protoctx = &ssn;
3685  f->proto = IPPROTO_TCP;
3686  f->alproto = ALPROTO_HTTP1;
3687 
3688  StreamTcpInitConfig(true);
3689 
3690  int r = AppLayerParserParse(
3691  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3692  FAIL_IF(r != 0);
3693 
3694  r = AppLayerParserParse(
3695  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3696  FAIL_IF(r != 0);
3697 
3698  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3699  FAIL_IF(r != 0);
3700 
3701  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3702  FAIL_IF(r != 0);
3703 
3704  r = AppLayerParserParse(
3705  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3706  FAIL_IF(r != 0);
3707 
3708  r = AppLayerParserParse(
3709  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3710  FAIL_IF(r != 0);
3711 
3712  HtpState *http_state = f->alstate;
3713  FAIL_IF_NULL(http_state);
3714 
3715  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3716  FAIL_IF_NULL(tx);
3717  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3718  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3719 
3720  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3721  FAIL_IF_NULL(h);
3722 
3723  FAIL_IF_NOT(tx->response_status_number == 200);
3724 
3726  StreamTcpFreeConfig(true);
3727  UTHFreeFlow(f);
3728  PASS;
3729 }
3730 
3731 /** \test Test proper chunked encoded response body
3732  */
3733 static int HTPParserTest06(void)
3734 {
3735  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3736  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3737  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3738  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3739  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3740  "GMT\r\n"
3741  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3742  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3743  "FrontPage/5.0.2.2510\r\n"
3744  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3745  "chunked\r\n"
3746  "Content-Type: text/html\r\n\r\n"
3747  "580\r\n"
3748  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3749  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3750  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3751  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3752  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3753  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3754  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3755  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3756  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3757  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3758  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3759  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3760  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3761  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3762  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3763  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3764  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3765  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3766  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3767  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3768  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3769  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3770  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3771  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3772  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3773  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3774  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3775  "aHA=\r\n0\r\n\r\n";
3776  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3777  TcpSession ssn;
3778 
3781 
3782  memset(&ssn, 0, sizeof(ssn));
3783 
3784  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3785  FAIL_IF_NULL(f);
3786  f->protoctx = &ssn;
3787  f->proto = IPPROTO_TCP;
3788  f->alproto = ALPROTO_HTTP1;
3789 
3790  StreamTcpInitConfig(true);
3791 
3792  int r = AppLayerParserParse(
3793  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3794  FAIL_IF(r != 0);
3795  r = AppLayerParserParse(
3796  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3797  FAIL_IF(r != 0);
3798 
3799  HtpState *http_state = f->alstate;
3800  FAIL_IF_NULL(http_state);
3801 
3802  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3803  FAIL_IF_NULL(tx);
3804 
3805  FAIL_IF(tx->request_method_number != HTP_M_GET);
3806  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3807 
3808  FAIL_IF(tx->response_status_number != 200);
3809  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3810 
3811  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3812  FAIL_IF_NULL(h);
3813 
3815  StreamTcpFreeConfig(true);
3816  UTHFreeFlow(f);
3817  PASS;
3818 }
3819 
3820 /** \test
3821  */
3822 static int HTPParserTest07(void)
3823 {
3824  int result = 0;
3825  Flow *f = NULL;
3826  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3827  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3828  TcpSession ssn;
3829  HtpState *htp_state = NULL;
3830  int r = 0;
3832 
3833  memset(&ssn, 0, sizeof(ssn));
3834 
3835  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3836  if (f == NULL)
3837  goto end;
3838  f->protoctx = &ssn;
3839  f->proto = IPPROTO_TCP;
3840  f->alproto = ALPROTO_HTTP1;
3841 
3842  StreamTcpInitConfig(true);
3843 
3844  uint32_t u;
3845  for (u = 0; u < httplen1; u++) {
3846  uint8_t flags = 0;
3847 
3848  if (u == 0)
3849  flags = STREAM_TOSERVER|STREAM_START;
3850  else if (u == (httplen1 - 1))
3851  flags = STREAM_TOSERVER|STREAM_EOF;
3852  else
3853  flags = STREAM_TOSERVER;
3854 
3855  FLOWLOCK_WRLOCK(f);
3856  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3857  if (r != 0) {
3858  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3859  " 0: ", u, r);
3860  FLOWLOCK_UNLOCK(f);
3861  goto end;
3862  }
3863  FLOWLOCK_UNLOCK(f);
3864  }
3865 
3866  htp_state = f->alstate;
3867  if (htp_state == NULL) {
3868  printf("no http state: ");
3869  goto end;
3870  }
3871 
3872  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3873  size_t reflen = sizeof(ref) - 1;
3874 
3875  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3876  if (tx == NULL)
3877  goto end;
3878  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3879  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3880  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3881  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3882  (uintmax_t)reflen,
3883  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3884  goto end;
3885  }
3886 
3887  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3888  bstr_len(tx_ud->request_uri_normalized)) != 0)
3889  {
3890  printf("normalized uri \"");
3891  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3892  printf("\" != \"");
3893  PrintRawUriFp(stdout, ref, reflen);
3894  printf("\": ");
3895  goto end;
3896  }
3897  }
3898 
3899  result = 1;
3900 end:
3901  if (alp_tctx != NULL)
3903  StreamTcpFreeConfig(true);
3904  UTHFreeFlow(f);
3905  return result;
3906 }
3907 
3908 #include "conf-yaml-loader.h"
3909 
3910 /** \test Abort
3911  */
3912 static int HTPParserTest08(void)
3913 {
3914  int result = 0;
3915  Flow *f = NULL;
3916  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3917  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3918  TcpSession ssn;
3920 
3921  char input[] = "\
3922 %YAML 1.1\n\
3923 ---\n\
3924 libhtp:\n\
3925 \n\
3926  default-config:\n\
3927  personality: IDS\n\
3928 ";
3929 
3931  ConfInit();
3933 
3934  ConfYamlLoadString(input, strlen(input));
3935  HTPConfigure();
3936 
3937  HtpState *htp_state = NULL;
3938  int r = 0;
3939  memset(&ssn, 0, sizeof(ssn));
3940 
3941  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3942  if (f == NULL)
3943  goto end;
3944  f->protoctx = &ssn;
3945  f->proto = IPPROTO_TCP;
3946  f->alproto = ALPROTO_HTTP1;
3947 
3948  StreamTcpInitConfig(true);
3949 
3950  uint8_t flags = 0;
3951  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3952 
3953  FLOWLOCK_WRLOCK(f);
3954  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3955  if (r != 0) {
3956  printf("toserver chunk returned %" PRId32 ", expected"
3957  " 0: ", r);
3958  result = 0;
3959  FLOWLOCK_UNLOCK(f);
3960  goto end;
3961  }
3962  FLOWLOCK_UNLOCK(f);
3963 
3964  htp_state = f->alstate;
3965  if (htp_state == NULL) {
3966  printf("no http state: ");
3967  result = 0;
3968  goto end;
3969  }
3970 
3971  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3972  if (tx == NULL)
3973  goto end;
3974  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3975  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3976  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3977  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3978  bstr_len(tx_ud->request_uri_normalized));
3979  }
3980 
3981  result = 1;
3982 end:
3983  if (alp_tctx != NULL)
3985  StreamTcpFreeConfig(true);
3986  HTPFreeConfig();
3987  ConfDeInit();
3990  UTHFreeFlow(f);
3991  return result;
3992 }
3993 
3994 /** \test Abort
3995  */
3996 static int HTPParserTest09(void)
3997 {
3998  int result = 0;
3999  Flow *f = NULL;
4000  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
4001  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4002  TcpSession ssn;
4004 
4005  char input[] = "\
4006 %YAML 1.1\n\
4007 ---\n\
4008 libhtp:\n\
4009 \n\
4010  default-config:\n\
4011  personality: Apache_2_2\n\
4012 ";
4013 
4015  ConfInit();
4017 
4018  ConfYamlLoadString(input, strlen(input));
4019  HTPConfigure();
4020 
4021  HtpState *htp_state = NULL;
4022  int r = 0;
4023 
4024  memset(&ssn, 0, sizeof(ssn));
4025 
4026  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4027  if (f == NULL)
4028  goto end;
4029  f->protoctx = &ssn;
4030  f->proto = IPPROTO_TCP;
4031  f->alproto = ALPROTO_HTTP1;
4032 
4033  StreamTcpInitConfig(true);
4034 
4035  uint8_t flags = 0;
4036  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
4037 
4038  FLOWLOCK_WRLOCK(f);
4039  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
4040  if (r != 0) {
4041  printf("toserver chunk returned %" PRId32 ", expected"
4042  " 0: ", r);
4043  FLOWLOCK_UNLOCK(f);
4044  goto end;
4045  }
4046  FLOWLOCK_UNLOCK(f);
4047 
4048  htp_state = f->alstate;
4049  if (htp_state == NULL) {
4050  printf("no http state: ");
4051  goto end;
4052  }
4053 
4054  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4055  if (tx == NULL)
4056  goto end;
4057  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4058  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4059  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4060  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4061  bstr_len(tx_ud->request_uri_normalized));
4062  }
4063 
4064  result = 1;
4065 end:
4066  if (alp_tctx != NULL)
4068  StreamTcpFreeConfig(true);
4069  HTPFreeConfig();
4070  ConfDeInit();
4073  UTHFreeFlow(f);
4074  return result;
4075 }
4076 
4077 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4078  */
4079 static int HTPParserTest10(void)
4080 {
4081  int result = 0;
4082  Flow *f = NULL;
4083  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4084  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4085  TcpSession ssn;
4086  HtpState *htp_state = NULL;
4087  int r = 0;
4089 
4090  memset(&ssn, 0, sizeof(ssn));
4091 
4092  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4093  if (f == NULL)
4094  goto end;
4095  f->protoctx = &ssn;
4096  f->proto = IPPROTO_TCP;
4097  f->alproto = ALPROTO_HTTP1;
4098 
4099  StreamTcpInitConfig(true);
4100 
4101  uint32_t u;
4102  for (u = 0; u < httplen1; u++) {
4103  uint8_t flags = 0;
4104 
4105  if (u == 0)
4106  flags = STREAM_TOSERVER|STREAM_START;
4107  else if (u == (httplen1 - 1))
4108  flags = STREAM_TOSERVER|STREAM_EOF;
4109  else
4110  flags = STREAM_TOSERVER;
4111 
4112  FLOWLOCK_WRLOCK(f);
4113  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4114  if (r != 0) {
4115  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4116  " 0: ", u, r);
4117  FLOWLOCK_UNLOCK(f);
4118  goto end;
4119  }
4120  FLOWLOCK_UNLOCK(f);
4121  }
4122 
4123  htp_state = f->alstate;
4124  if (htp_state == NULL) {
4125  printf("no http state: ");
4126  goto end;
4127  }
4128 
4129  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4130  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4131  if (h == NULL) {
4132  goto end;
4133  }
4134 
4135  char *name = bstr_util_strdup_to_c(h->name);
4136  if (name == NULL) {
4137  goto end;
4138  }
4139 
4140  if (strcmp(name, "Host") != 0) {
4141  printf("header name not \"Host\", instead \"%s\": ", name);
4142  free(name);
4143  goto end;
4144  }
4145  free(name);
4146 
4147  char *value = bstr_util_strdup_to_c(h->value);
4148  if (value == NULL) {
4149  goto end;
4150  }
4151 
4152  if (strcmp(value, "www.google.com") != 0) {
4153  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4154  free(value);
4155  goto end;
4156  }
4157  free(value);
4158 
4159  result = 1;
4160 end:
4161  if (alp_tctx != NULL)
4163  StreamTcpFreeConfig(true);
4164  UTHFreeFlow(f);
4165  return result;
4166 }
4167 
4168 /** \test double encoding in path
4169  */
4170 static int HTPParserTest11(void)
4171 {
4172  int result = 0;
4173  Flow *f = NULL;
4174  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4175  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4176  TcpSession ssn;
4177  HtpState *htp_state = NULL;
4178  int r = 0;
4180 
4181  memset(&ssn, 0, sizeof(ssn));
4182 
4183  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4184  if (f == NULL)
4185  goto end;
4186  f->protoctx = &ssn;
4187  f->proto = IPPROTO_TCP;
4188  f->alproto = ALPROTO_HTTP1;
4189 
4190  StreamTcpInitConfig(true);
4191 
4192  uint32_t u;
4193  for (u = 0; u < httplen1; u++) {
4194  uint8_t flags = 0;
4195 
4196  if (u == 0)
4197  flags = STREAM_TOSERVER|STREAM_START;
4198  else if (u == (httplen1 - 1))
4199  flags = STREAM_TOSERVER|STREAM_EOF;
4200  else
4201  flags = STREAM_TOSERVER;
4202 
4203  FLOWLOCK_WRLOCK(f);
4204  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4205  if (r != 0) {
4206  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4207  " 0: ", u, r);
4208  FLOWLOCK_UNLOCK(f);
4209  goto end;
4210  }
4211  FLOWLOCK_UNLOCK(f);
4212  }
4213 
4214  htp_state = f->alstate;
4215  if (htp_state == NULL) {
4216  printf("no http state: ");
4217  goto end;
4218  }
4219 
4220  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4221  if (tx == NULL)
4222  goto end;
4223  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4224  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4225  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4226  printf("normalized uri len should be 2, is %"PRIuMAX,
4227  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4228  goto end;
4229  }
4230 
4231  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4232  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4233  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4234  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4235  {
4236  printf("normalized uri \"");
4237  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4238  printf("\": ");
4239  goto end;
4240  }
4241  }
4242 
4243  result = 1;
4244 end:
4245  if (alp_tctx != NULL)
4247  StreamTcpFreeConfig(true);
4248  UTHFreeFlow(f);
4249  return result;
4250 }
4251 
4252 /** \test double encoding in query
4253  */
4254 static int HTPParserTest12(void)
4255 {
4256  int result = 0;
4257  Flow *f = NULL;
4258  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4259  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4260  TcpSession ssn;
4261  HtpState *htp_state = NULL;
4262  int r = 0;
4264 
4265  memset(&ssn, 0, sizeof(ssn));
4266 
4267  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4268  if (f == NULL)
4269  goto end;
4270  f->protoctx = &ssn;
4271  f->proto = IPPROTO_TCP;
4272  f->alproto = ALPROTO_HTTP1;
4273 
4274  StreamTcpInitConfig(true);
4275 
4276  uint32_t u;
4277  for (u = 0; u < httplen1; u++) {
4278  uint8_t flags = 0;
4279 
4280  if (u == 0)
4281  flags = STREAM_TOSERVER|STREAM_START;
4282  else if (u == (httplen1 - 1))
4283  flags = STREAM_TOSERVER|STREAM_EOF;
4284  else
4285  flags = STREAM_TOSERVER;
4286 
4287  FLOWLOCK_WRLOCK(f);
4288  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4289  if (r != 0) {
4290  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4291  " 0: ", u, r);
4292  FLOWLOCK_UNLOCK(f);
4293  goto end;
4294  }
4295  FLOWLOCK_UNLOCK(f);
4296  }
4297 
4298  htp_state = f->alstate;
4299  if (htp_state == NULL) {
4300  printf("no http state: ");
4301  goto end;
4302  }
4303 
4304  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4305  if (tx == NULL)
4306  goto end;
4307  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4308  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4309  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4310  printf("normalized uri len should be 5, is %"PRIuMAX,
4311  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4312  goto end;
4313  }
4314 
4315  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4316  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4317  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4318  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4319  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4320  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4321  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4322  {
4323  printf("normalized uri \"");
4324  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4325  printf("\": ");
4326  goto end;
4327  }
4328  }
4329 
4330  result = 1;
4331  end:
4332  if (alp_tctx != NULL)
4334  StreamTcpFreeConfig(true);
4335  UTHFreeFlow(f);
4336  return result;
4337 }
4338 
4339 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4340  */
4341 static int HTPParserTest13(void)
4342 {
4343  int result = 0;
4344  Flow *f = NULL;
4345  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4346  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4347  TcpSession ssn;
4348  HtpState *htp_state = NULL;
4349  int r = 0;
4351 
4352  memset(&ssn, 0, sizeof(ssn));
4353 
4354  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4355  if (f == NULL)
4356  goto end;
4357  f->protoctx = &ssn;
4358  f->proto = IPPROTO_TCP;
4359  f->alproto = ALPROTO_HTTP1;
4360 
4361  StreamTcpInitConfig(true);
4362 
4363  uint32_t u;
4364  for (u = 0; u < httplen1; u++) {
4365  uint8_t flags = 0;
4366 
4367  if (u == 0)
4368  flags = STREAM_TOSERVER|STREAM_START;
4369  else if (u == (httplen1 - 1))
4370  flags = STREAM_TOSERVER|STREAM_EOF;
4371  else
4372  flags = STREAM_TOSERVER;
4373 
4374  FLOWLOCK_WRLOCK(f);
4375  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4376  if (r != 0) {
4377  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4378  " 0: ", u, r);
4379  FLOWLOCK_UNLOCK(f);
4380  goto end;
4381  }
4382  FLOWLOCK_UNLOCK(f);
4383  }
4384 
4385  htp_state = f->alstate;
4386  if (htp_state == NULL) {
4387  printf("no http state: ");
4388  goto end;
4389  }
4390 
4391  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4392  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4393  if (h == NULL) {
4394  goto end;
4395  }
4396 
4397  char *name = bstr_util_strdup_to_c(h->name);
4398  if (name == NULL) {
4399  goto end;
4400  }
4401 
4402  if (strcmp(name, "Host") != 0) {
4403  printf("header name not \"Host\", instead \"%s\": ", name);
4404  free(name);
4405  goto end;
4406  }
4407  free(name);
4408 
4409  char *value = bstr_util_strdup_to_c(h->value);
4410  if (value == NULL) {
4411  goto end;
4412  }
4413 
4414  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4415  printf("header value not \"www.google.com\", instead \"");
4416  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4417  printf("\": ");
4418  free(value);
4419  goto end;
4420  }
4421  free(value);
4422 
4423  result = 1;
4424 end:
4425  if (alp_tctx != NULL)
4427  StreamTcpFreeConfig(true);
4428  UTHFreeFlow(f);
4429  return result;
4430 }
4431 
4432 /** \test Test basic config */
4433 static int HTPParserConfigTest01(void)
4434 {
4435  int ret = 0;
4436  char input[] = "\
4437 %YAML 1.1\n\
4438 ---\n\
4439 libhtp:\n\
4440 \n\
4441  default-config:\n\
4442  personality: IDS\n\
4443 \n\
4444  server-config:\n\
4445 \n\
4446  - apache-tomcat:\n\
4447  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4448  personality: Tomcat_6_0\n\
4449 \n\
4450  - iis7:\n\
4451  address: \n\
4452  - 192.168.0.0/24\n\
4453  - 192.168.10.0/24\n\
4454  personality: IIS_7_0\n\
4455 ";
4456 
4458  ConfInit();
4459 
4460  ConfYamlLoadString(input, strlen(input));
4461 
4462  ConfNode *outputs;
4463  outputs = ConfGetNode("libhtp.default-config.personality");
4464  if (outputs == NULL) {
4465  goto end;
4466  }
4467 
4468  outputs = ConfGetNode("libhtp.server-config");
4469  if (outputs == NULL) {
4470  goto end;
4471  }
4472 
4473  ConfNode *node = TAILQ_FIRST(&outputs->head);
4474  if (node == NULL) {
4475  goto end;
4476  }
4477  if (strcmp(node->name, "0") != 0) {
4478  goto end;
4479  }
4480  node = TAILQ_FIRST(&node->head);
4481  if (node == NULL) {
4482  goto end;
4483  }
4484  if (strcmp(node->name, "apache-tomcat") != 0) {
4485  goto end;
4486  }
4487 
4488  int i = 0;
4489  ConfNode *n;
4490 
4491  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4492  if (node2 == NULL) {
4493  goto end;
4494  }
4495  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4496  goto end;
4497  }
4498 
4499  node = ConfNodeLookupChild(node, "address");
4500  if (node == NULL) {
4501  goto end;
4502  }
4503  TAILQ_FOREACH(n, &node->head, next) {
4504  if (n == NULL) {
4505  goto end;
4506  }
4507 
4508  switch(i) {
4509  case 0:
4510  if (strcmp(n->name, "0") != 0) {
4511  goto end;
4512  }
4513  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4514  goto end;
4515  }
4516  break;
4517  case 1:
4518  if (strcmp(n->name, "1") != 0) {
4519  goto end;
4520  }
4521  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4522  goto end;
4523  }
4524  break;
4525  case 2:
4526  if (strcmp(n->name, "2") != 0) {
4527  goto end;
4528  }
4529  if (strcmp(n->val, "::1") != 0) {
4530  goto end;
4531  }
4532  break;
4533  default:
4534  goto end;
4535  }
4536  i++;
4537  }
4538 
4539  outputs = ConfGetNode("libhtp.server-config");
4540  if (outputs == NULL) {
4541  goto end;
4542  }
4543 
4544  node = TAILQ_FIRST(&outputs->head);
4545  node = TAILQ_NEXT(node, next);
4546  if (node == NULL) {
4547  goto end;
4548  }
4549  if (strcmp(node->name, "1") != 0) {
4550  goto end;
4551  }
4552  node = TAILQ_FIRST(&node->head);
4553  if (node == NULL) {
4554  goto end;
4555  }
4556  if (strcmp(node->name, "iis7") != 0) {
4557  goto end;
4558  }
4559 
4560  node2 = ConfNodeLookupChild(node, "personality");
4561  if (node2 == NULL) {
4562  goto end;
4563  }
4564  if (strcmp(node2->val, "IIS_7_0") != 0) {
4565  goto end;
4566  }
4567 
4568  node = ConfNodeLookupChild(node, "address");
4569  if (node == NULL) {
4570  goto end;
4571  }
4572 
4573  i = 0;
4574  TAILQ_FOREACH(n, &node->head, next) {
4575  if (n == NULL) {
4576  goto end;
4577  }
4578 
4579  switch(i) {
4580  case 0:
4581  if (strcmp(n->name, "0") != 0) {
4582  goto end;
4583  }
4584  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4585  goto end;
4586  }
4587  break;
4588  case 1:
4589  if (strcmp(n->name, "1") != 0) {
4590  goto end;
4591  }
4592  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4593  goto end;
4594  }
4595  break;
4596  default:
4597  goto end;
4598  }
4599  i++;
4600  }
4601 
4602  ret = 1;
4603 
4604 end:
4605  ConfDeInit();
4607 
4608  return ret;
4609 }
4610 
4611 /** \test Test config builds radix correctly */
4612 static int HTPParserConfigTest02(void)
4613 {
4614  int ret = 0;
4615  char input[] = "\
4616 %YAML 1.1\n\
4617 ---\n\
4618 libhtp:\n\
4619 \n\
4620  default-config:\n\
4621  personality: IDS\n\
4622 \n\
4623  server-config:\n\
4624 \n\
4625  - apache-tomcat:\n\
4626  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4627  personality: Tomcat_6_0\n\
4628 \n\
4629  - iis7:\n\
4630  address: \n\
4631  - 192.168.0.0/24\n\
4632  - 192.168.10.0/24\n\
4633  personality: IIS_7_0\n\
4634 ";
4635 
4637  ConfInit();
4639 
4640  ConfYamlLoadString(input, strlen(input));
4641 
4642  HTPConfigure();
4643 
4644  if (cfglist.cfg == NULL) {
4645  printf("No default config created.\n");
4646  goto end;
4647  }
4648 
4649  if (cfgtree == NULL) {
4650  printf("No config tree created.\n");
4651  goto end;
4652  }
4653 
4654  htp_cfg_t *htp = cfglist.cfg;
4655  uint8_t buf[128];
4656  const char *addr;
4657  void *user_data = NULL;
4658 
4659  addr = "192.168.10.42";
4660  if (inet_pton(AF_INET, addr, buf) == 1) {
4661  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4662  if (user_data != NULL) {
4663  HTPCfgRec *htp_cfg_rec = user_data;
4664  htp = htp_cfg_rec->cfg;
4665  SCLogDebug("LIBHTP using config: %p", htp);
4666  }
4667  if (htp == NULL) {
4668  printf("Could not get config for: %s\n", addr);
4669  goto end;
4670  }
4671  }
4672  else {
4673  printf("Failed to parse address: %s\n", addr);
4674  goto end;
4675  }
4676 
4677  user_data = NULL;
4678  addr = "::1";
4679  if (inet_pton(AF_INET6, addr, buf) == 1) {
4680  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4681  if (user_data != NULL) {
4682  HTPCfgRec *htp_cfg_rec = user_data;
4683  htp = htp_cfg_rec->cfg;
4684  SCLogDebug("LIBHTP using config: %p", htp);
4685  }
4686  if (htp == NULL) {
4687  printf("Could not get config for: %s\n", addr);
4688  goto end;
4689  }
4690  }
4691  else {
4692  printf("Failed to parse address: %s\n", addr);
4693  goto end;
4694  }
4695 
4696  ret = 1;
4697 
4698 end:
4699  HTPFreeConfig();
4700  ConfDeInit();
4703 
4704  return ret;
4705 }
4706 
4707 /** \test Test traffic is handled by the correct htp config */
4708 static int HTPParserConfigTest03(void)
4709 {
4710  int result = 1;
4711  Flow *f = NULL;
4712  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4713  " Data is c0oL!";
4714  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4715  TcpSession ssn;
4717 
4718  HtpState *htp_state = NULL;
4719  int r = 0;
4720  char input[] = "\
4721 %YAML 1.1\n\
4722 ---\n\
4723 libhtp:\n\
4724 \n\
4725  default-config:\n\
4726  personality: IDS\n\
4727 \n\
4728  server-config:\n\
4729 \n\
4730  - apache-tomcat:\n\
4731  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4732  personality: Tomcat_6_0\n\
4733 \n\
4734  - iis7:\n\
4735  address: \n\
4736  - 192.168.0.0/24\n\
4737  - 192.168.10.0/24\n\
4738  personality: IIS_7_0\n\
4739 ";
4740 
4742  ConfInit();
4744 
4745  ConfYamlLoadString(input, strlen(input));
4746 
4747  HTPConfigure();
4748 
4749  const char *addr = "192.168.10.42";
4750 
4751  memset(&ssn, 0, sizeof(ssn));
4752 
4753  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4754  if (f == NULL)
4755  goto end;
4756  f->protoctx = &ssn;
4757  f->proto = IPPROTO_TCP;
4758  f->alproto = ALPROTO_HTTP1;
4759 
4760  htp_cfg_t *htp = cfglist.cfg;
4761 
4762  void *user_data = NULL;
4763  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4764  if (user_data != NULL) {
4765  HTPCfgRec *htp_cfg_rec = user_data;
4766  htp = htp_cfg_rec->cfg;
4767  SCLogDebug("LIBHTP using config: %p", htp);
4768  }
4769  if (htp == NULL) {
4770  printf("Could not get config for: %s\n", addr);
4771  goto end;
4772  }
4773 
4774  StreamTcpInitConfig(true);
4775 
4776  uint32_t u;
4777  for (u = 0; u < httplen1; u++) {
4778  uint8_t flags = 0;
4779 
4780  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4781  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4782  else flags = STREAM_TOSERVER;
4783 
4784  FLOWLOCK_WRLOCK(f);
4785  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4786  if (r != 0) {
4787  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4788  " 0: ", u, r);
4789  result = 0;
4790  FLOWLOCK_UNLOCK(f);
4791  goto end;
4792  }
4793  FLOWLOCK_UNLOCK(f);
4794  }
4795 
4796  htp_state = f->alstate;
4797  if (htp_state == NULL) {
4798  printf("no http state: ");
4799  result = 0;
4800  goto end;
4801  }
4802 
4803  if (HTPStateGetTxCnt(htp_state) != 2) {
4804  printf("HTPStateGetTxCnt(htp_state) failure\n");
4805  goto end;
4806  }
4807 
4808  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4809  if (tx == NULL)
4810  goto end;
4811  if (tx->cfg != htp) {
4812  printf("wrong HTP config (%p instead of %p - default=%p): ",
4813  tx->cfg, htp, cfglist.cfg);
4814  goto end;
4815  }
4816  tx = HTPStateGetTx(htp_state, 1);
4817  if (tx == NULL)
4818  goto end;
4819  if (tx->cfg != htp) {
4820  printf("wrong HTP config (%p instead of %p - default=%p): ",
4821  tx->cfg, htp, cfglist.cfg);
4822  goto end;
4823  }
4824 
4825 end:
4826  if (alp_tctx != NULL)
4828  HTPFreeConfig();
4829  ConfDeInit();
4832 
4833  StreamTcpFreeConfig(true);
4834  UTHFreeFlow(f);
4835  return result;
4836 }
4837 
4838 /* disabled when we upgraded to libhtp 0.5.x */
4839 #if 0
4840 static int HTPParserConfigTest04(void)
4841 {
4842  int result = 0;
4843 
4844  char input[] = "\
4845 %YAML 1.1\n\
4846 ---\n\
4847 libhtp:\n\
4848 \n\
4849  default-config:\n\
4850  personality: IDS\n\
4851  path-control-char-handling: status_400\n\
4852  path-convert-utf8: yes\n\
4853  path-invalid-encoding-handling: remove_percent\n\
4854 \n\
4855  server-config:\n\
4856 \n\
4857  - apache-tomcat:\n\
4858  personality: Tomcat_6_0\n\
4859  path-invalid-utf8-handling: none\n\
4860  path-nul-encoded-handling: status_404\n\
4861  path-nul-raw-handling: status_400\n\
4862 \n\
4863  - iis7:\n\
4864  personality: IIS_7_0\n\
4865  path-replacement-char: o\n\
4866  path-unicode-mapping: status_400\n\
4867 ";
4868 
4870  ConfInit();
4872 
4873  ConfYamlLoadString(input, strlen(input));
4874 
4875  HTPConfigure();
4876 
4877  HTPCfgRec *cfg_rec = &cfglist;
4878  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4879  cfg_rec->cfg->path_convert_utf8 != 1 ||
4880  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4881  printf("failed 1\n");
4882  goto end;
4883  }
4884 
4885  cfg_rec = cfg_rec->next;
4886  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4887  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4888  printf("failed 2\n");
4889  goto end;
4890  }
4891 
4892  cfg_rec = cfg_rec->next;
4893  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4894  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4895  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4896  printf("failed 3\n");
4897  goto end;
4898  }
4899 
4900  result = 1;
4901 
4902 end:
4903  HTPFreeConfig();
4904  ConfDeInit();
4907 
4908  return result;
4909 }
4910 #endif
4911 
4912 /** \test Test %2f decoding in profile Apache_2_2
4913  *
4914  * %2f in path is left untouched
4915  * %2f in query string is normalized to %2F
4916  * %252f in query string is decoded/normalized to %2F
4917  */
4918 static int HTPParserDecodingTest01(void)
4919 {
4920  uint8_t httpbuf1[] =
4921  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4922  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4923  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4924  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4925  TcpSession ssn;
4928 
4929  char input[] = "\
4930 %YAML 1.1\n\
4931 ---\n\
4932 libhtp:\n\
4933 \n\
4934  default-config:\n\
4935  personality: Apache_2\n\
4936 ";
4937 
4939  ConfInit();
4941  ConfYamlLoadString(input, strlen(input));
4942  HTPConfigure();
4943  const char *addr = "4.3.2.1";
4944  memset(&ssn, 0, sizeof(ssn));
4945 
4946  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4947  FAIL_IF_NULL(f);
4948  f->protoctx = &ssn;
4949  f->proto = IPPROTO_TCP;
4950  f->alproto = ALPROTO_HTTP1;
4951 
4952  StreamTcpInitConfig(true);
4953 
4954  for (uint32_t u = 0; u < httplen1; u++) {
4955  uint8_t flags = 0;
4956  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4957  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4958  else flags = STREAM_TOSERVER;
4959 
4960  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4961  FAIL_IF(r != 0);
4962  }
4963 
4964  HtpState *htp_state = f->alstate;
4965  FAIL_IF_NULL(htp_state);
4966 
4967  uint8_t ref1[] = "/abc%2fdef";
4968  size_t reflen = sizeof(ref1) - 1;
4969 
4970  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4971  FAIL_IF_NULL(tx);
4972 
4973  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4974  FAIL_IF_NULL(tx_ud);
4976  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4977  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4978  bstr_len(tx_ud->request_uri_normalized)) != 0);
4979 
4980  uint8_t ref2[] = "/abc/def?ghi/jkl";
4981  reflen = sizeof(ref2) - 1;
4982 
4983  tx = HTPStateGetTx(htp_state, 1);
4984  FAIL_IF_NULL(tx);
4985  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4986  FAIL_IF_NULL(tx_ud);
4988  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4989 
4990  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4991  bstr_len(tx_ud->request_uri_normalized)) != 0);
4992 
4993  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4994  reflen = sizeof(ref3) - 1;
4995  tx = HTPStateGetTx(htp_state, 2);
4996  FAIL_IF_NULL(tx);
4997  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4998  FAIL_IF_NULL(tx_ud);
5000  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5001 
5002  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5003  bstr_len(tx_ud->request_uri_normalized)) != 0);
5004 
5006  HTPFreeConfig();
5007  ConfDeInit();
5010 
5011  StreamTcpFreeConfig(true);
5012  UTHFreeFlow(f);
5013  PASS;
5014 }
5015 
5016 static int HTPParserDecodingTest01a(void)
5017 {
5018  uint8_t httpbuf1[] = "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5019  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5020  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5021  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5022  TcpSession ssn;
5025 
5026  char input[] = "\
5027 %YAML 1.1\n\
5028 ---\n\
5029 libhtp:\n\
5030 \n\
5031  default-config:\n\
5032  personality: Apache_2\n\
5033 ";
5034 
5036  ConfInit();
5038  ConfYamlLoadString(input, strlen(input));
5039  HTPConfigure();
5040  const char *addr = "4.3.2.1";
5041  memset(&ssn, 0, sizeof(ssn));
5042 
5043  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5044  FAIL_IF_NULL(f);
5045  f->protoctx = &ssn;
5046  f->proto = IPPROTO_TCP;
5047  f->alproto = ALPROTO_HTTP1;
5048 
5049  StreamTcpInitConfig(true);
5050 
5051  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
5052  (STREAM_TOSERVER | STREAM_START | STREAM_EOF), httpbuf1, httplen1);
5053  FAIL_IF(r != 0);
5054 
5055  HtpState *htp_state = f->alstate;
5056  FAIL_IF_NULL(htp_state);
5057 
5058  uint8_t ref1[] = "/abc%2fdef";
5059  size_t reflen = sizeof(ref1) - 1;
5060 
5061  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5062  FAIL_IF_NULL(tx);
5063 
5064  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5065  FAIL_IF_NULL(tx_ud);
5067  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5068  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5069  bstr_len(tx_ud->request_uri_normalized)) != 0);
5070 
5071  uint8_t ref2[] = "/abc/def?ghi/jkl";
5072  reflen = sizeof(ref2) - 1;
5073 
5074  tx = HTPStateGetTx(htp_state, 1);
5075  FAIL_IF_NULL(tx);
5076  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5077  FAIL_IF_NULL(tx_ud);
5079  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5080 
5081  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5082  bstr_len(tx_ud->request_uri_normalized)) != 0);
5083 
5084  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5085  reflen = sizeof(ref3) - 1;
5086  tx = HTPStateGetTx(htp_state, 2);
5087  FAIL_IF_NULL(tx);
5088  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5089  FAIL_IF_NULL(tx_ud);
5091  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5092 
5093  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5094  bstr_len(tx_ud->request_uri_normalized)) != 0);
5095 
5097  HTPFreeConfig();
5098  ConfDeInit();
5101 
5102  StreamTcpFreeConfig(true);
5103  UTHFreeFlow(f);
5104  PASS;
5105 }
5106 
5107 /** \test Test %2f decoding in profile IDS
5108  *
5109  * %2f in path decoded to /
5110  * %2f in query string is decoded to /
5111  * %252f in query string is decoded to %2F
5112  */
5113 static int HTPParserDecodingTest02(void)
5114 {
5115  int result = 0;
5116  Flow *f = NULL;
5117  uint8_t httpbuf1[] =
5118  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5119  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5120  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5121  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5122  TcpSession ssn;
5124 
5125  HtpState *htp_state = NULL;
5126  int r = 0;
5127  char input[] = "\
5128 %YAML 1.1\n\
5129 ---\n\
5130 libhtp:\n\
5131 \n\
5132  default-config:\n\
5133  personality: IDS\n\
5134  double-decode-path: no\n\
5135  double-decode-query: no\n\
5136 ";
5137 
5139  ConfInit();
5141  ConfYamlLoadString(input, strlen(input));
5142  HTPConfigure();
5143  const char *addr = "4.3.2.1";
5144  memset(&ssn, 0, sizeof(ssn));
5145 
5146  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5147  if (f == NULL)
5148  goto end;
5149  f->protoctx = &ssn;
5150  f->proto = IPPROTO_TCP;
5151  f->alproto = ALPROTO_HTTP1;
5152 
5153  StreamTcpInitConfig(true);
5154 
5155  uint32_t u;
5156  for (u = 0; u < httplen1; u++) {
5157  uint8_t flags = 0;
5158 
5159  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5160  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5161  else flags = STREAM_TOSERVER;
5162 
5163  FLOWLOCK_WRLOCK(f);
5164  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5165  if (r != 0) {
5166  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5167  " 0: ", u, r);
5168  FLOWLOCK_UNLOCK(f);
5169  goto end;
5170  }
5171  FLOWLOCK_UNLOCK(f);
5172  }
5173 
5174  htp_state = f->alstate;
5175  if (htp_state == NULL) {
5176  printf("no http state: ");
5177  goto end;
5178  }
5179 
5180  uint8_t ref1[] = "/abc/def";
5181  size_t reflen = sizeof(ref1) - 1;
5182 
5183  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5184  if (tx == NULL)
5185  goto end;
5186  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5187  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5188  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5189  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5190  (uintmax_t)reflen,
5191  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5192  goto end;
5193  }
5194 
5195  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5196  bstr_len(tx_ud->request_uri_normalized)) != 0)
5197  {
5198  printf("normalized uri \"");
5199  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5200  printf("\" != \"");
5201  PrintRawUriFp(stdout, ref1, reflen);
5202  printf("\": ");
5203  goto end;
5204  }
5205  }
5206 
5207  uint8_t ref2[] = "/abc/def?ghi/jkl";
5208  reflen = sizeof(ref2) - 1;
5209 
5210  tx = HTPStateGetTx(htp_state, 1);
5211  if (tx == NULL)
5212  goto end;
5213  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5214  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5215  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5216  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5217  (uintmax_t)reflen,
5218  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5219  goto end;
5220  }
5221 
5222  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5223  bstr_len(tx_ud->request_uri_normalized)) != 0)
5224  {
5225  printf("normalized uri \"");
5226  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5227  printf("\" != \"");
5228  PrintRawUriFp(stdout, ref2, reflen);
5229  printf("\": ");
5230  goto end;
5231  }
5232  }
5233 
5234  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5235  reflen = sizeof(ref3) - 1;
5236  tx = HTPStateGetTx(htp_state, 2);
5237  if (tx == NULL)
5238  goto end;
5239  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5240  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5241  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5242  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5243  (uintmax_t)reflen,
5244  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5245  goto end;
5246  }
5247 
5248  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5249  bstr_len(tx_ud->request_uri_normalized)) != 0)
5250  {
5251  printf("normalized uri \"");
5252  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5253  printf("\" != \"");
5254  PrintRawUriFp(stdout, ref3, reflen);
5255  printf("\": ");
5256  goto end;
5257  }
5258  }
5259 
5260  result = 1;
5261 
5262 end:
5263  if (alp_tctx != NULL)
5265  HTPFreeConfig();
5266  ConfDeInit();
5269 
5270  StreamTcpFreeConfig(true);
5271  UTHFreeFlow(f);
5272  return result;
5273 }
5274 
5275 /** \test Test %2f decoding in profile IDS with double-decode-* options
5276  *
5277  * %252f in path decoded to /
5278  * %252f in query string is decoded to /
5279  */
5280 static int HTPParserDecodingTest03(void)
5281 {
5282  int result = 0;
5283  Flow *f = NULL;
5284  uint8_t httpbuf1[] =
5285  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5286  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5287  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5288  TcpSession ssn;
5290 
5291  HtpState *htp_state = NULL;
5292  int r = 0;
5293  char input[] = "\
5294 %YAML 1.1\n\
5295 ---\n\
5296 libhtp:\n\
5297 \n\
5298  default-config:\n\
5299  personality: IDS\n\
5300  double-decode-path: yes\n\
5301  double-decode-query: yes\n\
5302 ";
5303 
5305  ConfInit();
5307  ConfYamlLoadString(input, strlen(input));
5308  HTPConfigure();
5309  const char *addr = "4.3.2.1";
5310  memset(&ssn, 0, sizeof(ssn));
5311 
5312  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5313  if (f == NULL)
5314  goto end;
5315  f->protoctx = &ssn;
5316  f->proto = IPPROTO_TCP;
5317  f->alproto = ALPROTO_HTTP1;
5318 
5319  StreamTcpInitConfig(true);
5320 
5321  uint32_t u;
5322  for (u = 0; u < httplen1; u++) {
5323  uint8_t flags = 0;
5324 
5325  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5326  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5327  else flags = STREAM_TOSERVER;
5328 
5329  FLOWLOCK_WRLOCK(f);
5330  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5331  if (r != 0) {
5332  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5333  " 0: ", u, r);
5334  FLOWLOCK_UNLOCK(f);
5335  goto end;
5336  }
5337  FLOWLOCK_UNLOCK(f);
5338  }
5339 
5340  htp_state = f->alstate;
5341  if (htp_state == NULL) {
5342  printf("no http state: ");
5343  goto end;
5344  }
5345 
5346  uint8_t ref1[] = "/abc/def";
5347  size_t reflen = sizeof(ref1) - 1;
5348 
5349  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5350  if (tx == NULL)
5351  goto end;
5352  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5353  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5354  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5355  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5356  (uintmax_t)reflen,
5357  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5358  goto end;
5359  }
5360 
5361  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5362  bstr_len(tx_ud->request_uri_normalized)) != 0)
5363  {
5364  printf("normalized uri \"");
5365  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5366  printf("\" != \"");
5367  PrintRawUriFp(stdout, ref1, reflen);
5368  printf("\": ");
5369  goto end;
5370  }
5371  }
5372 
5373  uint8_t ref2[] = "/abc/def?ghi/jkl";
5374  reflen = sizeof(ref2) - 1;
5375 
5376  tx = HTPStateGetTx(htp_state, 1);
5377  if (tx == NULL)
5378  goto end;
5379  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5380  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5381  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5382  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5383  (uintmax_t)reflen,
5384  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5385  goto end;
5386  }
5387 
5388  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5389  bstr_len(tx_ud->request_uri_normalized)) != 0)
5390  {
5391  printf("normalized uri \"");
5392  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5393  printf("\" != \"");
5394  PrintRawUriFp(stdout, ref2, reflen);
5395  printf("\": ");
5396  goto end;
5397  }
5398  }
5399 
5400  result = 1;
5401 
5402 end:
5403  if (alp_tctx != NULL)
5405  HTPFreeConfig();
5406  ConfDeInit();
5409 
5410  StreamTcpFreeConfig(true);
5411  UTHFreeFlow(f);
5412  return result;
5413 }
5414 
5415 /** \test Test http:// in query profile IDS
5416  */
5417 static int HTPParserDecodingTest04(void)
5418 {
5419  int result = 0;
5420  Flow *f = NULL;
5421  uint8_t httpbuf1[] =
5422  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5423  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5424  TcpSession ssn;
5426 
5427  HtpState *htp_state = NULL;
5428  int r = 0;
5429  char input[] = "\
5430 %YAML 1.1\n\
5431 ---\n\
5432 libhtp:\n\
5433 \n\
5434  default-config:\n\
5435  personality: IDS\n\
5436  double-decode-path: yes\n\
5437  double-decode-query: yes\n\
5438 ";
5439 
5441  ConfInit();
5443  ConfYamlLoadString(input, strlen(input));
5444  HTPConfigure();
5445  const char *addr = "4.3.2.1";
5446  memset(&ssn, 0, sizeof(ssn));
5447 
5448  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5449  if (f == NULL)
5450  goto end;
5451  f->protoctx = &ssn;
5452  f->proto = IPPROTO_TCP;
5453  f->alproto = ALPROTO_HTTP1;
5454 
5455  StreamTcpInitConfig(true);
5456 
5457  uint32_t u;
5458  for (u = 0; u < httplen1; u++) {
5459  uint8_t flags = 0;
5460 
5461  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5462  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5463  else flags = STREAM_TOSERVER;
5464 
5465  FLOWLOCK_WRLOCK(f);
5466  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5467  if (r != 0) {
5468  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5469  " 0: ", u, r);
5470  FLOWLOCK_UNLOCK(f);
5471  goto end;
5472  }
5473  FLOWLOCK_UNLOCK(f);
5474  }
5475 
5476  htp_state = f->alstate;
5477  if (htp_state == NULL) {
5478  printf("no http state: ");
5479  goto end;
5480  }
5481 
5482  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5483  size_t reflen = sizeof(ref1) - 1;
5484 
5485  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5486  if (tx == NULL)
5487  goto end;
5488  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5489  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5490  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5491  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5492  (uintmax_t)reflen,
5493  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5494  goto end;
5495  }
5496 
5497  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5498  bstr_len(tx_ud->request_uri_normalized)) != 0)
5499  {
5500  printf("normalized uri \"");
5501  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5502  printf("\" != \"");
5503  PrintRawUriFp(stdout, ref1, reflen);
5504  printf("\": ");
5505  goto end;
5506  }
5507  }
5508 
5509  result = 1;
5510 
5511 end:
5512  if (alp_tctx != NULL)
5514  HTPFreeConfig();
5515  ConfDeInit();
5518 
5519  StreamTcpFreeConfig(true);
5520  UTHFreeFlow(f);
5521  return result;
5522 }
5523 
5524 /** \test Test \ char in query profile IDS. Bug 739
5525  */
5526 static int HTPParserDecodingTest05(void)
5527 {
5528  int result = 0;
5529  Flow *f = NULL;
5530  uint8_t httpbuf1[] =
5531  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5532  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5533  TcpSession ssn;
5535 
5536  HtpState *htp_state = NULL;
5537  int r = 0;
5538  char input[] = "\
5539 %YAML 1.1\n\
5540 ---\n\
5541 libhtp:\n\
5542 \n\
5543  default-config:\n\
5544  personality: IDS\n\
5545  double-decode-path: yes\n\
5546  double-decode-query: yes\n\
5547 ";
5548 
5550  ConfInit();
5552  ConfYamlLoadString(input, strlen(input));
5553  HTPConfigure();
5554  const char *addr = "4.3.2.1";
5555  memset(&ssn, 0, sizeof(ssn));
5556 
5557  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5558  if (f == NULL)
5559  goto end;
5560  f->protoctx = &ssn;
5561  f->proto = IPPROTO_TCP;
5562  f->alproto = ALPROTO_HTTP1;
5563 
5564  StreamTcpInitConfig(true);
5565 
5566  uint32_t u;
5567  for (u = 0; u < httplen1; u++) {
5568  uint8_t flags = 0;
5569 
5570  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5571  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5572  else flags = STREAM_TOSERVER;
5573 
5574  FLOWLOCK_WRLOCK(f);
5575  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5576  if (r != 0) {
5577  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5578  " 0: ", u, r);
5579  FLOWLOCK_UNLOCK(f);
5580  goto end;
5581  }
5582  FLOWLOCK_UNLOCK(f);
5583  }
5584 
5585  htp_state = f->alstate;
5586  if (htp_state == NULL) {
5587  printf("no http state: ");
5588  goto end;
5589  }
5590 
5591  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5592  size_t reflen = sizeof(ref1) - 1;
5593 
5594  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5595  if (tx == NULL)
5596  goto end;
5597  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5598  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5599  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5600  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5601  (uintmax_t)reflen,
5602  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5603  goto end;
5604  }
5605 
5606  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5607  bstr_len(tx_ud->request_uri_normalized)) != 0)
5608  {
5609  printf("normalized uri \"");
5610  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5611  printf("\" != \"");
5612  PrintRawUriFp(stdout, ref1, reflen);
5613  printf("\": ");
5614  goto end;
5615  }
5616  }
5617 
5618  result = 1;
5619 
5620 end:
5621  if (alp_tctx != NULL)
5623  HTPFreeConfig();
5624  ConfDeInit();
5627 
5628  StreamTcpFreeConfig(true);
5629  UTHFreeFlow(f);
5630  return result;
5631 }
5632 
5633 /** \test Test + char in query. Bug 1035
5634  */
5635 static int HTPParserDecodingTest06(void)
5636 {
5637  int result = 0;
5638  Flow *f = NULL;
5639  uint8_t httpbuf1[] =
5640  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5641  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5642  TcpSession ssn;
5644 
5645  HtpState *htp_state = NULL;
5646  int r = 0;
5647  char input[] = "\
5648 %YAML 1.1\n\
5649 ---\n\
5650 libhtp:\n\
5651 \n\
5652  default-config:\n\
5653  personality: IDS\n\
5654  double-decode-path: yes\n\
5655  double-decode-query: yes\n\
5656 ";
5657 
5659  ConfInit();
5661  ConfYamlLoadString(input, strlen(input));
5662  HTPConfigure();
5663  const char *addr = "4.3.2.1";
5664  memset(&ssn, 0, sizeof(ssn));
5665 
5666  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5667  if (f == NULL)
5668  goto end;
5669  f->protoctx = &ssn;
5670  f->proto = IPPROTO_TCP;
5671  f->alproto = ALPROTO_HTTP1;
5672 
5673  StreamTcpInitConfig(true);
5674 
5675  uint32_t u;
5676  for (u = 0; u < httplen1; u++) {
5677  uint8_t flags = 0;
5678 
5679  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5680  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5681  else flags = STREAM_TOSERVER;
5682 
5683  FLOWLOCK_WRLOCK(f);
5684  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5685  if (r != 0) {
5686  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5687  " 0: ", u, r);
5688  FLOWLOCK_UNLOCK(f);
5689  goto end;
5690  }
5691  FLOWLOCK_UNLOCK(f);
5692  }
5693 
5694  htp_state = f->alstate;
5695  if (htp_state == NULL) {
5696  printf("no http state: ");
5697  goto end;
5698  }
5699 
5700  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5701  size_t reflen = sizeof(ref1) - 1;
5702 
5703  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5704  if (tx == NULL)
5705  goto end;
5706  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5707  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5708  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5709  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5710  (uintmax_t)reflen,
5711  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5712  goto end;
5713  }
5714 
5715  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5716  bstr_len(tx_ud->request_uri_normalized)) != 0)
5717  {
5718  printf("normalized uri \"");
5719  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5720  printf("\" != \"");
5721  PrintRawUriFp(stdout, ref1, reflen);
5722  printf("\": ");
5723  goto end;
5724  }
5725  }
5726 
5727  result = 1;
5728 
5729 end:
5730  if (alp_tctx != NULL)
5732  HTPFreeConfig();
5733  ConfDeInit();
5736 
5737  StreamTcpFreeConfig(true);
5738  UTHFreeFlow(f);
5739  return result;
5740 }
5741 
5742 /** \test Test + char in query. Bug 1035
5743  */
5744 static int HTPParserDecodingTest07(void)
5745 {
5746  int result = 0;
5747  Flow *f = NULL;
5748  uint8_t httpbuf1[] =
5749  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5750  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5751  TcpSession ssn;
5753 
5754  HtpState *htp_state = NULL;
5755  int r = 0;
5756  char input[] = "\
5757 %YAML 1.1\n\
5758 ---\n\
5759 libhtp:\n\
5760 \n\
5761  default-config:\n\
5762  personality: IDS\n\
5763  double-decode-path: yes\n\
5764  double-decode-query: yes\n\
5765  query-plusspace-decode: yes\n\
5766 ";
5767 
5769  ConfInit();
5771  ConfYamlLoadString(input, strlen(input));
5772  HTPConfigure();
5773  const char *addr = "4.3.2.1";
5774  memset(&ssn, 0, sizeof(ssn));
5775 
5776  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5777  if (f == NULL)
5778  goto end;
5779  f->protoctx = &ssn;
5780  f->proto = IPPROTO_TCP;
5781  f->alproto = ALPROTO_HTTP1;
5782 
5783  StreamTcpInitConfig(true);
5784 
5785  uint32_t u;
5786  for (u = 0; u < httplen1; u++) {
5787  uint8_t flags = 0;
5788 
5789  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5790  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5791  else flags = STREAM_TOSERVER;
5792 
5793  FLOWLOCK_WRLOCK(f);
5794  r =