suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 #include "util-byte.h"
49 
50 #include "stream-tcp-private.h"
51 #include "stream-tcp-reassemble.h"
52 #include "stream-tcp.h"
53 #include "stream.h"
54 
55 #include "app-layer-protos.h"
56 #include "app-layer-parser.h"
57 
58 #include "app-layer.h"
59 #include "app-layer-htp.h"
60 #include "app-layer-htp-body.h"
61 #include "app-layer-htp-file.h"
62 #include "app-layer-htp-libhtp.h"
63 #include "app-layer-htp-xff.h"
64 
65 #include "util-spm.h"
66 #include "util-debug.h"
67 #include "util-time.h"
68 #include "util-misc.h"
69 
70 #include "util-unittest.h"
71 #include "util-unittest-helper.h"
72 #include "flow-util.h"
73 
74 #include "detect-engine.h"
75 #include "detect-engine-state.h"
76 #include "detect-parse.h"
77 
78 #include "decode-events.h"
79 
80 #include "util-memcmp.h"
81 #include "util-random.h"
82 #include "util-validate.h"
83 
84 //#define PRINT
85 
86 /** Fast lookup tree (radix) for the various HTP configurations */
87 static SCRadixTree *cfgtree;
88 /** List of HTP configurations. */
89 static HTPCfgRec cfglist;
90 
91 /** Limit to the number of libhtp messages that can be handled */
92 #define HTP_MAX_MESSAGES 512
93 
94 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
95 
96 #ifdef DEBUG
97 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
98 static uint64_t htp_state_memuse = 0;
99 static uint64_t htp_state_memcnt = 0;
100 #endif
101 
103  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
104  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
105  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
106  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
107  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
108  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
109  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
111  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
113  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
115  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
117  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
119  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
121  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
122  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
124  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
125  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
126  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
127  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
128  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
129  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
130  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
131  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
132  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
133  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
134  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
135  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
136  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
138  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
139  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
140  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
141  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
142  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
143  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
144  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
145  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
146  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
147  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
148  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
149  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
150  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
152  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
153  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
154  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
155  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
156 
157  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
158  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
159 
160  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
161 
162  /* suricata warnings/errors */
163  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
164  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
165  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
166 
167  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
168 
169  { NULL, -1 },
170 };
171 
172 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
173 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
174 static uint64_t HTPStateGetTxCnt(void *alstate);
175 #ifdef UNITTESTS
176 static void HTPParserRegisterTests(void);
177 #endif
178 
179 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
180 {
181  uint64_t id = HTPStateGetTxCnt(s);
182  BUG_ON(id == 0);
183  return id - 1;
184 }
185 
186 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
187 {
188  return s->transaction_cnt;
189 }
190 
191 #ifdef DEBUG
192 /**
193  * \internal
194  *
195  * \brief Lookup the HTP personality string from the numeric personality.
196  *
197  * \todo This needs to be a libhtp function.
198  */
199 static const char *HTPLookupPersonalityString(int p)
200 {
201 #define CASE_HTP_PERSONALITY_STRING(p) \
202  case HTP_SERVER_ ## p: return #p
203 
204  switch (p) {
205  CASE_HTP_PERSONALITY_STRING(MINIMAL);
206  CASE_HTP_PERSONALITY_STRING(GENERIC);
207  CASE_HTP_PERSONALITY_STRING(IDS);
208  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
209  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
210  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
211  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
212  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
213  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
214  CASE_HTP_PERSONALITY_STRING(APACHE_2);
215  }
216 
217  return NULL;
218 }
219 #endif /* DEBUG */
220 
221 /**
222  * \internal
223  *
224  * \brief Lookup the numeric HTP personality from a string.
225  *
226  * \todo This needs to be a libhtp function.
227  */
228 static int HTPLookupPersonality(const char *str)
229 {
230 #define IF_HTP_PERSONALITY_NUM(p) \
231  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
232 
233  IF_HTP_PERSONALITY_NUM(MINIMAL);
234  IF_HTP_PERSONALITY_NUM(GENERIC);
236  IF_HTP_PERSONALITY_NUM(IIS_4_0);
237  IF_HTP_PERSONALITY_NUM(IIS_5_0);
238  IF_HTP_PERSONALITY_NUM(IIS_5_1);
239  IF_HTP_PERSONALITY_NUM(IIS_6_0);
240  IF_HTP_PERSONALITY_NUM(IIS_7_0);
241  IF_HTP_PERSONALITY_NUM(IIS_7_5);
242  IF_HTP_PERSONALITY_NUM(APACHE_2);
243  if (strcasecmp("TOMCAT_6_0", str) == 0) {
244  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
245  "longer supported by libhtp.", str);
246  return -1;
247  } else if ((strcasecmp("APACHE", str) == 0) ||
248  (strcasecmp("APACHE_2_2", str) == 0))
249  {
250  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
251  "longer supported by libhtp, failing back to "
252  "Apache2 personality.", str);
253  return HTP_SERVER_APACHE_2;
254  }
255 
256  return -1;
257 }
258 
259 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
260  const uint8_t dir, const uint8_t e)
261 {
262  SCLogDebug("setting event %u", e);
263 
264  if (htud) {
265  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
266  s->events++;
267  return;
268  }
269 
270  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
271  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
272 
273  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
274  if (tx == NULL && tx_id > 0)
275  tx = HTPStateGetTx(s, tx_id - 1);
276  if (tx != NULL) {
277  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
278  if (htud != NULL) {
279  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
280  s->events++;
281  return;
282  }
283  }
284  SCLogDebug("couldn't set event %u", e);
285 }
286 
287 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
288  * connection parser to be used by the HTP library
289  */
290 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
291 {
292  SCEnter();
293 
294  HtpState *s = HTPMalloc(sizeof(HtpState));
295  if (unlikely(s == NULL)) {
296  SCReturnPtr(NULL, "void");
297  }
298 
299  memset(s, 0x00, sizeof(HtpState));
300 
301 #ifdef DEBUG
302  SCMutexLock(&htp_state_mem_lock);
303  htp_state_memcnt++;
304  htp_state_memuse += sizeof(HtpState);
305  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
306  SCMutexUnlock(&htp_state_mem_lock);
307 #endif
308 
309  SCReturnPtr((void *)s, "void");
310 }
311 
312 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
313 {
314  if (likely(htud)) {
315  HtpBodyFree(&htud->request_body);
316  HtpBodyFree(&htud->response_body);
317  bstr_free(htud->request_uri_normalized);
318  if (htud->request_headers_raw)
320  if (htud->response_headers_raw)
323  if (htud->boundary)
324  HTPFree(htud->boundary, htud->boundary_len);
325  if (htud->tx_data.de_state != NULL) {
326  DetectEngineStateFree(htud->tx_data.de_state);
327  }
328  HTPFree(htud, sizeof(HtpTxUserData));
329  }
330 }
331 
332 /** \brief Function to frees the HTTP state memory and also frees the HTTP
333  * connection parser memory which was used by the HTP library
334  */
335 void HTPStateFree(void *state)
336 {
337  SCEnter();
338 
339  HtpState *s = (HtpState *)state;
340  if (s == NULL) {
341  SCReturn;
342  }
343 
344  /* free the connection parser memory used by HTP library */
345  if (s->connp != NULL) {
346  SCLogDebug("freeing HTP state");
347 
348  uint64_t tx_id;
349  uint64_t total_txs = HTPStateGetTxCnt(state);
350  /* free the list of body chunks */
351  if (s->conn != NULL) {
352  for (tx_id = 0; tx_id < total_txs; tx_id++) {
353  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
354  if (tx != NULL) {
355  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
356  HtpTxUserDataFree(s, htud);
357  htp_tx_set_user_data(tx, NULL);
358  }
359  }
360  }
361  htp_connp_destroy_all(s->connp);
362  }
363 
364  if (s->file_range) {
365  HTPFileCloseHandleRange(s->files_tc, 0, s->file_range, NULL, 0);
367  }
368 
371  HTPFree(s, sizeof(HtpState));
372 
373 #ifdef DEBUG
374  SCMutexLock(&htp_state_mem_lock);
375  htp_state_memcnt--;
376  htp_state_memuse -= sizeof(HtpState);
377  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
378  SCMutexUnlock(&htp_state_mem_lock);
379 #endif
380 
381  SCReturn;
382 }
383 
384 /**
385  * \brief HTP transaction cleanup callback
386  *
387  * \warning We cannot actually free the transactions here. It seems that
388  * HTP only accepts freeing of transactions in the response callback.
389  */
390 static void HTPStateTransactionFree(void *state, uint64_t id)
391 {
392  SCEnter();
393 
394  HtpState *s = (HtpState *)state;
395 
396  SCLogDebug("state %p, id %"PRIu64, s, id);
397 
398  htp_tx_t *tx = HTPStateGetTx(s, id);
399  if (tx != NULL) {
400  /* This will remove obsolete body chunks */
401  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
402  HtpTxUserDataFree(s, htud);
403  htp_tx_set_user_data(tx, NULL);
404 
405  /* hack: even if libhtp considers the tx incomplete, we want to
406  * free it here. htp_tx_destroy however, will refuse to do this.
407  * As htp_tx_destroy_incomplete isn't available in the public API,
408  * we hack around it here. */
409  if (unlikely(!(
410  tx->request_progress == HTP_REQUEST_COMPLETE &&
411  tx->response_progress == HTP_RESPONSE_COMPLETE)))
412  {
413  tx->request_progress = HTP_REQUEST_COMPLETE;
414  tx->response_progress = HTP_RESPONSE_COMPLETE;
415  }
416  htp_tx_destroy(tx);
417  }
418 }
419 
420 /**
421  * \brief Sets a flag that informs the HTP app layer that some module in the
422  * engine needs the http request body data.
423  * \initonly
424  */
426 {
427  SCEnter();
428 
429  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
430  SCReturn;
431 }
432 
433 /**
434  * \brief Sets a flag that informs the HTP app layer that some module in the
435  * engine needs the http request body data.
436  * \initonly
437  */
439 {
440  SCEnter();
441 
442  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
443  SCReturn;
444 }
445 
446 /**
447  * \brief Sets a flag that informs the HTP app layer that some module in the
448  * engine needs the http request multi part header.
449  *
450  * \initonly
451  */
452 static void AppLayerHtpNeedMultipartHeader(void)
453 {
454  SCEnter();
456 
457  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
458  SCReturn;
459 }
460 
461 /**
462  * \brief Sets a flag that informs the HTP app layer that some module in the
463  * engine needs the http request file.
464  *
465  * \initonly
466  */
468 {
469  SCEnter();
470  AppLayerHtpNeedMultipartHeader();
473 
474  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
475  SCReturn;
476 }
477 
478 static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
479 {
480  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
481  if (tx_ud) {
482  if (flags & STREAM_TOCLIENT) {
483  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
484  } else {
485  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
486  }
487  }
488 }
489 
490 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
491 {
492  if (flags & HTP_STREAM_DEPTH_SET) {
493  uint32_t stream_depth = FileReassemblyDepth();
494  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
495  return true;
496  }
497  } else {
498  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
499  return true;
500  }
501  }
502  return false;
503 }
504 
505 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
506  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
507 {
508  uint32_t chunk_len = 0;
509  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
510  (content_len_so_far < (uint64_t)body_limit) &&
511  (content_len_so_far + (uint64_t)data_len) > body_limit)
512  {
513  chunk_len = body_limit - content_len_so_far;
514  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
515  (content_len_so_far < (uint64_t)stream_depth) &&
516  (content_len_so_far + (uint64_t)data_len) > stream_depth)
517  {
518  chunk_len = stream_depth - content_len_so_far;
519  }
520  SCLogDebug("len %u", chunk_len);
521  return (chunk_len == 0 ? data_len : chunk_len);
522 }
523 
524 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
525 struct {
526  const char *msg;
527  int de;
528 } htp_errors[] = {
529  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
530  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
531  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
532  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
533  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
534 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
535  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
536 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
537  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
538  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
539  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
540  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
541 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
542  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
543  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
544  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
545  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
546 };
547 
548 struct {
549  const char *msg;
550  int de;
551 } htp_warnings[] = {
552  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
553  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
554  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
555  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
556  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
557 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
558  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
559  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
560  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
561  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
562  * luckily, "Request server port=" is unique */
563 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
565  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
566  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
567  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
568  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
569  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
570  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
571  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
572  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
573  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
574  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
575  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
576  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
577  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
578  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
579  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
580  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
581  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
582  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
583 };
584 
585 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
586 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
587 
588 /**
589  * \internal
590  *
591  * \brief Get the warning id for the warning msg.
592  *
593  * \param msg warning message
594  *
595  * \retval id the id or 0 in case of not found
596  */
597 static int HTPHandleWarningGetId(const char *msg)
598 {
599  SCLogDebug("received warning \"%s\"", msg);
600  size_t idx;
601  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
602  if (strncmp(htp_warnings[idx].msg, msg,
603  strlen(htp_warnings[idx].msg)) == 0)
604  {
605  return htp_warnings[idx].de;
606  }
607  }
608 
609  return 0;
610 }
611 
612 /**
613  * \internal
614  *
615  * \brief Get the error id for the error msg.
616  *
617  * \param msg error message
618  *
619  * \retval id the id or 0 in case of not found
620  */
621 static int HTPHandleErrorGetId(const char *msg)
622 {
623  SCLogDebug("received error \"%s\"", msg);
624 
625  size_t idx;
626  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
627  if (strncmp(htp_errors[idx].msg, msg,
628  strlen(htp_errors[idx].msg)) == 0)
629  {
630  return htp_errors[idx].de;
631  }
632  }
633 
634  return 0;
635 }
636 
637 /**
638  * \internal
639  *
640  * \brief Check state for errors, warnings and add any as events
641  *
642  * \param s state
643  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
644  */
645 static void HTPHandleError(HtpState *s, const uint8_t dir)
646 {
647  if (s == NULL || s->conn == NULL ||
648  s->conn->messages == NULL) {
649  return;
650  }
651 
652  size_t size = htp_list_size(s->conn->messages);
653  size_t msg;
654  if(size >= HTP_MAX_MESSAGES) {
656  //only once per HtpState
657  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
659  //too noisy in fuzzing
660  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
661  }
662  // ignore further messages
663  return;
664  }
665 
666  for (msg = s->htp_messages_offset; msg < size; msg++) {
667  htp_log_t *log = htp_list_get(s->conn->messages, msg);
668  if (log == NULL)
669  continue;
670 
671  HtpTxUserData *htud = NULL;
672  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
673  if (tx != NULL)
674  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
675 
676  SCLogDebug("message %s", log->msg);
677 
678  int id = HTPHandleErrorGetId(log->msg);
679  if (id == 0) {
680  id = HTPHandleWarningGetId(log->msg);
681  if (id == 0)
683  }
684 
685  if (id > 0) {
686  HTPSetEvent(s, htud, dir, id);
687  }
688  }
689  s->htp_messages_offset = (uint16_t)msg;
690  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
691 }
692 
693 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
694 {
695 #ifdef DEBUG
696  BUG_ON(s == NULL || tx == NULL);
697 #endif
698  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
699  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
700  HTP_HOSTH_INVALID))
701  {
702  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
703  if (htud == NULL)
704  return;
705 
706  if (tx->flags & HTP_REQUEST_INVALID_T_E)
707  HTPSetEvent(s, htud, STREAM_TOSERVER,
709  if (tx->flags & HTP_REQUEST_INVALID_C_L)
710  HTPSetEvent(s, htud, STREAM_TOSERVER,
712  if (tx->flags & HTP_HOST_MISSING)
713  HTPSetEvent(s, htud, STREAM_TOSERVER,
715  if (tx->flags & HTP_HOST_AMBIGUOUS)
716  HTPSetEvent(s, htud, STREAM_TOSERVER,
718  if (tx->flags & HTP_HOSTU_INVALID)
719  HTPSetEvent(s, htud, STREAM_TOSERVER,
721  if (tx->flags & HTP_HOSTH_INVALID)
722  HTPSetEvent(s, htud, STREAM_TOSERVER,
724  }
725  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
726  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
727  if (htud == NULL)
728  return;
729  HTPSetEvent(s, htud, STREAM_TOSERVER,
731  }
732  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
733  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
734  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
735  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
736  if (htud == NULL)
737  return;
738  HTPSetEvent(s, htud, STREAM_TOSERVER,
740  }
741 }
742 
743 static int Setup(Flow *f, HtpState *hstate)
744 {
745  /* store flow ref in state so callbacks can access it */
746  hstate->f = f;
747 
748  HTPCfgRec *htp_cfg_rec = &cfglist;
749  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
750  void *user_data = NULL;
751 
752  if (FLOW_IS_IPV4(f)) {
753  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
754  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
755  }
756  else if (FLOW_IS_IPV6(f)) {
757  SCLogDebug("Looking up HTP config for ipv6");
758  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
759  }
760  else {
761  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
762  goto error;
763  }
764 
765  if (user_data != NULL) {
766  htp_cfg_rec = user_data;
767  htp = htp_cfg_rec->cfg;
768  SCLogDebug("LIBHTP using config: %p", htp);
769  } else {
770  SCLogDebug("Using default HTP config: %p", htp);
771  }
772 
773  if (NULL == htp) {
774 #ifdef DEBUG_VALIDATION
775  BUG_ON(htp == NULL);
776 #endif
777  /* should never happen if HTPConfigure is properly invoked */
778  goto error;
779  }
780 
781  hstate->connp = htp_connp_create(htp);
782  if (hstate->connp == NULL) {
783  goto error;
784  }
785 
786  hstate->conn = htp_connp_get_connection(hstate->connp);
787 
788  htp_connp_set_user_data(hstate->connp, (void *)hstate);
789  hstate->cfg = htp_cfg_rec;
790 
791  SCLogDebug("New hstate->connp %p", hstate->connp);
792 
793  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
794 
796  htp_cfg_rec->request.inspect_min_size);
798  htp_cfg_rec->response.inspect_min_size);
799  return 0;
800 error:
801  return -1;
802 }
803 
804 /**
805  * \brief Function to handle the reassembled data from client and feed it to
806  * the HTP library to process it.
807  *
808  * \param flow Pointer to the flow the data belong to
809  * \param htp_state Pointer the state in which the parsed value to be stored
810  * \param pstate Application layer parser state for this session
811  * \param input Pointer the received HTTP client data
812  * \param input_len Length in bytes of the received data
813  * \param output Pointer to the output (not used in this function)
814  *
815  * \retval On success returns 1 or on failure returns -1.
816  */
817 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state,
818  AppLayerParserState *pstate,
819  const uint8_t *input, uint32_t input_len,
820  void *local_data, const uint8_t flags)
821 {
822  SCEnter();
823  int ret = 0;
824  HtpState *hstate = (HtpState *)htp_state;
825 
826  /* On the first invocation, create the connection parser structure to
827  * be used by HTP library. This is looked up via IP in the radix
828  * tree. Failing that, the default HTP config is used.
829  */
830  if (NULL == hstate->conn) {
831  if (Setup(f, hstate) != 0) {
833  }
834  }
835  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
836 
837  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
838  /* pass the new data to the htp parser */
839  if (input_len > 0) {
840  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
841  switch (r) {
842  case HTP_STREAM_ERROR:
843  ret = -1;
844  break;
845  default:
846  break;
847  }
848  HTPHandleError(hstate, STREAM_TOSERVER);
849  }
850 
851  /* if the TCP connection is closed, then close the HTTP connection */
853  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
854  {
855  htp_connp_req_close(hstate->connp, &ts);
856  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
857  SCLogDebug("stream eof encountered, closing htp handle for ts");
858  }
859 
860  SCLogDebug("hstate->connp %p", hstate->connp);
861 
862  if (ret < 0) {
864  }
866 }
867 
868 /**
869  * \brief Function to handle the reassembled data from server and feed it to
870  * the HTP library to process it.
871  *
872  * \param flow Pointer to the flow the data belong to
873  * \param htp_state Pointer the state in which the parsed value to be stored
874  * \param pstate Application layer parser state for this session
875  * \param input Pointer the received HTTP server data
876  * \param input_len Length in bytes of the received data
877  * \param output Pointer to the output (not used in this function)
878  *
879  * \retval On success returns 1 or on failure returns -1
880  */
881 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state,
882  AppLayerParserState *pstate,
883  const uint8_t *input, uint32_t input_len,
884  void *local_data, const uint8_t flags)
885 {
886  SCEnter();
887  int ret = 0;
888  HtpState *hstate = (HtpState *)htp_state;
889 
890  /* On the first invocation, create the connection parser structure to
891  * be used by HTP library. This is looked up via IP in the radix
892  * tree. Failing that, the default HTP config is used.
893  */
894  if (NULL == hstate->conn) {
895  if (Setup(f, hstate) != 0) {
897  }
898  }
899  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
900 
901  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
902  htp_tx_t *tx = NULL;
903  size_t consumed = 0;
904  if (input_len > 0) {
905  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
906  switch (r) {
907  case HTP_STREAM_ERROR:
908  ret = -1;
909  break;
910  case HTP_STREAM_TUNNEL:
911  tx = htp_connp_get_out_tx(hstate->connp);
912  if (tx != NULL && tx->response_status_number == 101) {
913  htp_header_t *h =
914  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
915  if (h == NULL || bstr_cmp_c(h->value, "h2c") != 0) {
916  break;
917  }
919  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
920  break;
921  }
922  uint16_t dp = 0;
923  if (tx->request_port_number != -1) {
924  dp = (uint16_t)tx->request_port_number;
925  }
926  consumed = htp_connp_res_data_consumed(hstate->connp);
928  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
929  // and we need to parser the remaining part with HTTP2
930  if (consumed > 0 && consumed < input_len) {
931  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
932  }
934  }
935  break;
936  default:
937  break;
938  }
939  HTPHandleError(hstate, STREAM_TOCLIENT);
940  }
941 
942  /* if we the TCP connection is closed, then close the HTTP connection */
944  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
945  {
946  htp_connp_close(hstate->connp, &ts);
947  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
948  }
949 
950  SCLogDebug("hstate->connp %p", hstate->connp);
951 
952  if (ret < 0) {
954  }
956 }
957 
958 /**
959  * \param name /Lowercase/ version of the variable name
960  */
961 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
962  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
963 {
964 #ifdef PRINT
965  printf("DATA START: \n");
966  PrintRawDataFp(stdout, data, len);
967  printf("DATA END: \n");
968 #endif
969  size_t x;
970  int quote = 0;
971 
972  for (x = 0; x < len; x++) {
973  if (!(isspace(data[x])))
974  break;
975  }
976 
977  if (x >= len)
978  return 0;
979 
980  uint8_t *line = data+x;
981  size_t line_len = len-x;
982  size_t offset = 0;
983 #ifdef PRINT
984  printf("LINE START: \n");
985  PrintRawDataFp(stdout, line, line_len);
986  printf("LINE END: \n");
987 #endif
988  for (x = 0 ; x < line_len; x++) {
989  if (x > 0) {
990  if (line[x - 1] != '\\' && line[x] == '\"') {
991  quote++;
992  }
993 
994  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
995  uint8_t *token = line + offset;
996  size_t token_len = x - offset;
997 
998  if ((x + 1) == line_len) {
999  token_len++;
1000  }
1001 
1002  offset = x + 1;
1003 
1004  while (offset < line_len && isspace(line[offset])) {
1005  x++;
1006  offset++;
1007  }
1008 #ifdef PRINT
1009  printf("TOKEN START: \n");
1010  PrintRawDataFp(stdout, token, token_len);
1011  printf("TOKEN END: \n");
1012 #endif
1013  if (token_len > name_len) {
1014  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1015  uint8_t *value = token + name_len;
1016  size_t value_len = token_len - name_len;
1017 
1018  if (value[0] == '\"') {
1019  value++;
1020  value_len--;
1021  }
1022  if (value[value_len-1] == '\"') {
1023  value_len--;
1024  }
1025 #ifdef PRINT
1026  printf("VALUE START: \n");
1027  PrintRawDataFp(stdout, value, value_len);
1028  printf("VALUE END: \n");
1029 #endif
1030  *retptr = value;
1031  *retlen = value_len;
1032  return 1;
1033  }
1034  }
1035  }
1036  }
1037  }
1038 
1039  return 0;
1040 }
1041 
1042 /**
1043  * \param name /Lowercase/ version of the variable name
1044  */
1045 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1046  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1047 {
1048  SCEnter();
1049 #ifdef PRINT
1050  printf("DATA START: \n");
1051  PrintRawDataFp(stdout, data, len);
1052  printf("DATA END: \n");
1053 #endif
1054  size_t x;
1055  int quote = 0;
1056 
1057  for (x = 0; x < len; x++) {
1058  if (!(isspace(data[x])))
1059  break;
1060  }
1061 
1062  if (x >= len) {
1063  SCReturnInt(0);
1064  }
1065 
1066  uint8_t *line = data+x;
1067  size_t line_len = len-x;
1068  size_t offset = 0;
1069 #ifdef PRINT
1070  printf("LINE START: \n");
1071  PrintRawDataFp(stdout, line, line_len);
1072  printf("LINE END: \n");
1073 #endif
1074  for (x = 0 ; x < line_len; x++) {
1075  if (x > 0) {
1076  if (line[x - 1] != '\\' && line[x] == '\"') {
1077  quote++;
1078  }
1079 
1080  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1081  uint8_t *token = line + offset;
1082  size_t token_len = x - offset;
1083 
1084  if ((x + 1) == line_len) {
1085  token_len++;
1086  }
1087 
1088  offset = x + 1;
1089 
1090  while (offset < line_len && isspace(line[offset])) {
1091  x++;
1092  offset++;
1093  }
1094 #ifdef PRINT
1095  printf("TOKEN START: \n");
1096  PrintRawDataFp(stdout, token, token_len);
1097  printf("TOKEN END: \n");
1098 #endif
1099  if (token_len > name_len) {
1100  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1101  uint8_t *value = token + name_len;
1102  size_t value_len = token_len - name_len;
1103 
1104  if (value[0] == '\"') {
1105  value++;
1106  value_len--;
1107  }
1108  if (value[value_len-1] == '\"') {
1109  value_len--;
1110  }
1111 #ifdef PRINT
1112  printf("VALUE START: \n");
1113  PrintRawDataFp(stdout, value, value_len);
1114  printf("VALUE END: \n");
1115 #endif
1116  *retptr = value;
1117  *retlen = value_len;
1118  SCReturnInt(1);
1119  }
1120  }
1121  }
1122  }
1123  }
1124 
1125  SCReturnInt(0);
1126 }
1127 
1128 /**
1129  * \brief setup multipart parsing: extract boundary and store it
1130  *
1131  * \param d HTTP transaction
1132  * \param htud transaction userdata
1133  *
1134  * \retval 1 ok, multipart set up
1135  * \retval 0 ok, not multipart though
1136  * \retval -1 error: problem with the boundary
1137  *
1138  * If the request contains a multipart message, this function will
1139  * set the HTP_BOUNDARY_SET in the transaction.
1140  */
1141 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1142 {
1143  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1144  "Content-Type");
1145  if (h != NULL && bstr_len(h->value) > 0) {
1146  uint8_t *boundary = NULL;
1147  size_t boundary_len = 0;
1148 
1149  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1150  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1151  &boundary, &boundary_len);
1152  if (r == 1) {
1153 #ifdef PRINT
1154  printf("BOUNDARY START: \n");
1155  PrintRawDataFp(stdout, boundary, boundary_len);
1156  printf("BOUNDARY END: \n");
1157 #endif
1158  if (boundary_len < HTP_BOUNDARY_MAX) {
1159  htud->boundary = HTPMalloc(boundary_len);
1160  if (htud->boundary == NULL) {
1161  return -1;
1162  }
1163  htud->boundary_len = (uint8_t)boundary_len;
1164  memcpy(htud->boundary, boundary, boundary_len);
1165 
1166  htud->tsflags |= HTP_BOUNDARY_SET;
1167  } else {
1168  SCLogDebug("invalid boundary");
1169  return -1;
1170  }
1171  SCReturnInt(1);
1172  }
1173  //SCReturnInt(1);
1174  }
1175  SCReturnInt(0);
1176 }
1177 
1178 #define C_D_HDR "content-disposition:"
1179 #define C_D_HDR_LEN 20
1180 #define C_T_HDR "content-type:"
1181 #define C_T_HDR_LEN 13
1183 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1184  HtpTxUserData *htud,
1185  uint8_t *header, uint32_t header_len,
1186  uint8_t **filename, uint16_t *filename_len,
1187  uint8_t **filetype, uint16_t *filetype_len)
1188 {
1189  uint8_t *fn = NULL;
1190  size_t fn_len = 0;
1191  uint8_t *ft = NULL;
1192  size_t ft_len = 0;
1193 
1194 #ifdef PRINT
1195  printf("HEADER START: \n");
1196  PrintRawDataFp(stdout, header, header_len);
1197  printf("HEADER END: \n");
1198 #endif
1199 
1200  while (header_len > 0) {
1201  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1202  uint8_t *line = header;
1203  uint32_t line_len;
1204 
1205  if (next_line == NULL) {
1206  line_len = header_len;
1207  } else {
1208  line_len = next_line - header;
1209  }
1210  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1211  if (sc == NULL) {
1212  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1214  /* if the : we found is the final char, it means we have
1215  * no value */
1216  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1217  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1219  } else {
1220 #ifdef PRINT
1221  printf("LINE START: \n");
1222  PrintRawDataFp(stdout, line, line_len);
1223  printf("LINE END: \n");
1224 #endif
1225  if (line_len >= C_D_HDR_LEN &&
1226  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1227  uint8_t *value = line + C_D_HDR_LEN;
1228  uint32_t value_len = line_len - C_D_HDR_LEN;
1229 
1230  /* parse content-disposition */
1231  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1232  value, value_len, &fn, &fn_len);
1233  } else if (line_len >= C_T_HDR_LEN &&
1234  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1235  SCLogDebug("content-type line");
1236  uint8_t *value = line + C_T_HDR_LEN;
1237  uint32_t value_len = line_len - C_T_HDR_LEN;
1238 
1239  (void)HTTPParseContentTypeHeader(NULL, 0,
1240  value, value_len, &ft, &ft_len);
1241  }
1242  }
1243 
1244  if (next_line == NULL) {
1245  SCLogDebug("no next_line");
1246  break;
1247  }
1248  header_len -= ((next_line + 2) - header);
1249  header = next_line + 2;
1250  } /* while (header_len > 0) */
1251 
1252  if (fn_len > USHRT_MAX)
1253  fn_len = USHRT_MAX;
1254  if (ft_len > USHRT_MAX)
1255  ft_len = USHRT_MAX;
1256 
1257  *filename = fn;
1258  *filename_len = fn_len;
1259  *filetype = ft;
1260  *filetype_len = ft_len;
1261 }
1262 
1263 /**
1264  * \brief Create a single buffer from the HtpBodyChunks in our list
1265  *
1266  * \param htud transaction user data
1267  * \param chunks_buffers pointer to pass back the buffer to the caller
1268  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1269  */
1270 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1271  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1272 {
1274  chunks_buffer, chunks_buffer_len,
1275  htud->request_body.body_parsed);
1276 }
1277 
1278 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1279 {
1280  SCEnter();
1281  if (tx && tx->tx_data.de_state) {
1282  if (dir == STREAM_TOSERVER) {
1283  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1284  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1285  } else if (STREAM_TOCLIENT) {
1286  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1287  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1288  }
1289  }
1290 }
1291 
1292 /**
1293  * \brief Setup boundary buffers
1294  */
1295 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1296  uint8_t *boundary, uint32_t boundary_len)
1297 {
1298  memset(boundary, '-', boundary_len);
1299  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1300 }
1301 
1302 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1303  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1304 {
1305  int result = 0;
1306  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1307  uint32_t expected_boundary_len = htud->boundary_len + 2;
1308  uint32_t expected_boundary_end_len = htud->boundary_len + 4;
1309  int tx_progress = 0;
1310 
1311 #ifdef PRINT
1312  printf("CHUNK START: \n");
1313  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1314  printf("CHUNK END: \n");
1315 #endif
1316 
1317  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1318 
1319  /* search for the header start, header end and form end */
1320  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1321  boundary, expected_boundary_len);
1322  /* end of the multipart form */
1323  const uint8_t *form_end = NULL;
1324  /* end marker belonging to header_start */
1325  const uint8_t *header_end = NULL;
1326  if (header_start != NULL) {
1327  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1328  (uint8_t *)"\r\n\r\n", 4);
1329  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1330  boundary, expected_boundary_end_len);
1331  }
1332 
1333  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1334  header_end, form_end);
1335 
1336  /* we currently only handle multipart for ts. When we support it for tc,
1337  * we will need to supply right direction */
1338  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1339  /* if we're in the file storage process, deal with that now */
1340  if (htud->tsflags & HTP_FILENAME_SET) {
1341  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1342  SCLogDebug("reached the end of the file");
1343 
1344  const uint8_t *filedata = chunks_buffer;
1345  uint32_t filedata_len = 0;
1346  uint8_t flags = 0;
1347 
1348  if (header_start != NULL) {
1349  if (header_start == filedata + 2) {
1350  /* last chunk had all data, but not the boundary */
1351  SCLogDebug("last chunk had all data, but not the boundary");
1352  filedata_len = 0;
1353  } else if (header_start > filedata + 2) {
1354  SCLogDebug("some data from last file before the boundary");
1355  /* some data from last file before the boundary */
1356  filedata_len = header_start - filedata - 2;
1357  }
1358  }
1359  /* body parsing done, we did not get our form end. Use all data
1360  * we still have and signal to files API we have an issue. */
1361  if (tx_progress > HTP_REQUEST_BODY) {
1362  filedata_len = chunks_buffer_len;
1364  }
1365 
1366  if (filedata_len > chunks_buffer_len) {
1367  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1369  goto end;
1370  }
1371 #ifdef PRINT
1372  printf("FILEDATA (final chunk) START: \n");
1373  PrintRawDataFp(stdout, filedata, filedata_len);
1374  printf("FILEDATA (final chunk) END: \n");
1375 #endif
1376  if (!(htud->tsflags & HTP_DONTSTORE)) {
1377  if (HTPFileClose(hstate, filedata, filedata_len, flags,
1378  STREAM_TOSERVER) == -1)
1379  {
1380  goto end;
1381  }
1382  }
1383 
1384  htud->tsflags &=~ HTP_FILENAME_SET;
1385 
1386  /* fall through */
1387  } else {
1388  SCLogDebug("not yet at the end of the file");
1389 
1390  if (chunks_buffer_len > expected_boundary_end_len) {
1391  const uint8_t *filedata = chunks_buffer;
1392  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1393 #ifdef PRINT
1394  printf("FILEDATA (part) START: \n");
1395  PrintRawDataFp(stdout, filedata, filedata_len);
1396  printf("FILEDATA (part) END: \n");
1397 #endif
1398 
1399  if (!(htud->tsflags & HTP_DONTSTORE)) {
1400  result = HTPFileStoreChunk(hstate, filedata,
1401  filedata_len, STREAM_TOSERVER);
1402  if (result == -1) {
1403  goto end;
1404  } else if (result == -2) {
1405  /* we know for sure we're not storing the file */
1406  htud->tsflags |= HTP_DONTSTORE;
1407  }
1408  }
1409 
1410  htud->request_body.body_parsed += filedata_len;
1411  } else {
1412  SCLogDebug("chunk too small to already process in part");
1413  }
1414 
1415  goto end;
1416  }
1417  }
1418 
1419  while (header_start != NULL && header_end != NULL &&
1420  header_end != form_end &&
1421  header_start < (chunks_buffer + chunks_buffer_len) &&
1422  header_end < (chunks_buffer + chunks_buffer_len) &&
1423  header_start < header_end)
1424  {
1425  uint8_t *filename = NULL;
1426  uint16_t filename_len = 0;
1427  uint8_t *filetype = NULL;
1428  uint16_t filetype_len = 0;
1429 
1430  uint32_t header_len = header_end - header_start;
1431  SCLogDebug("header_len %u", header_len);
1432  uint8_t *header = (uint8_t *)header_start;
1433 
1434  /* skip empty records */
1435  if (expected_boundary_len == header_len) {
1436  goto next;
1437  } else if ((expected_boundary_len + 2) <= header_len) {
1438  header_len -= (expected_boundary_len + 2);
1439  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1440  }
1441 
1442  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1443  &filename, &filename_len, &filetype, &filetype_len);
1444 
1445  if (filename != NULL) {
1446  const uint8_t *filedata = NULL;
1447  uint32_t filedata_len = 0;
1448 
1449  SCLogDebug("we have a filename");
1450 
1451  htud->tsflags |= HTP_FILENAME_SET;
1452  htud->tsflags &= ~HTP_DONTSTORE;
1453 
1454  SCLogDebug("header_end %p", header_end);
1455  SCLogDebug("form_end %p", form_end);
1456 
1457  /* everything until the final boundary is the file */
1458  if (form_end != NULL) {
1459  SCLogDebug("have form_end");
1460 
1461  filedata = header_end + 4;
1462  if (form_end == filedata) {
1463  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1465  goto end;
1466  } else if (form_end < filedata) {
1467  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1469  goto end;
1470  }
1471 
1472  filedata_len = form_end - (header_end + 4 + 2);
1473  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1474 
1475  /* or is it? */
1476  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1477  boundary, expected_boundary_len);
1478  if (header_next != NULL) {
1479  filedata_len -= (form_end - header_next);
1480  }
1481 
1482  if (filedata_len > chunks_buffer_len) {
1483  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1485  goto end;
1486  }
1487  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1488 #ifdef PRINT
1489  printf("FILEDATA START: \n");
1490  PrintRawDataFp(stdout, filedata, filedata_len);
1491  printf("FILEDATA END: \n");
1492 #endif
1493 
1494  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata, filedata_len,
1495  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1496  if (result == -1) {
1497  goto end;
1498  } else if (result == -2) {
1499  htud->tsflags |= HTP_DONTSTORE;
1500  } else {
1501  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1502  goto end;
1503  }
1504  }
1505  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1506 
1507  htud->request_body.body_parsed += (header_end - chunks_buffer);
1508  htud->tsflags &= ~HTP_FILENAME_SET;
1509  } else {
1510  SCLogDebug("chunk doesn't contain form end");
1511 
1512  filedata = header_end + 4;
1513  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1514  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1515 
1516  if (filedata_len > chunks_buffer_len) {
1517  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1519  goto end;
1520  }
1521 
1522 #ifdef PRINT
1523  printf("FILEDATA START: \n");
1524  PrintRawDataFp(stdout, filedata, filedata_len);
1525  printf("FILEDATA END: \n");
1526 #endif
1527  /* form doesn't end in this chunk, but the part might. Lets
1528  * see if have another coming up */
1529  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1530  boundary, expected_boundary_len);
1531  SCLogDebug("header_next %p", header_next);
1532  if (header_next == NULL) {
1533  SCLogDebug("more file data to come");
1534 
1535  uint32_t offset = (header_end + 4) - chunks_buffer;
1536  SCLogDebug("offset %u", offset);
1537  htud->request_body.body_parsed += offset;
1538 
1539  if (filedata_len >= (expected_boundary_len + 2)) {
1540  filedata_len -= (expected_boundary_len + 2 - 1);
1541  SCLogDebug("opening file with partial data");
1542  } else {
1543  filedata = NULL;
1544  filedata_len = 0;
1545  }
1546  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1547  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1548  if (result == -1) {
1549  goto end;
1550  } else if (result == -2) {
1551  htud->tsflags |= HTP_DONTSTORE;
1552  }
1553  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1554  htud->request_body.body_parsed += filedata_len;
1555  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1556 
1557  } else if (header_next - filedata > 2) {
1558  filedata_len = header_next - filedata - 2;
1559  SCLogDebug("filedata_len %u", filedata_len);
1560 
1561  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1562  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1563  if (result == -1) {
1564  goto end;
1565  } else if (result == -2) {
1566  htud->tsflags |= HTP_DONTSTORE;
1567  } else {
1568  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1569  goto end;
1570  }
1571  }
1572  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1573 
1574  htud->tsflags &= ~HTP_FILENAME_SET;
1575  htud->request_body.body_parsed += (header_end - chunks_buffer);
1576  }
1577  }
1578  }
1579 next:
1580  SCLogDebug("header_start %p, header_end %p, form_end %p",
1581  header_start, header_end, form_end);
1582 
1583  /* Search next boundary entry after the start of body */
1584  uint32_t cursizeread = header_end - chunks_buffer;
1585  header_start = Bs2bmSearch(header_end + 4,
1586  chunks_buffer_len - (cursizeread + 4),
1587  boundary, expected_boundary_len);
1588  if (header_start != NULL) {
1589  header_end = Bs2bmSearch(header_end + 4,
1590  chunks_buffer_len - (cursizeread + 4),
1591  (uint8_t *) "\r\n\r\n", 4);
1592  }
1593  }
1594 
1595  /* if we're parsing the multipart and we're not currently processing a
1596  * file, we move the body pointer forward. */
1597  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1598  if (chunks_buffer_len > expected_boundary_end_len) {
1599  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1600 
1601  htud->request_body.body_parsed += move;
1602  SCLogDebug("form not ready, file not set, parsing non-file "
1603  "record: moved %u", move);
1604  }
1605  }
1606 
1607 end:
1608  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1609  return 0;
1610 }
1611 
1612 /** \internal
1613  * \brief Handle POST or PUT, no multipart body data
1614  */
1615 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1616  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1617 {
1618  int result = 0;
1619 
1620  /* see if we need to open the file */
1621  if (!(htud->tsflags & HTP_FILENAME_SET))
1622  {
1623  uint8_t *filename = NULL;
1624  size_t filename_len = 0;
1625 
1626  /* get the name */
1627  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1628  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1629  filename_len = bstr_len(tx->parsed_uri->path);
1630  }
1631 
1632  if (filename != NULL) {
1633  result = HTPFileOpen(hstate, htud, filename, (uint32_t)filename_len, data, data_len,
1634  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1635  if (result == -1) {
1636  goto end;
1637  } else if (result == -2) {
1638  htud->tsflags |= HTP_DONTSTORE;
1639  } else {
1640  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1641  htud->tsflags |= HTP_FILENAME_SET;
1642  htud->tsflags &= ~HTP_DONTSTORE;
1643  }
1644  }
1645  }
1646  else
1647  {
1648  /* otherwise, just store the data */
1649 
1650  if (!(htud->tsflags & HTP_DONTSTORE)) {
1651  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1652  if (result == -1) {
1653  goto end;
1654  } else if (result == -2) {
1655  /* we know for sure we're not storing the file */
1656  htud->tsflags |= HTP_DONTSTORE;
1657  }
1658  }
1659  }
1660 
1661  return 0;
1662 end:
1663  return -1;
1664 }
1665 
1666 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1667  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1668 {
1669  SCEnter();
1670 
1671  int result = 0;
1672 
1673  /* see if we need to open the file
1674  * we check for tx->response_line in case of junk
1675  * interpreted as body before response line
1676  */
1677  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1678  (tx->response_line != NULL || tx->is_protocol_0_9))
1679  {
1680  SCLogDebug("setting up file name");
1681 
1682  uint8_t *filename = NULL;
1683  size_t filename_len = 0;
1684 
1685  /* try Content-Disposition header first */
1686  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1687  "Content-Disposition");
1688  if (h != NULL && bstr_len(h->value) > 0) {
1689  /* parse content-disposition */
1690  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1691  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1692  }
1693 
1694  /* fall back to name from the uri */
1695  if (filename == NULL) {
1696  /* get the name */
1697  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1698  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1699  filename_len = bstr_len(tx->parsed_uri->path);
1700  }
1701  }
1702 
1703  if (filename != NULL) {
1704  // set range if present
1705  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1706  if (h_content_range != NULL) {
1707  result = HTPFileOpenWithRange(hstate, htud, filename, (uint32_t)filename_len, data,
1708  data_len, HtpGetActiveResponseTxID(hstate), h_content_range->value, htud);
1709  } else {
1710  result = HTPFileOpen(hstate, htud, filename, (uint32_t)filename_len, data, data_len,
1711  HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1712  }
1713  SCLogDebug("result %d", result);
1714  if (result == -1) {
1715  goto end;
1716  } else if (result == -2) {
1717  htud->tcflags |= HTP_DONTSTORE;
1718  } else {
1719  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1720  htud->tcflags |= HTP_FILENAME_SET;
1721  htud->tcflags &= ~HTP_DONTSTORE;
1722  }
1723  }
1724  }
1725  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1726  {
1727  /* otherwise, just store the data */
1728 
1729  if (!(htud->tcflags & HTP_DONTSTORE)) {
1730  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1731  SCLogDebug("result %d", result);
1732  if (result == -1) {
1733  goto end;
1734  } else if (result == -2) {
1735  /* we know for sure we're not storing the file */
1736  htud->tcflags |= HTP_DONTSTORE;
1737  }
1738  }
1739  }
1740 
1741  htud->response_body.body_parsed += data_len;
1742  return 0;
1743 end:
1744  return -1;
1745 }
1746 
1747 /**
1748  * \brief Function callback to append chunks for Requests
1749  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1750  * \retval int HTP_OK if all goes well
1751  */
1752 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1753 {
1754  SCEnter();
1755 
1756  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1757  SCReturnInt(HTP_OK);
1758 
1759  if (d->len == 0)
1760  SCReturnInt(HTP_OK);
1761 
1762 #ifdef PRINT
1763  printf("HTPBODY START: \n");
1764  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1765  printf("HTPBODY END: \n");
1766 #endif
1767 
1768  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1769  if (hstate == NULL) {
1770  SCReturnInt(HTP_ERROR);
1771  }
1772 
1773  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1774  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1775 
1776  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1777  if (tx_ud == NULL) {
1778  SCReturnInt(HTP_OK);
1779  }
1780  if (!tx_ud->response_body_init) {
1781  tx_ud->response_body_init = 1;
1782 
1783  if (d->tx->request_method_number == HTP_M_POST) {
1784  SCLogDebug("POST");
1785  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1786  if (r == 1) {
1788  } else if (r == 0) {
1790  SCLogDebug("not multipart");
1791  }
1792  } else if (d->tx->request_method_number == HTP_M_PUT) {
1794  }
1795  }
1796 
1797  /* see if we can get rid of htp body chunks */
1798  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1799 
1800  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1801  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1802 
1803  /* within limits, add the body chunk to the state. */
1804  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1805  uint32_t stream_depth = FileReassemblyDepth();
1806  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1807  hstate->cfg->request.body_limit,
1808  stream_depth,
1809  tx_ud->tsflags,
1810  (uint32_t)d->len);
1811  BUG_ON(len > (uint32_t)d->len);
1812 
1813  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1814 
1815  const uint8_t *chunks_buffer = NULL;
1816  uint32_t chunks_buffer_len = 0;
1817 
1819  /* multi-part body handling starts here */
1820  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1821  goto end;
1822  }
1823 
1824  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1825  if (chunks_buffer == NULL) {
1826  goto end;
1827  }
1828 #ifdef PRINT
1829  printf("REASSCHUNK START: \n");
1830  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1831  printf("REASSCHUNK END: \n");
1832 #endif
1833 
1834  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1835 
1836  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1838  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1839  }
1840 
1841  } else {
1842  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1843  SCLogDebug("closing file that was being stored");
1844  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1845  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1846  }
1847  }
1848 
1849 end:
1850  if (hstate->conn != NULL) {
1851  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1853  hstate->cfg->request.inspect_min_size,
1854  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1855 
1856  /* if we reach the inspect_min_size we'll trigger inspection,
1857  * so make sure that raw stream is also inspected. Set the
1858  * data to be used to the amount of raw bytes we've seen to
1859  * get here. */
1860  if (tx_ud->request_body.body_inspected == 0 &&
1862  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1863  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1864  {
1865  uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1866 
1867  /* body still in progress, but due to min inspect size we need to inspect now */
1870  }
1871  /* after the start of the body, disable the depth logic */
1872  } else if (tx_ud->request_body.body_inspected > 0) {
1874  }
1875  }
1876  SCReturnInt(HTP_OK);
1877 }
1878 
1879 /**
1880  * \brief Function callback to append chunks for Responses
1881  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1882  * \retval int HTP_OK if all goes well
1883  */
1884 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1885 {
1886  SCEnter();
1887 
1888  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1889  SCReturnInt(HTP_OK);
1890 
1891  if (d->len == 0)
1892  SCReturnInt(HTP_OK);
1893 
1894  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1895  if (hstate == NULL) {
1896  SCReturnInt(HTP_ERROR);
1897  }
1898 
1899  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1900  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1901 
1902  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1903  if (tx_ud == NULL) {
1904  SCReturnInt(HTP_OK);
1905  }
1906  if (!tx_ud->request_body_init) {
1907  tx_ud->request_body_init = 1;
1908  }
1909 
1910  /* see if we can get rid of htp body chunks */
1911  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1912 
1913  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1914  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1915 
1916  /* within limits, add the body chunk to the state. */
1917  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1918  uint32_t stream_depth = FileReassemblyDepth();
1919  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1920  hstate->cfg->response.body_limit,
1921  stream_depth,
1922  tx_ud->tcflags,
1923  (uint32_t)d->len);
1924  BUG_ON(len > (uint32_t)d->len);
1925 
1926  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1927 
1928  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1929  } else {
1930  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1931  SCLogDebug("closing file that was being stored");
1932  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1933  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1934  }
1935  }
1936 
1937  if (hstate->conn != NULL) {
1938  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1940  hstate->cfg->response.inspect_min_size,
1941  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
1942  /* if we reach the inspect_min_size we'll trigger inspection,
1943  * so make sure that raw stream is also inspected. Set the
1944  * data to be used to the amount of raw bytes we've seen to
1945  * get here. */
1946  if (tx_ud->response_body.body_inspected == 0 &&
1948  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
1949  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
1950  {
1951  uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
1952 
1953  /* body still in progress, but due to min inspect size we need to inspect now */
1956  }
1957  /* after the start of the body, disable the depth logic */
1958  } else if (tx_ud->response_body.body_inspected > 0) {
1960  }
1961  }
1962  SCReturnInt(HTP_OK);
1963 }
1964 
1965 /**
1966  * \brief Print the stats of the HTTP requests
1967  */
1969 {
1970 #ifdef DEBUG
1971  SCEnter();
1972  SCMutexLock(&htp_state_mem_lock);
1973  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
1974  htp_state_memcnt, htp_state_memuse);
1975  SCMutexUnlock(&htp_state_mem_lock);
1976  SCReturn;
1977 #endif
1978 }
1979 
1980 /** \brief Clears the HTTP server configuration memory used by HTP library */
1981 void HTPFreeConfig(void)
1982 {
1983  SCEnter();
1984 
1985  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
1986  !AppLayerParserConfParserEnabled("tcp", "http"))
1987  {
1988  SCReturn;
1989  }
1990 
1991  HTPCfgRec *nextrec = cfglist.next;
1992  SCRadixReleaseRadixTree(cfgtree);
1993  cfgtree = NULL;
1994  htp_config_destroy(cfglist.cfg);
1995  while (nextrec != NULL) {
1996  HTPCfgRec *htprec = nextrec;
1997  nextrec = nextrec->next;
1998 
1999  htp_config_destroy(htprec->cfg);
2000  SCFree(htprec);
2001  }
2002  SCReturn;
2003 }
2004 
2005 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2006 {
2007  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2008  if (htud != NULL) {
2009  htud->request_has_trailers = 1;
2010  }
2011  return HTP_OK;
2012 }
2013 
2014 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2015 {
2016  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2017  if (htud != NULL) {
2018  htud->response_has_trailers = 1;
2019  }
2020  return HTP_OK;
2021 }
2022 
2023 /**\internal
2024  * \brief called at start of request
2025  * Set min inspect size.
2026  */
2027 static int HTPCallbackRequestStart(htp_tx_t *tx)
2028 {
2029  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2030  if (hstate == NULL) {
2031  SCReturnInt(HTP_ERROR);
2032  }
2033 
2034  if (hstate->cfg)
2036  hstate->cfg->request.inspect_min_size);
2037 
2038  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2039  if (tx_ud == NULL) {
2040  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2041  if (unlikely(tx_ud == NULL)) {
2042  SCReturnInt(HTP_OK);
2043  }
2044  htp_tx_set_user_data(tx, tx_ud);
2045  }
2046  SCReturnInt(HTP_OK);
2047 }
2048 
2049 /**\internal
2050  * \brief called at start of response
2051  * Set min inspect size.
2052  */
2053 static int HTPCallbackResponseStart(htp_tx_t *tx)
2054 {
2055  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2056  if (hstate == NULL) {
2057  SCReturnInt(HTP_ERROR);
2058  }
2059 
2060  if (hstate->cfg)
2062  hstate->cfg->response.inspect_min_size);
2063 
2064  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2065  if (tx_ud == NULL) {
2066  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2067  if (unlikely(tx_ud == NULL)) {
2068  SCReturnInt(HTP_OK);
2069  }
2070  htp_tx_set_user_data(tx, tx_ud);
2071  }
2072  SCReturnInt(HTP_OK);
2073 }
2074 
2075 
2076 /**
2077  * \brief callback for request to store the recent incoming request
2078  in to the recent_in_tx for the given htp state
2079  * \param connp pointer to the current connection parser which has the htp
2080  * state in it as user data
2081  */
2082 static int HTPCallbackRequest(htp_tx_t *tx)
2083 {
2084  SCEnter();
2085 
2086  if (tx == NULL) {
2087  SCReturnInt(HTP_ERROR);
2088  }
2089 
2090  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2091  if (hstate == NULL) {
2092  SCReturnInt(HTP_ERROR);
2093  }
2094 
2095  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2096  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2097 
2098  SCLogDebug("HTTP request completed");
2099 
2100  HTPErrorCheckTxRequestFlags(hstate, tx);
2101 
2102  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2103  if (htud != NULL) {
2104  if (htud->tsflags & HTP_FILENAME_SET) {
2105  SCLogDebug("closing file that was being stored");
2106  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
2107  htud->tsflags &= ~HTP_FILENAME_SET;
2109  (uint32_t)hstate->conn->in_data_counter);
2110  }
2111  }
2112 
2113  hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
2114  /* request done, do raw reassembly now to inspect state and stream
2115  * at the same time. */
2117  SCReturnInt(HTP_OK);
2118 }
2119 
2120 /**
2121  * \brief callback for response to remove the recent received requests
2122  from the recent_in_tx for the given htp state
2123  * \param connp pointer to the current connection parser which has the htp
2124  * state in it as user data
2125  */
2126 static int HTPCallbackResponse(htp_tx_t *tx)
2127 {
2128  SCEnter();
2129 
2130  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2131  if (hstate == NULL) {
2132  SCReturnInt(HTP_ERROR);
2133  }
2134 
2135  /* we have one whole transaction now */
2136  hstate->transaction_cnt++;
2137 
2138  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2139  if (htud != NULL) {
2140  if (htud->tcflags & HTP_FILENAME_SET) {
2141  SCLogDebug("closing file that was being stored");
2142  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
2143  htud->tcflags &= ~HTP_FILENAME_SET;
2144  }
2145  }
2146 
2147  /* response done, do raw reassembly now to inspect state and stream
2148  * at the same time. */
2150 
2151  /* handle HTTP CONNECT */
2152  if (tx->request_method_number == HTP_M_CONNECT) {
2153  /* any 2XX status response implies that the connection will become
2154  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2155  if ((tx->response_status_number >= 200) &&
2156  (tx->response_status_number < 300) &&
2157  (hstate->transaction_cnt == 1)) {
2158  uint16_t dp = 0;
2159  if (tx->request_port_number != -1) {
2160  dp = (uint16_t)tx->request_port_number;
2161  }
2162  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2164  tx->request_progress = HTP_REQUEST_COMPLETE;
2165  tx->response_progress = HTP_RESPONSE_COMPLETE;
2166  }
2167  }
2168 
2169  hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
2170  SCReturnInt(HTP_OK);
2171 }
2172 
2173 static int HTPCallbackRequestLine(htp_tx_t *tx)
2174 {
2175  HtpTxUserData *tx_ud;
2176  bstr *request_uri_normalized;
2177  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2178  const HTPCfgRec *cfg = hstate->cfg;
2179 
2180  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2181  if (request_uri_normalized == NULL)
2182  return HTP_OK;
2183 
2184  tx_ud = htp_tx_get_user_data(tx);
2185  if (likely(tx_ud == NULL)) {
2186  tx_ud = HTPMalloc(sizeof(*tx_ud));
2187  if (unlikely(tx_ud == NULL)) {
2188  bstr_free(request_uri_normalized);
2189  return HTP_OK;
2190  }
2191  memset(tx_ud, 0, sizeof(*tx_ud));
2192  htp_tx_set_user_data(tx, tx_ud);
2193  }
2194  if (unlikely(tx_ud->request_uri_normalized != NULL))
2195  bstr_free(tx_ud->request_uri_normalized);
2196  tx_ud->request_uri_normalized = request_uri_normalized;
2197 
2198  if (tx->flags) {
2199  HTPErrorCheckTxRequestFlags(hstate, tx);
2200  }
2201  return HTP_OK;
2202 }
2203 
2204 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2205 {
2206  if (part == NULL)
2207  return HTP_OK;
2208 
2209  uint64_t flags = 0;
2210  size_t prevlen = bstr_len(part);
2211  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2212  // shorter string means that uri was encoded
2213  if (res == HTP_OK && prevlen > bstr_len(part)) {
2214  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2215  if (htud == NULL)
2216  return HTP_OK;
2217  HtpState *s = htp_connp_get_user_data(tx->connp);
2218  if (s == NULL)
2219  return HTP_OK;
2220  HTPSetEvent(s, htud, STREAM_TOSERVER,
2222  }
2223 
2224  return HTP_OK;
2225 }
2226 
2227 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2228 {
2229  if (tx->parsed_uri == NULL)
2230  return HTP_OK;
2231 
2232  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2233 }
2234 
2235 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2236 {
2237  if (tx->parsed_uri == NULL)
2238  return HTP_OK;
2239 
2240  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2241 }
2242 
2243 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2244 {
2245  void *ptmp;
2246  if (tx_data->len == 0 || tx_data->tx == NULL)
2247  return HTP_OK;
2248 
2249  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2250  if (tx_ud == NULL) {
2251  return HTP_OK;
2252  }
2253  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2254  tx_ud->request_headers_raw_len,
2255  tx_ud->request_headers_raw_len + tx_data->len);
2256  if (ptmp == NULL) {
2257  return HTP_OK;
2258  }
2259  tx_ud->request_headers_raw = ptmp;
2260 
2261  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2262  tx_data->data, tx_data->len);
2263  tx_ud->request_headers_raw_len += tx_data->len;
2264 
2265  if (tx_data->tx && tx_data->tx->flags) {
2266  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2267  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2268  }
2269  return HTP_OK;
2270 }
2271 
2272 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2273 {
2274  void *ptmp;
2275  if (tx_data->len == 0 || tx_data->tx == NULL)
2276  return HTP_OK;
2277 
2278  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2279  if (tx_ud == NULL) {
2280  return HTP_OK;
2281  }
2282  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2283  tx_ud->response_headers_raw_len,
2284  tx_ud->response_headers_raw_len + tx_data->len);
2285  if (ptmp == NULL) {
2286  return HTP_OK;
2287  }
2288  tx_ud->response_headers_raw = ptmp;
2289 
2290  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2291  tx_data->data, tx_data->len);
2292  tx_ud->response_headers_raw_len += tx_data->len;
2293 
2294  return HTP_OK;
2295 }
2296 
2297 /*
2298  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2299  */
2300 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2301 {
2302  cfg_prec->uri_include_all = FALSE;
2309 
2310  if (!g_disable_randomness) {
2312  } else {
2313  cfg_prec->randomize = 0;
2314  }
2316 
2317  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2318  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2319  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2320  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2321 
2322  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2323  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2324 
2325  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2326  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2327 
2328  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2329  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
2330 
2331  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2332  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
2333 
2334  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2335 
2336  /* don't convert + to space by default */
2337  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2338  // enables request decompression
2339  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2340 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2341  // disable by default
2342  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2343 #endif
2344 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2345  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2347 #endif
2348 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2349  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2351 #endif
2352 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2353  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2354 #endif
2355  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2356  * only the hard limit. So we set both here to the (current) htp defaults.
2357  * The reason we do this is that if the user sets the hard limit in the
2358  * config, we have to set the soft limit as well. If libhtp starts using
2359  * the soft limit in the future, we at least make sure we control what
2360  * it's value is. */
2361  htp_config_set_field_limits(cfg_prec->cfg,
2364  return;
2365 }
2366 
2367 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2368  * but we can get both <0 and >RAND_MAX values from RandomGet
2369  */
2370 static int RandomGetWrap(void)
2371 {
2372  unsigned long r;
2373 
2374  do {
2375  r = RandomGet();
2376  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2377 
2378  return r % RAND_MAX;
2379 }
2380 
2381 /*
2382  * We have this splitup so that in case double decoding has been enabled
2383  * for query and path, they would be called first on the callback queue,
2384  * before the callback set by Phase2() is called. We need this, since
2385  * the callback in Phase2() generates the normalized uri which utilizes
2386  * the query and path. */
2387 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2388 {
2389  /* randomize inspection size if needed */
2390  if (cfg_prec->randomize) {
2391  int rdrange = cfg_prec->randomize_range;
2392 
2393  long int r = RandomGetWrap();
2394  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2395  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2396 
2397  r = RandomGetWrap();
2398  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2399  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2400  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2401  " %d and 'request-body-inspect-window' set to %d after"
2402  " randomization.",
2403  name,
2404  cfg_prec->request.inspect_min_size,
2405  cfg_prec->request.inspect_window);
2406 
2407 
2408  r = RandomGetWrap();
2409  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2410  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2411 
2412  r = RandomGetWrap();
2413  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2414  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2415 
2416  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2417  " %d and 'response-body-inspect-window' set to %d after"
2418  " randomization.",
2419  name,
2420  cfg_prec->response.inspect_min_size,
2421  cfg_prec->response.inspect_window);
2422  }
2423 
2424  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2425 
2426  cfg_prec->request.sbcfg.flags = 0;
2427  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2428  cfg_prec->request.inspect_window : 256;
2429  cfg_prec->request.sbcfg.buf_slide = 0;
2430  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2431  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2432  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2433  cfg_prec->request.sbcfg.Free = HTPFree;
2434 
2435  cfg_prec->response.sbcfg.flags = 0;
2436  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2437  cfg_prec->response.inspect_window : 256;
2438  cfg_prec->response.sbcfg.buf_slide = 0;
2439  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2440  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2441  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2442  cfg_prec->response.sbcfg.Free = HTPFree;
2443  return;
2444 }
2445 
2446 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2447  SCRadixTree *tree)
2448 {
2449  if (cfg_prec == NULL || s == NULL || tree == NULL)
2450  return;
2451 
2452  ConfNode *p = NULL;
2453 
2454  /* Default Parameters */
2455  TAILQ_FOREACH(p, &s->head, next) {
2456 
2457  if (strcasecmp("address", p->name) == 0) {
2458  ConfNode *pval;
2459  /* Addresses */
2460  TAILQ_FOREACH(pval, &p->head, next) {
2461  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2462  pval->val);
2463 
2464  /* IPV6 or IPV4? */
2465  if (strchr(pval->val, ':') != NULL) {
2466  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2467  s->name, pval->val, cfg_prec->cfg);
2468  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2469  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2470  "add ipv6 server %s, ignoring", pval->val);
2471  }
2472  } else {
2473  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2474  s->name, pval->val, cfg_prec->cfg);
2475  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2476  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2477  "to add ipv4 server %s, ignoring",
2478  pval->val);
2479  }
2480  } /* else - if (strchr(pval->val, ':') != NULL) */
2481  } /* TAILQ_FOREACH(pval, &p->head, next) */
2482 
2483  } else if (strcasecmp("personality", p->name) == 0) {
2484  /* Personalities */
2485  int personality = HTPLookupPersonality(p->val);
2486  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2487  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2488 
2489  if (personality >= 0) {
2490  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2491  personality);
2492  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2493  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2494  "personality \"%s\", ignoring", p->val);
2495  } else {
2496  SCLogDebug("LIBHTP personality set to %s",
2497  HTPLookupPersonalityString(personality));
2498  }
2499 
2500  /* The IDS personality by default converts the path (and due to
2501  * our query string callback also the query string) to lowercase.
2502  * Signatures do not expect this, so override it. */
2503  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2504  } else {
2505  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2506  "\"%s\", ignoring", p->val);
2507  continue;
2508  }
2509 
2510  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2511  strcasecmp("request_body_limit", p->name) == 0) {
2512  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2513  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2514  "from conf file - %s. Killing engine", p->val);
2515  exit(EXIT_FAILURE);
2516  }
2517 
2518  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2519  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2520  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2521  "from conf file - %s. Killing engine", p->val);
2522  exit(EXIT_FAILURE);
2523  }
2524 
2525  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2526  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2527  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2528  "from conf file - %s. Killing engine", p->val);
2529  exit(EXIT_FAILURE);
2530  }
2531 
2532  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2533  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2534  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2535  "from conf file - %s. Killing engine", p->val);
2536  exit(EXIT_FAILURE);
2537  }
2538 
2539  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2540  if (ConfValIsTrue(p->val)) {
2541  htp_config_register_request_line(cfg_prec->cfg,
2542  HTPCallbackDoubleDecodeQuery);
2543  }
2544 
2545  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2546  if (ConfValIsTrue(p->val)) {
2547  htp_config_register_request_line(cfg_prec->cfg,
2548  HTPCallbackDoubleDecodePath);
2549  }
2550 
2551  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2552  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2553  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2554  "from conf file - %s. Killing engine", p->val);
2555  exit(EXIT_FAILURE);
2556  }
2557 
2558  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2559  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2560  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2561  "from conf file - %s. Killing engine", p->val);
2562  exit(EXIT_FAILURE);
2563  }
2564 
2565  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2566  uint32_t value = 2;
2567  if (ParseSizeStringU32(p->val, &value) < 0) {
2568  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2569  "from conf file - %s. Killing engine", p->val);
2570  exit(EXIT_FAILURE);
2571  }
2572 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2573  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2574 #else
2575  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2576  "to %u, libhtp version too old", value);
2577 #endif
2578  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2579  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2580  HTP_DECODER_URL_PATH,
2581  ConfValIsTrue(p->val));
2582  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2583  if (strlen(p->val) == 1) {
2584  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2585  HTP_DECODER_URL_PATH,
2586  p->val[0]);
2587  } else {
2588  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2589  "for libhtp param path-bestfit-replacement-char");
2590  }
2591  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2592  htp_config_set_convert_lowercase(cfg_prec->cfg,
2593  HTP_DECODER_URL_PATH,
2594  ConfValIsTrue(p->val));
2595  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2596  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2597  HTP_DECODER_URL_PATH,
2598  ConfValIsTrue(p->val));
2599  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2600  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2601  HTP_DECODER_URL_PATH,
2602  ConfValIsTrue(p->val));
2603  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2604  htp_config_set_path_separators_compress(cfg_prec->cfg,
2605  HTP_DECODER_URL_PATH,
2606  ConfValIsTrue(p->val));
2607  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2608  htp_config_set_path_separators_decode(cfg_prec->cfg,
2609  HTP_DECODER_URL_PATH,
2610  ConfValIsTrue(p->val));
2611  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2612  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2613  HTP_DECODER_URL_PATH,
2614  ConfValIsTrue(p->val));
2615  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2616  enum htp_url_encoding_handling_t handling;
2617  if (strcasecmp(p->val, "preserve_percent") == 0) {
2618  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2619  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2620  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2621  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2622  handling = HTP_URL_DECODE_PROCESS_INVALID;
2623  } else {
2624  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2625  "for libhtp param path-url-encoding-invalid-handling");
2626  return;
2627  }
2628  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2629  HTP_DECODER_URL_PATH,
2630  handling);
2631  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2632  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2633  HTP_DECODER_URL_PATH,
2634  ConfValIsTrue(p->val));
2635  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2636  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2637  SCLogDebug("uri-include-all %s",
2638  cfg_prec->uri_include_all ? "enabled" : "disabled");
2639  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2640  htp_config_set_plusspace_decode(cfg_prec->cfg,
2641  HTP_DECODER_URLENCODED,
2642  ConfValIsTrue(p->val));
2643  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2644  uint32_t limit = 0;
2645  if (ParseSizeStringU32(p->val, &limit) < 0) {
2646  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2647  "from conf file - %s. Killing engine", p->val);
2648  exit(EXIT_FAILURE);
2649  }
2650  if (limit == 0) {
2651  FatalError(SC_ERR_FATAL, "Error meta-field-limit "
2652  "from conf file cannot be 0. Killing engine");
2653  }
2654  /* set default soft-limit with our new hard limit */
2655  htp_config_set_field_limits(cfg_prec->cfg,
2657  (size_t)limit);
2658 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2659  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2660  uint32_t limit = 0;
2661  if (ParseSizeStringU32(p->val, &limit) < 0) {
2662  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'lzma-memlimit' "
2663  "from conf file - %s.", p->val);
2664  }
2665  if (limit == 0) {
2666  FatalError(SC_ERR_SIZE_PARSE, "'lzma-memlimit' "
2667  "from conf file cannot be 0.");
2668  }
2669  /* set default soft-limit with our new hard limit */
2670  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2671  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2672 #endif
2673 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2674  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2675  if (ConfValIsTrue(p->val)) {
2676  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2677  } else if (!ConfValIsFalse(p->val)) {
2678  int8_t limit;
2679  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2681  "failed to parse 'lzma-enabled' "
2682  "from conf file - %s.",
2683  p->val);
2684  }
2685  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2686  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2687  }
2688 #endif
2689 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2690  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2691  uint32_t limit = 0;
2692  if (ParseSizeStringU32(p->val, &limit) < 0) {
2693  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'compression-bomb-limit' "
2694  "from conf file - %s.", p->val);
2695  }
2696  if (limit == 0) {
2697  FatalError(SC_ERR_SIZE_PARSE, "'compression-bomb-limit' "
2698  "from conf file cannot be 0.");
2699  }
2700  /* set default soft-limit with our new hard limit */
2701  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2702  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2703 #endif
2704 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2705  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2706  uint32_t limit = 0;
2707  // between 1 usec and 1 second
2708  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2710  "failed to parse 'decompression-time-limit' "
2711  "from conf file - %s.",
2712  p->val);
2713  }
2714  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2715  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2716 #endif
2717  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2718  if (!g_disable_randomness) {
2719  cfg_prec->randomize = ConfValIsTrue(p->val);
2720  }
2721  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2722  uint32_t range;
2723  if (StringParseU32RangeCheck(&range, 10, 0,
2724  (const char *)p->val, 0, 100) < 0) {
2725  SCLogError(SC_ERR_INVALID_VALUE, "Invalid value for randomize"
2726  "-inspection-range setting from conf file - \"%s\"."
2727  " It should be a valid integer less than or equal to 100."
2728  " Killing engine",
2729  p->val);
2730  exit(EXIT_FAILURE);
2731  }
2732  cfg_prec->randomize_range = range;
2733  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2734  if (ConfValIsTrue(p->val)) {
2735  cfg_prec->http_body_inline = 1;
2736  } else if (ConfValIsFalse(p->val)) {
2737  cfg_prec->http_body_inline = 0;
2738  } else {
2739  if (strcmp("auto", p->val) != 0) {
2740  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2741  }
2742  if (EngineModeIsIPS()) {
2743  cfg_prec->http_body_inline = 1;
2744  } else {
2745  cfg_prec->http_body_inline = 0;
2746  }
2747  }
2748  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2749  ConfNode *pval;
2750 
2751  TAILQ_FOREACH(pval, &p->head, next) {
2752  if (strcasecmp("enabled", pval->name) == 0) {
2753  if (ConfValIsTrue(pval->val)) {
2754  cfg_prec->swf_decompression_enabled = 1;
2755  } else if (ConfValIsFalse(pval->val)) {
2756  cfg_prec->swf_decompression_enabled = 0;
2757  } else {
2758  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2759  }
2760  } else if (strcasecmp("type", pval->name) == 0) {
2761  if (strcasecmp("no", pval->val) == 0) {
2763  } else if (strcasecmp("deflate", pval->val) == 0) {
2765  } else if (strcasecmp("lzma", pval->val) == 0) {
2767  } else if (strcasecmp("both", pval->val) == 0) {
2769  } else {
2771  "Invalid entry for "
2772  "swf-decompression.type: %s - "
2773  "Killing engine", pval->val);
2774  exit(EXIT_FAILURE);
2775  }
2776  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2777  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2779  "Error parsing swf-decompression.compression-depth "
2780  "from conf file - %s. Killing engine", p->val);
2781  exit(EXIT_FAILURE);
2782  }
2783  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2784  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2786  "Error parsing swf-decompression.decompression-depth "
2787  "from conf file - %s. Killing engine", p->val);
2788  exit(EXIT_FAILURE);
2789  }
2790  } else {
2791  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2792  }
2793  }
2794  } else {
2795  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2796  "default config: %s", p->name);
2797  }
2798  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2799 
2800  return;
2801 }
2802 
2803 void HTPConfigure(void)
2804 {
2805  SCEnter();
2806 
2807  cfglist.next = NULL;
2808 
2809  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2810  if (NULL == cfgtree)
2811  exit(EXIT_FAILURE);
2812 
2813  /* Default Config */
2814  cfglist.cfg = htp_config_create();
2815  if (NULL == cfglist.cfg) {
2816  FatalError(SC_ERR_FATAL, "Failed to create HTP default config");
2817  }
2818  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2819  HTPConfigSetDefaultsPhase1(&cfglist);
2820  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2821  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2822  cfgtree);
2823  } else {
2824  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2825  }
2826  HTPConfigSetDefaultsPhase2("default", &cfglist);
2827 
2828  HTPParseMemcap();
2829 
2830  /* Read server config and create a parser for each IP in radix tree */
2831  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2832  if (server_config == NULL) {
2833  server_config = ConfGetNode("libhtp.server-config");
2834  if (server_config == NULL) {
2835  SCLogDebug("LIBHTP Configuring %p", server_config);
2836  SCReturn;
2837  }
2838  }
2839  SCLogDebug("LIBHTP Configuring %p", server_config);
2840 
2841  ConfNode *si;
2842  /* Server Nodes */
2843  TAILQ_FOREACH(si, &server_config->head, next) {
2844  /* Need the named node, not the index */
2845  ConfNode *s = TAILQ_FIRST(&si->head);
2846  if (NULL == s) {
2847  SCLogDebug("LIBHTP s NULL");
2848  continue;
2849  }
2850 
2851  SCLogDebug("LIBHTP server %s", s->name);
2852 
2853  HTPCfgRec *nextrec = cfglist.next;
2854  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2855  if (NULL == htprec)
2856  exit(EXIT_FAILURE);
2857  memset(htprec, 0x00, sizeof(*htprec));
2858 
2859  cfglist.next = htprec;
2860 
2861  cfglist.next->next = nextrec;
2862  cfglist.next->cfg = htp_config_create();
2863  if (NULL == cfglist.next->cfg) {
2864  FatalError(SC_ERR_FATAL, "Failed to create HTP server config");
2865  }
2866 
2867  HTPConfigSetDefaultsPhase1(htprec);
2868  HTPConfigParseParameters(htprec, s, cfgtree);
2869  HTPConfigSetDefaultsPhase2(s->name, htprec);
2870  }
2871 
2872  SCReturn;
2873 }
2874 
2876 {
2877 #ifdef DEBUG
2878  SCMutexLock(&htp_state_mem_lock);
2879  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2880  SCMutexUnlock(&htp_state_mem_lock);
2881 #endif
2882 }
2883 
2884 /** \internal
2885  * \brief get files callback
2886  * \param state state ptr
2887  * \param direction flow direction
2888  * \retval files files ptr
2889  */
2890 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
2891 {
2892  if (state == NULL)
2893  return NULL;
2894 
2895  HtpState *http_state = (HtpState *)state;
2896 
2897  if (direction & STREAM_TOCLIENT) {
2898  SCReturnPtr(http_state->files_tc, "FileContainer");
2899  } else {
2900  SCReturnPtr(http_state->files_ts, "FileContainer");
2901  }
2902 }
2903 
2904 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2905 {
2906  if (direction & STREAM_TOSERVER)
2907  return ((htp_tx_t *)tx)->request_progress;
2908  else
2909  return ((htp_tx_t *)tx)->response_progress;
2910 }
2911 
2912 static uint64_t HTPStateGetTxCnt(void *alstate)
2913 {
2914  HtpState *http_state = (HtpState *)alstate;
2915 
2916  if (http_state != NULL && http_state->conn != NULL) {
2917  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
2918  if (size < 0)
2919  return 0ULL;
2920  SCLogDebug("size %"PRIu64, size);
2921  return (uint64_t)size;
2922  } else {
2923  return 0ULL;
2924  }
2925 }
2926 
2927 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2928 {
2929  HtpState *http_state = (HtpState *)alstate;
2930 
2931  if (http_state != NULL && http_state->conn != NULL)
2932  return htp_list_get(http_state->conn->transactions, tx_id);
2933  else
2934  return NULL;
2935 }
2936 
2937 void *HtpGetTxForH2(void *alstate)
2938 {
2939  // gets last transaction
2940  HtpState *http_state = (HtpState *)alstate;
2941  if (http_state != NULL && http_state->conn != NULL) {
2942  size_t txid = htp_list_array_size(http_state->conn->transactions);
2943  if (txid > 0) {
2944  return htp_list_get(http_state->conn->transactions, txid - 1);
2945  }
2946  }
2947  return NULL;
2948 }
2949 
2950 static int HTPStateGetEventInfo(const char *event_name,
2951  int *event_id, AppLayerEventType *event_type)
2952 {
2953  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
2954  if (*event_id == -1) {
2955  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
2956  "http's enum map table.", event_name);
2957  /* this should be treated as fatal */
2958  return -1;
2959  }
2960 
2961  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2962 
2963  return 0;
2964 }
2965 
2966 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
2967  AppLayerEventType *event_type)
2968 {
2969  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
2970  if (*event_name == NULL) {
2971  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
2972  "http's enum map table.", event_id);
2973  /* this should be treated as fatal */
2974  return -1;
2975  }
2976 
2977  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2978 
2979  return 0;
2980 }
2981 
2982 static void HTPStateTruncate(void *state, uint8_t direction)
2983 {
2984  FileContainer *fc = HTPStateGetFiles(state, direction);
2985  if (fc != NULL) {
2987  }
2988 }
2989 
2990 static AppLayerTxData *HTPGetTxData(void *vtx)
2991 {
2992  htp_tx_t *tx = (htp_tx_t *)vtx;
2993  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2994  if (tx_ud) {
2995  return &tx_ud->tx_data;
2996  }
2997  return NULL;
2998 }
2999 
3000 static int HTPRegisterPatternsForProtocolDetection(void)
3001 {
3002  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3003  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3004  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3005  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3006  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3007  const char *spacings[] = { "|20|", "|09|", NULL };
3008  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3009 
3010  int methods_pos;
3011  int spacings_pos;
3012  int versions_pos;
3013  int register_result;
3014  char method_buffer[32] = "";
3015 
3016  /* Loop through all the methods ands spacings and register the patterns */
3017  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3018  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3019 
3020  /* Combine the method name and the spacing */
3021  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3022 
3023  /* Register the new method+spacing pattern
3024  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3025  * but the pattern matching should only be one char
3026  */
3027  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3028  method_buffer, strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3029  if (register_result < 0) {
3030  return -1;
3031  }
3032  }
3033  }
3034 
3035  /* Loop through all the http verions patterns that are TO_CLIENT */
3036  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3037  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3038  versions[versions_pos], strlen(versions[versions_pos]), 0, STREAM_TOCLIENT);
3039  if (register_result < 0) {
3040  return -1;
3041  }
3042  }
3043 
3044  return 0;
3045 }
3046 
3047 /**
3048  * \brief Register the HTTP protocol and state handling functions to APP layer
3049  * of the engine.
3050  */
3052 {
3053  SCEnter();
3054 
3055  const char *proto_name = "http";
3056 
3057  /** HTTP */
3058  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3060  if (HTPRegisterPatternsForProtocolDetection() < 0)
3061  return;
3062  } else {
3063  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3064  proto_name);
3065  return;
3066  }
3067 
3068  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3069  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3070  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3071  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetFiles);
3073  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3074  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3075  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3076 
3078  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3079  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3081  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3082 
3083  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTruncate);
3084  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3085 
3087  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3088 
3090  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3092  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3093  SC_ATOMIC_INIT(htp_config_flags);
3094  /* This parser accepts gaps. */
3099  HTPConfigure();
3100  } else {
3101  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3102  "still on.", proto_name);
3103  }
3104 #ifdef UNITTESTS
3105  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3106 #endif
3107 
3108  SCReturn;
3109 }
3110 
3111 #ifdef UNITTESTS
3112 static HTPCfgRec cfglist_backup;
3113 
3115 {
3116  cfglist_backup = cfglist;
3117 
3118  return;
3119 }
3120 
3122 {
3123  cfglist = cfglist_backup;
3124 
3125  return;
3126 }
3127 
3128 /** \test Test case where chunks are sent in smaller chunks and check the
3129  * response of the parser from HTP library. */
3130 static int HTPParserTest01(void)
3131 {
3132  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3133  " Data is c0oL!";
3134  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3135 
3136  TcpSession ssn;
3137  memset(&ssn, 0, sizeof(ssn));
3138 
3141 
3142  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3143  FAIL_IF_NULL(f);
3144  f->protoctx = &ssn;
3145  f->proto = IPPROTO_TCP;
3146  f->alproto = ALPROTO_HTTP1;
3147 
3148  StreamTcpInitConfig(true);
3149 
3150  uint32_t u;
3151  for (u = 0; u < httplen1; u++) {
3152  uint8_t flags = 0;
3153 
3154  if (u == 0)
3156  else if (u == (httplen1 - 1))
3158  else
3160 
3161  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3162  FAIL_IF(r != 0);
3163  }
3164 
3165  HtpState *htp_state = f->alstate;
3166  FAIL_IF_NULL(htp_state);
3167 
3168  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3169  FAIL_IF_NULL(tx);
3170 
3171  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3172  FAIL_IF_NULL(h);
3173 
3174  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3175  FAIL_IF(tx->request_method_number != HTP_M_POST);
3176  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3177 
3179  StreamTcpFreeConfig(true);
3180  UTHFreeFlow(f);
3181  PASS;
3182 }
3183 
3184 /** \test Test folding in 1 read case */
3185 static int HTPParserTest01b(void)
3186 {
3187  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3188  " Data is c0oL!";
3189  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3190 
3191  TcpSession ssn;
3192  memset(&ssn, 0, sizeof(ssn));
3193 
3196 
3197  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3198  FAIL_IF_NULL(f);
3199  f->protoctx = &ssn;
3200  f->proto = IPPROTO_TCP;
3201  f->alproto = ALPROTO_HTTP1;
3202 
3203  StreamTcpInitConfig(true);
3204 
3206  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3207  FAIL_IF(r != 0);
3208 
3209  HtpState *htp_state = f->alstate;
3210  FAIL_IF_NULL(htp_state);
3211 
3212  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3213  FAIL_IF_NULL(tx);
3214 
3215  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3216  FAIL_IF_NULL(h);
3217 
3218  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3219  FAIL_IF(tx->request_method_number != HTP_M_POST);
3220  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3221 
3223  StreamTcpFreeConfig(true);
3224  UTHFreeFlow(f);
3225  PASS;
3226 }
3227 
3228 /** \test Test folding in 1byte per read case */
3229 static int HTPParserTest01c(void)
3230 {
3231  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3232  " Data is c0oL!";
3233  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3234 
3235  TcpSession ssn;
3236  memset(&ssn, 0, sizeof(ssn));
3237 
3240 
3241  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3242  FAIL_IF_NULL(f);
3243  f->protoctx = &ssn;
3244  f->proto = IPPROTO_TCP;
3245  f->alproto = ALPROTO_HTTP1;
3246 
3247  StreamTcpInitConfig(true);
3248 
3249  uint32_t u;
3250  for (u = 0; u < httplen1; u++) {
3251  uint8_t flags = 0;
3252 
3253  if (u == 0)
3255  else if (u == (httplen1 - 1))
3257  else
3259 
3260  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3261  FAIL_IF(r != 0);
3262  }
3263 
3264  HtpState *htp_state = f->alstate;
3265  FAIL_IF_NULL(htp_state);
3266 
3267  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3268  FAIL_IF_NULL(tx);
3269 
3270  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3271  FAIL_IF_NULL(h);
3272 
3273  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3274  FAIL_IF(tx->request_method_number != HTP_M_POST);
3275  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3276 
3278  StreamTcpFreeConfig(true);
3279  UTHFreeFlow(f);
3280  PASS;
3281 }
3282 
3283 /** \test Test case where chunks are sent in smaller chunks and check the
3284  * response of the parser from HTP library. */
3285 static int HTPParserTest01a(void)
3286 {
3287  int result = 0;
3288  Flow *f = NULL;
3289  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3290  " Data is c0oL!";
3291  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3292  TcpSession ssn;
3293  HtpState *htp_state = NULL;
3294  int r = 0;
3296 
3297  memset(&ssn, 0, sizeof(ssn));
3298 
3299  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3300  if (f == NULL)
3301  goto end;
3302  f->protoctx = &ssn;
3303  f->proto = IPPROTO_TCP;
3304  f->alproto = ALPROTO_HTTP1;
3305 
3306  StreamTcpInitConfig(true);
3307 
3308  uint32_t u;
3309  for (u = 0; u < httplen1; u++) {
3310  uint8_t flags = 0;
3311 
3312  if (u == 0)
3314  else if (u == (httplen1 - 1))
3316  else
3318 
3319  FLOWLOCK_WRLOCK(f);
3320  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3321  if (r != 0) {
3322  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3323  " 0: ", u, r);
3324  FLOWLOCK_UNLOCK(f);
3325  goto end;
3326  }
3327  FLOWLOCK_UNLOCK(f);
3328  }
3329 
3330  htp_state = f->alstate;
3331  if (htp_state == NULL) {
3332  printf("no http state: ");
3333  goto end;
3334  }
3335 
3336  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3337  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3338  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3339  || tx->request_method_number != HTP_M_POST ||
3340  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3341  {
3342  printf("expected header value: Victor/1.0 and got %s: and expected"
3343  " method: POST and got %s, expected protocol number HTTP/1.0"
3344  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3345  bstr_util_strdup_to_c(tx->request_method),
3346  bstr_util_strdup_to_c(tx->request_protocol));
3347  goto end;
3348  }
3349  result = 1;
3350 end:
3351  if (alp_tctx != NULL)
3353  StreamTcpFreeConfig(true);
3354  UTHFreeFlow(f);
3355  return result;
3356 }
3357 
3358 /** \test See how it deals with an incomplete request. */
3359 static int HTPParserTest02(void)
3360 {
3361  int result = 0;
3362  Flow *f = NULL;
3363  uint8_t httpbuf1[] = "POST";
3364  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3365  TcpSession ssn;
3366  HtpState *http_state = NULL;
3368 
3369  memset(&ssn, 0, sizeof(ssn));
3370 
3371  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3372  if (f == NULL)
3373  goto end;
3374  f->protoctx = &ssn;
3375  f->proto = IPPROTO_TCP;
3376  f->alproto = ALPROTO_HTTP1;
3377 
3378  StreamTcpInitConfig(true);
3379 
3380  FLOWLOCK_WRLOCK(f);
3381  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3382  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3383  if (r != 0) {
3384  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3385  FLOWLOCK_UNLOCK(f);
3386  goto end;
3387  }
3388  FLOWLOCK_UNLOCK(f);
3389 
3390  http_state = f->alstate;
3391  if (http_state == NULL) {
3392  printf("no http state: ");
3393  goto end;
3394  }
3395 
3396  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3397  FAIL_IF_NULL(tx);
3398  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3399  FAIL_IF_NOT_NULL(h);
3400 
3401  FAIL_IF_NULL(tx->request_method);
3402  char *method = bstr_util_strdup_to_c(tx->request_method);
3403  FAIL_IF_NULL(method);
3404 
3405  FAIL_IF(strcmp(method, "POST") != 0);
3406  SCFree(method);
3407 
3408  result = 1;
3409 end:
3410  if (alp_tctx != NULL)
3412  StreamTcpFreeConfig(true);
3413  UTHFreeFlow(f);
3414  return result;
3415 }
3416 
3417 /** \test Test case where method is invalid and data is sent in smaller chunks
3418  * and check the response of the parser from HTP library. */
3419 static int HTPParserTest03(void)
3420 {
3421  int result = 0;
3422  Flow *f = NULL;
3423  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3424  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3425  TcpSession ssn;
3426  HtpState *htp_state = NULL;
3427  int r = 0;
3429 
3430  memset(&ssn, 0, sizeof(ssn));
3431 
3432  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3433  if (f == NULL)
3434  goto end;
3435  f->protoctx = &ssn;
3436  f->proto = IPPROTO_TCP;
3437  f->alproto = ALPROTO_HTTP1;
3438 
3439  StreamTcpInitConfig(true);
3440 
3441  uint32_t u;
3442  for (u = 0; u < httplen1; u++) {
3443  uint8_t flags = 0;
3444 
3445  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3446  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3447  else flags = STREAM_TOSERVER;
3448 
3449  FLOWLOCK_WRLOCK(f);
3450  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3451  if (r != 0) {
3452  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3453  " 0: ", u, r);
3454  FLOWLOCK_UNLOCK(f);
3455  goto end;
3456  }
3457  FLOWLOCK_UNLOCK(f);
3458  }
3459  htp_state = f->alstate;
3460  if (htp_state == NULL) {
3461  printf("no http state: ");
3462  goto end;
3463  }
3464 
3465  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3466 
3467  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3468  if (tx->request_method_number != HTP_M_UNKNOWN ||
3469  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3470  {
3471  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3472  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3473  bstr_util_strdup_to_c(tx->request_protocol));
3474  goto end;
3475  }
3476  result = 1;
3477 end:
3478  if (alp_tctx != NULL)
3480  StreamTcpFreeConfig(true);
3481  UTHFreeFlow(f);
3482  return result;
3483 }
3484 
3485 /** \test Test case where invalid data is sent and check the response of the
3486  * parser from HTP library. */
3487 static int HTPParserTest04(void)
3488 {
3489  int result = 0;
3490  Flow *f = NULL;
3491  HtpState *htp_state = NULL;
3492  uint8_t httpbuf1[] = "World!\r\n";
3493  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3494  TcpSession ssn;
3495  int r = 0;
3497 
3498  memset(&ssn, 0, sizeof(ssn));
3499 
3500  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3501  if (f == NULL)
3502  goto end;
3503  f->protoctx = &ssn;
3504  f->proto = IPPROTO_TCP;
3505  f->alproto = ALPROTO_HTTP1;
3506 
3507  StreamTcpInitConfig(true);
3508 
3509  FLOWLOCK_WRLOCK(f);
3511  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3512  if (r != 0) {
3513  FLOWLOCK_UNLOCK(f);
3514  goto end;
3515  }
3516  FLOWLOCK_UNLOCK(f);
3517 
3518  htp_state = f->alstate;
3519  if (htp_state == NULL) {
3520  printf("no http state: ");
3521  goto end;
3522  }
3523 
3524  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3525  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3526  if (tx->request_method_number != HTP_M_UNKNOWN ||
3527  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3528  {
3529  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3530  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3531  bstr_util_strdup_to_c(tx->request_protocol));
3532  goto end;
3533  }
3534  result = 1;
3535 end:
3536  if (alp_tctx != NULL)
3538  StreamTcpFreeConfig(true);
3539  UTHFreeFlow(f);
3540  return result;
3541 }
3542 
3543 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3544  * properly parsed them and also keeps them separated. */
3545 static int HTPParserTest05(void)
3546 {
3547  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3548  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3549  uint8_t httpbuf2[] = "Post D";
3550  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3551  uint8_t httpbuf3[] = "ata is c0oL!";
3552  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3553 
3554  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3555  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3556  uint8_t httpbuf5[] = "post R";
3557  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3558  uint8_t httpbuf6[] = "esults are tha bomb!";
3559  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3560 
3561  TcpSession ssn;
3562  memset(&ssn, 0, sizeof(ssn));
3563 
3566 
3567  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3568  FAIL_IF_NULL(f);
3569  f->protoctx = &ssn;
3570  f->proto = IPPROTO_TCP;
3571  f->alproto = ALPROTO_HTTP1;
3572 
3573  StreamTcpInitConfig(true);
3574 
3575  int r = AppLayerParserParse(
3576  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3577  FAIL_IF(r != 0);
3578 
3579  r = AppLayerParserParse(
3580  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3581  FAIL_IF(r != 0);
3582 
3583  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3584  FAIL_IF(r != 0);
3585 
3586  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3587  FAIL_IF(r != 0);
3588 
3589  r = AppLayerParserParse(
3590  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3591  FAIL_IF(r != 0);
3592 
3593  r = AppLayerParserParse(
3594  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3595  FAIL_IF(r != 0);
3596 
3597  HtpState *http_state = f->alstate;
3598  FAIL_IF_NULL(http_state);
3599 
3600  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3601  FAIL_IF_NULL(tx);
3602  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3603  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3604 
3605  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3606  FAIL_IF_NULL(h);
3607 
3608  FAIL_IF_NOT(tx->response_status_number == 200);
3609 
3611  StreamTcpFreeConfig(true);
3612  UTHFreeFlow(f);
3613  PASS;
3614 }
3615 
3616 /** \test Test proper chunked encoded response body
3617  */
3618 static int HTPParserTest06(void)
3619 {
3620  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3621  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3622  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3623  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3624  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3625  "GMT\r\n"
3626  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3627  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3628  "FrontPage/5.0.2.2510\r\n"
3629  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3630  "chunked\r\n"
3631  "Content-Type: text/html\r\n\r\n"
3632  "580\r\n"
3633  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3634  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3635  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3636  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3637  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3638  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3639  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3640  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3641  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3642  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3643  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3644  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3645  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3646  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3647  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3648  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3649  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3650  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3651  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3652  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3653  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3654  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3655  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3656  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3657  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3658  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3659  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3660  "aHA=\r\n0\r\n\r\n";
3661  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3662  TcpSession ssn;
3663 
3666 
3667  memset(&ssn, 0, sizeof(ssn));
3668 
3669  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3670  FAIL_IF_NULL(f);
3671  f->protoctx = &ssn;
3672  f->proto = IPPROTO_TCP;
3673  f->alproto = ALPROTO_HTTP1;
3674 
3675  StreamTcpInitConfig(true);
3676 
3677  int r = AppLayerParserParse(
3678  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3679  FAIL_IF(r != 0);
3680  r = AppLayerParserParse(
3681  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3682  FAIL_IF(r != 0);
3683 
3684  HtpState *http_state = f->alstate;
3685  FAIL_IF_NULL(http_state);
3686 
3687  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3688  FAIL_IF_NULL(tx);
3689 
3690  FAIL_IF(tx->request_method_number != HTP_M_GET);
3691  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3692 
3693  FAIL_IF(tx->response_status_number != 200);
3694  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3695 
3696  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3697  FAIL_IF_NULL(h);
3698 
3700  StreamTcpFreeConfig(true);
3701  UTHFreeFlow(f);
3702  PASS;
3703 }
3704 
3705 /** \test
3706  */
3707 static int HTPParserTest07(void)
3708 {
3709  int result = 0;
3710  Flow *f = NULL;
3711  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3712  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3713  TcpSession ssn;
3714  HtpState *htp_state = NULL;
3715  int r = 0;
3717 
3718  memset(&ssn, 0, sizeof(ssn));
3719 
3720  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3721  if (f == NULL)
3722  goto end;
3723  f->protoctx = &ssn;
3724  f->proto = IPPROTO_TCP;
3725  f->alproto = ALPROTO_HTTP1;
3726 
3727  StreamTcpInitConfig(true);
3728 
3729  uint32_t u;
3730  for (u = 0; u < httplen1; u++) {
3731  uint8_t flags = 0;
3732 
3733  if (u == 0)
3735  else if (u == (httplen1 - 1))
3737  else
3739 
3740  FLOWLOCK_WRLOCK(f);
3741  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3742  if (r != 0) {
3743  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3744  " 0: ", u, r);
3745  FLOWLOCK_UNLOCK(f);
3746  goto end;
3747  }
3748  FLOWLOCK_UNLOCK(f);
3749  }
3750 
3751  htp_state = f->alstate;
3752  if (htp_state == NULL) {
3753  printf("no http state: ");
3754  goto end;
3755  }
3756 
3757  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3758  size_t reflen = sizeof(ref) - 1;
3759 
3760  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3761  if (tx == NULL)
3762  goto end;
3763  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3764  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3765  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3766  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3767  (uintmax_t)reflen,
3768  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3769  goto end;
3770  }
3771 
3772  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3773  bstr_len(tx_ud->request_uri_normalized)) != 0)
3774  {
3775  printf("normalized uri \"");
3776  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3777  printf("\" != \"");
3778  PrintRawUriFp(stdout, ref, reflen);
3779  printf("\": ");
3780  goto end;
3781  }
3782  }
3783 
3784  result = 1;
3785 end:
3786  if (alp_tctx != NULL)
3788  StreamTcpFreeConfig(true);
3789  UTHFreeFlow(f);
3790  return result;
3791 }
3792 
3793 #include "conf-yaml-loader.h"
3794 
3795 /** \test Abort
3796  */
3797 static int HTPParserTest08(void)
3798 {
3799  int result = 0;
3800  Flow *f = NULL;
3801  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3802  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3803  TcpSession ssn;
3805 
3806  char input[] = "\
3807 %YAML 1.1\n\
3808 ---\n\
3809 libhtp:\n\
3810 \n\
3811  default-config:\n\
3812  personality: IDS\n\
3813 ";
3814 
3816  ConfInit();
3818 
3819  ConfYamlLoadString(input, strlen(input));
3820  HTPConfigure();
3821 
3822  HtpState *htp_state = NULL;
3823  int r = 0;
3824  memset(&ssn, 0, sizeof(ssn));
3825 
3826  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3827  if (f == NULL)
3828  goto end;
3829  f->protoctx = &ssn;
3830  f->proto = IPPROTO_TCP;
3831  f->alproto = ALPROTO_HTTP1;
3832 
3833  StreamTcpInitConfig(true);
3834 
3835  uint8_t flags = 0;
3837 
3838  FLOWLOCK_WRLOCK(f);
3839  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3840  if (r != 0) {
3841  printf("toserver chunk returned %" PRId32 ", expected"
3842  " 0: ", r);
3843  result = 0;
3844  FLOWLOCK_UNLOCK(f);
3845  goto end;
3846  }
3847  FLOWLOCK_UNLOCK(f);
3848 
3849  htp_state = f->alstate;
3850  if (htp_state == NULL) {
3851  printf("no http state: ");
3852  result = 0;
3853  goto end;
3854  }
3855 
3856  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3857  if (tx == NULL)
3858  goto end;
3859  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3860  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3861  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3862  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3863  bstr_len(tx_ud->request_uri_normalized));
3864  }
3865 
3866  result = 1;
3867 end:
3868  if (alp_tctx != NULL)
3870  StreamTcpFreeConfig(true);
3871  HTPFreeConfig();
3872  ConfDeInit();
3875  UTHFreeFlow(f);
3876  return result;
3877 }
3878 
3879 /** \test Abort
3880  */
3881 static int HTPParserTest09(void)
3882 {
3883  int result = 0;
3884  Flow *f = NULL;
3885  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3886  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3887  TcpSession ssn;
3889 
3890  char input[] = "\
3891 %YAML 1.1\n\
3892 ---\n\
3893 libhtp:\n\
3894 \n\
3895  default-config:\n\
3896  personality: Apache_2_2\n\
3897 ";
3898 
3900  ConfInit();
3902 
3903  ConfYamlLoadString(input, strlen(input));
3904  HTPConfigure();
3905 
3906  HtpState *htp_state = NULL;
3907  int r = 0;
3908 
3909  memset(&ssn, 0, sizeof(ssn));
3910 
3911  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3912  if (f == NULL)
3913  goto end;
3914  f->protoctx = &ssn;
3915  f->proto = IPPROTO_TCP;
3916  f->alproto = ALPROTO_HTTP1;
3917 
3918  StreamTcpInitConfig(true);
3919 
3920  uint8_t flags = 0;
3922 
3923  FLOWLOCK_WRLOCK(f);
3924  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3925  if (r != 0) {
3926  printf("toserver chunk returned %" PRId32 ", expected"
3927  " 0: ", r);
3928  FLOWLOCK_UNLOCK(f);
3929  goto end;
3930  }
3931  FLOWLOCK_UNLOCK(f);
3932 
3933  htp_state = f->alstate;
3934  if (htp_state == NULL) {
3935  printf("no http state: ");
3936  goto end;
3937  }
3938 
3939  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3940  if (tx == NULL)
3941  goto end;
3942  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3943  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3944  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3945  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3946  bstr_len(tx_ud->request_uri_normalized));
3947  }
3948 
3949  result = 1;
3950 end:
3951  if (alp_tctx != NULL)
3953  StreamTcpFreeConfig(true);
3954  HTPFreeConfig();
3955  ConfDeInit();
3958  UTHFreeFlow(f);
3959  return result;
3960 }
3961 
3962 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
3963  */
3964 static int HTPParserTest10(void)
3965 {
3966  int result = 0;
3967  Flow *f = NULL;
3968  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
3969  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3970  TcpSession ssn;
3971  HtpState *htp_state = NULL;
3972  int r = 0;
3974 
3975  memset(&ssn, 0, sizeof(ssn));
3976 
3977  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3978  if (f == NULL)
3979  goto end;
3980  f->protoctx = &ssn;
3981  f->proto = IPPROTO_TCP;
3982  f->alproto = ALPROTO_HTTP1;
3983 
3984  StreamTcpInitConfig(true);
3985 
3986  uint32_t u;
3987  for (u = 0; u < httplen1; u++) {
3988  uint8_t flags = 0;
3989 
3990  if (u == 0)
3992  else if (u == (httplen1 - 1))
3994  else
3996 
3997  FLOWLOCK_WRLOCK(f);
3998  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3999  if (r != 0) {
4000  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4001  " 0: ", u, r);
4002  FLOWLOCK_UNLOCK(f);
4003  goto end;
4004  }
4005  FLOWLOCK_UNLOCK(f);
4006  }
4007 
4008  htp_state = f->alstate;
4009  if (htp_state == NULL) {
4010  printf("no http state: ");
4011  goto end;
4012  }
4013 
4014  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4015  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4016  if (h == NULL) {
4017  goto end;
4018  }
4019 
4020  char *name = bstr_util_strdup_to_c(h->name);
4021  if (name == NULL) {
4022  goto end;
4023  }
4024 
4025  if (strcmp(name, "Host") != 0) {
4026  printf("header name not \"Host\", instead \"%s\": ", name);
4027  free(name);
4028  goto end;
4029  }
4030  free(name);
4031 
4032  char *value = bstr_util_strdup_to_c(h->value);
4033  if (value == NULL) {
4034  goto end;
4035  }
4036 
4037  if (strcmp(value, "www.google.com") != 0) {
4038  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4039  free(value);
4040  goto end;
4041  }
4042  free(value);
4043 
4044  result = 1;
4045 end:
4046  if (alp_tctx != NULL)
4048  StreamTcpFreeConfig(true);
4049  UTHFreeFlow(f);
4050  return result;
4051 }
4052 
4053 /** \test double encoding in path
4054  */
4055 static int HTPParserTest11(void)
4056 {
4057  int result = 0;
4058  Flow *f = NULL;
4059  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4060  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4061  TcpSession ssn;
4062  HtpState *htp_state = NULL;
4063  int r = 0;
4065 
4066  memset(&ssn, 0, sizeof(ssn));
4067 
4068  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4069  if (f == NULL)
4070  goto end;
4071  f->protoctx = &ssn;
4072  f->proto = IPPROTO_TCP;
4073  f->alproto = ALPROTO_HTTP1;
4074 
4075  StreamTcpInitConfig(true);
4076 
4077  uint32_t u;
4078  for (u = 0; u < httplen1; u++) {
4079  uint8_t flags = 0;
4080 
4081  if (u == 0)
4083  else if (u == (httplen1 - 1))
4085  else
4087 
4088  FLOWLOCK_WRLOCK(f);
4089  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4090  if (r != 0) {
4091  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4092  " 0: ", u, r);
4093  FLOWLOCK_UNLOCK(f);
4094  goto end;
4095  }
4096  FLOWLOCK_UNLOCK(f);
4097  }
4098 
4099  htp_state = f->alstate;
4100  if (htp_state == NULL) {
4101  printf("no http state: ");
4102  goto end;
4103  }
4104 
4105  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4106  if (tx == NULL)
4107  goto end;
4108  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4109  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4110  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4111  printf("normalized uri len should be 2, is %"PRIuMAX,
4112  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4113  goto end;
4114  }
4115 
4116  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4117  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4118  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4119  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4120  {
4121  printf("normalized uri \"");
4122  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4123  printf("\": ");
4124  goto end;
4125  }
4126  }
4127 
4128  result = 1;
4129 end:
4130  if (alp_tctx != NULL)
4132  StreamTcpFreeConfig(true);
4133  UTHFreeFlow(f);
4134  return result;
4135 }
4136 
4137 /** \test double encoding in query
4138  */
4139 static int HTPParserTest12(void)
4140 {
4141  int result = 0;
4142  Flow *f = NULL;
4143  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4144  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4145  TcpSession ssn;
4146  HtpState *htp_state = NULL;
4147  int r = 0;
4149 
4150  memset(&ssn, 0, sizeof(ssn));
4151 
4152  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4153  if (f == NULL)
4154  goto end;
4155  f->protoctx = &ssn;
4156  f->proto = IPPROTO_TCP;
4157  f->alproto = ALPROTO_HTTP1;
4158 
4159  StreamTcpInitConfig(true);
4160 
4161  uint32_t u;
4162  for (u = 0; u < httplen1; u++) {
4163  uint8_t flags = 0;
4164 
4165  if (u == 0)
4167  else if (u == (httplen1 - 1))
4169  else
4171 
4172  FLOWLOCK_WRLOCK(f);
4173  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4174  if (r != 0) {
4175  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4176  " 0: ", u, r);
4177  FLOWLOCK_UNLOCK(f);
4178  goto end;
4179  }
4180  FLOWLOCK_UNLOCK(f);
4181  }
4182 
4183  htp_state = f->alstate;
4184  if (htp_state == NULL) {
4185  printf("no http state: ");
4186  goto end;
4187  }
4188 
4189  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4190  if (tx == NULL)
4191  goto end;
4192  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4193  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4194  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4195  printf("normalized uri len should be 5, is %"PRIuMAX,
4196  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4197  goto end;
4198  }
4199 
4200  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4201  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4202  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4203  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4204  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4205  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4206  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4207  {
4208  printf("normalized uri \"");
4209  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4210  printf("\": ");
4211  goto end;
4212  }
4213  }
4214 
4215  result = 1;
4216  end:
4217  if (alp_tctx != NULL)
4219  StreamTcpFreeConfig(true);
4220  UTHFreeFlow(f);
4221  return result;
4222 }
4223 
4224 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4225  */
4226 static int HTPParserTest13(void)
4227 {
4228  int result = 0;
4229  Flow *f = NULL;
4230  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4231  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4232  TcpSession ssn;
4233  HtpState *htp_state = NULL;
4234  int r = 0;
4236 
4237  memset(&ssn, 0, sizeof(ssn));
4238 
4239  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4240  if (f == NULL)
4241  goto end;
4242  f->protoctx = &ssn;
4243  f->proto = IPPROTO_TCP;
4244  f->alproto = ALPROTO_HTTP1;
4245 
4246  StreamTcpInitConfig(true);
4247 
4248  uint32_t u;
4249  for (u = 0; u < httplen1; u++) {
4250  uint8_t flags = 0;
4251 
4252  if (u == 0)
4254  else if (u == (httplen1 - 1))
4256  else
4258 
4259  FLOWLOCK_WRLOCK(f);
4260  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4261  if (r != 0) {
4262  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4263  " 0: ", u, r);
4264  FLOWLOCK_UNLOCK(f);
4265  goto end;
4266  }
4267  FLOWLOCK_UNLOCK(f);
4268  }
4269 
4270  htp_state = f->alstate;
4271  if (htp_state == NULL) {
4272  printf("no http state: ");
4273  goto end;
4274  }
4275 
4276  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4277  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4278  if (h == NULL) {
4279  goto end;
4280  }
4281 
4282  char *name = bstr_util_strdup_to_c(h->name);
4283  if (name == NULL) {
4284  goto end;
4285  }
4286 
4287  if (strcmp(name, "Host") != 0) {
4288  printf("header name not \"Host\", instead \"%s\": ", name);
4289  free(name);
4290  goto end;
4291  }
4292  free(name);
4293 
4294  char *value = bstr_util_strdup_to_c(h->value);
4295  if (value == NULL) {
4296  goto end;
4297  }
4298 
4299  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4300  printf("header value not \"www.google.com\", instead \"");
4301  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4302  printf("\": ");
4303  free(value);
4304  goto end;
4305  }
4306  free(value);
4307 
4308  result = 1;
4309 end:
4310  if (alp_tctx != NULL)
4312  StreamTcpFreeConfig(true);
4313  UTHFreeFlow(f);
4314  return result;
4315 }
4316 
4317 /** \test Test basic config */
4318 static int HTPParserConfigTest01(void)
4319 {
4320  int ret = 0;
4321  char input[] = "\
4322 %YAML 1.1\n\
4323 ---\n\
4324 libhtp:\n\
4325 \n\
4326  default-config:\n\
4327  personality: IDS\n\
4328 \n\
4329  server-config:\n\
4330 \n\
4331  - apache-tomcat:\n\
4332  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4333  personality: Tomcat_6_0\n\
4334 \n\
4335  - iis7:\n\
4336  address: \n\
4337  - 192.168.0.0/24\n\
4338  - 192.168.10.0/24\n\
4339  personality: IIS_7_0\n\
4340 ";
4341 
4343  ConfInit();
4344 
4345  ConfYamlLoadString(input, strlen(input));
4346 
4347  ConfNode *outputs;
4348  outputs = ConfGetNode("libhtp.default-config.personality");
4349  if (outputs == NULL) {
4350  goto end;
4351  }
4352 
4353  outputs = ConfGetNode("libhtp.server-config");
4354  if (outputs == NULL) {
4355  goto end;
4356  }
4357 
4358  ConfNode *node = TAILQ_FIRST(&outputs->head);
4359  if (node == NULL) {
4360  goto end;
4361  }
4362  if (strcmp(node->name, "0") != 0) {
4363  goto end;
4364  }
4365  node = TAILQ_FIRST(&node->head);
4366  if (node == NULL) {
4367  goto end;
4368  }
4369  if (strcmp(node->name, "apache-tomcat") != 0) {
4370  goto end;
4371  }
4372 
4373  int i = 0;
4374  ConfNode *n;
4375 
4376  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4377  if (node2 == NULL) {
4378  goto end;
4379  }
4380  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4381  goto end;
4382  }
4383 
4384  node = ConfNodeLookupChild(node, "address");
4385  if (node == NULL) {
4386  goto end;
4387  }
4388  TAILQ_FOREACH(n, &node->head, next) {
4389  if (n == NULL) {
4390  goto end;
4391  }
4392 
4393  switch(i) {
4394  case 0:
4395  if (strcmp(n->name, "0") != 0) {
4396  goto end;
4397  }
4398  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4399  goto end;
4400  }
4401  break;
4402  case 1:
4403  if (strcmp(n->name, "1") != 0) {
4404  goto end;
4405  }
4406  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4407  goto end;
4408  }
4409  break;
4410  case 2:
4411  if (strcmp(n->name, "2") != 0) {
4412  goto end;
4413  }
4414  if (strcmp(n->val, "::1") != 0) {
4415  goto end;
4416  }
4417  break;
4418  default:
4419  goto end;
4420  }
4421  i++;
4422  }
4423 
4424  outputs = ConfGetNode("libhtp.server-config");
4425  if (outputs == NULL) {
4426  goto end;
4427  }
4428 
4429  node = TAILQ_FIRST(&outputs->head);
4430  node = TAILQ_NEXT(node, next);
4431  if (node == NULL) {
4432  goto end;
4433  }
4434  if (strcmp(node->name, "1") != 0) {
4435  goto end;
4436  }
4437  node = TAILQ_FIRST(&node->head);
4438  if (node == NULL) {
4439  goto end;
4440  }
4441  if (strcmp(node->name, "iis7") != 0) {
4442  goto end;
4443  }
4444 
4445  node2 = ConfNodeLookupChild(node, "personality");
4446  if (node2 == NULL) {
4447  goto end;
4448  }
4449  if (strcmp(node2->val, "IIS_7_0") != 0) {
4450  goto end;
4451  }
4452 
4453  node = ConfNodeLookupChild(node, "address");
4454  if (node == NULL) {
4455  goto end;
4456  }
4457 
4458  i = 0;
4459  TAILQ_FOREACH(n, &node->head, next) {
4460  if (n == NULL) {
4461  goto end;
4462  }
4463 
4464  switch(i) {
4465  case 0:
4466  if (strcmp(n->name, "0") != 0) {
4467  goto end;
4468  }
4469  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4470  goto end;
4471  }
4472  break;
4473  case 1:
4474  if (strcmp(n->name, "1") != 0) {
4475  goto end;
4476  }
4477  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4478  goto end;
4479  }
4480  break;
4481  default:
4482  goto end;
4483  }
4484  i++;
4485  }
4486 
4487  ret = 1;
4488 
4489 end:
4490  ConfDeInit();
4492 
4493  return ret;
4494 }
4495 
4496 /** \test Test config builds radix correctly */
4497 static int HTPParserConfigTest02(void)
4498 {
4499  int ret = 0;
4500  char input[] = "\
4501 %YAML 1.1\n\
4502 ---\n\
4503 libhtp:\n\
4504 \n\
4505  default-config:\n\
4506  personality: IDS\n\
4507 \n\
4508  server-config:\n\
4509 \n\
4510  - apache-tomcat:\n\
4511  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4512  personality: Tomcat_6_0\n\
4513 \n\
4514  - iis7:\n\
4515  address: \n\
4516  - 192.168.0.0/24\n\
4517  - 192.168.10.0/24\n\
4518  personality: IIS_7_0\n\
4519 ";
4520 
4522  ConfInit();
4524 
4525  ConfYamlLoadString(input, strlen(input));
4526 
4527  HTPConfigure();
4528 
4529  if (cfglist.cfg == NULL) {
4530  printf("No default config created.\n");
4531  goto end;
4532  }
4533 
4534  if (cfgtree == NULL) {
4535  printf("No config tree created.\n");
4536  goto end;
4537  }
4538 
4539  htp_cfg_t *htp = cfglist.cfg;
4540  uint8_t buf[128];
4541  const char *addr;
4542  void *user_data = NULL;
4543 
4544  addr = "192.168.10.42";
4545  if (inet_pton(AF_INET, addr, buf) == 1) {
4546  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4547  if (user_data != NULL) {
4548  HTPCfgRec *htp_cfg_rec = user_data;
4549  htp = htp_cfg_rec->cfg;
4550  SCLogDebug("LIBHTP using config: %p", htp);
4551  }
4552  if (htp == NULL) {
4553  printf("Could not get config for: %s\n", addr);
4554  goto end;
4555  }
4556  }
4557  else {
4558  printf("Failed to parse address: %s\n", addr);
4559  goto end;
4560  }
4561 
4562  user_data = NULL;
4563  addr = "::1";
4564  if (inet_pton(AF_INET6, addr, buf) == 1) {
4565  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4566  if (user_data != NULL) {
4567  HTPCfgRec *htp_cfg_rec = user_data;
4568  htp = htp_cfg_rec->cfg;
4569  SCLogDebug("LIBHTP using config: %p", htp);
4570  }
4571  if (htp == NULL) {
4572  printf("Could not get config for: %s\n", addr);
4573  goto end;
4574  }
4575  }
4576  else {
4577  printf("Failed to parse address: %s\n", addr);
4578  goto end;
4579  }
4580 
4581  ret = 1;
4582 
4583 end:
4584  HTPFreeConfig();
4585  ConfDeInit();
4588 
4589  return ret;
4590 }
4591 
4592 /** \test Test traffic is handled by the correct htp config */
4593 static int HTPParserConfigTest03(void)
4594 {
4595  int result = 1;
4596  Flow *f = NULL;
4597  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4598  " Data is c0oL!";
4599  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4600  TcpSession ssn;
4602 
4603  HtpState *htp_state = NULL;
4604  int r = 0;
4605  char input[] = "\
4606 %YAML 1.1\n\
4607 ---\n\
4608 libhtp:\n\
4609 \n\
4610  default-config:\n\
4611  personality: IDS\n\
4612 \n\
4613  server-config:\n\
4614 \n\
4615  - apache-tomcat:\n\
4616  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4617  personality: Tomcat_6_0\n\
4618 \n\
4619  - iis7:\n\
4620  address: \n\
4621  - 192.168.0.0/24\n\
4622  - 192.168.10.0/24\n\
4623  personality: IIS_7_0\n\
4624 ";
4625 
4627  ConfInit();
4629 
4630  ConfYamlLoadString(input, strlen(input));
4631 
4632  HTPConfigure();
4633 
4634  const char *addr = "192.168.10.42";
4635 
4636  memset(&ssn, 0, sizeof(ssn));
4637 
4638  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4639  if (f == NULL)
4640  goto end;
4641  f->protoctx = &ssn;
4642  f->proto = IPPROTO_TCP;
4643  f->alproto = ALPROTO_HTTP1;
4644 
4645  htp_cfg_t *htp = cfglist.cfg;
4646 
4647  void *user_data = NULL;
4648  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4649  if (user_data != NULL) {
4650  HTPCfgRec *htp_cfg_rec = user_data;
4651  htp = htp_cfg_rec->cfg;
4652  SCLogDebug("LIBHTP using config: %p", htp);
4653  }
4654  if (htp == NULL) {
4655  printf("Could not get config for: %s\n", addr);
4656  goto end;
4657  }
4658 
4659  StreamTcpInitConfig(true);
4660 
4661  uint32_t u;
4662  for (u = 0; u < httplen1; u++) {
4663  uint8_t flags = 0;
4664 
4665  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4666  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4667  else flags = STREAM_TOSERVER;
4668 
4669  FLOWLOCK_WRLOCK(f);
4670  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4671  if (r != 0) {
4672  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4673  " 0: ", u, r);
4674  result = 0;
4675  FLOWLOCK_UNLOCK(f);
4676  goto end;
4677  }
4678  FLOWLOCK_UNLOCK(f);
4679  }
4680 
4681  htp_state = f->alstate;
4682  if (htp_state == NULL) {
4683  printf("no http state: ");
4684  result = 0;
4685  goto end;
4686  }
4687 
4688  if (HTPStateGetTxCnt(htp_state) != 2) {
4689  printf("HTPStateGetTxCnt(htp_state) failure\n");
4690  goto end;
4691  }
4692 
4693  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4694  if (tx == NULL)
4695  goto end;
4696  if (tx->cfg != htp) {
4697  printf("wrong HTP config (%p instead of %p - default=%p): ",
4698  tx->cfg, htp, cfglist.cfg);
4699  goto end;
4700  }
4701  tx = HTPStateGetTx(htp_state, 1);
4702  if (tx == NULL)
4703  goto end;
4704  if (tx->cfg != htp) {
4705  printf("wrong HTP config (%p instead of %p - default=%p): ",
4706  tx->cfg, htp, cfglist.cfg);
4707  goto end;
4708  }
4709 
4710 end:
4711  if (alp_tctx != NULL)
4713  HTPFreeConfig();
4714  ConfDeInit();
4717 
4718  StreamTcpFreeConfig(true);
4719  UTHFreeFlow(f);
4720  return result;
4721 }
4722 
4723 /* disabled when we upgraded to libhtp 0.5.x */
4724 #if 0
4725 static int HTPParserConfigTest04(void)
4726 {
4727  int result = 0;
4728 
4729  char input[] = "\
4730 %YAML 1.1\n\
4731 ---\n\
4732 libhtp:\n\
4733 \n\
4734  default-config:\n\
4735  personality: IDS\n\
4736  path-control-char-handling: status_400\n\
4737  path-convert-utf8: yes\n\
4738  path-invalid-encoding-handling: remove_percent\n\
4739 \n\
4740  server-config:\n\
4741 \n\
4742  - apache-tomcat:\n\
4743  personality: Tomcat_6_0\n\
4744  path-invalid-utf8-handling: none\n\
4745  path-nul-encoded-handling: status_404\n\
4746  path-nul-raw-handling: status_400\n\
4747 \n\
4748  - iis7:\n\
4749  personality: IIS_7_0\n\
4750  path-replacement-char: o\n\
4751  path-unicode-mapping: status_400\n\
4752 ";
4753 
4755  ConfInit();
4757 
4758  ConfYamlLoadString(input, strlen(input));
4759 
4760  HTPConfigure();
4761 
4762  HTPCfgRec *cfg_rec = &cfglist;
4763  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4764  cfg_rec->cfg->path_convert_utf8 != 1 ||
4765  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4766  printf("failed 1\n");
4767  goto end;
4768  }
4769 
4770  cfg_rec = cfg_rec->next;
4771  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4772  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4773  printf("failed 2\n");
4774  goto end;
4775  }
4776 
4777  cfg_rec = cfg_rec->next;
4778  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4779  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4780  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4781  printf("failed 3\n");
4782  goto end;
4783  }
4784 
4785  result = 1;
4786 
4787 end:
4788  HTPFreeConfig();
4789  ConfDeInit();
4792 
4793  return result;
4794 }
4795 #endif
4796 
4797 /** \test Test %2f decoding in profile Apache_2_2
4798  *
4799  * %2f in path is left untouched
4800  * %2f in query string is normalized to %2F
4801  * %252f in query string is decoded/normalized to %2F
4802  */
4803 static int HTPParserDecodingTest01(void)
4804 {
4805  int result = 0;
4806  Flow *f = NULL;
4807  uint8_t httpbuf1[] =
4808  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4809  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4810  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4811  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4812  TcpSession ssn;
4814 
4815  HtpState *htp_state = NULL;
4816  int r = 0;
4817  char input[] = "\
4818 %YAML 1.1\n\
4819 ---\n\
4820 libhtp:\n\
4821 \n\
4822  default-config:\n\
4823  personality: Apache_2\n\
4824 ";
4825 
4827  ConfInit();
4829  ConfYamlLoadString(input, strlen(input));
4830  HTPConfigure();
4831  const char *addr = "4.3.2.1";
4832  memset(&ssn, 0, sizeof(ssn));
4833 
4834  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4835  if (f == NULL)
4836  goto end;
4837  f->protoctx = &ssn;
4838  f->proto = IPPROTO_TCP;
4839  f->alproto = ALPROTO_HTTP1;
4840 
4841  StreamTcpInitConfig(true);
4842 
4843  uint32_t u;
4844  for (u = 0; u < httplen1; u++) {
4845  uint8_t flags = 0;
4846 
4847  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4848  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4849  else flags = STREAM_TOSERVER;
4850 
4851  FLOWLOCK_WRLOCK(f);
4852  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4853  if (r != 0) {
4854  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4855  " 0: ", u, r);
4856  FLOWLOCK_UNLOCK(f);
4857  goto end;
4858  }
4859  FLOWLOCK_UNLOCK(f);
4860  }
4861 
4862  htp_state = f->alstate;
4863  if (htp_state == NULL) {
4864  printf("no http state: ");
4865  goto end;
4866  }
4867 
4868  uint8_t ref1[] = "/abc%2fdef";
4869  size_t reflen = sizeof(ref1) - 1;
4870 
4871  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4872  if (tx == NULL)
4873  goto end;
4874  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4875  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4876  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4877  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4878  (uintmax_t)reflen,
4879  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4880  goto end;
4881  }
4882 
4883  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4884  bstr_len(tx_ud->request_uri_normalized)) != 0)
4885  {
4886  printf("normalized uri \"");
4887  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4888  printf("\" != \"");
4889  PrintRawUriFp(stdout, ref1, reflen);
4890  printf("\": ");
4891  goto end;
4892  }
4893  }
4894 
4895  uint8_t ref2[] = "/abc/def?ghi/jkl";
4896  reflen = sizeof(ref2) - 1;
4897 
4898  tx = HTPStateGetTx(htp_state, 1);
4899  if (tx == NULL)
4900  goto end;
4901  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4902  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4903  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4904  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4905  (uintmax_t)reflen,
4906  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4907  goto end;
4908  }
4909 
4910  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4911  bstr_len(tx_ud->request_uri_normalized)) != 0)
4912  {
4913  printf("normalized uri \"");
4914  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4915  printf("\" != \"");
4916  PrintRawUriFp(stdout, ref2, reflen);
4917  printf("\": ");
4918  goto end;
4919  }
4920  }
4921 
4922  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4923  reflen = sizeof(ref3) - 1;
4924  tx = HTPStateGetTx(htp_state, 2);
4925  if (tx == NULL)
4926  goto end;
4927  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4928  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4929  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4930  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4931  (uintmax_t)reflen,
4932  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4933  goto end;
4934  }
4935 
4936  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4937  bstr_len(tx_ud->request_uri_normalized)) != 0)
4938  {
4939  printf("normalized uri \"");
4940  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4941  printf("\" != \"");
4942  PrintRawUriFp(stdout, ref3, reflen);
4943  printf("\": ");
4944  goto end;
4945  }
4946  }
4947 
4948  result = 1;
4949 
4950 end:
4951  if (alp_tctx != NULL)
4953  HTPFreeConfig();
4954  ConfDeInit();
4957 
4958  StreamTcpFreeConfig(true);
4959  UTHFreeFlow(f);
4960  return result;
4961 }
4962 
4963 /** \test Test %2f decoding in profile IDS
4964  *
4965  * %2f in path decoded to /
4966  * %2f in query string is decoded to /
4967  * %252f in query string is decoded to %2F
4968  */
4969 static int HTPParserDecodingTest02(void)
4970 {
4971  int result = 0;
4972  Flow *f = NULL;
4973  uint8_t httpbuf1[] =
4974  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4975  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4976  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4977  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4978  TcpSession ssn;
4980 
4981  HtpState *htp_state = NULL;
4982  int r = 0;
4983  char input[] = "\
4984 %YAML 1.1\n\
4985 ---\n\
4986 libhtp:\n\
4987 \n\
4988  default-config:\n\
4989  personality: IDS\n\
4990  double-decode-path: no\n\
4991  double-decode-query: no\n\
4992 ";
4993 
4995  ConfInit();
4997  ConfYamlLoadString(input, strlen(input));
4998  HTPConfigure();
4999  const char *addr = "4.3.2.1";
5000  memset(&ssn, 0, sizeof(ssn));
5001 
5002  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5003  if (f == NULL)
5004  goto end;
5005  f->protoctx = &ssn;
5006  f->proto = IPPROTO_TCP;
5007  f->alproto = ALPROTO_HTTP1;
5008 
5009  StreamTcpInitConfig(true);
5010 
5011  uint32_t u;
5012  for (u = 0; u < httplen1; u++) {
5013  uint8_t flags = 0;
5014 
5015  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5016  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5017  else flags = STREAM_TOSERVER;
5018 
5019  FLOWLOCK_WRLOCK(f);
5020  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5021  if (r != 0) {
5022  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5023  " 0: ", u, r);
5024  FLOWLOCK_UNLOCK(f);
5025  goto end;
5026  }
5027  FLOWLOCK_UNLOCK(f);
5028  }
5029 
5030  htp_state = f->alstate;
5031  if (htp_state == NULL) {
5032  printf("no http state: ");
5033  goto end;
5034  }
5035 
5036  uint8_t ref1[] = "/abc/def";
5037  size_t reflen = sizeof(ref1) - 1;
5038 
5039  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5040  if (tx == NULL)
5041  goto end;
5042  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5043  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5044  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5045  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5046  (uintmax_t)reflen,
5047  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5048  goto end;
5049  }
5050 
5051  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5052  bstr_len(tx_ud->request_uri_normalized)) != 0)
5053  {
5054  printf("normalized uri \"");
5055  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5056  printf("\" != \"");
5057  PrintRawUriFp(stdout, ref1, reflen);
5058  printf("\": ");
5059  goto end;
5060  }
5061  }
5062 
5063  uint8_t ref2[] = "/abc/def?ghi/jkl";
5064  reflen = sizeof(ref2) - 1;
5065 
5066  tx = HTPStateGetTx(htp_state, 1);
5067  if (tx == NULL)
5068  goto end;
5069  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5070  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5071  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5072  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5073  (uintmax_t)reflen,
5074  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5075  goto end;
5076  }
5077 
5078  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5079  bstr_len(tx_ud->request_uri_normalized)) != 0)
5080  {
5081  printf("normalized uri \"");
5082  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5083  printf("\" != \"");
5084  PrintRawUriFp(stdout, ref2, reflen);
5085  printf("\": ");
5086  goto end;
5087  }
5088  }
5089 
5090  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5091  reflen = sizeof(ref3) - 1;
5092  tx = HTPStateGetTx(htp_state, 2);
5093  if (tx == NULL)
5094  goto end;
5095  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5096  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5097  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5098  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5099  (uintmax_t)reflen,
5100  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5101  goto end;
5102  }
5103 
5104  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5105  bstr_len(tx_ud->request_uri_normalized)) != 0)
5106  {
5107  printf("normalized uri \"");
5108  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5109  printf("\" != \"");
5110  PrintRawUriFp(stdout, ref3, reflen);
5111  printf("\": ");
5112  goto end;
5113  }
5114  }
5115 
5116  result = 1;
5117 
5118 end:
5119  if (alp_tctx != NULL)
5121  HTPFreeConfig();
5122  ConfDeInit();
5125 
5126  StreamTcpFreeConfig(true);
5127  UTHFreeFlow(f);
5128  return result;
5129 }
5130 
5131 /** \test Test %2f decoding in profile IDS with double-decode-* options
5132  *
5133  * %252f in path decoded to /
5134  * %252f in query string is decoded to /
5135  */
5136 static int HTPParserDecodingTest03(void)
5137 {
5138  int result = 0;
5139  Flow *f = NULL;
5140  uint8_t httpbuf1[] =
5141  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5142  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5143  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5144  TcpSession ssn;
5146 
5147  HtpState *htp_state = NULL;
5148  int r = 0;
5149  char input[] = "\
5150 %YAML 1.1\n\
5151 ---\n\
5152 libhtp:\n\
5153 \n\
5154  default-config:\n\
5155  personality: IDS\n\
5156  double-decode-path: yes\n\
5157  double-decode-query: yes\n\
5158 ";
5159 
5161  ConfInit();
5163  ConfYamlLoadString(input, strlen(input));
5164  HTPConfigure();
5165  const char *addr = "4.3.2.1";
5166  memset(&ssn, 0, sizeof(ssn));
5167 
5168  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5169  if (f == NULL)
5170  goto end;
5171  f->protoctx = &ssn;
5172  f->proto = IPPROTO_TCP;
5173  f->alproto = ALPROTO_HTTP1;
5174 
5175  StreamTcpInitConfig(true);
5176 
5177  uint32_t u;
5178  for (u = 0; u < httplen1; u++) {
5179  uint8_t flags = 0;
5180 
5181  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5182  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5183  else flags = STREAM_TOSERVER;
5184 
5185  FLOWLOCK_WRLOCK(f);
5186  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5187  if (r != 0) {
5188  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5189  " 0: ", u, r);
5190  FLOWLOCK_UNLOCK(f);
5191  goto end;
5192  }
5193  FLOWLOCK_UNLOCK(f);
5194  }
5195 
5196  htp_state = f->alstate;
5197  if (htp_state == NULL) {
5198  printf("no http state: ");
5199  goto end;
5200  }
5201 
5202  uint8_t ref1[] = "/abc/def";
5203  size_t reflen = sizeof(ref1) - 1;
5204 
5205  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5206  if (tx == NULL)
5207  goto end;
5208  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5209  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5210  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5211  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5212  (uintmax_t)reflen,
5213  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5214  goto end;
5215  }
5216 
5217  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5218  bstr_len(tx_ud->request_uri_normalized)) != 0)
5219  {
5220  printf("normalized uri \"");
5221  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5222  printf("\" != \"");
5223  PrintRawUriFp(stdout, ref1, reflen);
5224  printf("\": ");
5225  goto end;
5226  }
5227  }
5228 
5229  uint8_t ref2[] = "/abc/def?ghi/jkl";
5230  reflen = sizeof(ref2) - 1;
5231 
5232  tx = HTPStateGetTx(htp_state, 1);
5233  if (tx == NULL)
5234  goto end;
5235  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5236  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5237  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5238  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5239  (uintmax_t)reflen,
5240  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5241  goto end;
5242  }
5243 
5244  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5245  bstr_len(tx_ud->request_uri_normalized)) != 0)
5246  {
5247  printf("normalized uri \"");
5248  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5249  printf("\" != \"");
5250  PrintRawUriFp(stdout, ref2, reflen);
5251  printf("\": ");
5252  goto end;
5253  }
5254  }
5255 
5256  result = 1;
5257 
5258 end:
5259  if (alp_tctx != NULL)
5261  HTPFreeConfig();
5262  ConfDeInit();
5265 
5266  StreamTcpFreeConfig(true);
5267  UTHFreeFlow(f);
5268  return result;
5269 }
5270 
5271 /** \test Test http:// in query profile IDS
5272  */
5273 static int HTPParserDecodingTest04(void)
5274 {
5275  int result = 0;
5276  Flow *f = NULL;
5277  uint8_t httpbuf1[] =
5278  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5279  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5280  TcpSession ssn;
5282 
5283  HtpState *htp_state = NULL;
5284  int r = 0;
5285  char input[] = "\
5286 %YAML 1.1\n\
5287 ---\n\
5288 libhtp:\n\
5289 \n\
5290  default-config:\n\
5291  personality: IDS\n\
5292  double-decode-path: yes\n\
5293  double-decode-query: yes\n\
5294 ";
5295 
5297  ConfInit();
5299  ConfYamlLoadString(input, strlen(input));
5300  HTPConfigure();
5301  const char *addr = "4.3.2.1";
5302  memset(&ssn, 0, sizeof(ssn));
5303 
5304  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5305  if (f == NULL)
5306  goto end;
5307  f->protoctx = &ssn;
5308  f->proto = IPPROTO_TCP;
5309  f->alproto = ALPROTO_HTTP1;
5310 
5311  StreamTcpInitConfig(true);
5312 
5313  uint32_t u;
5314  for (u = 0; u < httplen1; u++) {
5315  uint8_t flags = 0;
5316 
5317  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5318  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5319  else flags = STREAM_TOSERVER;
5320 
5321  FLOWLOCK_WRLOCK(f);
5322  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5323  if (r != 0) {
5324  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5325  " 0: ", u, r);
5326  FLOWLOCK_UNLOCK(f);
5327  goto end;
5328  }
5329  FLOWLOCK_UNLOCK(f);
5330  }
5331 
5332  htp_state = f->alstate;
5333  if (htp_state == NULL) {
5334  printf("no http state: ");
5335  goto end;
5336  }
5337 
5338  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5339  size_t reflen = sizeof(ref1) - 1;
5340 
5341  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5342  if (tx == NULL)
5343  goto end;
5344  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5345  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5346  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5347  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5348  (uintmax_t)reflen,
5349  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5350  goto end;
5351  }
5352 
5353  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5354  bstr_len(tx_ud->request_uri_normalized)) != 0)
5355  {
5356  printf("normalized uri \"");
5357  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5358  printf("\" != \"");
5359  PrintRawUriFp(stdout, ref1, reflen);
5360  printf("\": ");
5361  goto end;
5362  }
5363  }
5364 
5365  result = 1;
5366 
5367 end:
5368  if (alp_tctx != NULL)
5370  HTPFreeConfig();
5371  ConfDeInit();
5374 
5375  StreamTcpFreeConfig(true);
5376  UTHFreeFlow(f);
5377  return result;
5378 }
5379 
5380 /** \test Test \ char in query profile IDS. Bug 739
5381  */
5382 static int HTPParserDecodingTest05(void)
5383 {
5384  int result = 0;
5385  Flow *f = NULL;
5386  uint8_t httpbuf1[] =
5387  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5388  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5389  TcpSession ssn;
5391 
5392  HtpState *htp_state = NULL;
5393  int r = 0;
5394  char input[] = "\
5395 %YAML 1.1\n\
5396 ---\n\
5397 libhtp:\n\
5398 \n\
5399  default-config:\n\
5400  personality: IDS\n\
5401  double-decode-path: yes\n\
5402  double-decode-query: yes\n\
5403 ";
5404 
5406  ConfInit();
5408  ConfYamlLoadString(input, strlen(input));
5409  HTPConfigure();
5410  const char *addr = "4.3.2.1";
5411  memset(&ssn, 0, sizeof(ssn));
5412 
5413  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5414  if (f == NULL)
5415  goto end;
5416  f->protoctx = &ssn;
5417  f->proto = IPPROTO_TCP;
5418  f->alproto = ALPROTO_HTTP1;
5419 
5420  StreamTcpInitConfig(true);
5421 
5422  uint32_t u;
5423  for (u = 0; u < httplen1; u++) {
5424  uint8_t flags = 0;
5425 
5426  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5427  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5428  else flags = STREAM_TOSERVER;
5429 
5430  FLOWLOCK_WRLOCK(f);
5431  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5432  if (r != 0) {
5433  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5434  " 0: ", u, r);
5435  FLOWLOCK_UNLOCK(f);
5436  goto end;
5437  }
5438  FLOWLOCK_UNLOCK(f);
5439  }
5440 
5441  htp_state = f->alstate;
5442  if (htp_state == NULL) {
5443  printf("no http state: ");
5444  goto end;
5445  }
5446 
5447  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5448  size_t reflen = sizeof(ref1) - 1;
5449 
5450  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5451  if (tx == NULL)
5452  goto end;
5453  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5454  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5455  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5456  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5457  (uintmax_t)reflen,
5458  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5459  goto end;
5460  }
5461 
5462  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5463  bstr_len(tx_ud->request_uri_normalized)) != 0)
5464  {
5465  printf("normalized uri \"");
5466  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5467  printf("\" != \"");
5468  PrintRawUriFp(stdout, ref1, reflen);
5469  printf("\": ");
5470  goto end;
5471  }
5472  }
5473 
5474  result = 1;
5475 
5476 end:
5477  if (alp_tctx != NULL)
5479  HTPFreeConfig();
5480  ConfDeInit();
5483 
5484  StreamTcpFreeConfig(true);
5485  UTHFreeFlow(f);
5486  return result;
5487 }
5488 
5489 /** \test Test + char in query. Bug 1035
5490  */
5491 static int HTPParserDecodingTest06(void)
5492 {
5493  int result = 0;
5494  Flow *f = NULL;
5495  uint8_t httpbuf1[] =
5496  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5497  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5498  TcpSession ssn;
5500 
5501  HtpState *htp_state = NULL;
5502  int r = 0;
5503  char input[] = "\
5504 %YAML 1.1\n\
5505 ---\n\
5506 libhtp:\n\
5507 \n\
5508  default-config:\n\
5509  personality: IDS\n\
5510  double-decode-path: yes\n\
5511  double-decode-query: yes\n\
5512 ";
5513 
5515  ConfInit();
5517  ConfYamlLoadString(input, strlen(input));
5518  HTPConfigure();
5519  const char *addr = "4.3.2.1";
5520  memset(&ssn, 0, sizeof(ssn));
5521 
5522  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5523  if (f == NULL)
5524  goto end;
5525  f->protoctx = &ssn;
5526  f->proto = IPPROTO_TCP;
5527  f->alproto = ALPROTO_HTTP1;
5528 
5529  StreamTcpInitConfig(true);
5530 
5531  uint32_t u;
5532  for (u = 0; u < httplen1; u++) {
5533  uint8_t flags = 0;
5534 
5535  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5536  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5537  else flags = STREAM_TOSERVER;
5538 
5539  FLOWLOCK_WRLOCK(f);
5540  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5541  if (r != 0) {
5542  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5543  " 0: ", u, r);
5544  FLOWLOCK_UNLOCK(f);
5545  goto end;
5546  }
5547  FLOWLOCK_UNLOCK(f);
5548  }
5549 
5550  htp_state = f->alstate;
5551  if (htp_state == NULL) {
5552  printf("no http state: ");
5553  goto end;
5554  }
5555 
5556  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5557  size_t reflen = sizeof(ref1) - 1;
5558 
5559  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5560  if (tx == NULL)
5561  goto end;
5562  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5563  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5564  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5565  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5566  (uintmax_t)reflen,
5567  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5568  goto end;
5569  }
5570 
5571  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5572  bstr_len(tx_ud->request_uri_normalized)) != 0)
5573  {
5574  printf("normalized uri \"");
5575  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5576  printf("\" != \"");
5577  PrintRawUriFp(stdout, ref1, reflen);
5578  printf("\": ");
5579  goto end;
5580  }
5581  }
5582 
5583  result = 1;
5584 
5585 end:
5586  if (alp_tctx != NULL)
5588  HTPFreeConfig();
5589  ConfDeInit();
5592 
5593  StreamTcpFreeConfig(true);
5594  UTHFreeFlow(f);
5595  return result;
5596 }
5597 
5598 /** \test Test + char in query. Bug 1035
5599  */
5600 static int HTPParserDecodingTest07(void)
5601 {
5602  int result = 0;
5603  Flow *f = NULL;
5604  uint8_t httpbuf1[] =
5605  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5606  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5607  TcpSession ssn;
5609 
5610  HtpState *htp_state = NULL;
5611  int r = 0;
5612  char input[] = "\
5613 %YAML 1.1\n\
5614 ---\n\
5615 libhtp:\n\
5616 \n\
5617  default-config:\n\
5618  personality: IDS\n\
5619  double-decode-path: yes\n\
5620  double-decode-query: yes\n\
5621  query-plusspace-decode: yes\n\
5622 ";
5623 
5625  ConfInit();
5627  ConfYamlLoadString(input, strlen(input));
5628  HTPConfigure();
5629  const char *addr = "4.3.2.1";
5630  memset(&ssn, 0, sizeof(ssn));
5631 
5632  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5633  if (f == NULL)
5634  goto end;
5635  f->protoctx = &ssn;
5636  f->proto = IPPROTO_TCP;
5637  f->alproto = ALPROTO_HTTP1;
5638 
5639  StreamTcpInitConfig(true);
5640 
5641  uint32_t u;
5642  for (u = 0; u < httplen1; u++) {
5643  uint8_t flags = 0;
5644 
5645  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5646  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5647  else flags = STREAM_TOSERVER;
5648 
5649  FLOWLOCK_WRLOCK(f);
5650  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5651  if (r != 0) {
5652  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5653  " 0: ", u, r);
5654  FLOWLOCK_UNLOCK(f);
5655  goto end;
5656  }
5657  FLOWLOCK_UNLOCK(f);
5658  }
5659 
5660  htp_state = f->alstate;
5661  if (htp_state == NULL) {
5662  printf("no http state: ");
5663  goto end;
5664  }
5665 
5666  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5667  size_t reflen = sizeof(ref1) - 1;
5668 
5669  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5670  if (tx == NULL)
5671  goto end;
5672  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5673  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5674  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5675  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5676  (uintmax_t)reflen,
5677  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5678  goto end;
5679  }
5680 
5681  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5682  bstr_len(tx_ud->request_uri_normalized)) != 0)
5683  {
5684  printf("normalized uri \"");
5685  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5686  printf("\" != \"");
5687  PrintRawUriFp(stdout, ref1, reflen);
5688  printf("\": ");
5689  goto end;
5690  }
5691  }
5692 
5693  result = 1;
5694 
5695 end:
5696  if (alp_tctx != NULL)
5698  HTPFreeConfig();
5699  ConfDeInit();
5702 
5703  StreamTcpFreeConfig(true);
5704  UTHFreeFlow(f);
5705  return result;
5706 }
5707 
5708 /** \test Test 'proxy' URI normalization. Ticket 1008
5709  */
5710 static int HTPParserDecodingTest08(void)
5711 {
5712  int result = 0;
5713  Flow *f = NULL;
5714  uint8_t httpbuf1[] =
5715  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5716  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5717  TcpSession ssn;
5719 
5720  HtpState *htp_state = NULL;
5721  int r = 0;
5722  char input[] = "\
5723 %YAML 1.1\n\
5724 ---\n\
5725 libhtp:\n\
5726 \n\
5727  default-config:\n\
5728  personality: IDS\n\
5729 ";
5730 
5732  ConfInit();
5734  ConfYamlLoadString(input, strlen(input));
5735  HTPConfigure();
5736  const char *addr = "4.3.2.1";
5737  memset(&ssn, 0, sizeof(ssn));
5738 
5739  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5740  if (f == NULL)
5741  goto end;
5742  f->protoctx = &ssn;
5743  f->proto = IPPROTO_TCP;
5744  f->alproto = ALPROTO_HTTP1;
5745 
5746  StreamTcpInitConfig(true);
5747 
5748  uint32_t u;
5749  for (u = 0; u < httplen1; u++) {
5750  uint8_t flags = 0;
5751 
5752  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5753  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5754  else