suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2024 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "decode.h"
40 #include "threads.h"
41 #include "counters.h"
42 
43 #include "util-print.h"
44 #include "util-pool.h"
45 #include "util-radix-tree.h"
46 #include "util-file.h"
47 #include "util-byte.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 #include "app-layer-expectation.h"
57 
58 #include "app-layer.h"
59 #include "app-layer-detect-proto.h"
60 #include "app-layer-frames.h"
61 #include "app-layer-htp.h"
62 #include "app-layer-htp-body.h"
63 #include "app-layer-htp-file.h"
64 #include "app-layer-htp-libhtp.h"
65 #include "app-layer-htp-xff.h"
66 #include "app-layer-htp-range.h"
67 #include "app-layer-htp-mem.h"
68 
69 #include "util-spm.h"
70 #include "util-debug.h"
71 #include "util-time.h"
72 #include "util-misc.h"
73 #include "util-enum.h"
74 
75 #include "util-unittest.h"
76 #include "util-unittest-helper.h"
77 #include "flow-util.h"
78 
79 #include "detect-engine.h"
80 #include "detect-engine-build.h"
81 #include "detect-engine-state.h"
82 #include "detect-parse.h"
83 
84 #include "decode-events.h"
85 
86 #include "util-memcmp.h"
87 #include "util-random.h"
88 #include "util-validate.h"
89 
90 //#define PRINT
91 
92 /** Fast lookup tree (radix) for the various HTP configurations */
93 static SCRadixTree *cfgtree;
94 /** List of HTP configurations. */
95 static HTPCfgRec cfglist;
96 
98 
99 /** Limit to the number of libhtp messages that can be handled */
100 #define HTP_MAX_MESSAGES 512
101 
102 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
103 
104 #ifdef DEBUG
105 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
106 static uint64_t htp_state_memuse = 0;
107 static uint64_t htp_state_memcnt = 0;
108 #endif
109 
111  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
112  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
113  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
114  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
115  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
116  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
117  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
119  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
121  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
123  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
125  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
127  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
129  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
130  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
132  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
133  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
134  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
135  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
136  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
137  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
138  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
139  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
140  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
141  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
142  { "FILE_NAME_TOO_LONG", HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG },
143  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
144  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
145  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
147  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
148  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
149  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
150  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
151  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
152  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
153  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
154  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
155  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
156  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
157  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
158  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
159  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
161  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
162  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
163  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
164  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
165 
166  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
167  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
168 
169  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
170  { "REQUEST_CHUNK_EXTENSION", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
171  { "REQUEST_LINE_MISSING_PROTOCOL", HTTP_DECODER_EVENT_REQUEST_LINE_MISSING_PROTOCOL },
172 
173  /* suricata warnings/errors */
174  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
175  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
176  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
177 
178  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
179  { "FAILED_PROTOCOL_CHANGE", HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE },
180 
181  { NULL, -1 },
182 };
183 
184 /* app-layer-frame-documentation tag start: HttpFrameTypes */
188 };
189 
191  {
192  "request",
194  },
195  {
196  "response",
198  },
199  { NULL, -1 },
200 };
201 /* app-layer-frame-documentation tag end: HttpFrameTypes */
202 
203 static int HTTPGetFrameIdByName(const char *frame_name)
204 {
205  int id = SCMapEnumNameToValue(frame_name, http_frame_table);
206  if (id < 0) {
207  return -1;
208  }
209  return id;
210 }
211 
212 static const char *HTTPGetFrameNameById(const uint8_t frame_id)
213 {
214  const char *name = SCMapEnumValueToName(frame_id, http_frame_table);
215  return name;
216 }
217 
218 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
219 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
220 static uint64_t HTPStateGetTxCnt(void *alstate);
221 #ifdef UNITTESTS
222 static void HTPParserRegisterTests(void);
223 #endif
224 
225 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
226 {
227  uint64_t id = HTPStateGetTxCnt(s);
228  BUG_ON(id == 0);
229  return id - 1;
230 }
231 
232 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
233 {
234  return s->transaction_cnt;
235 }
236 
237 #ifdef DEBUG
238 /**
239  * \internal
240  *
241  * \brief Lookup the HTP personality string from the numeric personality.
242  *
243  * \todo This needs to be a libhtp function.
244  */
245 static const char *HTPLookupPersonalityString(int p)
246 {
247 #define CASE_HTP_PERSONALITY_STRING(p) \
248  case HTP_SERVER_ ## p: return #p
249 
250  switch (p) {
251  CASE_HTP_PERSONALITY_STRING(MINIMAL);
252  CASE_HTP_PERSONALITY_STRING(GENERIC);
253  CASE_HTP_PERSONALITY_STRING(IDS);
254  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
255  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
256  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
257  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
258  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
259  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
260  CASE_HTP_PERSONALITY_STRING(APACHE_2);
261  }
262 
263  return NULL;
264 }
265 #endif /* DEBUG */
266 
267 /**
268  * \internal
269  *
270  * \brief Lookup the numeric HTP personality from a string.
271  *
272  * \todo This needs to be a libhtp function.
273  */
274 static int HTPLookupPersonality(const char *str)
275 {
276 #define IF_HTP_PERSONALITY_NUM(p) \
277  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
278 
279  IF_HTP_PERSONALITY_NUM(MINIMAL);
280  IF_HTP_PERSONALITY_NUM(GENERIC);
282  IF_HTP_PERSONALITY_NUM(IIS_4_0);
283  IF_HTP_PERSONALITY_NUM(IIS_5_0);
284  IF_HTP_PERSONALITY_NUM(IIS_5_1);
285  IF_HTP_PERSONALITY_NUM(IIS_6_0);
286  IF_HTP_PERSONALITY_NUM(IIS_7_0);
287  IF_HTP_PERSONALITY_NUM(IIS_7_5);
288  IF_HTP_PERSONALITY_NUM(APACHE_2);
289  if (strcasecmp("TOMCAT_6_0", str) == 0) {
290  SCLogError("Personality %s no "
291  "longer supported by libhtp.",
292  str);
293  return -1;
294  } else if ((strcasecmp("APACHE", str) == 0) ||
295  (strcasecmp("APACHE_2_2", str) == 0))
296  {
297  SCLogWarning("Personality %s no "
298  "longer supported by libhtp, failing back to "
299  "Apache2 personality.",
300  str);
301  return HTP_SERVER_APACHE_2;
302  }
303 
304  return -1;
305 }
306 
307 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
308  const uint8_t dir, const uint8_t e)
309 {
310  SCLogDebug("setting event %u", e);
311 
312  if (htud) {
313  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
314  s->events++;
315  return;
316  }
317 
318  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
319  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
320 
321  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
322  if (tx == NULL && tx_id > 0)
323  tx = HTPStateGetTx(s, tx_id - 1);
324  if (tx != NULL) {
325  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
326  if (htud != NULL) {
327  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
328  s->events++;
329  return;
330  }
331  }
332  SCLogDebug("couldn't set event %u", e);
333 }
334 
335 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
336  * connection parser to be used by the HTP library
337  */
338 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
339 {
340  SCEnter();
341 
342  HtpState *s = HTPMalloc(sizeof(HtpState));
343  if (unlikely(s == NULL)) {
344  SCReturnPtr(NULL, "void");
345  }
346 
347  memset(s, 0x00, sizeof(HtpState));
348 
349 #ifdef DEBUG
350  SCMutexLock(&htp_state_mem_lock);
351  htp_state_memcnt++;
352  htp_state_memuse += sizeof(HtpState);
353  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
354  SCMutexUnlock(&htp_state_mem_lock);
355 #endif
356 
357  SCReturnPtr((void *)s, "void");
358 }
359 
360 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
361 {
362  if (likely(htud)) {
363  HtpBodyFree(&htud->request_body);
364  HtpBodyFree(&htud->response_body);
365  bstr_free(htud->request_uri_normalized);
366  if (htud->request_headers_raw)
368  if (htud->response_headers_raw)
371  if (htud->mime_state)
372  SCMimeStateFree(htud->mime_state);
373  if (htud->tx_data.de_state != NULL) {
374  DetectEngineStateFree(htud->tx_data.de_state);
375  }
376  if (htud->file_range) {
377  HTPFileCloseHandleRange(&htp_sbcfg, &htud->files_tc, 0, htud->file_range, NULL, 0);
379  }
382  HTPFree(htud, sizeof(HtpTxUserData));
383  }
384 }
385 
386 /** \brief Function to frees the HTTP state memory and also frees the HTTP
387  * connection parser memory which was used by the HTP library
388  */
389 void HTPStateFree(void *state)
390 {
391  SCEnter();
392 
393  HtpState *s = (HtpState *)state;
394  if (s == NULL) {
395  SCReturn;
396  }
397 
398  /* free the connection parser memory used by HTP library */
399  if (s->connp != NULL) {
400  SCLogDebug("freeing HTP state");
401 
402  uint64_t tx_id;
403  uint64_t total_txs = HTPStateGetTxCnt(state);
404  /* free the list of body chunks */
405  if (s->conn != NULL) {
406  for (tx_id = s->tx_freed; tx_id < total_txs; tx_id++) {
407  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
408  if (tx != NULL) {
409  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
410  HtpTxUserDataFree(s, htud);
411  htp_tx_set_user_data(tx, NULL);
412  }
413  }
414  }
415  htp_connp_destroy_all(s->connp);
416  }
417 
418  HTPFree(s, sizeof(HtpState));
419 
420 #ifdef DEBUG
421  SCMutexLock(&htp_state_mem_lock);
422  htp_state_memcnt--;
423  htp_state_memuse -= sizeof(HtpState);
424  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
425  SCMutexUnlock(&htp_state_mem_lock);
426 #endif
427 
428  SCReturn;
429 }
430 
431 /**
432  * \brief HTP transaction cleanup callback
433  *
434  * \warning We cannot actually free the transactions here. It seems that
435  * HTP only accepts freeing of transactions in the response callback.
436  */
437 static void HTPStateTransactionFree(void *state, uint64_t id)
438 {
439  SCEnter();
440 
441  HtpState *s = (HtpState *)state;
442 
443  SCLogDebug("state %p, id %"PRIu64, s, id);
444 
445  htp_tx_t *tx = HTPStateGetTx(s, id);
446  if (tx != NULL) {
447  /* This will remove obsolete body chunks */
448  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
449  HtpTxUserDataFree(s, htud);
450  htp_tx_set_user_data(tx, NULL);
451 
452  /* hack: even if libhtp considers the tx incomplete, we want to
453  * free it here. htp_tx_destroy however, will refuse to do this.
454  * As htp_tx_destroy_incomplete isn't available in the public API,
455  * we hack around it here. */
456  if (unlikely(!(
457  tx->request_progress == HTP_REQUEST_COMPLETE &&
458  tx->response_progress == HTP_RESPONSE_COMPLETE)))
459  {
460  tx->request_progress = HTP_REQUEST_COMPLETE;
461  tx->response_progress = HTP_RESPONSE_COMPLETE;
462  }
463  // replaces tx in the s->conn->transactions list by NULL
464  htp_tx_destroy(tx);
465  }
466  s->tx_freed += htp_connp_tx_freed(s->connp);
467 }
468 
469 /**
470  * \brief Sets a flag that informs the HTP app layer that some module in the
471  * engine needs the http request body data.
472  * \initonly
473  */
475 {
476  SCEnter();
477 
478  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
479  SCReturn;
480 }
481 
482 /**
483  * \brief Sets a flag that informs the HTP app layer that some module in the
484  * engine needs the http request body data.
485  * \initonly
486  */
488 {
489  SCEnter();
490 
491  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
492  SCReturn;
493 }
494 
495 /**
496  * \brief Sets a flag that informs the HTP app layer that some module in the
497  * engine needs the http request file.
498  *
499  * \initonly
500  */
502 {
503  SCEnter();
506 
507  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
508  SCReturn;
509 }
510 
511 static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags)
512 {
513  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
514  if (tx_ud) {
515  SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags);
516  if (flags & STREAM_TOCLIENT) {
517  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
518  } else {
519  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
520  }
521  }
522 }
523 
524 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
525 {
526  SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64,
528  if (flags & HTP_STREAM_DEPTH_SET) {
529  uint32_t stream_depth = FileReassemblyDepth();
530  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
531  SCLogDebug("true");
532  return true;
533  }
534  } else {
535  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
536  return true;
537  }
538  }
539  SCLogDebug("false");
540  return false;
541 }
542 
543 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
544  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
545 {
546  uint32_t chunk_len = 0;
547  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
548  (content_len_so_far < (uint64_t)body_limit) &&
549  (content_len_so_far + (uint64_t)data_len) > body_limit)
550  {
551  chunk_len = (uint32_t)(body_limit - content_len_so_far);
552  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
553  (content_len_so_far < (uint64_t)stream_depth) &&
554  (content_len_so_far + (uint64_t)data_len) > stream_depth)
555  {
556  chunk_len = (uint32_t)(stream_depth - content_len_so_far);
557  }
558  SCLogDebug("len %u", chunk_len);
559  return (chunk_len == 0 ? data_len : chunk_len);
560 }
561 
562 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
563 struct {
564  const char *msg;
565  uint8_t de;
566 } htp_errors[] = {
567  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
568  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
569  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
570  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
571  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
572 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
573  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
574 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
575  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
576  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
577  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
578  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
579 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
580  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
581  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
582  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
583  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
584 };
585 
586 struct {
587  const char *msg;
588  uint8_t de;
589 } htp_warnings[] = {
590  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
591  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
592  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
593  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
594  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
595  /* { "Host information in request headers required by HTTP/1.1",
596  HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING { "Host information
597  ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
598  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
599  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
600  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number
601  * differs from the actual TCP port=%d", port, connp->conn->server_port); luckily, "Request
602  * server port=" is unique */
603  /* { "Request server port number differs from the actual TCP port",
604  HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
606  { "Request line: URI contains non-compliant delimiter",
608  { "Request line: non-compliant delimiter between Method and URI",
610  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
611  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
612  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
613  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
614  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
615  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
616  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
617  { "Transfer-encoding has abnormal chunked value",
619  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0",
621  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
622  { "Invalid response line: invalid response status",
624  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
625  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
626  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
627  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST },
628  { "Ambiguous response C-L value",
630  { "Request chunk extension", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
631  { "Request line: missing protocol", HTTP_DECODER_EVENT_REQUEST_LINE_MISSING_PROTOCOL },
632 };
633 
634 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
635 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
636 
637 /**
638  * \internal
639  *
640  * \brief Get the warning id for the warning msg.
641  *
642  * \param msg warning message
643  *
644  * \retval id the id or 0 in case of not found
645  */
646 static uint8_t HTPHandleWarningGetId(const char *msg)
647 {
648  SCLogDebug("received warning \"%s\"", msg);
649  size_t idx;
650  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
651  if (strncmp(htp_warnings[idx].msg, msg,
652  strlen(htp_warnings[idx].msg)) == 0)
653  {
654  return htp_warnings[idx].de;
655  }
656  }
657 
658  return 0;
659 }
660 
661 /**
662  * \internal
663  *
664  * \brief Get the error id for the error msg.
665  *
666  * \param msg error message
667  *
668  * \retval id the id or 0 in case of not found
669  */
670 static uint8_t HTPHandleErrorGetId(const char *msg)
671 {
672  SCLogDebug("received error \"%s\"", msg);
673 
674  size_t idx;
675  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
676  if (strncmp(htp_errors[idx].msg, msg,
677  strlen(htp_errors[idx].msg)) == 0)
678  {
679  return htp_errors[idx].de;
680  }
681  }
682 
683  return 0;
684 }
685 
686 /**
687  * \internal
688  *
689  * \brief Check state for errors, warnings and add any as events
690  *
691  * \param s state
692  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
693  */
694 static void HTPHandleError(HtpState *s, const uint8_t dir)
695 {
696  if (s == NULL || s->conn == NULL ||
697  s->conn->messages == NULL) {
698  return;
699  }
700 
701  size_t size = htp_list_size(s->conn->messages);
702  size_t msg;
703  if(size >= HTP_MAX_MESSAGES) {
705  //only once per HtpState
706  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
708  //too noisy in fuzzing
709  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
710  }
711  // ignore further messages
712  return;
713  }
714 
715  for (msg = s->htp_messages_offset; msg < size; msg++) {
716  htp_log_t *log = htp_list_get(s->conn->messages, msg);
717  if (log == NULL)
718  continue;
719 
720  HtpTxUserData *htud = NULL;
721  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
722  if (tx != NULL)
723  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
724 
725  SCLogDebug("message %s", log->msg);
726 
727  uint8_t id = HTPHandleErrorGetId(log->msg);
728  if (id == 0) {
729  id = HTPHandleWarningGetId(log->msg);
730  if (id == 0)
732  }
733 
734  if (id > 0) {
735  HTPSetEvent(s, htud, dir, id);
736  }
737  }
738  s->htp_messages_offset = (uint16_t)msg;
739  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
740 }
741 
742 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
743 {
744 #ifdef DEBUG
745  BUG_ON(s == NULL || tx == NULL);
746 #endif
747  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
748  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
749  HTP_HOSTH_INVALID))
750  {
751  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
752  if (htud == NULL)
753  return;
754 
755  if (tx->flags & HTP_REQUEST_INVALID_T_E)
756  HTPSetEvent(s, htud, STREAM_TOSERVER,
758  if (tx->flags & HTP_REQUEST_INVALID_C_L)
759  HTPSetEvent(s, htud, STREAM_TOSERVER,
761  if (tx->flags & HTP_HOST_MISSING)
762  HTPSetEvent(s, htud, STREAM_TOSERVER,
764  if (tx->flags & HTP_HOST_AMBIGUOUS)
765  HTPSetEvent(s, htud, STREAM_TOSERVER,
767  if (tx->flags & HTP_HOSTU_INVALID)
768  HTPSetEvent(s, htud, STREAM_TOSERVER,
770  if (tx->flags & HTP_HOSTH_INVALID)
771  HTPSetEvent(s, htud, STREAM_TOSERVER,
773  }
774  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
775  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
776  if (htud == NULL)
777  return;
778  HTPSetEvent(s, htud, STREAM_TOSERVER,
780  }
781  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
782  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
783  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
784  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
785  if (htud == NULL)
786  return;
787  HTPSetEvent(s, htud, STREAM_TOSERVER,
789  }
790 }
791 
792 static int Setup(Flow *f, HtpState *hstate)
793 {
794  /* store flow ref in state so callbacks can access it */
795  hstate->f = f;
796 
797  HTPCfgRec *htp_cfg_rec = &cfglist;
798  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
799  void *user_data = NULL;
800 
801  if (FLOW_IS_IPV4(f)) {
802  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
803  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
804  }
805  else if (FLOW_IS_IPV6(f)) {
806  SCLogDebug("Looking up HTP config for ipv6");
807  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
808  }
809  else {
810  SCLogError("unknown address family, bug!");
811  goto error;
812  }
813 
814  if (user_data != NULL) {
815  htp_cfg_rec = user_data;
816  htp = htp_cfg_rec->cfg;
817  SCLogDebug("LIBHTP using config: %p", htp);
818  } else {
819  SCLogDebug("Using default HTP config: %p", htp);
820  }
821 
822  if (NULL == htp) {
823 #ifdef DEBUG_VALIDATION
824  BUG_ON(htp == NULL);
825 #endif
826  /* should never happen if HTPConfigure is properly invoked */
827  goto error;
828  }
829 
830  hstate->connp = htp_connp_create(htp);
831  if (hstate->connp == NULL) {
832  goto error;
833  }
834 
835  hstate->conn = htp_connp_get_connection(hstate->connp);
836 
837  htp_connp_set_user_data(hstate->connp, (void *)hstate);
838  hstate->cfg = htp_cfg_rec;
839 
840  SCLogDebug("New hstate->connp %p", hstate->connp);
841 
842  struct timeval tv = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
843  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &tv);
844 
846  htp_cfg_rec->request.inspect_min_size);
848  htp_cfg_rec->response.inspect_min_size);
849  return 0;
850 error:
851  return -1;
852 }
853 
854 /**
855  * \brief Function to handle the reassembled data from client and feed it to
856  * the HTP library to process it.
857  *
858  * \param flow Pointer to the flow the data belong to
859  * \param htp_state Pointer the state in which the parsed value to be stored
860  * \param pstate Application layer parser state for this session
861  *
862  * \retval On success returns 1 or on failure returns -1.
863  */
864 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state, AppLayerParserState *pstate,
865  StreamSlice stream_slice, void *local_data)
866 {
867  SCEnter();
868  int ret = 0;
869  HtpState *hstate = (HtpState *)htp_state;
870 
871  /* On the first invocation, create the connection parser structure to
872  * be used by HTP library. This is looked up via IP in the radix
873  * tree. Failing that, the default HTP config is used.
874  */
875  if (NULL == hstate->conn) {
876  if (Setup(f, hstate) != 0) {
878  }
879  }
880  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
881  hstate->slice = &stream_slice;
882 
883  const uint8_t *input = StreamSliceGetData(&stream_slice);
884  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
885 
886  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
887  /* pass the new data to the htp parser */
888  if (input_len > 0) {
889  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
890  switch (r) {
891  case HTP_STREAM_ERROR:
892  ret = -1;
893  break;
894  default:
895  break;
896  }
897  HTPHandleError(hstate, STREAM_TOSERVER);
898  }
899 
900  /* if the TCP connection is closed, then close the HTTP connection */
902  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
903  {
904  htp_connp_req_close(hstate->connp, &ts);
905  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
906  SCLogDebug("stream eof encountered, closing htp handle for ts");
907  }
908 
909  SCLogDebug("hstate->connp %p", hstate->connp);
910  hstate->slice = NULL;
911 
912  if (ret < 0) {
914  }
916 }
917 
918 /**
919  * \brief Function to handle the reassembled data from server and feed it to
920  * the HTP library to process it.
921  *
922  * \param flow Pointer to the flow the data belong to
923  * \param htp_state Pointer the state in which the parsed value to be stored
924  * \param pstate Application layer parser state for this session
925  * \param input Pointer the received HTTP server data
926  * \param input_len Length in bytes of the received data
927  * \param output Pointer to the output (not used in this function)
928  *
929  * \retval On success returns 1 or on failure returns -1
930  */
931 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state, AppLayerParserState *pstate,
932  StreamSlice stream_slice, void *local_data)
933 {
934  SCEnter();
935  int ret = 0;
936  HtpState *hstate = (HtpState *)htp_state;
937 
938  const uint8_t *input = StreamSliceGetData(&stream_slice);
939  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
940 
941  /* On the first invocation, create the connection parser structure to
942  * be used by HTP library. This is looked up via IP in the radix
943  * tree. Failing that, the default HTP config is used.
944  */
945  if (NULL == hstate->conn) {
946  if (Setup(f, hstate) != 0) {
948  }
949  }
950  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
951  hstate->slice = &stream_slice;
952 
953  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
954  htp_tx_t *tx = NULL;
955  uint32_t consumed = 0;
956  if (input_len > 0) {
957  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
958  switch (r) {
959  case HTP_STREAM_ERROR:
960  ret = -1;
961  break;
962  case HTP_STREAM_TUNNEL:
963  tx = htp_connp_get_out_tx(hstate->connp);
964  if (tx != NULL && tx->response_status_number == 101) {
965  htp_header_t *h =
966  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
967  if (h == NULL) {
968  break;
969  }
970  uint16_t dp = 0;
971  if (tx->request_port_number != -1) {
972  dp = (uint16_t)tx->request_port_number;
973  }
974  consumed = (uint32_t)htp_connp_res_data_consumed(hstate->connp);
975  if (bstr_cmp_c(h->value, "h2c") == 0) {
977  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
978  break;
979  }
980  hstate->slice = NULL;
981  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_HTTP2)) {
982  HTPSetEvent(hstate, NULL, STREAM_TOCLIENT,
984  }
985  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
986  // and we need to parser the remaining part with HTTP2
987  if (consumed > 0 && consumed < input_len) {
988  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
989  }
991  } else if (bstr_cmp_c_nocase(h->value, "WebSocket") == 0) {
993  // if WS is disabled, keep the HTP_STREAM_TUNNEL mode
994  break;
995  }
996  hstate->slice = NULL;
998  HTPSetEvent(hstate, NULL, STREAM_TOCLIENT,
1000  }
1001  // During WS upgrade, we may consume the HTTP1 part of the data
1002  // and we need to parser the remaining part with WS
1003  if (consumed > 0 && consumed < input_len) {
1004  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
1005  }
1007  }
1008  }
1009  break;
1010  default:
1011  break;
1012  }
1013  HTPHandleError(hstate, STREAM_TOCLIENT);
1014  }
1015 
1016  /* if we the TCP connection is closed, then close the HTTP connection */
1018  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
1019  {
1020  htp_connp_close(hstate->connp, &ts);
1021  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
1022  }
1023 
1024  SCLogDebug("hstate->connp %p", hstate->connp);
1025  hstate->slice = NULL;
1026 
1027  if (ret < 0) {
1029  }
1031 }
1032 
1033 /**
1034  * \param name /Lowercase/ version of the variable name
1035  */
1036 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1037  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1038 {
1039 #ifdef PRINT
1040  printf("DATA START: \n");
1041  PrintRawDataFp(stdout, data, len);
1042  printf("DATA END: \n");
1043 #endif
1044  size_t x;
1045  int quote = 0;
1046 
1047  for (x = 0; x < len; x++) {
1048  if (!(isspace(data[x])))
1049  break;
1050  }
1051 
1052  if (x >= len)
1053  return 0;
1054 
1055  uint8_t *line = data+x;
1056  size_t line_len = len-x;
1057  size_t offset = 0;
1058 #ifdef PRINT
1059  printf("LINE START: \n");
1060  PrintRawDataFp(stdout, line, line_len);
1061  printf("LINE END: \n");
1062 #endif
1063  for (x = 0 ; x < line_len; x++) {
1064  if (x > 0) {
1065  if (line[x - 1] != '\\' && line[x] == '\"') {
1066  quote++;
1067  }
1068 
1069  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1070  uint8_t *token = line + offset;
1071  size_t token_len = x - offset;
1072 
1073  if ((x + 1) == line_len) {
1074  token_len++;
1075  }
1076 
1077  offset = x + 1;
1078 
1079  while (offset < line_len && isspace(line[offset])) {
1080  x++;
1081  offset++;
1082  }
1083 #ifdef PRINT
1084  printf("TOKEN START: \n");
1085  PrintRawDataFp(stdout, token, token_len);
1086  printf("TOKEN END: \n");
1087 #endif
1088  if (token_len > name_len) {
1089  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1090  uint8_t *value = token + name_len;
1091  size_t value_len = token_len - name_len;
1092 
1093  if (value[0] == '\"') {
1094  value++;
1095  value_len--;
1096  }
1097  if (value[value_len-1] == '\"') {
1098  value_len--;
1099  }
1100 #ifdef PRINT
1101  printf("VALUE START: \n");
1102  PrintRawDataFp(stdout, value, value_len);
1103  printf("VALUE END: \n");
1104 #endif
1105  *retptr = value;
1106  *retlen = value_len;
1107  return 1;
1108  }
1109  }
1110  }
1111  }
1112  }
1113 
1114  return 0;
1115 }
1116 
1117 /**
1118  * \brief setup multipart parsing: extract boundary and store it
1119  *
1120  * \param d HTTP transaction
1121  * \param htud transaction userdata
1122  *
1123  * \retval 1 ok, multipart set up
1124  * \retval 0 ok, not multipart though
1125  * \retval -1 error: problem with the boundary
1126  *
1127  * If the request contains a multipart message, this function will
1128  * set the HTP_BOUNDARY_SET in the transaction.
1129  */
1130 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1131 {
1132  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1133  "Content-Type");
1134  if (h != NULL && bstr_len(h->value) > 0) {
1135  htud->mime_state = SCMimeStateInit(bstr_ptr(h->value), (uint32_t)bstr_len(h->value));
1136  if (htud->mime_state) {
1137  htud->tsflags |= HTP_BOUNDARY_SET;
1138  SCReturnInt(1);
1139  }
1140  }
1141  SCReturnInt(0);
1142 }
1143 
1144 /**
1145  * \brief Create a single buffer from the HtpBodyChunks in our list
1146  *
1147  * \param htud transaction user data
1148  * \param chunks_buffers pointer to pass back the buffer to the caller
1149  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1150  */
1151 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1152  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1153 {
1155  chunks_buffer, chunks_buffer_len,
1156  htud->request_body.body_parsed);
1157 }
1158 
1159 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1160 {
1161  SCEnter();
1162  if (tx && tx->tx_data.de_state) {
1163  if (dir == STREAM_TOSERVER) {
1164  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1165  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1166  } else if (STREAM_TOCLIENT) {
1167  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1168  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1169  }
1170  }
1171 }
1172 
1173 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1174  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len, bool eof)
1175 {
1176 #ifdef PRINT
1177  printf("CHUNK START: \n");
1178  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1179  printf("CHUNK END: \n");
1180 #endif
1181 
1182  // libhtp will not call us back too late
1183  // should libhtp send a callback eof for 0 chunked ?
1185  STREAM_TOSERVER) >= HTP_REQUEST_COMPLETE);
1186 
1187  const uint8_t *cur_buf = chunks_buffer;
1188  uint32_t cur_buf_len = chunks_buffer_len;
1189 
1190  if (eof) {
1191  // abrupt end of connection
1192  if (htud->tsflags & HTP_FILENAME_SET && !(htud->tsflags & HTP_DONTSTORE)) {
1193  /* we currently only handle multipart for ts. When we support it for tc,
1194  * we will need to supply right direction */
1195  HTPFileClose(htud, cur_buf, cur_buf_len, FILE_TRUNCATED, STREAM_TOSERVER);
1196  }
1197  htud->tsflags &= ~HTP_FILENAME_SET;
1198  goto end;
1199  }
1200 
1201  uint32_t consumed;
1202  uint32_t warnings;
1203  int result = 0;
1204  const uint8_t *filename = NULL;
1205  uint16_t filename_len = 0;
1206 
1207  // keep parsing mime and use callbacks when needed
1208  while (cur_buf_len > 0) {
1209  MimeParserResult r =
1210  SCMimeParse(htud->mime_state, cur_buf, cur_buf_len, &consumed, &warnings);
1211  DEBUG_VALIDATE_BUG_ON(consumed > cur_buf_len);
1212  htud->request_body.body_parsed += consumed;
1213  if (warnings) {
1214  if (warnings & MIME_EVENT_FLAG_INVALID_HEADER) {
1215  HTPSetEvent(
1216  hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER);
1217  }
1218  if (warnings & MIME_EVENT_FLAG_NO_FILEDATA) {
1219  HTPSetEvent(
1220  hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA);
1221  }
1222  }
1223  switch (r) {
1224  case MimeNeedsMore:
1225  // there is not enough data, wait for more next time
1226  goto end;
1227  case MimeFileOpen:
1228  // get filename owned by mime state
1229  SCMimeStateGetFilename(htud->mime_state, &filename, &filename_len);
1230  if (filename_len > 0) {
1231  htud->tsflags |= HTP_FILENAME_SET;
1232  htud->tsflags &= ~HTP_DONTSTORE;
1233  result = HTPFileOpen(
1234  hstate, htud, filename, filename_len, NULL, 0, STREAM_TOSERVER);
1235  if (result == -1) {
1236  goto end;
1237  } else if (result == -2) {
1238  htud->tsflags |= HTP_DONTSTORE;
1239  }
1240  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1241  }
1242  break;
1243  case MimeFileChunk:
1244  if (htud->tsflags & HTP_FILENAME_SET && !(htud->tsflags & HTP_DONTSTORE)) {
1245  result = HTPFileStoreChunk(htud, cur_buf, consumed, STREAM_TOSERVER);
1246  if (result == -1) {
1247  goto end;
1248  } else if (result == -2) {
1249  /* we know for sure we're not storing the file */
1250  htud->tsflags |= HTP_DONTSTORE;
1251  }
1252  }
1253  break;
1254  case MimeFileClose:
1255  if (htud->tsflags & HTP_FILENAME_SET && !(htud->tsflags & HTP_DONTSTORE)) {
1256  uint32_t lastsize = consumed;
1257  if (lastsize > 0 && cur_buf[lastsize - 1] == '\n') {
1258  lastsize--;
1259  if (lastsize > 0 && cur_buf[lastsize - 1] == '\r') {
1260  lastsize--;
1261  }
1262  }
1263  HTPFileClose(htud, cur_buf, lastsize, 0, STREAM_TOSERVER);
1264  }
1265  htud->tsflags &= ~HTP_FILENAME_SET;
1266  break;
1267  }
1268  cur_buf += consumed;
1269  cur_buf_len -= consumed;
1270  }
1271 
1272 end:
1273  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1274  return 0;
1275 }
1276 
1277 /** \internal
1278  * \brief Handle POST or PUT, no multipart body data
1279  */
1280 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1281  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1282 {
1283  int result = 0;
1284 
1285  /* see if we need to open the file */
1286  if (!(htud->tsflags & HTP_FILENAME_SET))
1287  {
1288  uint8_t *filename = NULL;
1289  size_t filename_len = 0;
1290 
1291  /* get the name */
1292  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1293  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1294  filename_len = bstr_len(tx->parsed_uri->path);
1295  }
1296 
1297  if (filename != NULL) {
1298  if (filename_len > SC_FILENAME_MAX) {
1299  // explicitly truncate the file name if too long
1300  filename_len = SC_FILENAME_MAX;
1301  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1302  }
1303  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1304  STREAM_TOSERVER);
1305  if (result == -1) {
1306  goto end;
1307  } else if (result == -2) {
1308  htud->tsflags |= HTP_DONTSTORE;
1309  } else {
1310  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1311  htud->tsflags |= HTP_FILENAME_SET;
1312  htud->tsflags &= ~HTP_DONTSTORE;
1313  }
1314  }
1315  }
1316  else
1317  {
1318  /* otherwise, just store the data */
1319 
1320  if (!(htud->tsflags & HTP_DONTSTORE)) {
1321  result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOSERVER);
1322  if (result == -1) {
1323  goto end;
1324  } else if (result == -2) {
1325  /* we know for sure we're not storing the file */
1326  htud->tsflags |= HTP_DONTSTORE;
1327  }
1328  }
1329  }
1330 
1331  return 0;
1332 end:
1333  return -1;
1334 }
1335 
1336 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1337  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1338 {
1339  SCEnter();
1340 
1341  int result = 0;
1342 
1343  /* see if we need to open the file
1344  * we check for tx->response_line in case of junk
1345  * interpreted as body before response line
1346  */
1347  if (!(htud->tcflags & HTP_FILENAME_SET)) {
1348  SCLogDebug("setting up file name");
1349 
1350  uint8_t *filename = NULL;
1351  size_t filename_len = 0;
1352 
1353  /* try Content-Disposition header first */
1354  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1355  "Content-Disposition");
1356  if (h != NULL && bstr_len(h->value) > 0) {
1357  /* parse content-disposition */
1358  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1359  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1360  }
1361 
1362  /* fall back to name from the uri */
1363  if (filename == NULL) {
1364  /* get the name */
1365  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1366  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1367  filename_len = bstr_len(tx->parsed_uri->path);
1368  }
1369  }
1370 
1371  if (filename != NULL) {
1372  // set range if present
1373  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1374  if (filename_len > SC_FILENAME_MAX) {
1375  // explicitly truncate the file name if too long
1376  filename_len = SC_FILENAME_MAX;
1377  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1378  }
1379  if (h_content_range != NULL) {
1380  result = HTPFileOpenWithRange(hstate, htud, filename, (uint16_t)filename_len, data,
1381  data_len, tx, h_content_range->value, htud);
1382  } else {
1383  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1384  STREAM_TOCLIENT);
1385  }
1386  SCLogDebug("result %d", result);
1387  if (result == -1) {
1388  goto end;
1389  } else if (result == -2) {
1390  htud->tcflags |= HTP_DONTSTORE;
1391  } else {
1392  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1393  htud->tcflags |= HTP_FILENAME_SET;
1394  htud->tcflags &= ~HTP_DONTSTORE;
1395  }
1396  }
1397  } else {
1398  /* otherwise, just store the data */
1399 
1400  if (!(htud->tcflags & HTP_DONTSTORE)) {
1401  result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOCLIENT);
1402  SCLogDebug("result %d", result);
1403  if (result == -1) {
1404  goto end;
1405  } else if (result == -2) {
1406  /* we know for sure we're not storing the file */
1407  htud->tcflags |= HTP_DONTSTORE;
1408  }
1409  }
1410  }
1411 
1412  htud->response_body.body_parsed += data_len;
1413  return 0;
1414 end:
1415  return -1;
1416 }
1417 
1418 /**
1419  * \brief Function callback to append chunks for Requests
1420  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1421  * \retval int HTP_OK if all goes well
1422  */
1423 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1424 {
1425  SCEnter();
1426 
1427  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1428  SCReturnInt(HTP_OK);
1429 
1430  if (d->len == 0)
1431  SCReturnInt(HTP_OK);
1432 
1433 #ifdef PRINT
1434  printf("HTPBODY START: \n");
1435  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1436  printf("HTPBODY END: \n");
1437 #endif
1438 
1439  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1440  if (hstate == NULL) {
1441  SCReturnInt(HTP_ERROR);
1442  }
1443 
1444  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1445  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1446 
1447  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1448  if (tx_ud == NULL) {
1449  SCReturnInt(HTP_OK);
1450  }
1451  SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags);
1452 
1453  if (!tx_ud->response_body_init) {
1454  tx_ud->response_body_init = 1;
1455 
1456  if (d->tx->request_method_number == HTP_M_POST) {
1457  SCLogDebug("POST");
1458  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1459  if (r == 1) {
1461  } else if (r == 0) {
1463  SCLogDebug("not multipart");
1464  }
1465  } else if (d->tx->request_method_number == HTP_M_PUT) {
1467  }
1468  }
1469 
1470  /* see if we can get rid of htp body chunks */
1471  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1472 
1473  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1474  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1475 
1476  /* within limits, add the body chunk to the state. */
1477  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1478  uint32_t stream_depth = FileReassemblyDepth();
1479  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1480  hstate->cfg->request.body_limit,
1481  stream_depth,
1482  tx_ud->tsflags,
1483  (uint32_t)d->len);
1484  BUG_ON(len > (uint32_t)d->len);
1485 
1486  HtpBodyAppendChunk(&tx_ud->request_body, d->data, len);
1487 
1488  const uint8_t *chunks_buffer = NULL;
1489  uint32_t chunks_buffer_len = 0;
1490 
1492  /* multi-part body handling starts here */
1493  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1494  goto end;
1495  }
1496 
1497  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1498  if (chunks_buffer == NULL) {
1499  goto end;
1500  }
1501 #ifdef PRINT
1502  printf("REASSCHUNK START: \n");
1503  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1504  printf("REASSCHUNK END: \n");
1505 #endif
1506 
1507  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len,
1508  (d->data == NULL && d->len == 0));
1509 
1510  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1512  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1513  }
1514 
1515  } else {
1516  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1517  SCLogDebug("closing file that was being stored");
1518  (void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1519  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1520  }
1521  }
1522 
1523 end:
1524  if (hstate->conn != NULL) {
1525  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1527  hstate->cfg->request.inspect_min_size,
1528  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1529 
1530  /* if we reach the inspect_min_size we'll trigger inspection,
1531  * so make sure that raw stream is also inspected. Set the
1532  * data to be used to the amount of raw bytes we've seen to
1533  * get here. */
1534  if (tx_ud->request_body.body_inspected == 0 &&
1536  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1537  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1538  {
1539  const uint32_t data_size = (uint32_t)(
1540  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1541  const uint32_t depth = MIN(data_size, hstate->cfg->request.inspect_min_size);
1542 
1543  /* body still in progress, but due to min inspect size we need to inspect now */
1544  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, depth);
1545  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1546  }
1547  /* after the start of the body, disable the depth logic */
1548  } else if (tx_ud->request_body.body_inspected > 0) {
1549  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1550  }
1551  }
1552  SCReturnInt(HTP_OK);
1553 }
1554 
1555 /**
1556  * \brief Function callback to append chunks for Responses
1557  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1558  * \retval int HTP_OK if all goes well
1559  */
1560 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1561 {
1562  SCEnter();
1563 
1564  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1565  SCReturnInt(HTP_OK);
1566 
1567  if (d->len == 0)
1568  SCReturnInt(HTP_OK);
1569 
1570  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1571  if (hstate == NULL) {
1572  SCReturnInt(HTP_ERROR);
1573  }
1574 
1575  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1576  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1577 
1578  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1579  if (tx_ud == NULL) {
1580  SCReturnInt(HTP_OK);
1581  }
1582  SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags);
1583  if (!tx_ud->request_body_init) {
1584  tx_ud->request_body_init = 1;
1585  }
1586 
1587  /* see if we can get rid of htp body chunks */
1588  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1589 
1590  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1591  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1592 
1593  /* within limits, add the body chunk to the state. */
1594  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1595  uint32_t stream_depth = FileReassemblyDepth();
1596  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1597  hstate->cfg->response.body_limit,
1598  stream_depth,
1599  tx_ud->tcflags,
1600  (uint32_t)d->len);
1601  BUG_ON(len > (uint32_t)d->len);
1602 
1603  HtpBodyAppendChunk(&tx_ud->response_body, d->data, len);
1604 
1605  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1606  } else {
1607  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1608  SCLogDebug("closing file that was being stored");
1609  (void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1610  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1611  }
1612  }
1613 
1614  if (hstate->conn != NULL) {
1615  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1617  hstate->cfg->response.inspect_min_size,
1618  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
1619  /* if we reach the inspect_min_size we'll trigger inspection,
1620  * so make sure that raw stream is also inspected. Set the
1621  * data to be used to the amount of raw bytes we've seen to
1622  * get here. */
1623  if (tx_ud->response_body.body_inspected == 0 &&
1625  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
1626  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
1627  {
1628  const uint32_t data_size = (uint32_t)((uint64_t)hstate->conn->out_data_counter -
1629  hstate->last_response_data_stamp);
1630  const uint32_t depth = MIN(data_size, hstate->cfg->response.inspect_min_size);
1631 
1632  /* body still in progress, but due to min inspect size we need to inspect now */
1633  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, depth);
1634  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
1635  }
1636  /* after the start of the body, disable the depth logic */
1637  } else if (tx_ud->response_body.body_inspected > 0) {
1638  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
1639  }
1640  }
1641  SCReturnInt(HTP_OK);
1642 }
1643 
1644 /**
1645  * \brief Print the stats of the HTTP requests
1646  */
1648 {
1649 #ifdef DEBUG
1650  SCEnter();
1651  SCMutexLock(&htp_state_mem_lock);
1652  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
1653  htp_state_memcnt, htp_state_memuse);
1654  SCMutexUnlock(&htp_state_mem_lock);
1655  SCReturn;
1656 #endif
1657 }
1658 
1659 /** \brief Clears the HTTP server configuration memory used by HTP library */
1660 void HTPFreeConfig(void)
1661 {
1662  SCEnter();
1663 
1664  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
1665  !AppLayerParserConfParserEnabled("tcp", "http"))
1666  {
1667  SCReturn;
1668  }
1669 
1670  HTPCfgRec *nextrec = cfglist.next;
1671  SCRadixReleaseRadixTree(cfgtree);
1672  cfgtree = NULL;
1673  htp_config_destroy(cfglist.cfg);
1674  while (nextrec != NULL) {
1675  HTPCfgRec *htprec = nextrec;
1676  nextrec = nextrec->next;
1677 
1678  htp_config_destroy(htprec->cfg);
1679  SCFree(htprec);
1680  }
1681  SCReturn;
1682 }
1683 
1684 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
1685 {
1686  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
1687  if (htud != NULL) {
1688  htud->request_has_trailers = 1;
1689  }
1690  return HTP_OK;
1691 }
1692 
1693 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
1694 {
1695  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
1696  if (htud != NULL) {
1697  htud->response_has_trailers = 1;
1698  }
1699  return HTP_OK;
1700 }
1701 
1702 /**\internal
1703  * \brief called at start of request
1704  * Set min inspect size.
1705  */
1706 static int HTPCallbackRequestStart(htp_tx_t *tx)
1707 {
1708  HtpState *hstate = htp_connp_get_user_data(tx->connp);
1709  if (hstate == NULL) {
1710  SCReturnInt(HTP_ERROR);
1711  }
1712 
1713  uint64_t consumed = hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
1714  SCLogDebug("HTTP request start: data offset %" PRIu64 ", in_data_counter %" PRIu64, consumed,
1715  (uint64_t)hstate->conn->in_data_counter);
1716 
1717  /* app-layer-frame-documentation tag start: frame registration http request */
1719  hstate->f, hstate->slice, consumed, -1, 0, HTTP_FRAME_REQUEST);
1720  if (frame) {
1721  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
1722  hstate->request_frame_id = frame->id;
1723  AppLayerFrameSetTxId(frame, HtpGetActiveRequestTxID(hstate));
1724  }
1725  /* app-layer-frame-documentation tag end: frame registration http request */
1726 
1727  if (hstate->cfg)
1728  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
1729  hstate->cfg->request.inspect_min_size);
1730 
1731  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
1732  if (tx_ud == NULL) {
1733  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
1734  if (unlikely(tx_ud == NULL)) {
1735  SCReturnInt(HTP_OK);
1736  }
1737  tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files
1738  htp_tx_set_user_data(tx, tx_ud);
1739  }
1740  SCReturnInt(HTP_OK);
1741 }
1742 
1743 /**\internal
1744  * \brief called at start of response
1745  * Set min inspect size.
1746  */
1747 static int HTPCallbackResponseStart(htp_tx_t *tx)
1748 {
1749  HtpState *hstate = htp_connp_get_user_data(tx->connp);
1750  if (hstate == NULL) {
1751  SCReturnInt(HTP_ERROR);
1752  }
1753 
1754  uint64_t consumed = hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
1755  SCLogDebug("HTTP response start: data offset %" PRIu64 ", out_data_counter %" PRIu64, consumed,
1756  (uint64_t)hstate->conn->out_data_counter);
1757 
1759  hstate->f, hstate->slice, consumed, -1, 1, HTTP_FRAME_RESPONSE);
1760  if (frame) {
1761  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
1762  hstate->response_frame_id = frame->id;
1763  AppLayerFrameSetTxId(frame, HtpGetActiveResponseTxID(hstate));
1764  }
1765 
1766  if (hstate->cfg)
1767  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
1768  hstate->cfg->response.inspect_min_size);
1769 
1770  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
1771  if (tx_ud == NULL) {
1772  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
1773  if (unlikely(tx_ud == NULL)) {
1774  SCReturnInt(HTP_OK);
1775  }
1776  tx_ud->tx_data.file_tx =
1777  STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed.
1778  htp_tx_set_user_data(tx, tx_ud);
1779  }
1780  SCReturnInt(HTP_OK);
1781 }
1782 
1783 /**
1784  * \brief callback for request to store the recent incoming request
1785  into the recent_in_tx for the given htp state
1786  * \param connp pointer to the current connection parser which has the htp
1787  * state in it as user data
1788  */
1789 static int HTPCallbackRequestComplete(htp_tx_t *tx)
1790 {
1791  SCEnter();
1792 
1793  if (tx == NULL) {
1794  SCReturnInt(HTP_ERROR);
1795  }
1796 
1797  HtpState *hstate = htp_connp_get_user_data(tx->connp);
1798  if (hstate == NULL) {
1799  SCReturnInt(HTP_ERROR);
1800  }
1801 
1802  const uint64_t abs_right_edge =
1803  hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
1804 
1805  /* app-layer-frame-documentation tag start: updating frame->len */
1806  if (hstate->request_frame_id > 0) {
1807  Frame *frame = AppLayerFrameGetById(hstate->f, 0, hstate->request_frame_id);
1808  if (frame) {
1809  const uint64_t request_size = abs_right_edge - hstate->last_request_data_stamp;
1810 
1811  SCLogDebug("HTTP request complete: data offset %" PRIu64 ", request_size %" PRIu64,
1812  hstate->last_request_data_stamp, request_size);
1813  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
1814  request_size);
1815  frame->len = (int64_t)request_size;
1816  /* app-layer-frame-documentation tag end: updating frame->len */
1817  }
1818  hstate->request_frame_id = 0;
1819  }
1820 
1821  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
1822  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
1823 
1824  SCLogDebug("HTTP request completed");
1825 
1826  HTPErrorCheckTxRequestFlags(hstate, tx);
1827 
1828  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
1829  if (htud != NULL) {
1830  if (htud->tsflags & HTP_FILENAME_SET) {
1831  SCLogDebug("closing file that was being stored");
1832  (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER);
1833  htud->tsflags &= ~HTP_FILENAME_SET;
1834  if (abs_right_edge < (uint64_t)UINT32_MAX) {
1836  hstate->f->protoctx, STREAM_TOSERVER, (uint32_t)abs_right_edge);
1837  }
1838  }
1839  }
1840 
1841  hstate->last_request_data_stamp = abs_right_edge;
1842  /* request done, do raw reassembly now to inspect state and stream
1843  * at the same time. */
1844  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1845  SCReturnInt(HTP_OK);
1846 }
1847 
1848 /**
1849  * \brief callback for response to remove the recent received requests
1850  from the recent_in_tx for the given htp state
1851  * \param connp pointer to the current connection parser which has the htp
1852  * state in it as user data
1853  */
1854 static int HTPCallbackResponseComplete(htp_tx_t *tx)
1855 {
1856  SCEnter();
1857 
1858  HtpState *hstate = htp_connp_get_user_data(tx->connp);
1859  if (hstate == NULL) {
1860  SCReturnInt(HTP_ERROR);
1861  }
1862 
1863  /* we have one whole transaction now */
1864  hstate->transaction_cnt++;
1865 
1866  const uint64_t abs_right_edge =
1867  hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
1868 
1869  if (hstate->response_frame_id > 0) {
1870  Frame *frame = AppLayerFrameGetById(hstate->f, 1, hstate->response_frame_id);
1871  if (frame) {
1872  const uint64_t response_size = abs_right_edge - hstate->last_response_data_stamp;
1873 
1874  SCLogDebug("HTTP response complete: data offset %" PRIu64 ", response_size %" PRIu64,
1875  hstate->last_response_data_stamp, response_size);
1876  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
1877  response_size);
1878  frame->len = (int64_t)response_size;
1879  }
1880  hstate->response_frame_id = 0;
1881  }
1882 
1883  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
1884  if (htud != NULL) {
1885  if (htud->tcflags & HTP_FILENAME_SET) {
1886  SCLogDebug("closing file that was being stored");
1887  (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOCLIENT);
1888  htud->tcflags &= ~HTP_FILENAME_SET;
1889  }
1890  }
1891 
1892  /* response done, do raw reassembly now to inspect state and stream
1893  * at the same time. */
1894  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
1895 
1896  /* handle HTTP CONNECT */
1897  if (tx->request_method_number == HTP_M_CONNECT) {
1898  /* any 2XX status response implies that the connection will become
1899  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
1900  if ((tx->response_status_number >= 200) &&
1901  (tx->response_status_number < 300) &&
1902  (hstate->transaction_cnt == 1)) {
1903  uint16_t dp = 0;
1904  if (tx->request_port_number != -1) {
1905  dp = (uint16_t)tx->request_port_number;
1906  }
1907  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
1908  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN)) {
1909  HTPSetEvent(
1910  hstate, htud, STREAM_TOCLIENT, HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE);
1911  }
1912  tx->request_progress = HTP_REQUEST_COMPLETE;
1913  tx->response_progress = HTP_RESPONSE_COMPLETE;
1914  }
1915  }
1916 
1917  hstate->last_response_data_stamp = abs_right_edge;
1918  SCReturnInt(HTP_OK);
1919 }
1920 
1921 static int HTPCallbackRequestLine(htp_tx_t *tx)
1922 {
1923  HtpTxUserData *tx_ud;
1924  bstr *request_uri_normalized;
1925  HtpState *hstate = htp_connp_get_user_data(tx->connp);
1926  const HTPCfgRec *cfg = hstate->cfg;
1927 
1928  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
1929  if (request_uri_normalized == NULL)
1930  return HTP_OK;
1931 
1932  tx_ud = htp_tx_get_user_data(tx);
1933  if (unlikely(tx_ud == NULL)) {
1934  bstr_free(request_uri_normalized);
1935  return HTP_OK;
1936  }
1937  if (unlikely(tx_ud->request_uri_normalized != NULL))
1938  bstr_free(tx_ud->request_uri_normalized);
1939  tx_ud->request_uri_normalized = request_uri_normalized;
1940 
1941  if (tx->flags) {
1942  HTPErrorCheckTxRequestFlags(hstate, tx);
1943  }
1944  return HTP_OK;
1945 }
1946 
1947 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
1948 {
1949  if (part == NULL)
1950  return HTP_OK;
1951 
1952  uint64_t flags = 0;
1953  size_t prevlen = bstr_len(part);
1954  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
1955  // shorter string means that uri was encoded
1956  if (res == HTP_OK && prevlen > bstr_len(part)) {
1957  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
1958  if (htud == NULL)
1959  return HTP_OK;
1960  HtpState *s = htp_connp_get_user_data(tx->connp);
1961  if (s == NULL)
1962  return HTP_OK;
1963  HTPSetEvent(s, htud, STREAM_TOSERVER,
1965  }
1966 
1967  return HTP_OK;
1968 }
1969 
1970 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
1971 {
1972  if (tx->parsed_uri == NULL)
1973  return HTP_OK;
1974 
1975  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
1976 }
1977 
1978 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
1979 {
1980  if (tx->parsed_uri == NULL)
1981  return HTP_OK;
1982 
1983  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
1984 }
1985 
1986 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
1987 {
1988  void *ptmp;
1989  if (tx_data->len == 0 || tx_data->tx == NULL)
1990  return HTP_OK;
1991 
1992  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
1993  if (tx_ud == NULL) {
1994  return HTP_OK;
1995  }
1996  ptmp = HTPRealloc(tx_ud->request_headers_raw,
1997  tx_ud->request_headers_raw_len,
1998  tx_ud->request_headers_raw_len + tx_data->len);
1999  if (ptmp == NULL) {
2000  return HTP_OK;
2001  }
2002  tx_ud->request_headers_raw = ptmp;
2003 
2004  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2005  tx_data->data, tx_data->len);
2006  tx_ud->request_headers_raw_len += tx_data->len;
2007 
2008  if (tx_data->tx && tx_data->tx->flags) {
2009  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2010  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2011  }
2012  return HTP_OK;
2013 }
2014 
2015 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2016 {
2017  void *ptmp;
2018  if (tx_data->len == 0 || tx_data->tx == NULL)
2019  return HTP_OK;
2020 
2021  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2022  if (tx_ud == NULL) {
2023  return HTP_OK;
2024  }
2025  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2026  tx_ud->response_headers_raw_len,
2027  tx_ud->response_headers_raw_len + tx_data->len);
2028  if (ptmp == NULL) {
2029  return HTP_OK;
2030  }
2031  tx_ud->response_headers_raw = ptmp;
2032 
2033  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2034  tx_data->data, tx_data->len);
2035  tx_ud->response_headers_raw_len += tx_data->len;
2036 
2037  return HTP_OK;
2038 }
2039 
2040 /*
2041  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2042  */
2043 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2044 {
2045  cfg_prec->uri_include_all = false;
2052 
2053  if (!g_disable_randomness) {
2055  } else {
2056  cfg_prec->randomize = 0;
2057  }
2059 
2060  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2061  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2062  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2063  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2064 
2065  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2066  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2067 
2068  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2069  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2070 
2071  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2072  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequestComplete);
2073 
2074  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2075  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponseComplete);
2076 
2077  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2078 #ifdef HAVE_HTP_CONFIG_SET_ALLOW_SPACE_URI
2079  htp_config_set_allow_space_uri(cfg_prec->cfg, 1);
2080 #endif
2081 
2082  /* don't convert + to space by default */
2083  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2084  // enables request decompression
2085  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2086 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2087  // disable by default
2088  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2089 #endif
2090 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2091  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2093 #endif
2094 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2095  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2097 #endif
2098 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2099  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2100 #endif
2101 #ifdef HAVE_HTP_CONFIG_SET_MAX_TX
2102 #define HTP_CONFIG_DEFAULT_MAX_TX_LIMIT 512
2103  htp_config_set_max_tx(cfg_prec->cfg, HTP_CONFIG_DEFAULT_MAX_TX_LIMIT);
2104 #endif
2105  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2106  * only the hard limit. So we set both here to the (current) htp defaults.
2107  * The reason we do this is that if the user sets the hard limit in the
2108  * config, we have to set the soft limit as well. If libhtp starts using
2109  * the soft limit in the future, we at least make sure we control what
2110  * it's value is. */
2111  htp_config_set_field_limits(cfg_prec->cfg, (size_t)HTP_CONFIG_DEFAULT_FIELD_LIMIT_SOFT,
2113 }
2114 
2115 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2116  * but we can get both <0 and >RAND_MAX values from RandomGet
2117  */
2118 static int RandomGetWrap(void)
2119 {
2120  unsigned long r;
2121 
2122  do {
2123  r = RandomGet();
2124  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2125 
2126  return r % RAND_MAX;
2127 }
2128 
2129 /*
2130  * We have this splitup so that in case double decoding has been enabled
2131  * for query and path, they would be called first on the callback queue,
2132  * before the callback set by Phase2() is called. We need this, since
2133  * the callback in Phase2() generates the normalized uri which utilizes
2134  * the query and path. */
2135 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2136 {
2137  /* randomize inspection size if needed */
2138  if (cfg_prec->randomize) {
2139  int rdrange = cfg_prec->randomize_range;
2140 
2141  long int r = RandomGetWrap();
2142  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2143  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2144 
2145  r = RandomGetWrap();
2146  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2147  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2148  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2149  " %u and 'request-body-inspect-window' set to %u after"
2150  " randomization.",
2151  name, cfg_prec->request.inspect_min_size, cfg_prec->request.inspect_window);
2152 
2153  r = RandomGetWrap();
2154  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2155  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2156 
2157  r = RandomGetWrap();
2158  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2159  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2160 
2161  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2162  " %u and 'response-body-inspect-window' set to %u after"
2163  " randomization.",
2164  name, cfg_prec->response.inspect_min_size, cfg_prec->response.inspect_window);
2165  }
2166 
2167  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2168 }
2169 
2170 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2171  SCRadixTree *tree)
2172 {
2173  if (cfg_prec == NULL || s == NULL || tree == NULL)
2174  return;
2175 
2176  ConfNode *p = NULL;
2177 
2178  /* Default Parameters */
2179  TAILQ_FOREACH(p, &s->head, next) {
2180 
2181  if (strcasecmp("address", p->name) == 0) {
2182  ConfNode *pval;
2183  /* Addresses */
2184  TAILQ_FOREACH(pval, &p->head, next) {
2185  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2186  pval->val);
2187 
2188  /* IPV6 or IPV4? */
2189  if (strchr(pval->val, ':') != NULL) {
2190  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2191  s->name, pval->val, cfg_prec->cfg);
2192  if (!SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec)) {
2193  SCLogWarning("LIBHTP failed to "
2194  "add ipv6 server %s, ignoring",
2195  pval->val);
2196  }
2197  } else {
2198  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2199  s->name, pval->val, cfg_prec->cfg);
2200  if (!SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec)) {
2201  SCLogWarning("LIBHTP failed "
2202  "to add ipv4 server %s, ignoring",
2203  pval->val);
2204  }
2205  } /* else - if (strchr(pval->val, ':') != NULL) */
2206  } /* TAILQ_FOREACH(pval, &p->head, next) */
2207 
2208  } else if (strcasecmp("personality", p->name) == 0) {
2209  /* Personalities */
2210  int personality = HTPLookupPersonality(p->val);
2211  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2212  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2213 
2214  if (personality >= 0) {
2215  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2216  personality);
2217  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2218  SCLogWarning("LIBHTP Failed adding "
2219  "personality \"%s\", ignoring",
2220  p->val);
2221  } else {
2222  SCLogDebug("LIBHTP personality set to %s",
2223  HTPLookupPersonalityString(personality));
2224  }
2225 
2226  /* The IDS personality by default converts the path (and due to
2227  * our query string callback also the query string) to lowercase.
2228  * Signatures do not expect this, so override it. */
2229  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2230  } else {
2231  SCLogWarning("LIBHTP Unknown personality "
2232  "\"%s\", ignoring",
2233  p->val);
2234  continue;
2235  }
2236 
2237  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2238  strcasecmp("request_body_limit", p->name) == 0) {
2239  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2240  SCLogError("Error parsing request-body-limit "
2241  "from conf file - %s. Killing engine",
2242  p->val);
2243  exit(EXIT_FAILURE);
2244  }
2245 
2246  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2247  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2248  SCLogError("Error parsing response-body-limit "
2249  "from conf file - %s. Killing engine",
2250  p->val);
2251  exit(EXIT_FAILURE);
2252  }
2253 
2254  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2255  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2256  SCLogError("Error parsing request-body-minimal-inspect-size "
2257  "from conf file - %s. Killing engine",
2258  p->val);
2259  exit(EXIT_FAILURE);
2260  }
2261 
2262  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2263  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2264  SCLogError("Error parsing request-body-inspect-window "
2265  "from conf file - %s. Killing engine",
2266  p->val);
2267  exit(EXIT_FAILURE);
2268  }
2269 
2270  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2271  if (ConfValIsTrue(p->val)) {
2272  htp_config_register_request_line(cfg_prec->cfg,
2273  HTPCallbackDoubleDecodeQuery);
2274  }
2275 
2276  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2277  if (ConfValIsTrue(p->val)) {
2278  htp_config_register_request_line(cfg_prec->cfg,
2279  HTPCallbackDoubleDecodePath);
2280  }
2281 
2282  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2283  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2284  SCLogError("Error parsing response-body-minimal-inspect-size "
2285  "from conf file - %s. Killing engine",
2286  p->val);
2287  exit(EXIT_FAILURE);
2288  }
2289 
2290  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2291  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2292  SCLogError("Error parsing response-body-inspect-window "
2293  "from conf file - %s. Killing engine",
2294  p->val);
2295  exit(EXIT_FAILURE);
2296  }
2297 
2298  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2299  uint32_t value = 2;
2300  if (ParseSizeStringU32(p->val, &value) < 0) {
2301  SCLogError("Error parsing response-body-inspect-window "
2302  "from conf file - %s. Killing engine",
2303  p->val);
2304  exit(EXIT_FAILURE);
2305  }
2306 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2307  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2308 #else
2309  SCLogWarning("can't set response-body-decompress-layer-limit "
2310  "to %u, libhtp version too old",
2311  value);
2312 #endif
2313  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2314  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2315  HTP_DECODER_URL_PATH,
2316  ConfValIsTrue(p->val));
2317  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2318  if (strlen(p->val) == 1) {
2319  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2320  HTP_DECODER_URL_PATH,
2321  p->val[0]);
2322  } else {
2323  SCLogError("Invalid entry "
2324  "for libhtp param path-bestfit-replacement-char");
2325  }
2326  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2327  htp_config_set_convert_lowercase(cfg_prec->cfg,
2328  HTP_DECODER_URL_PATH,
2329  ConfValIsTrue(p->val));
2330  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2331  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2332  HTP_DECODER_URL_PATH,
2333  ConfValIsTrue(p->val));
2334  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2335  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2336  HTP_DECODER_URL_PATH,
2337  ConfValIsTrue(p->val));
2338  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2339  htp_config_set_path_separators_compress(cfg_prec->cfg,
2340  HTP_DECODER_URL_PATH,
2341  ConfValIsTrue(p->val));
2342  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2343  htp_config_set_path_separators_decode(cfg_prec->cfg,
2344  HTP_DECODER_URL_PATH,
2345  ConfValIsTrue(p->val));
2346  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2347  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2348  HTP_DECODER_URL_PATH,
2349  ConfValIsTrue(p->val));
2350  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2351  enum htp_url_encoding_handling_t handling;
2352  if (strcasecmp(p->val, "preserve_percent") == 0) {
2353  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2354  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2355  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2356  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2357  handling = HTP_URL_DECODE_PROCESS_INVALID;
2358  } else {
2359  SCLogError("Invalid entry "
2360  "for libhtp param path-url-encoding-invalid-handling");
2361  return;
2362  }
2363  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2364  HTP_DECODER_URL_PATH,
2365  handling);
2366  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2367  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2368  HTP_DECODER_URL_PATH,
2369  ConfValIsTrue(p->val));
2370  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2371  cfg_prec->uri_include_all = (1 == ConfValIsTrue(p->val));
2372  SCLogDebug("uri-include-all %s",
2373  cfg_prec->uri_include_all ? "enabled" : "disabled");
2374  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2375  htp_config_set_plusspace_decode(cfg_prec->cfg,
2376  HTP_DECODER_URLENCODED,
2377  ConfValIsTrue(p->val));
2378  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2379  uint32_t limit = 0;
2380  if (ParseSizeStringU32(p->val, &limit) < 0) {
2381  SCLogError("Error meta-field-limit "
2382  "from conf file - %s. Killing engine",
2383  p->val);
2384  exit(EXIT_FAILURE);
2385  }
2386  if (limit == 0) {
2387  FatalError("Error meta-field-limit "
2388  "from conf file cannot be 0. Killing engine");
2389  }
2390  /* set default soft-limit with our new hard limit */
2391  htp_config_set_field_limits(cfg_prec->cfg,
2393  (size_t)limit);
2394 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2395  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2396  uint32_t limit = 0;
2397  if (ParseSizeStringU32(p->val, &limit) < 0) {
2398  FatalError("failed to parse 'lzma-memlimit' "
2399  "from conf file - %s.",
2400  p->val);
2401  }
2402  if (limit == 0) {
2403  FatalError("'lzma-memlimit' "
2404  "from conf file cannot be 0.");
2405  }
2406  /* set default soft-limit with our new hard limit */
2407  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2408  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2409 #endif
2410 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2411  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2412  if (ConfValIsTrue(p->val)) {
2413  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2414  } else if (!ConfValIsFalse(p->val)) {
2415  int8_t limit;
2416  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2417  FatalError("failed to parse 'lzma-enabled' "
2418  "from conf file - %s.",
2419  p->val);
2420  }
2421  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2422  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2423  }
2424 #endif
2425 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2426  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2427  uint32_t limit = 0;
2428  if (ParseSizeStringU32(p->val, &limit) < 0) {
2429  FatalError("failed to parse 'compression-bomb-limit' "
2430  "from conf file - %s.",
2431  p->val);
2432  }
2433  if (limit == 0) {
2434  FatalError("'compression-bomb-limit' "
2435  "from conf file cannot be 0.");
2436  }
2437  /* set default soft-limit with our new hard limit */
2438  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2439  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2440 #endif
2441 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2442  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2443  uint32_t limit = 0;
2444  // between 1 usec and 1 second
2445  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2446  FatalError("failed to parse 'decompression-time-limit' "
2447  "from conf file - %s.",
2448  p->val);
2449  }
2450  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2451  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2452 #endif
2453 #ifdef HAVE_HTP_CONFIG_SET_MAX_TX
2454  } else if (strcasecmp("max-tx", p->name) == 0) {
2455  uint32_t limit = 0;
2456  if (ParseSizeStringU32(p->val, &limit) < 0) {
2457  FatalError("failed to parse 'max-tx' "
2458  "from conf file - %s.",
2459  p->val);
2460  }
2461  /* set default soft-limit with our new hard limit */
2462  SCLogConfig("Setting HTTP max-tx limit to %" PRIu32 " bytes", limit);
2463  htp_config_set_max_tx(cfg_prec->cfg, limit);
2464 #endif
2465  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2466  if (!g_disable_randomness) {
2467  cfg_prec->randomize = ConfValIsTrue(p->val);
2468  }
2469  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2470  uint32_t range;
2471  if (StringParseU32RangeCheck(&range, 10, 0,
2472  (const char *)p->val, 0, 100) < 0) {
2473  SCLogError("Invalid value for randomize"
2474  "-inspection-range setting from conf file - \"%s\"."
2475  " It should be a valid integer less than or equal to 100."
2476  " Killing engine",
2477  p->val);
2478  exit(EXIT_FAILURE);
2479  }
2480  cfg_prec->randomize_range = range;
2481  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2482  if (ConfValIsTrue(p->val)) {
2483  cfg_prec->http_body_inline = 1;
2484  } else if (ConfValIsFalse(p->val)) {
2485  cfg_prec->http_body_inline = 0;
2486  } else {
2487  if (strcmp("auto", p->val) != 0) {
2488  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2489  }
2490  if (EngineModeIsIPS()) {
2491  cfg_prec->http_body_inline = 1;
2492  } else {
2493  cfg_prec->http_body_inline = 0;
2494  }
2495  }
2496  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2497  ConfNode *pval;
2498 
2499  TAILQ_FOREACH(pval, &p->head, next) {
2500  if (strcasecmp("enabled", pval->name) == 0) {
2501  if (ConfValIsTrue(pval->val)) {
2502  cfg_prec->swf_decompression_enabled = 1;
2503  } else if (ConfValIsFalse(pval->val)) {
2504  cfg_prec->swf_decompression_enabled = 0;
2505  } else {
2506  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2507  }
2508  } else if (strcasecmp("type", pval->name) == 0) {
2509  if (strcasecmp("no", pval->val) == 0) {
2511  } else if (strcasecmp("deflate", pval->val) == 0) {
2513  } else if (strcasecmp("lzma", pval->val) == 0) {
2515  } else if (strcasecmp("both", pval->val) == 0) {
2517  } else {
2518  SCLogError("Invalid entry for "
2519  "swf-decompression.type: %s - "
2520  "Killing engine",
2521  pval->val);
2522  exit(EXIT_FAILURE);
2523  }
2524  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2525  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2526  SCLogError("Error parsing swf-decompression.compression-depth "
2527  "from conf file - %s. Killing engine",
2528  p->val);
2529  exit(EXIT_FAILURE);
2530  }
2531  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2532  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2533  SCLogError("Error parsing swf-decompression.decompression-depth "
2534  "from conf file - %s. Killing engine",
2535  p->val);
2536  exit(EXIT_FAILURE);
2537  }
2538  } else {
2539  SCLogWarning("Ignoring unknown param %s", pval->name);
2540  }
2541  }
2542  } else {
2543  SCLogWarning("LIBHTP Ignoring unknown "
2544  "default config: %s",
2545  p->name);
2546  }
2547  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2548 }
2549 
2550 void HTPConfigure(void)
2551 {
2552  SCEnter();
2553 
2554  cfglist.next = NULL;
2555 
2559 
2560  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2561  if (NULL == cfgtree)
2562  exit(EXIT_FAILURE);
2563 
2564  /* Default Config */
2565  cfglist.cfg = htp_config_create();
2566  if (NULL == cfglist.cfg) {
2567  FatalError("Failed to create HTP default config");
2568  }
2569  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2570  HTPConfigSetDefaultsPhase1(&cfglist);
2571  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2572  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2573  cfgtree);
2574  } else {
2575  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2576  }
2577  HTPConfigSetDefaultsPhase2("default", &cfglist);
2578 
2579  HTPParseMemcap();
2580 
2581  /* Read server config and create a parser for each IP in radix tree */
2582  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2583  if (server_config == NULL) {
2584  server_config = ConfGetNode("libhtp.server-config");
2585  if (server_config == NULL) {
2586  SCLogDebug("LIBHTP Configuring %p", server_config);
2587  SCReturn;
2588  }
2589  }
2590  SCLogDebug("LIBHTP Configuring %p", server_config);
2591 
2592  ConfNode *si;
2593  /* Server Nodes */
2594  TAILQ_FOREACH(si, &server_config->head, next) {
2595  /* Need the named node, not the index */
2596  ConfNode *s = TAILQ_FIRST(&si->head);
2597  if (NULL == s) {
2598  SCLogDebug("LIBHTP s NULL");
2599  continue;
2600  }
2601 
2602  SCLogDebug("LIBHTP server %s", s->name);
2603 
2604  HTPCfgRec *nextrec = cfglist.next;
2605  HTPCfgRec *htprec = SCCalloc(1, sizeof(HTPCfgRec));
2606  if (NULL == htprec)
2607  exit(EXIT_FAILURE);
2608 
2609  cfglist.next = htprec;
2610 
2611  cfglist.next->next = nextrec;
2612  cfglist.next->cfg = htp_config_create();
2613  if (NULL == cfglist.next->cfg) {
2614  FatalError("Failed to create HTP server config");
2615  }
2616 
2617  HTPConfigSetDefaultsPhase1(htprec);
2618  HTPConfigParseParameters(htprec, s, cfgtree);
2619  HTPConfigSetDefaultsPhase2(s->name, htprec);
2620  }
2621 
2622  SCReturn;
2623 }
2624 
2626 {
2627 #ifdef DEBUG
2628  SCMutexLock(&htp_state_mem_lock);
2629  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2630  SCMutexUnlock(&htp_state_mem_lock);
2631 #endif
2632 }
2633 
2634 /** \internal
2635  * \brief get files callback
2636  * \param state state ptr
2637  * \param direction flow direction
2638  * \retval files files ptr
2639  */
2640 static AppLayerGetFileState HTPGetTxFiles(void *txv, uint8_t direction)
2641 {
2642  AppLayerGetFileState files = { .fc = NULL, .cfg = &htp_sbcfg };
2643  htp_tx_t *tx = (htp_tx_t *)txv;
2644  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2645  if (tx_ud) {
2646  if (direction & STREAM_TOCLIENT) {
2647  files.fc = &tx_ud->files_tc;
2648  } else {
2649  files.fc = &tx_ud->files_ts;
2650  }
2651  }
2652  return files;
2653 }
2654 
2655 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2656 {
2657  if (direction & STREAM_TOSERVER)
2658  return ((htp_tx_t *)tx)->request_progress;
2659  else
2660  return ((htp_tx_t *)tx)->response_progress;
2661 }
2662 
2663 static uint64_t HTPStateGetTxCnt(void *alstate)
2664 {
2665  HtpState *http_state = (HtpState *)alstate;
2666 
2667  if (http_state != NULL && http_state->conn != NULL) {
2668  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
2669  if (size < 0)
2670  return 0ULL;
2671  SCLogDebug("size %"PRIu64, size);
2672  return (uint64_t)size + http_state->tx_freed;
2673  } else {
2674  return 0ULL;
2675  }
2676 }
2677 
2678 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2679 {
2680  HtpState *http_state = (HtpState *)alstate;
2681 
2682  if (http_state != NULL && http_state->conn != NULL && tx_id >= http_state->tx_freed)
2683  return htp_list_get(http_state->conn->transactions, tx_id - http_state->tx_freed);
2684  else
2685  return NULL;
2686 }
2687 
2688 void *HtpGetTxForH2(void *alstate)
2689 {
2690  // gets last transaction
2691  HtpState *http_state = (HtpState *)alstate;
2692  if (http_state != NULL && http_state->conn != NULL) {
2693  size_t txid = HTPStateGetTxCnt(http_state);
2694  if (txid > http_state->tx_freed) {
2695  return htp_list_get(http_state->conn->transactions, txid - http_state->tx_freed - 1);
2696  }
2697  }
2698  return NULL;
2699 }
2700 
2701 static int HTPStateGetEventInfo(const char *event_name,
2702  int *event_id, AppLayerEventType *event_type)
2703 {
2704  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
2705  if (*event_id == -1) {
2706  SCLogError("event \"%s\" not present in "
2707  "http's enum map table.",
2708  event_name);
2709  /* this should be treated as fatal */
2710  return -1;
2711  }
2712 
2713  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2714 
2715  return 0;
2716 }
2717 
2718 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
2719  AppLayerEventType *event_type)
2720 {
2721  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
2722  if (*event_name == NULL) {
2723  SCLogError("event \"%d\" not present in "
2724  "http's enum map table.",
2725  event_id);
2726  /* this should be treated as fatal */
2727  return -1;
2728  }
2729 
2730  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2731 
2732  return 0;
2733 }
2734 
2735 static AppLayerTxData *HTPGetTxData(void *vtx)
2736 {
2737  htp_tx_t *tx = (htp_tx_t *)vtx;
2738  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2739  if (tx_ud) {
2740  return &tx_ud->tx_data;
2741  }
2742  return NULL;
2743 }
2744 
2745 static AppLayerStateData *HTPGetStateData(void *vstate)
2746 {
2747  HtpState *s = vstate;
2748  return &s->state_data;
2749 }
2750 
2751 static int HTPRegisterPatternsForProtocolDetection(void)
2752 {
2753  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
2754  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
2755  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
2756  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
2757  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
2758  const char *spacings[] = { "|20|", "|09|", NULL };
2759  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
2760 
2761  int methods_pos;
2762  int spacings_pos;
2763  int versions_pos;
2764  int register_result;
2765  char method_buffer[32] = "";
2766 
2767  /* Loop through all the methods ands spacings and register the patterns */
2768  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
2769  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
2770 
2771  /* Combine the method name and the spacing */
2772  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
2773 
2774  /* Register the new method+spacing pattern
2775  * 3 is subtracted from the length since the spacing is hex typed as |xx|
2776  * but the pattern matching should only be one char
2777  */
2778  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
2779  method_buffer, (uint16_t)strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
2780  if (register_result < 0) {
2781  return -1;
2782  }
2783  }
2784  }
2785 
2786  /* Loop through all the http version patterns that are TO_CLIENT */
2787  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
2788  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
2789  versions[versions_pos], (uint16_t)strlen(versions[versions_pos]), 0,
2790  STREAM_TOCLIENT);
2791  if (register_result < 0) {
2792  return -1;
2793  }
2794  }
2795 
2796  return 0;
2797 }
2798 
2799 /**
2800  * \brief Register the HTTP protocol and state handling functions to APP layer
2801  * of the engine.
2802  */
2804 {
2805  SCEnter();
2806 
2807  const char *proto_name = "http";
2808 
2809  /** HTTP */
2810  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
2812  if (HTPRegisterPatternsForProtocolDetection() < 0)
2813  return;
2814  } else {
2815  SCLogInfo("Protocol detection and parser disabled for %s protocol",
2816  proto_name);
2817  return;
2818  }
2819 
2820  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
2821  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
2822  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
2823  AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles);
2825  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
2826  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
2827  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
2828 
2830  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
2831  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
2833  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
2834 
2835  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
2836  AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData);
2837 
2839  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
2840 
2842  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
2844  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
2845  SC_ATOMIC_INIT(htp_config_flags);
2846  /* This parser accepts gaps. */
2850  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_TOCLIENT);
2851  /* app-layer-frame-documentation tag start: registering relevant callbacks */
2853  IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById);
2854  /* app-layer-frame-documentation tag end: registering relevant callbacks */
2855  HTPConfigure();
2856  } else {
2857  SCLogInfo("Parser disabled for %s protocol. Protocol detection still on.", proto_name);
2858  }
2859 #ifdef UNITTESTS
2860  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
2861 #endif
2862 
2863  SCReturn;
2864 }
2865 
2866 #ifdef UNITTESTS
2867 #include "detect-engine-alert.h"
2868 
2869 static HTPCfgRec cfglist_backup;
2870 
2872 {
2873  cfglist_backup = cfglist;
2874 }
2875 
2877 {
2878  cfglist = cfglist_backup;
2879 }
2880 
2881 /** \test Test case where chunks are sent in smaller chunks and check the
2882  * response of the parser from HTP library. */
2883 static int HTPParserTest01(void)
2884 {
2885  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
2886  " Data is c0oL!";
2887  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2888 
2889  TcpSession ssn;
2890  memset(&ssn, 0, sizeof(ssn));
2891 
2894 
2895  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
2896  FAIL_IF_NULL(f);
2897  f->protoctx = &ssn;
2898  f->proto = IPPROTO_TCP;
2899  f->alproto = ALPROTO_HTTP1;
2900 
2901  StreamTcpInitConfig(true);
2902 
2903  uint32_t u;
2904  for (u = 0; u < httplen1; u++) {
2905  uint8_t flags = 0;
2906 
2907  if (u == 0)
2908  flags = STREAM_TOSERVER|STREAM_START;
2909  else if (u == (httplen1 - 1))
2910  flags = STREAM_TOSERVER|STREAM_EOF;
2911  else
2912  flags = STREAM_TOSERVER;
2913 
2914  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
2915  FAIL_IF(r != 0);
2916  }
2917 
2918  HtpState *htp_state = f->alstate;
2919  FAIL_IF_NULL(htp_state);
2920 
2921  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
2922  FAIL_IF_NULL(tx);
2923 
2924  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
2925  FAIL_IF_NULL(h);
2926 
2927  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
2928  FAIL_IF(tx->request_method_number != HTP_M_POST);
2929  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
2930 
2932  StreamTcpFreeConfig(true);
2933  UTHFreeFlow(f);
2934  PASS;
2935 }
2936 
2937 /** \test Test folding in 1 read case */
2938 static int HTPParserTest01b(void)
2939 {
2940  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
2941  " Data is c0oL!";
2942  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2943 
2944  TcpSession ssn;
2945  memset(&ssn, 0, sizeof(ssn));
2946 
2949 
2950  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
2951  FAIL_IF_NULL(f);
2952  f->protoctx = &ssn;
2953  f->proto = IPPROTO_TCP;
2954  f->alproto = ALPROTO_HTTP1;
2955 
2956  StreamTcpInitConfig(true);
2957 
2958  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
2959  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
2960  FAIL_IF(r != 0);
2961 
2962  HtpState *htp_state = f->alstate;
2963  FAIL_IF_NULL(htp_state);
2964 
2965  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
2966  FAIL_IF_NULL(tx);
2967 
2968  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
2969  FAIL_IF_NULL(h);
2970 
2971  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
2972  FAIL_IF(tx->request_method_number != HTP_M_POST);
2973  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
2974 
2976  StreamTcpFreeConfig(true);
2977  UTHFreeFlow(f);
2978  PASS;
2979 }
2980 
2981 /** \test Test folding in 1byte per read case */
2982 static int HTPParserTest01c(void)
2983 {
2984  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
2985  " Data is c0oL!";
2986  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2987 
2988  TcpSession ssn;
2989  memset(&ssn, 0, sizeof(ssn));
2990 
2993 
2994  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
2995  FAIL_IF_NULL(f);
2996  f->protoctx = &ssn;
2997  f->proto = IPPROTO_TCP;
2998  f->alproto = ALPROTO_HTTP1;
2999 
3000  StreamTcpInitConfig(true);
3001 
3002  uint32_t u;
3003  for (u = 0; u < httplen1; u++) {
3004  uint8_t flags = 0;
3005 
3006  if (u == 0)
3007  flags = STREAM_TOSERVER|STREAM_START;
3008  else if (u == (httplen1 - 1))
3009  flags = STREAM_TOSERVER|STREAM_EOF;
3010  else
3011  flags = STREAM_TOSERVER;
3012 
3013  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3014  FAIL_IF(r != 0);
3015  }
3016 
3017  HtpState *htp_state = f->alstate;
3018  FAIL_IF_NULL(htp_state);
3019 
3020  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3021  FAIL_IF_NULL(tx);
3022 
3023  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3024  FAIL_IF_NULL(h);
3025 
3026  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3027  FAIL_IF(tx->request_method_number != HTP_M_POST);
3028  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3029 
3031  StreamTcpFreeConfig(true);
3032  UTHFreeFlow(f);
3033  PASS;
3034 }
3035 
3036 /** \test Test case where chunks are sent in smaller chunks and check the
3037  * response of the parser from HTP library. */
3038 static int HTPParserTest01a(void)
3039 {
3040  int result = 0;
3041  Flow *f = NULL;
3042  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3043  " Data is c0oL!";
3044  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3045  TcpSession ssn;
3046  HtpState *htp_state = NULL;
3047  int r = 0;
3049 
3050  memset(&ssn, 0, sizeof(ssn));
3051 
3052  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3053  if (f == NULL)
3054  goto end;
3055  f->protoctx = &ssn;
3056  f->proto = IPPROTO_TCP;
3057  f->alproto = ALPROTO_HTTP1;
3058 
3059  StreamTcpInitConfig(true);
3060 
3061  uint32_t u;
3062  for (u = 0; u < httplen1; u++) {
3063  uint8_t flags = 0;
3064 
3065  if (u == 0)
3066  flags = STREAM_TOSERVER|STREAM_START;
3067  else if (u == (httplen1 - 1))
3068  flags = STREAM_TOSERVER|STREAM_EOF;
3069  else
3070  flags = STREAM_TOSERVER;
3071 
3072  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3073  if (r != 0) {
3074  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3075  " 0: ", u, r);
3076  goto end;
3077  }
3078  }
3079 
3080  htp_state = f->alstate;
3081  if (htp_state == NULL) {
3082  printf("no http state: ");
3083  goto end;
3084  }
3085 
3086  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3087  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3088  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3089  || tx->request_method_number != HTP_M_POST ||
3090  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3091  {
3092  printf("expected header value: Victor/1.0 and got %s: and expected"
3093  " method: POST and got %s, expected protocol number HTTP/1.0"
3094  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3095  bstr_util_strdup_to_c(tx->request_method),
3096  bstr_util_strdup_to_c(tx->request_protocol));
3097  goto end;
3098  }
3099  result = 1;
3100 end:
3101  if (alp_tctx != NULL)
3103  StreamTcpFreeConfig(true);
3104  UTHFreeFlow(f);
3105  return result;
3106 }
3107 
3108 /** \test See how it deals with an incomplete request. */
3109 static int HTPParserTest02(void)
3110 {
3111  int result = 0;
3112  Flow *f = NULL;
3113  uint8_t httpbuf1[] = "POST";
3114  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3115  TcpSession ssn;
3116  HtpState *http_state = NULL;
3118 
3119  memset(&ssn, 0, sizeof(ssn));
3120 
3121  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3122  if (f == NULL)
3123  goto end;
3124  f->protoctx = &ssn;
3125  f->proto = IPPROTO_TCP;
3126  f->alproto = ALPROTO_HTTP1;
3127 
3128  StreamTcpInitConfig(true);
3129 
3130  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3131  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3132  if (r != 0) {
3133  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3134  goto end;
3135  }
3136 
3137  http_state = f->alstate;
3138  if (http_state == NULL) {
3139  printf("no http state: ");
3140  goto end;
3141  }
3142 
3143  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3144  FAIL_IF_NULL(tx);
3145  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3146  FAIL_IF_NOT_NULL(h);
3147 
3148  FAIL_IF_NULL(tx->request_method);
3149  char *method = bstr_util_strdup_to_c(tx->request_method);
3150  FAIL_IF_NULL(method);
3151 
3152  FAIL_IF(strcmp(method, "POST") != 0);
3153  SCFree(method);
3154 
3155  result = 1;
3156 end:
3157  if (alp_tctx != NULL)
3159  StreamTcpFreeConfig(true);
3160  UTHFreeFlow(f);
3161  return result;
3162 }
3163 
3164 /** \test Test case where method is invalid and data is sent in smaller chunks
3165  * and check the response of the parser from HTP library. */
3166 static int HTPParserTest03(void)
3167 {
3168  int result = 0;
3169  Flow *f = NULL;
3170  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3171  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3172  TcpSession ssn;
3173  HtpState *htp_state = NULL;
3174  int r = 0;
3176 
3177  memset(&ssn, 0, sizeof(ssn));
3178 
3179  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3180  if (f == NULL)
3181  goto end;
3182  f->protoctx = &ssn;
3183  f->proto = IPPROTO_TCP;
3184  f->alproto = ALPROTO_HTTP1;
3185 
3186  StreamTcpInitConfig(true);
3187 
3188  uint32_t u;
3189  for (u = 0; u < httplen1; u++) {
3190  uint8_t flags = 0;
3191 
3192  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3193  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3194  else flags = STREAM_TOSERVER;
3195 
3196  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3197  if (r != 0) {
3198  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3199  " 0: ", u, r);
3200  goto end;
3201  }
3202  }
3203  htp_state = f->alstate;
3204  if (htp_state == NULL) {
3205  printf("no http state: ");
3206  goto end;
3207  }
3208 
3209  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3210 
3211  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3212  if (tx->request_method_number != HTP_M_UNKNOWN ||
3213  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3214  {
3215  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3216  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3217  bstr_util_strdup_to_c(tx->request_protocol));
3218  goto end;
3219  }
3220  result = 1;
3221 end:
3222  if (alp_tctx != NULL)
3224  StreamTcpFreeConfig(true);
3225  UTHFreeFlow(f);
3226  return result;
3227 }
3228 
3229 /** \test Test case where invalid data is sent and check the response of the
3230  * parser from HTP library. */
3231 static int HTPParserTest04(void)
3232 {
3233  int result = 0;
3234  Flow *f = NULL;
3235  HtpState *htp_state = NULL;
3236  uint8_t httpbuf1[] = "World!\r\n";
3237  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3238  TcpSession ssn;
3239  int r = 0;
3241 
3242  memset(&ssn, 0, sizeof(ssn));
3243 
3244  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3245  if (f == NULL)
3246  goto end;
3247  f->protoctx = &ssn;
3248  f->proto = IPPROTO_TCP;
3249  f->alproto = ALPROTO_HTTP1;
3250 
3251  StreamTcpInitConfig(true);
3252 
3254  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3255  if (r != 0) {
3256  goto end;
3257  }
3258 
3259  htp_state = f->alstate;
3260  if (htp_state == NULL) {
3261  printf("no http state: ");
3262  goto end;
3263  }
3264 
3265  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3266  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3267  if (tx->request_method_number != HTP_M_UNKNOWN ||
3268  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3269  {
3270  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3271  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3272  bstr_util_strdup_to_c(tx->request_protocol));
3273  goto end;
3274  }
3275  result = 1;
3276 end:
3277  if (alp_tctx != NULL)
3279  StreamTcpFreeConfig(true);
3280  UTHFreeFlow(f);
3281  return result;
3282 }
3283 
3284 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3285  * properly parsed them and also keeps them separated. */
3286 static int HTPParserTest05(void)
3287 {
3288  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3289  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3290  uint8_t httpbuf2[] = "Post D";
3291  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3292  uint8_t httpbuf3[] = "ata is c0oL!";
3293  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3294 
3295  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3296  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3297  uint8_t httpbuf5[] = "post R";
3298  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3299  uint8_t httpbuf6[] = "esults are tha bomb!";
3300  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3301 
3302  TcpSession ssn;
3303  memset(&ssn, 0, sizeof(ssn));
3304 
3307 
3308  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3309  FAIL_IF_NULL(f);
3310  f->protoctx = &ssn;
3311  f->proto = IPPROTO_TCP;
3312  f->alproto = ALPROTO_HTTP1;
3313 
3314  StreamTcpInitConfig(true);
3315 
3316  int r = AppLayerParserParse(
3317  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3318  FAIL_IF(r != 0);
3319 
3320  r = AppLayerParserParse(
3321  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3322  FAIL_IF(r != 0);
3323 
3324  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3325  FAIL_IF(r != 0);
3326 
3327  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3328  FAIL_IF(r != 0);
3329 
3330  r = AppLayerParserParse(
3331  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3332  FAIL_IF(r != 0);
3333 
3334  r = AppLayerParserParse(
3335  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3336  FAIL_IF(r != 0);
3337 
3338  HtpState *http_state = f->alstate;
3339  FAIL_IF_NULL(http_state);
3340 
3341  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3342  FAIL_IF_NULL(tx);
3343  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3344  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3345 
3346  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3347  FAIL_IF_NULL(h);
3348 
3349  FAIL_IF_NOT(tx->response_status_number == 200);
3350 
3352  StreamTcpFreeConfig(true);
3353  UTHFreeFlow(f);
3354  PASS;
3355 }
3356 
3357 /** \test Test proper chunked encoded response body
3358  */
3359 static int HTPParserTest06(void)
3360 {
3361  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3362  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3363  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3364  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3365  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3366  "GMT\r\n"
3367  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3368  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3369  "FrontPage/5.0.2.2510\r\n"
3370  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3371  "chunked\r\n"
3372  "Content-Type: text/html\r\n\r\n"
3373  "580\r\n"
3374  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3375  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3376  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3377  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3378  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3379  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3380  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3381  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3382  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3383  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3384  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3385  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3386  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3387  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3388  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3389  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3390  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3391  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3392  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3393  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3394  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3395  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3396  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3397  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3398  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3399  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3400  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3401  "aHA=\r\n0\r\n\r\n";
3402  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3403  TcpSession ssn;
3404 
3407 
3408  memset(&ssn, 0, sizeof(ssn));
3409 
3410  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3411  FAIL_IF_NULL(f);
3412  f->protoctx = &ssn;
3413  f->proto = IPPROTO_TCP;
3414  f->alproto = ALPROTO_HTTP1;
3415 
3416  StreamTcpInitConfig(true);
3417 
3418  int r = AppLayerParserParse(
3419  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3420  FAIL_IF(r != 0);
3421  r = AppLayerParserParse(
3422  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3423  FAIL_IF(r != 0);
3424 
3425  HtpState *http_state = f->alstate;
3426  FAIL_IF_NULL(http_state);
3427 
3428  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3429  FAIL_IF_NULL(tx);
3430 
3431  FAIL_IF(tx->request_method_number != HTP_M_GET);
3432  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3433 
3434  FAIL_IF(tx->response_status_number != 200);
3435  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3436 
3437  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3438  FAIL_IF_NULL(h);
3439 
3441  StreamTcpFreeConfig(true);
3442  UTHFreeFlow(f);
3443  PASS;
3444 }
3445 
3446 /** \test
3447  */
3448 static int HTPParserTest07(void)
3449 {
3450  int result = 0;
3451  Flow *f = NULL;
3452  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3453  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3454  TcpSession ssn;
3455  HtpState *htp_state = NULL;
3456  int r = 0;
3458 
3459  memset(&ssn, 0, sizeof(ssn));
3460 
3461  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3462  if (f == NULL)
3463  goto end;
3464  f->protoctx = &ssn;
3465  f->proto = IPPROTO_TCP;
3466  f->alproto = ALPROTO_HTTP1;
3467 
3468  StreamTcpInitConfig(true);
3469 
3470  uint32_t u;
3471  for (u = 0; u < httplen1; u++) {
3472  uint8_t flags = 0;
3473 
3474  if (u == 0)
3475  flags = STREAM_TOSERVER|STREAM_START;
3476  else if (u == (httplen1 - 1))
3477  flags = STREAM_TOSERVER|STREAM_EOF;
3478  else
3479  flags = STREAM_TOSERVER;
3480 
3481  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3482  if (r != 0) {
3483  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3484  " 0: ", u, r);
3485  goto end;
3486  }
3487  }
3488 
3489  htp_state = f->alstate;
3490  if (htp_state == NULL) {
3491  printf("no http state: ");
3492  goto end;
3493  }
3494 
3495  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3496  size_t reflen = sizeof(ref) - 1;
3497 
3498  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3499  if (tx == NULL)
3500  goto end;
3501  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3502  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3503  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3504  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3505  (uintmax_t)reflen,
3506  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3507  goto end;
3508  }
3509 
3510  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3511  bstr_len(tx_ud->request_uri_normalized)) != 0)
3512  {
3513  printf("normalized uri \"");
3514  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3515  printf("\" != \"");
3516  PrintRawUriFp(stdout, ref, reflen);
3517  printf("\": ");
3518  goto end;
3519  }
3520  }
3521 
3522  result = 1;
3523 end:
3524  if (alp_tctx != NULL)
3526  StreamTcpFreeConfig(true);
3527  UTHFreeFlow(f);
3528  return result;
3529 }
3530 
3531 #include "conf-yaml-loader.h"
3532 
3533 /** \test Abort
3534  */
3535 static int HTPParserTest08(void)
3536 {
3537  int result = 0;
3538  Flow *f = NULL;
3539  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3540  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3541  TcpSession ssn;
3543 
3544  char input[] = "\
3545 %YAML 1.1\n\
3546 ---\n\
3547 libhtp:\n\
3548 \n\
3549  default-config:\n\
3550  personality: IDS\n\
3551 ";
3552 
3554  ConfInit();
3556 
3557  ConfYamlLoadString(input, strlen(input));
3558  HTPConfigure();
3559 
3560  HtpState *htp_state = NULL;
3561  int r = 0;
3562  memset(&ssn, 0, sizeof(ssn));
3563 
3564  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3565  if (f == NULL)
3566  goto end;
3567  f->protoctx = &ssn;
3568  f->proto = IPPROTO_TCP;
3569  f->alproto = ALPROTO_HTTP1;
3570 
3571  StreamTcpInitConfig(true);
3572 
3573  uint8_t flags = 0;
3574  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3575 
3576  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3577  if (r != 0) {
3578  printf("toserver chunk returned %" PRId32 ", expected"
3579  " 0: ", r);
3580  result = 0;
3581  goto end;
3582  }
3583 
3584  htp_state = f->alstate;
3585  if (htp_state == NULL) {
3586  printf("no http state: ");
3587  result = 0;
3588  goto end;
3589  }
3590 
3591  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3592  if (tx == NULL)
3593  goto end;
3594  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3595  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3596  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3597  bstr_len(tx_ud->request_uri_normalized));
3598  }
3599 
3600  result = 1;
3601 end:
3602  if (alp_tctx != NULL)
3604  StreamTcpFreeConfig(true);
3605  HTPFreeConfig();
3606  ConfDeInit();
3609  UTHFreeFlow(f);
3610  return result;
3611 }
3612 
3613 /** \test Abort
3614  */
3615 static int HTPParserTest09(void)
3616 {
3617  int result = 0;
3618  Flow *f = NULL;
3619  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3620  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3621  TcpSession ssn;
3623 
3624  char input[] = "\
3625 %YAML 1.1\n\
3626 ---\n\
3627 libhtp:\n\
3628 \n\
3629  default-config:\n\
3630  personality: Apache_2_2\n\
3631 ";
3632 
3634  ConfInit();
3636 
3637  ConfYamlLoadString(input, strlen(input));
3638  HTPConfigure();
3639 
3640  HtpState *htp_state = NULL;
3641  int r = 0;
3642 
3643  memset(&ssn, 0, sizeof(ssn));
3644 
3645  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3646  if (f == NULL)
3647  goto end;
3648  f->protoctx = &ssn;
3649  f->proto = IPPROTO_TCP;
3650  f->alproto = ALPROTO_HTTP1;
3651 
3652  StreamTcpInitConfig(true);
3653 
3654  uint8_t flags = 0;
3655  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3656 
3657  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3658  if (r != 0) {
3659  printf("toserver chunk returned %" PRId32 ", expected"
3660  " 0: ", r);
3661  goto end;
3662  }
3663 
3664  htp_state = f->alstate;
3665  if (htp_state == NULL) {
3666  printf("no http state: ");
3667  goto end;
3668  }
3669 
3670  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3671  if (tx == NULL)
3672  goto end;
3673  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3674  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3675  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3676  bstr_len(tx_ud->request_uri_normalized));
3677  }
3678 
3679  result = 1;
3680 end:
3681  if (alp_tctx != NULL)
3683  StreamTcpFreeConfig(true);
3684  HTPFreeConfig();
3685  ConfDeInit();
3688  UTHFreeFlow(f);
3689  return result;
3690 }
3691 
3692 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
3693  */
3694 static int HTPParserTest10(void)
3695 {
3696  int result = 0;
3697  Flow *f = NULL;
3698  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
3699  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3700  TcpSession ssn;
3701  HtpState *htp_state = NULL;
3702  int r = 0;
3704 
3705  memset(&ssn, 0, sizeof(ssn));
3706 
3707  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3708  if (f == NULL)
3709  goto end;
3710  f->protoctx = &ssn;
3711  f->proto = IPPROTO_TCP;
3712  f->alproto = ALPROTO_HTTP1;
3713 
3714  StreamTcpInitConfig(true);
3715 
3716  uint32_t u;
3717  for (u = 0; u < httplen1; u++) {
3718  uint8_t flags = 0;
3719 
3720  if (u == 0)
3721  flags = STREAM_TOSERVER|STREAM_START;
3722  else if (u == (httplen1 - 1))
3723  flags = STREAM_TOSERVER|STREAM_EOF;
3724  else
3725  flags = STREAM_TOSERVER;
3726 
3727  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3728  if (r != 0) {
3729  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3730  " 0: ", u, r);
3731  goto end;
3732  }
3733  }
3734 
3735  htp_state = f->alstate;
3736  if (htp_state == NULL) {
3737  printf("no http state: ");
3738  goto end;
3739  }
3740 
3741  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3742  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3743  if (h == NULL) {
3744  goto end;
3745  }
3746 
3747  char *name = bstr_util_strdup_to_c(h->name);
3748  if (name == NULL) {
3749  goto end;
3750  }
3751 
3752  if (strcmp(name, "Host") != 0) {
3753  printf("header name not \"Host\", instead \"%s\": ", name);
3754  free(name);
3755  goto end;
3756  }
3757  free(name);
3758 
3759  char *value = bstr_util_strdup_to_c(h->value);
3760  if (value == NULL) {
3761  goto end;
3762  }
3763 
3764  if (strcmp(value, "www.google.com") != 0) {
3765  printf("header value not \"www.google.com\", instead \"%s\": ", value);
3766  free(value);
3767  goto end;
3768  }
3769  free(value);
3770 
3771  result = 1;
3772 end:
3773  if (alp_tctx != NULL)
3775  StreamTcpFreeConfig(true);
3776  UTHFreeFlow(f);
3777  return result;
3778 }
3779 
3780 /** \test double encoding in path
3781  */
3782 static int HTPParserTest11(void)
3783 {
3784  int result = 0;
3785  Flow *f = NULL;
3786  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
3787  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3788  TcpSession ssn;
3789  HtpState *htp_state = NULL;
3790  int r = 0;
3792 
3793  memset(&ssn, 0, sizeof(ssn));
3794 
3795  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3796  if (f == NULL)
3797  goto end;
3798  f->protoctx = &ssn;
3799  f->proto = IPPROTO_TCP;
3800  f->alproto = ALPROTO_HTTP1;
3801 
3802  StreamTcpInitConfig(true);
3803 
3804  uint32_t u;
3805  for (u = 0; u < httplen1; u++) {
3806  uint8_t flags = 0;
3807 
3808  if (u == 0)
3809  flags = STREAM_TOSERVER|STREAM_START;
3810  else if (u == (httplen1 - 1))
3811  flags = STREAM_TOSERVER|STREAM_EOF;
3812  else
3813  flags = STREAM_TOSERVER;
3814 
3815  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3816  if (r != 0) {
3817  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3818  " 0: ", u, r);
3819  goto end;
3820  }
3821  }
3822 
3823  htp_state = f->alstate;
3824  if (htp_state == NULL) {
3825  printf("no http state: ");
3826  goto end;
3827  }
3828 
3829  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3830  if (tx == NULL)
3831  goto end;
3832  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
3833  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3834  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
3835  printf("normalized uri len should be 2, is %"PRIuMAX,
3836  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3837  goto end;
3838  }
3839 
3840  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
3841  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
3842  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
3843  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
3844  {
3845  printf("normalized uri \"");
3846  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3847  printf("\": ");
3848  goto end;
3849  }
3850  }
3851 
3852  result = 1;
3853 end:
3854  if (alp_tctx != NULL)
3856  StreamTcpFreeConfig(true);
3857  UTHFreeFlow(f);
3858  return result;
3859 }
3860 
3861 /** \test double encoding in query
3862  */
3863 static int HTPParserTest12(void)
3864 {
3865  int result = 0;
3866  Flow *f = NULL;
3867  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
3868  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3869  TcpSession ssn;
3870  HtpState *htp_state = NULL;
3871  int r = 0;
3873 
3874  memset(&ssn, 0, sizeof(ssn));
3875 
3876  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3877  if (f == NULL)
3878  goto end;
3879  f->protoctx = &ssn;
3880  f->proto = IPPROTO_TCP;
3881  f->alproto = ALPROTO_HTTP1;
3882 
3883  StreamTcpInitConfig(true);
3884 
3885  uint32_t u;
3886  for (u = 0; u < httplen1; u++) {
3887  uint8_t flags = 0;
3888 
3889  if (u == 0)
3890  flags = STREAM_TOSERVER|STREAM_START;
3891  else if (u == (httplen1 - 1))
3892  flags = STREAM_TOSERVER|STREAM_EOF;
3893  else
3894  flags = STREAM_TOSERVER;
3895 
3896  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3897  if (r != 0) {
3898  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3899  " 0: ", u, r);
3900  goto end;
3901  }
3902  }
3903 
3904  htp_state = f->alstate;
3905  if (htp_state == NULL) {
3906  printf("no http state: ");
3907  goto end;
3908  }
3909 
3910  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3911  if (tx == NULL)
3912  goto end;
3913  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3914  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3915  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
3916  printf("normalized uri len should be 5, is %"PRIuMAX,
3917  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3918  goto end;
3919  }
3920 
3921  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
3922  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
3923  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
3924  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
3925  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
3926  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
3927  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
3928  {
3929  printf("normalized uri \"");
3930  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3931  printf("\": ");
3932  goto end;
3933  }
3934  }
3935 
3936  result = 1;
3937  end:
3938  if (alp_tctx != NULL)
3940  StreamTcpFreeConfig(true);
3941  UTHFreeFlow(f);
3942  return result;
3943 }
3944 
3945 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
3946  */
3947 static int HTPParserTest13(void)
3948 {
3949  int result = 0;
3950  Flow *f = NULL;
3951  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
3952  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3953  TcpSession ssn;
3954  HtpState *htp_state = NULL;
3955  int r = 0;
3957 
3958  memset(&ssn, 0, sizeof(ssn));
3959 
3960  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3961  if (f == NULL)
3962  goto end;
3963  f->protoctx = &ssn;
3964  f->proto = IPPROTO_TCP;
3965  f->alproto = ALPROTO_HTTP1;
3966 
3967  StreamTcpInitConfig(true);
3968 
3969  uint32_t u;
3970  for (u = 0; u < httplen1; u++) {
3971  uint8_t flags = 0;
3972 
3973  if (u == 0)
3974  flags = STREAM_TOSERVER|STREAM_START;
3975  else if (u == (httplen1 - 1))
3976  flags = STREAM_TOSERVER|STREAM_EOF;
3977  else
3978  flags = STREAM_TOSERVER;
3979 
3980  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3981  if (r != 0) {
3982  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3983  " 0: ", u, r);
3984  goto end;
3985  }
3986  }
3987 
3988  htp_state = f->alstate;
3989  if (htp_state == NULL) {
3990  printf("no http state: ");
3991  goto end;
3992  }
3993 
3994  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3995  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3996  if (h == NULL) {
3997  goto end;
3998  }
3999 
4000  char *name = bstr_util_strdup_to_c(h->name);
4001  if (name == NULL) {
4002  goto end;
4003  }
4004 
4005  if (strcmp(name, "Host") != 0) {
4006  printf("header name not \"Host\", instead \"%s\": ", name);
4007  free(name);
4008  goto end;
4009  }
4010  free(name);
4011 
4012  char *value = bstr_util_strdup_to_c(h->value);
4013  if (value == NULL) {
4014  goto end;
4015  }
4016 
4017  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4018  printf("header value not \"www.google.com\", instead \"");
4019  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4020  printf("\": ");
4021  free(value);
4022  goto end;
4023  }
4024  free(value);
4025 
4026  result = 1;
4027 end:
4028  if (alp_tctx != NULL)
4030  StreamTcpFreeConfig(true);
4031  UTHFreeFlow(f);
4032  return result;
4033 }
4034 
4035 /** \test Test basic config */
4036 static int HTPParserConfigTest01(void)
4037 {
4038  int ret = 0;
4039  char input[] = "\
4040 %YAML 1.1\n\
4041 ---\n\
4042 libhtp:\n\
4043 \n\
4044  default-config:\n\
4045  personality: IDS\n\
4046 \n\
4047  server-config:\n\
4048 \n\
4049  - apache-tomcat:\n\
4050  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4051  personality: Tomcat_6_0\n\
4052 \n\
4053  - iis7:\n\
4054  address: \n\
4055  - 192.168.0.0/24\n\
4056  - 192.168.10.0/24\n\
4057  personality: IIS_7_0\n\
4058 ";
4059 
4061  ConfInit();
4062 
4063  ConfYamlLoadString(input, strlen(input));
4064 
4065  ConfNode *outputs;
4066  outputs = ConfGetNode("libhtp.default-config.personality");
4067  if (outputs == NULL) {
4068  goto end;
4069  }
4070 
4071  outputs = ConfGetNode("libhtp.server-config");
4072  if (outputs == NULL) {
4073  goto end;
4074  }
4075 
4076  ConfNode *node = TAILQ_FIRST(&outputs->head);
4077  if (node == NULL) {
4078  goto end;
4079  }
4080  if (strcmp(node->name, "0") != 0) {
4081  goto end;
4082  }
4083  node = TAILQ_FIRST(&node->head);
4084  if (node == NULL) {
4085  goto end;
4086  }
4087  if (strcmp(node->name, "apache-tomcat") != 0) {
4088  goto end;
4089  }
4090 
4091  int i = 0;
4092  ConfNode *n;
4093 
4094  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4095  if (node2 == NULL) {
4096  goto end;
4097  }
4098  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4099  goto end;
4100  }
4101 
4102  node = ConfNodeLookupChild(node, "address");
4103  if (node == NULL) {
4104  goto end;
4105  }
4106  TAILQ_FOREACH(n, &node->head, next) {
4107  if (n == NULL) {
4108  goto end;
4109  }
4110 
4111  switch(i) {
4112  case 0:
4113  if (strcmp(n->name, "0") != 0) {
4114  goto end;
4115  }
4116  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4117  goto end;
4118  }
4119  break;
4120  case 1:
4121  if (strcmp(n->name, "1") != 0) {
4122  goto end;
4123  }
4124  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4125  goto end;
4126  }
4127  break;
4128  case 2:
4129  if (strcmp(n->name, "2") != 0) {
4130  goto end;
4131  }
4132  if (strcmp(n->val, "::1") != 0) {
4133  goto end;
4134  }
4135  break;
4136  default:
4137  goto end;
4138  }
4139  i++;
4140  }
4141 
4142  outputs = ConfGetNode("libhtp.server-config");
4143  if (outputs == NULL) {
4144  goto end;
4145  }
4146 
4147  node = TAILQ_FIRST(&outputs->head);
4148  node = TAILQ_NEXT(node, next);
4149  if (node == NULL) {
4150  goto end;
4151  }
4152  if (strcmp(node->name, "1") != 0) {
4153  goto end;
4154  }
4155  node = TAILQ_FIRST(&node->head);
4156  if (node == NULL) {
4157  goto end;
4158  }
4159  if (strcmp(node->name, "iis7") != 0) {
4160  goto end;
4161  }
4162 
4163  node2 = ConfNodeLookupChild(node, "personality");
4164  if (node2 == NULL) {
4165  goto end;
4166  }
4167  if (strcmp(node2->val, "IIS_7_0") != 0) {
4168  goto end;
4169  }
4170 
4171  node = ConfNodeLookupChild(node, "address");
4172  if (node == NULL) {
4173  goto end;
4174  }
4175 
4176  i = 0;
4177  TAILQ_FOREACH(n, &node->head, next) {
4178  if (n == NULL) {
4179  goto end;
4180  }
4181 
4182  switch(i) {
4183  case 0:
4184  if (strcmp(n->name, "0") != 0) {
4185  goto end;
4186  }
4187  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4188  goto end;
4189  }
4190  break;
4191  case 1:
4192  if (strcmp(n->name, "1") != 0) {
4193  goto end;
4194  }
4195  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4196  goto end;
4197  }
4198  break;
4199  default:
4200  goto end;
4201  }
4202  i++;
4203  }
4204 
4205  ret = 1;
4206 
4207 end:
4208  ConfDeInit();
4210 
4211  return ret;
4212 }
4213 
4214 /** \test Test config builds radix correctly */
4215 static int HTPParserConfigTest02(void)
4216 {
4217  int ret = 0;
4218  char input[] = "\
4219 %YAML 1.1\n\
4220 ---\n\
4221 libhtp:\n\
4222 \n\
4223  default-config:\n\
4224  personality: IDS\n\
4225 \n\
4226  server-config:\n\
4227 \n\
4228  - apache-tomcat:\n\
4229  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4230  personality: Tomcat_6_0\n\
4231 \n\
4232  - iis7:\n\
4233  address: \n\
4234  - 192.168.0.0/24\n\
4235  - 192.168.10.0/24\n\
4236  personality: IIS_7_0\n\
4237 ";
4238 
4240  ConfInit();
4242 
4243  ConfYamlLoadString(input, strlen(input));
4244 
4245  HTPConfigure();
4246 
4247  if (cfglist.cfg == NULL) {
4248  printf("No default config created.\n");
4249  goto end;
4250  }
4251 
4252  if (cfgtree == NULL) {
4253  printf("No config tree created.\n");
4254  goto end;
4255  }
4256 
4257  htp_cfg_t *htp = cfglist.cfg;
4258  uint8_t buf[128];
4259  const char *addr;
4260  void *user_data = NULL;
4261 
4262  addr = "192.168.10.42";
4263  if (inet_pton(AF_INET, addr, buf) == 1) {
4264  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4265  if (user_data != NULL) {
4266  HTPCfgRec *htp_cfg_rec = user_data;
4267  htp = htp_cfg_rec->cfg;
4268  SCLogDebug("LIBHTP using config: %p", htp);
4269  }
4270  if (htp == NULL) {
4271  printf("Could not get config for: %s\n", addr);
4272  goto end;
4273  }
4274  }
4275  else {
4276  printf("Failed to parse address: %s\n", addr);
4277  goto end;
4278  }
4279 
4280  user_data = NULL;
4281  addr = "::1";
4282  if (inet_pton(AF_INET6, addr, buf) == 1) {
4283  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4284  if (user_data != NULL) {
4285  HTPCfgRec *htp_cfg_rec = user_data;
4286  htp = htp_cfg_rec->cfg;
4287  SCLogDebug("LIBHTP using config: %p", htp);
4288  }
4289  if (htp == NULL) {
4290  printf("Could not get config for: %s\n", addr);
4291  goto end;
4292  }
4293  }
4294  else {
4295  printf("Failed to parse address: %s\n", addr);
4296  goto end;
4297  }
4298 
4299  ret = 1;
4300 
4301 end:
4302  HTPFreeConfig();
4303  ConfDeInit();
4306 
4307  return ret;
4308 }
4309 
4310 /** \test Test traffic is handled by the correct htp config */
4311 static int HTPParserConfigTest03(void)
4312 {
4313  int result = 1;
4314  Flow *f = NULL;
4315  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4316  " Data is c0oL!";
4317  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4318  TcpSession ssn;
4320 
4321  HtpState *htp_state = NULL;
4322  int r = 0;
4323  char input[] = "\
4324 %YAML 1.1\n\
4325 ---\n\
4326 libhtp:\n\
4327 \n\
4328  default-config:\n\
4329  personality: IDS\n\
4330 \n\
4331  server-config:\n\
4332 \n\
4333  - apache-tomcat:\n\
4334  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4335  personality: Tomcat_6_0\n\
4336 \n\
4337  - iis7:\n\
4338  address: \n\
4339  - 192.168.0.0/24\n\
4340  - 192.168.10.0/24\n\
4341  personality: IIS_7_0\n\
4342 ";
4343 
4345  ConfInit();
4347 
4348  ConfYamlLoadString(input, strlen(input));
4349 
4350  HTPConfigure();
4351 
4352  const char *addr = "192.168.10.42";
4353 
4354  memset(&ssn, 0, sizeof(ssn));
4355 
4356  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4357  if (f == NULL)
4358  goto end;
4359  f->protoctx = &ssn;
4360  f->proto = IPPROTO_TCP;
4361  f->alproto = ALPROTO_HTTP1;
4362 
4363  htp_cfg_t *htp = cfglist.cfg;
4364 
4365  void *user_data = NULL;
4366  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4367  if (user_data != NULL) {
4368  HTPCfgRec *htp_cfg_rec = user_data;
4369  htp = htp_cfg_rec->cfg;
4370  SCLogDebug("LIBHTP using config: %p", htp);
4371  }
4372  if (htp == NULL) {
4373  printf("Could not get config for: %s\n", addr);
4374  goto end;
4375  }
4376 
4377  StreamTcpInitConfig(true);
4378 
4379  uint32_t u;
4380  for (u = 0; u < httplen1; u++) {
4381  uint8_t flags = 0;
4382 
4383  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4384  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4385  else flags = STREAM_TOSERVER;
4386 
4387  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4388  if (r != 0) {
4389  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4390  " 0: ", u, r);
4391  result = 0;
4392  goto end;
4393  }
4394  }
4395 
4396  htp_state = f->alstate;
4397  if (htp_state == NULL) {
4398  printf("no http state: ");
4399  result = 0;
4400  goto end;
4401  }
4402 
4403  if (HTPStateGetTxCnt(htp_state) != 2) {
4404  printf("HTPStateGetTxCnt(htp_state) failure\n");
4405  goto end;
4406  }
4407 
4408  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4409  if (tx == NULL)
4410  goto end;
4411  if (tx->cfg != htp) {
4412  printf("wrong HTP config (%p instead of %p - default=%p): ",
4413  tx->cfg, htp, cfglist.cfg);
4414  goto end;
4415  }
4416  tx = HTPStateGetTx(htp_state, 1);
4417  if (tx == NULL)
4418  goto end;
4419  if (tx->cfg != htp) {
4420  printf("wrong HTP config (%p instead of %p - default=%p): ",
4421  tx->cfg, htp, cfglist.cfg);
4422  goto end;
4423  }
4424 
4425 end:
4426  if (alp_tctx != NULL)
4428  HTPFreeConfig();
4429  ConfDeInit();
4432 
4433  StreamTcpFreeConfig(true);
4434  UTHFreeFlow(f);
4435  return result;
4436 }
4437 
4438 /** \test Test %2f decoding in profile Apache_2_2
4439  *
4440  * %2f in path is left untouched
4441  * %2f in query string is normalized to %2F
4442  * %252f in query string is decoded/normalized to %2F
4443  */
4444 static int HTPParserDecodingTest01(void)
4445 {
4446  uint8_t httpbuf1[] =
4447  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4448  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4449  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4450  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4451  TcpSession ssn;
4454 
4455  char input[] = "\
4456 %YAML 1.1\n\
4457 ---\n\
4458 libhtp:\n\
4459 \n\
4460  default-config:\n\
4461  personality: Apache_2\n\
4462 ";
4463 
4465  ConfInit();
4467  ConfYamlLoadString(input, strlen(input));
4468  HTPConfigure();
4469  const char *addr = "4.3.2.1";
4470  memset(&ssn, 0, sizeof(ssn));
4471 
4472  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4473  FAIL_IF_NULL(f);
4474  f->protoctx = &ssn;
4475  f->proto = IPPROTO_TCP;
4476  f->alproto = ALPROTO_HTTP1;
4477 
4478  StreamTcpInitConfig(true);
4479 
4480  for (uint32_t u = 0; u < httplen1; u++) {
4481  uint8_t flags = 0;
4482  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4483  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4484  else flags = STREAM_TOSERVER;
4485 
4486  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4487  FAIL_IF(r != 0);
4488  }
4489 
4490  HtpState *htp_state = f->alstate;
4491  FAIL_IF_NULL(htp_state);
4492 
4493  uint8_t ref1[] = "/abc%2fdef";
4494  size_t reflen = sizeof(ref1) - 1;
4495 
4496  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4497  FAIL_IF_NULL(tx);
4498 
4499  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4500  FAIL_IF_NULL(tx_ud);
4502  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4503  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4504  bstr_len(tx_ud->request_uri_normalized)) != 0);
4505 
4506  uint8_t ref2[] = "/abc/def?ghi/jkl";
4507  reflen = sizeof(ref2) - 1;
4508 
4509  tx = HTPStateGetTx(htp_state, 1);
4510  FAIL_IF_NULL(tx);
4511  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4512  FAIL_IF_NULL(tx_ud);
4514  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4515 
4516  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4517  bstr_len(tx_ud->request_uri_normalized)) != 0);
4518 
4519  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4520  reflen = sizeof(ref3) - 1;
4521  tx = HTPStateGetTx(htp_state, 2);
4522  FAIL_IF_NULL(tx);
4523  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4524  FAIL_IF_NULL(tx_ud);
4526  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4527 
4528  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4529  bstr_len(tx_ud->request_uri_normalized)) != 0);
4530 
4532  HTPFreeConfig();
4533  ConfDeInit();
4536 
4537  StreamTcpFreeConfig(true);
4538  UTHFreeFlow(f);
4539  PASS;
4540 }
4541 
4542 static int HTPParserDecodingTest01a(void)
4543 {
4544  uint8_t httpbuf1[] = "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4545  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4546  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4547  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4548  TcpSession ssn;
4551 
4552  char input[] = "\
4553 %YAML 1.1\n\
4554 ---\n\
4555 libhtp:\n\
4556 \n\
4557  default-config:\n\
4558  personality: Apache_2\n\
4559 ";
4560 
4562  ConfInit();
4564  ConfYamlLoadString(input, strlen(input));
4565  HTPConfigure();
4566  const char *addr = "4.3.2.1";
4567  memset(&ssn, 0, sizeof(ssn));
4568 
4569  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4570  FAIL_IF_NULL(f);
4571  f->protoctx = &ssn;
4572  f->proto = IPPROTO_TCP;
4573  f->alproto = ALPROTO_HTTP1;
4574 
4575  StreamTcpInitConfig(true);
4576 
4577  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
4578  (STREAM_TOSERVER | STREAM_START | STREAM_EOF), httpbuf1, httplen1);
4579  FAIL_IF(r != 0);
4580 
4581  HtpState *htp_state = f->alstate;
4582  FAIL_IF_NULL(htp_state);
4583 
4584  uint8_t ref1[] = "/abc%2fdef";
4585  size_t reflen = sizeof(ref1) - 1;
4586 
4587  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4588  FAIL_IF_NULL(tx);
4589 
4590  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4591  FAIL_IF_NULL(tx_ud);
4593  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4594  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4595  bstr_len(tx_ud->request_uri_normalized)) != 0);
4596 
4597  uint8_t ref2[] = "/abc/def?ghi/jkl";
4598  reflen = sizeof(ref2) - 1;
4599 
4600  tx = HTPStateGetTx(htp_state, 1);
4601  FAIL_IF_NULL(tx);
4602  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4603  FAIL_IF_NULL(tx_ud);
4605  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4606 
4607  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4608  bstr_len(tx_ud->request_uri_normalized)) != 0);
4609 
4610  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4611  reflen = sizeof(ref3) - 1;
4612  tx = HTPStateGetTx(htp_state, 2);
4613  FAIL_IF_NULL(tx);
4614  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4615  FAIL_IF_NULL(tx_ud);
4617  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4618 
4619  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4620  bstr_len(tx_ud->request_uri_normalized)) != 0);
4621 
4623  HTPFreeConfig();
4624  ConfDeInit();
4627 
4628  StreamTcpFreeConfig(true);
4629  UTHFreeFlow(f);
4630  PASS;
4631 }
4632 
4633 /** \test Test %2f decoding in profile IDS
4634  *
4635  * %2f in path decoded to /
4636  * %2f in query string is decoded to /
4637  * %252f in query string is decoded to %2F
4638  */
4639 static int HTPParserDecodingTest02(void)
4640 {
4641  int result = 0;
4642  Flow *f = NULL;
4643  uint8_t httpbuf1[] =
4644  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4645  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4646  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4647  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4648  TcpSession ssn;
4650 
4651  HtpState *htp_state = NULL;
4652  int r = 0;
4653  char input[] = "\
4654 %YAML 1.1\n\
4655 ---\n\
4656 libhtp:\n\
4657 \n\
4658  default-config:\n\
4659  personality: IDS\n\
4660  double-decode-path: no\n\
4661  double-decode-query: no\n\
4662 ";
4663 
4665  ConfInit();
4667  ConfYamlLoadString(input, strlen(input));
4668  HTPConfigure();
4669  const char *addr = "4.3.2.1";
4670  memset(&ssn, 0, sizeof(ssn));
4671 
4672  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4673  if (f == NULL)
4674  goto end;
4675  f->protoctx = &ssn;
4676  f->proto = IPPROTO_TCP;
4677  f->alproto = ALPROTO_HTTP1;
4678 
4679  StreamTcpInitConfig(true);
4680 
4681  uint32_t u;
4682  for (u = 0; u < httplen1; u++) {
4683  uint8_t flags = 0;
4684 
4685  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4686  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4687  else flags = STREAM_TOSERVER;
4688 
4689  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4690  if (r != 0) {
4691  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4692  " 0: ", u, r);
4693  goto end;
4694  }
4695  }
4696 
4697  htp_state = f->alstate;
4698  if (htp_state == NULL) {
4699  printf("no http state: ");
4700  goto end;
4701  }
4702 
4703  uint8_t ref1[] = "/abc/def";
4704  size_t reflen = sizeof(ref1) - 1;
4705 
4706  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4707  if (tx == NULL)
4708  goto end;
4709  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4710  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4711  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4712  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4713  (uintmax_t)reflen,
4714  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4715  goto end;
4716  }
4717 
4718  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4719  bstr_len(tx_ud->request_uri_normalized)) != 0)
4720  {
4721  printf("normalized uri \"");
4722  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4723  printf("\" != \"");
4724  PrintRawUriFp(stdout, ref1, reflen);
4725  printf("\": ");
4726  goto end;
4727  }
4728  }
4729 
4730  uint8_t ref2[] = "/abc/def?ghi/jkl";
4731  reflen = sizeof(ref2) - 1;
4732 
4733  tx = HTPStateGetTx(htp_state, 1);
4734  if (tx == NULL)
4735  goto end;
4736  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4737  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4738  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4739  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4740  (uintmax_t)reflen,
4741  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4742  goto end;
4743  }
4744 
4745  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4746  bstr_len(tx_ud->request_uri_normalized)) != 0)
4747  {
4748  printf("normalized uri \"");
4749  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4750  printf("\" != \"");
4751  PrintRawUriFp(stdout, ref2, reflen);
4752  printf("\": ");
4753  goto end;
4754  }
4755  }
4756 
4757  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4758  reflen = sizeof(ref3) - 1;
4759  tx = HTPStateGetTx(htp_state, 2);
4760  if (tx == NULL)
4761  goto end;
4762  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4763  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4764  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4765  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
4766  (uintmax_t)reflen,
4767  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4768  goto end;
4769  }
4770 
4771  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4772  bstr_len(tx_ud->request_uri_normalized)) != 0)
4773  {
4774  printf("normalized uri \"");
4775  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4776  printf("\" != \"");
4777  PrintRawUriFp(stdout, ref3, reflen);
4778  printf("\": ");
4779  goto end;
4780  }
4781  }
4782 
4783  result = 1;
4784 
4785 end:
4786  if (alp_tctx != NULL)
4788  HTPFreeConfig();
4789  ConfDeInit();
4792 
4793  StreamTcpFreeConfig(true);
4794  UTHFreeFlow(f);
4795  return result;
4796 }
4797 
4798 /** \test Test %2f decoding in profile IDS with double-decode-* options
4799  *
4800  * %252f in path decoded to /
4801  * %252f in query string is decoded to /
4802  */
4803 static int HTPParserDecodingTest03(void)
4804 {
4805  int result = 0;
4806  Flow *f = NULL;
4807  uint8_t httpbuf1[] =
4808  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4809  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4810  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4811  TcpSession ssn;
4813 
4814  HtpState *htp_state = NULL;
4815  int r = 0;
4816  char input[] = "\
4817 %YAML 1.1\n\
4818 ---\n\
4819 libhtp:\n\
4820 \n\
4821  default-config:\n\
4822  personality: IDS\n\
4823  double-decode-path: yes\n\
4824  double-decode-query: yes\n\
4825 ";
4826 
4828  ConfInit();
4830  ConfYamlLoadString(input, strlen(input));
4831  HTPConfigure();
4832  const char *addr = "4.3.2.1";
4833  memset(&ssn, 0, sizeof(ssn));
4834 
4835  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4836  if (f == NULL)
4837  goto end;
4838  f->protoctx = &ssn;
4839  f->proto = IPPROTO_TCP;
4840  f->alproto = ALPROTO_HTTP1;
4841 
4842  StreamTcpInitConfig(true);
4843 
4844  uint32_t u;
4845  for (u = 0; u < httplen1; u++) {
4846  uint8_t flags = 0;
4847 
4848  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4849  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4850  else flags = STREAM_TOSERVER;
4851 
4852  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4853  if (r != 0) {
4854  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4855  " 0: ", u, r);
4856  goto end;
4857  }
4858  }
4859 
4860  htp_state = f->alstate;
4861  if (htp_state == NULL) {
4862  printf("no http state: ");
4863  goto end;
4864  }
4865 
4866  uint8_t ref1[] = "/abc/def";
4867  size_t reflen = sizeof(ref1) - 1;
4868 
4869  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4870  if (tx == NULL)
4871  goto end;
4872  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4873  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4874  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4875  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4876  (uintmax_t)reflen,
4877  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4878  goto end;
4879  }
4880 
4881  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4882  bstr_len(tx_ud->request_uri_normalized)) != 0)
4883  {
4884  printf("normalized uri \"");
4885  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4886  printf("\" != \"");
4887  PrintRawUriFp(stdout, ref1, reflen);
4888  printf("\": ");
4889  goto end;
4890  }
4891  }
4892 
4893  uint8_t ref2[] = "/abc/def?ghi/jkl";
4894  reflen = sizeof(ref2) - 1;
4895 
4896  tx = HTPStateGetTx(htp_state, 1);
4897  if (tx == NULL)
4898  goto end;
4899  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4900  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4901  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4902  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4903  (uintmax_t)reflen,
4904  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4905  goto end;
4906  }
4907 
4908  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4909  bstr_len(tx_ud->request_uri_normalized)) != 0)
4910  {
4911  printf("normalized uri \"");
4912  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4913  printf("\" != \"");
4914  PrintRawUriFp(stdout, ref2, reflen);
4915  printf("\": ");
4916  goto end;
4917  }
4918  }
4919 
4920  result = 1;
4921 
4922 end:
4923  if (alp_tctx != NULL)
4925  HTPFreeConfig();
4926  ConfDeInit();
4929 
4930  StreamTcpFreeConfig(true);
4931  UTHFreeFlow(f);
4932  return result;
4933 }
4934 
4935 /** \test Test http:// in query profile IDS
4936  */
4937 static int HTPParserDecodingTest04(void)
4938 {
4939  int result = 0;
4940  Flow *f = NULL;
4941  uint8_t httpbuf1[] =
4942  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4943  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4944  TcpSession ssn;
4946 
4947  HtpState *htp_state = NULL;
4948  int r = 0;
4949  char input[] = "\
4950 %YAML 1.1\n\
4951 ---\n\
4952 libhtp:\n\
4953 \n\
4954  default-config:\n\
4955  personality: IDS\n\
4956  double-decode-path: yes\n\
4957  double-decode-query: yes\n\
4958 ";
4959 
4961  ConfInit();
4963  ConfYamlLoadString(input, strlen(input));
4964  HTPConfigure();
4965  const char *addr = "4.3.2.1";
4966  memset(&ssn, 0, sizeof(ssn));
4967 
4968  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4969  if (f == NULL)
4970  goto end;
4971  f->protoctx = &ssn;
4972  f->proto = IPPROTO_TCP;
4973  f->alproto = ALPROTO_HTTP1;
4974 
4975  StreamTcpInitConfig(true);
4976 
4977  uint32_t u;
4978  for (u = 0; u < httplen1; u++) {
4979  uint8_t flags = 0;
4980 
4981  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4982  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4983  else flags = STREAM_TOSERVER;
4984 
4985  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4986  if (r != 0) {
4987  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4988  " 0: ", u, r);
4989  goto end;
4990  }
4991  }
4992 
4993  htp_state = f->alstate;
4994  if (htp_state == NULL) {
4995  printf("no http state: ");
4996  goto end;
4997  }
4998 
4999  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5000  size_t reflen = sizeof(ref1) - 1;
5001 
5002  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5003  if (tx == NULL)
5004  goto end;
5005  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5006  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5007  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5008  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5009  (uintmax_t)reflen,
5010  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5011  goto end;
5012  }
5013 
5014  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5015  bstr_len(tx_ud->request_uri_normalized)) != 0)
5016  {
5017  printf("normalized uri \"");
5018  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5019  printf("\" != \"");
5020  PrintRawUriFp(stdout, ref1, reflen);
5021  printf("\": ");
5022  goto end;
5023  }
5024  }
5025 
5026  result = 1;
5027 
5028 end:
5029  if (alp_tctx != NULL)
5031  HTPFreeConfig();
5032  ConfDeInit();
5035 
5036  StreamTcpFreeConfig(true);
5037  UTHFreeFlow(f);
5038  return result;
5039 }
5040 
5041 /** \test Test \ char in query profile IDS. Bug 739
5042  */
5043 static int HTPParserDecodingTest05(void)
5044 {
5045  int result = 0;
5046  Flow *f = NULL;
5047  uint8_t httpbuf1[] =
5048  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5049  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5050  TcpSession ssn;
5052 
5053  HtpState *htp_state = NULL;
5054  int r = 0;
5055  char input[] = "\
5056 %YAML 1.1\n\
5057 ---\n\
5058 libhtp:\n\
5059 \n\
5060  default-config:\n\
5061  personality: IDS\n\
5062  double-decode-path: yes\n\
5063  double-decode-query: yes\n\
5064 ";
5065 
5067  ConfInit();
5069  ConfYamlLoadString(input, strlen(input));
5070  HTPConfigure();
5071  const char *addr = "4.3.2.1";
5072  memset(&ssn, 0, sizeof(ssn));
5073 
5074  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5075  if (f == NULL)
5076  goto end;
5077  f->protoctx = &ssn;
5078  f->proto = IPPROTO_TCP;
5079  f->alproto = ALPROTO_HTTP1;
5080 
5081  StreamTcpInitConfig(true);
5082 
5083  uint32_t u;
5084  for (u = 0; u < httplen1; u++) {
5085  uint8_t flags = 0;
5086 
5087  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5088  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5089  else flags = STREAM_TOSERVER;
5090 
5091  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5092  if (r != 0) {
5093  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5094  " 0: ", u, r);
5095  goto end;
5096  }
5097  }
5098 
5099  htp_state = f->alstate;
5100  if (htp_state == NULL) {
5101  printf("no http state: ");
5102  goto end;
5103  }
5104 
5105  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5106  size_t reflen = sizeof(ref1) - 1;
5107 
5108  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5109  if (tx == NULL)
5110  goto end;
5111  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5112  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5113  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5114  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5115  (uintmax_t)reflen,
5116  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5117  goto end;
5118  }
5119 
5120  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5121  bstr_len(tx_ud->request_uri_normalized)) != 0)
5122  {
5123  printf("normalized uri \"");
5124  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5125  printf("\" != \"");
5126  PrintRawUriFp(stdout, ref1, reflen);
5127  printf("\": ");
5128  goto end;
5129  }
5130  }
5131 
5132  result = 1;
5133 
5134 end:
5135  if (alp_tctx != NULL)
5137  HTPFreeConfig();
5138  ConfDeInit();
5141 
5142  StreamTcpFreeConfig(true);
5143  UTHFreeFlow(f);
5144  return result;
5145 }
5146 
5147 /** \test Test + char in query. Bug 1035
5148  */
5149 static int HTPParserDecodingTest06(void)
5150 {
5151  int result = 0;
5152  Flow *f = NULL;
5153  uint8_t httpbuf1[] =
5154  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5155  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5156  TcpSession ssn;
5158 
5159  HtpState *htp_state = NULL;
5160  int r = 0;
5161  char input[] = "\
5162 %YAML 1.1\n\
5163 ---\n\
5164 libhtp:\n\
5165 \n\
5166  default-config:\n\
5167  personality: IDS\n\
5168  double-decode-path: yes\n\
5169  double-decode-query: yes\n\
5170 ";
5171 
5173  ConfInit();
5175  ConfYamlLoadString(input, strlen(input));
5176  HTPConfigure();
5177  const char *addr = "4.3.2.1";
5178  memset(&ssn, 0, sizeof(ssn));
5179 
5180  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5181  if (f == NULL)
5182  goto end;
5183  f->protoctx = &ssn;
5184  f->proto = IPPROTO_TCP;
5185  f->alproto = ALPROTO_HTTP1;
5186 
5187  StreamTcpInitConfig(true);
5188 
5189  uint32_t u;
5190  for (u = 0; u < httplen1; u++) {
5191  uint8_t flags = 0;
5192 
5193  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5194  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5195  else flags = STREAM_TOSERVER;
5196 
5197  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5198  if (r != 0) {
5199  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5200  " 0: ", u, r);
5201  goto end;
5202  }
5203  }
5204 
5205  htp_state = f->alstate;
5206  if (htp_state == NULL) {
5207  printf("no http state: ");
5208  goto end;
5209  }
5210 
5211  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5212  size_t reflen = sizeof(ref1) - 1;
5213 
5214  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5215  if (tx == NULL)
5216  goto end;
5217  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5218  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5219  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5220  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5221  (uintmax_t)reflen,
5222  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5223  goto end;
5224  }
5225 
5226  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5227  bstr_len(tx_ud->request_uri_normalized)) != 0)
5228  {
5229  printf("normalized uri \"");
5230  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5231  printf("\" != \"");
5232  PrintRawUriFp(stdout, ref1, reflen);
5233  printf("\": ");
5234  goto end;
5235  }
5236  }
5237 
5238  result = 1;
5239 
5240 end:
5241  if (alp_tctx != NULL)
5243  HTPFreeConfig();
5244  ConfDeInit();
5247 
5248  StreamTcpFreeConfig(true);
5249  UTHFreeFlow(f);
5250  return result;
5251 }
5252 
5253 /** \test Test + char in query. Bug 1035
5254  */
5255 static int HTPParserDecodingTest07(void)
5256 {
5257  int result = 0;
5258  Flow *f = NULL;
5259  uint8_t httpbuf1[] =
5260  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5261  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5262  TcpSession ssn;
5264 
5265  HtpState *htp_state = NULL;
5266  int r = 0;
5267  char input[] = "\
5268 %YAML 1.1\n\
5269 ---\n\
5270 libhtp:\n\
5271 \n\
5272  default-config:\n\
5273  personality: IDS\n\
5274  double-decode-path: yes\n\
5275  double-decode-query: yes\n\
5276  query-plusspace-decode: yes\n\
5277 ";
5278 
5280  ConfInit();
5282  ConfYamlLoadString(input, strlen(input));
5283  HTPConfigure();
5284  const char *addr = "4.3.2.1";
5285  memset(&ssn, 0, sizeof(ssn));
5286 
5287  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5288  if (f == NULL)
5289  goto end;
5290  f->protoctx = &ssn;
5291  f->proto = IPPROTO_TCP;
5292  f->alproto = ALPROTO_HTTP1;
5293 
5294  StreamTcpInitConfig(true);
5295 
5296  uint32_t u;
5297  for (u = 0; u < httplen1; u++) {
5298  uint8_t flags = 0;
5299 
5300  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5301  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5302  else flags = STREAM_TOSERVER;
5303 
5304  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5305  if (r != 0) {
5306  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5307  " 0: ", u, r);
5308  goto end;
5309  }
5310  }
5311 
5312  htp_state = f->alstate;
5313  if (htp_state == NULL) {
5314  printf("no http state: ");
5315  goto end;
5316  }
5317 
5318  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5319  size_t reflen = sizeof(ref1) - 1;
5320 
5321  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5322  if (tx == NULL)
5323  goto end;
5324  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5325  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5326  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5327  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5328  (uintmax_t)reflen,
5329  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5330  goto end;
5331  }
5332 
5333  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5334  bstr_len(tx_ud->request_uri_normalized)) != 0)
5335  {
5336  printf("normalized uri \"");
5337  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5338  printf("\" != \"");
5339  PrintRawUriFp(stdout, ref1, reflen);
5340  printf("\": ");
5341  goto end;
5342  }
5343  }
5344 
5345  result = 1;
5346 
5347 end:
5348  if (alp_tctx != NULL)
5350  HTPFreeConfig();
5351  ConfDeInit();
5354 
5355  StreamTcpFreeConfig(true);
5356  UTHFreeFlow(f);
5357  return result;
5358 }
5359 
5360 /** \test Test 'proxy' URI normalization. Ticket 1008
5361  */
5362 static int HTPParserDecodingTest08(void)
5363 {
5364  int result = 0;
5365  Flow *f = NULL;
5366  uint8_t httpbuf1[] =
5367  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5368  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5369  TcpSession ssn;
5371 
5372  HtpState *htp_state = NULL;
5373  int r = 0;
5374  char input[] = "\
5375 %YAML 1.1\n\
5376 ---\n\
5377 libhtp:\n\
5378 \n\
5379  default-config:\n\
5380  personality: IDS\n\
5381 ";
5382 
5384  ConfInit();
5386  ConfYamlLoadString(input, strlen(input));
5387  HTPConfigure();
5388  const char *addr = "4.3.2.1";
5389  memset(&ssn, 0, sizeof(ssn));
5390 
5391  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5392  if (f == NULL)
5393  goto end;
5394  f->protoctx = &ssn;
5395  f->proto = IPPROTO_TCP;
5396  f->alproto = ALPROTO_HTTP1;
5397 
5398  StreamTcpInitConfig(true);
5399 
5400  uint32_t u;
5401  for (u = 0; u < httplen1; u++) {
5402  uint8_t flags = 0;
5403 
5404  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5405  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5406  else flags = STREAM_TOSERVER;
5407 
5408  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5409  if (r != 0) {
5410  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5411  " 0: ", u, r);
5412  goto end;
5413  }
5414  }
5415 
5416  htp_state = f->alstate;
5417  if (htp_state == NULL) {
5418  printf("no http state: ");
5419  goto end;
5420  }
5421 
5422  uint8_t ref1[] = "/blah/";
5423  size_t reflen = sizeof(ref1) - 1;
5424 
5425  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5426  if (tx == NULL)
5427  goto end;
5428  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5429  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5430  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5431  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5432  (uintmax_t)reflen,
5433  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5434  goto end;
5435  }
5436 
5437  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5438  bstr_len(tx_ud->request_uri_normalized)) != 0)
5439  {
5440  printf("normalized uri \"");
5441  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5442  printf("\" != \"");
5443  PrintRawUriFp(stdout, ref1, reflen);
5444  printf("\": ");
5445  goto end;
5446  }
5447  }
5448 
5449  result = 1;
5450 
5451 end:
5452  if (alp_tctx != NULL)
5454  HTPFreeConfig();
5455  ConfDeInit();
5458 
5459  StreamTcpFreeConfig(true);
5460  UTHFreeFlow(f);
5461  return result;
5462 }
5463 
5464 /** \test Test 'proxy' URI normalization. Ticket 1008
5465  */
5466 static int HTPParserDecodingTest09(void)
5467 {
5468  int result = 0;
5469  Flow *f = NULL;
5470  uint8_t httpbuf1[] =
5471  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5472  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5473  TcpSession ssn;
5475 
5476  HtpState *htp_state = NULL;
5477  int r = 0;
5478  char input[] = "\
5479 %YAML 1.1\n\
5480 ---\n\
5481 libhtp:\n\
5482 \n\
5483  default-config:\n\
5484  personality: IDS\n\
5485  uri-include-all: true\n\
5486 ";
5487 
5489  ConfInit();
5491  ConfYamlLoadString(input, strlen(input));
5492  HTPConfigure();
5493  const char *addr = "4.3.2.1";
5494  memset(&ssn, 0, sizeof(ssn));
5495 
5496  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5497  if (f == NULL)
5498  goto end;
5499  f->protoctx = &ssn;
5500  f->proto = IPPROTO_TCP;
5501  f->alproto = ALPROTO_HTTP1;
5502 
5503  StreamTcpInitConfig(true);
5504 
5505  uint32_t u;
5506  for (u = 0; u < httplen1; u++) {
5507  uint8_t flags = 0;
5508 
5509  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5510  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5511  else flags = STREAM_TOSERVER;
5512 
5513  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5514  if (r != 0) {
5515  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5516  " 0: ", u, r);
5517  goto end;
5518  }
5519  }
5520 
5521  htp_state = f->alstate;
5522  if (htp_state == NULL) {
5523  printf("no http state: ");
5524  goto end;
5525  }
5526 
5527  uint8_t ref1[] = "http://suricata-ids.org/blah/";
5528  size_t reflen = sizeof(ref1) - 1;
5529 
5530  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5531  if (tx == NULL)
5532  goto end;
5533  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5534  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5535  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5536  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5537  (uintmax_t)reflen,
5538  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5539  goto end;
5540  }
5541 
5542  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5543  bstr_len(tx_ud->request_uri_normalized)) != 0)
5544  {
5545  printf("normalized uri \"");
5546  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5547  printf("\" != \"");
5548  PrintRawUriFp(stdout, ref1, reflen);
5549  printf("\": ");
5550  goto end;
5551  }
5552  }
5553 
5554  result = 1;
5555 
5556 end:
5557  if (alp_tctx != NULL)
5559  HTPFreeConfig();
5560  ConfDeInit();
5563 
5564  StreamTcpFreeConfig(true);
5565  UTHFreeFlow(f);
5566  return result;
5567 }
5568 
5569 /** \test BG box crash -- chunks are messed up. Observed for real. */
5570 static int HTPBodyReassemblyTest01(void)
5571 {
5572  int result = 0;
5573  HtpTxUserData htud;
5574  memset(&htud, 0x00, sizeof(htud));
5575  HtpState hstate;
5576  memset(&hstate, 0x00, sizeof(hstate));
5577  Flow flow;
5578  memset(&flow, 0x00, sizeof(flow));
5580  htp_tx_t tx;
5581  memset(&tx, 0, sizeof(tx));
5582 
5583  hstate.f = &flow;
5584  flow.alparser = parser;
5585 
5586  uint8_t chunk1[] = "--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5587  uint8_t chunk2[] = "POST /uri HTTP/1.1\r\nHost: hostname.com\r\nKeep-Alive: 115\r\nAccept-Charset: utf-8\r\nUser-Agent: Mozilla/5.0 (X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nConnection: keep-alive\r\nContent-length: 68102\r\nReferer: http://otherhost.com\r\nAccept-Encoding: gzip\r\nContent-Type: multipart/form-data; boundary=e5a320f21416a02493a0a6f561b1c494\r\nCookie: blah\r\nAccept-Language: us\r\n\r\n--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5588 
5589  int r = HtpBodyAppendChunk(&htud.request_body, chunk1, sizeof(chunk1) - 1);
5590  BUG_ON(r != 0);
5591  r = HtpBodyAppendChunk(&htud.request_body, chunk2, sizeof(chunk2) - 1);
5592  BUG_ON(r != 0);
5593 
5594  const uint8_t *chunks_buffer = NULL;
5595  uint32_t chunks_buffer_len = 0;
5596 
5597  HtpRequestBodyReassemble(&htud, &chunks_buffer, &chunks_buffer_len);
5598  if (chunks_buffer == NULL) {
5599  goto end;
5600  }
5601 #ifdef PRINT
5602  printf("REASSCHUNK START: \n");
5603  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
5604  printf("REASSCHUNK END: \n");
5605 #endif
5606 
5607  htud.mime_state = SCMimeStateInit((const uint8_t *)"multipart/form-data; boundary=toto",
5608  strlen("multipart/form-data; boundary=toto"));
5609  FAIL_IF_NULL(htud.mime_state);
5610  htud.tsflags |= HTP_BOUNDARY_SET;
5611  HtpRequestBodyHandleMultipart(&hstate, &htud, &tx, chunks_buffer, chunks_buffer_len, false);
5612 
5613  if (htud.request_body.content_len_so_far != 669) {
5614  printf("htud.request_body.content_len_so_far %"PRIu64": ", htud.request_body.content_len_so_far);
5615  goto end;
5616  }
5617 
5619 
5620  result = 1;
5621 end:
5622  return result;
5623 }
5624 
5625 /** \test BG crash */
5626 static int HTPSegvTest01(void)
5627 {
5628  int result = 0;
5629  Flow *f = NULL;
5630  uint8_t httpbuf1[] = "POST /uri HTTP/1.1\r\nHost: hostname.com\r\nKeep-Alive: 115\r\nAccept-Charset: utf-8\r\nUser-Agent: Mozilla/5.0 (X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nConnection: keep-alive\r\nContent-length: 68102\r\nReferer: http://otherhost.com\r\nAccept-Encoding: gzip\r\nContent-Type: multipart/form-data; boundary=e5a320f21416a02493a0a6f561b1c494\r\nCookie: blah\r\nAccept-Language: us\r\n\r\n--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5631  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5632  char input[] = "\
5633 %YAML 1.1\n\
5634 ---\n\
5635 libhtp:\n\
5636 \n\
5637  default-config:\n\
5638  personality: IDS\n\
5639  double-decode-path: no\n\
5640  double-decode-query: no\n\
5641  request-body-limit: 0\n\
5642  response-body-limit: 0\n\
5643 ";
5644 
5646  ConfInit();
5648  ConfYamlLoadString(input, strlen(input));
5649  HTPConfigure();
5650 
5651  TcpSession ssn;
5652  HtpState *http_state = NULL;
5654 
5655  memset(&ssn, 0, sizeof(ssn));
5656 
5657  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
5658  if (f == NULL)
5659  goto end;
5660  f->protoctx = &ssn;
5661  f->proto = IPPROTO_TCP;
5662  f->alproto = ALPROTO_HTTP1;
5663 
5664  StreamTcpInitConfig(true);
5665 
5666  SCLogDebug("\n>>>> processing chunk 1 <<<<\n");
5667  int r = AppLayerParserParse(
5668  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
5669  if (r != 0) {
5670  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
5671  goto end;
5672  }
5673  SCLogDebug("\n>>>> processing chunk 1 again <<<<\n");
5674  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf1, httplen1);
5675  if (r != 0) {
5676  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
5677  goto end;
5678  }
5679 
5680  http_state = f->alstate;
5681  if (http_state == NULL) {
5682  printf("no http state: ");
5683  goto end;
5684  }
5685 
5687  if (decoder_events != NULL) {
5688  printf("app events: ");
5689  goto end;
5690  }
5691  result = 1;
5692 end:
5693  if (alp_tctx != NULL)
5695  HTPFreeConfig();
5696  ConfDeInit();
5699  StreamTcpFreeConfig(true);
5700  UTHFreeFlow(f);
5701  return result;
5702 }
5703 
5704 /** \test Test really long request, this should result in HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG */
5705 static int HTPParserTest14(void)
5706 {
5707  size_t len = 18887;
5708  TcpSession ssn;
5709  char input[] = "\
5710 %YAML 1.1\n\
5711 ---\n\
5712 libhtp:\n\
5713 \n\
5714  default-config:\n\
5715  personality: IDS\n\
5716  double-decode-path: no\n\
5717  double-decode-query: no\n\
5718  request-body-limit: 0\n\
5719  response-body-limit: 0\n\
5720 ";
5723 
5724  memset(&ssn, 0, sizeof(ssn));
5725 
5727  ConfInit();
5729  ConfYamlLoadString(input, strlen(input));
5730  HTPConfigure();
5731 
5732  char *httpbuf = SCMalloc(len);
5733  FAIL_IF_NULL(httpbuf);
5734  memset(httpbuf, 0x00, len);
5735 
5736  /* create the request with a longer than 18k cookie */
5737  strlcpy(httpbuf, "GET /blah/ HTTP/1.1\r\n"
5738  "Host: myhost.lan\r\n"
5739  "Connection: keep-alive\r\n"
5740  "Accept: */*\r\n"
5741  "User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36\r\n"
5742  "Referer: http://blah.lan/\r\n"
5743  "Accept-Encoding: gzip,deflate,sdch\r\nAccept-Language: en-US,en;q=0.8\r\n"
5744  "Cookie: ", len);
5745  size_t o = strlen(httpbuf);
5746  for ( ; o < len - 4; o++) {
5747  httpbuf[o] = 'A';
5748  }
5749  httpbuf[len - 4] = '\r';
5750  httpbuf[len - 3] = '\n';
5751  httpbuf[len - 2] = '\r';
5752  httpbuf[len - 1] = '\n';
5753 
5754  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
5755  FAIL_IF_NULL(f);
5756  f->protoctx = &ssn;
5757  f->alproto = ALPROTO_HTTP1;
5758  f->proto = IPPROTO_TCP;
5759 
5760  StreamTcpInitConfig(true);
5761 
5762  uint32_t u;
5763  for (u = 0; u < len; u++) {
5764  uint8_t flags = 0;
5765 
5766  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5767  else if (u == (len - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5768  else flags = STREAM_TOSERVER;
5769 
5770  (void)AppLayerParserParse(
5771  NULL, alp_tctx, f, ALPROTO_HTTP1, flags, (uint8_t *)&httpbuf[u], 1);
5772  }
5773  HtpState *htp_state = f->alstate;
5774  FAIL_IF_NULL(htp_state);
5775 
5776  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5777  FAIL_IF_NULL(tx);
5778  FAIL_IF(tx->request_method_number != HTP_M_GET);
5779  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
5780 
5781  void *txtmp = AppLayerParserGetTx(IPPROTO_TCP,