suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "decode.h"
40 #include "threads.h"
41 #include "counters.h"
42 
43 #include "util-print.h"
44 #include "util-pool.h"
45 #include "util-radix-tree.h"
46 #include "util-file.h"
47 #include "util-byte.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-detect-proto.h"
59 #include "app-layer-frames.h"
60 #include "app-layer-htp.h"
61 #include "app-layer-htp-body.h"
62 #include "app-layer-htp-file.h"
63 #include "app-layer-htp-libhtp.h"
64 #include "app-layer-htp-xff.h"
65 #include "app-layer-htp-range.h"
66 #include "app-layer-htp-mem.h"
67 
68 #include "util-spm.h"
69 #include "util-debug.h"
70 #include "util-time.h"
71 #include "util-misc.h"
72 #include "util-enum.h"
73 
74 #include "util-unittest.h"
75 #include "util-unittest-helper.h"
76 #include "flow-util.h"
77 
78 #include "detect-engine.h"
79 #include "detect-engine-build.h"
80 #include "detect-engine-state.h"
81 #include "detect-parse.h"
82 
83 #include "decode-events.h"
84 
85 #include "util-memcmp.h"
86 #include "util-random.h"
87 #include "util-validate.h"
88 
89 //#define PRINT
90 
91 /** Fast lookup tree (radix) for the various HTP configurations */
92 static SCRadixTree *cfgtree;
93 /** List of HTP configurations. */
94 static HTPCfgRec cfglist;
95 
97 
98 /** Limit to the number of libhtp messages that can be handled */
99 #define HTP_MAX_MESSAGES 512
100 
101 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
102 
103 #ifdef DEBUG
104 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
105 static uint64_t htp_state_memuse = 0;
106 static uint64_t htp_state_memcnt = 0;
107 #endif
108 
110  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
111  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
112  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
113  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
114  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
115  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
116  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
118  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
120  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
122  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
124  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
126  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
128  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
129  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
131  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
132  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
133  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
134  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
135  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
136  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
137  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
138  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
139  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
140  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
141  { "FILE_NAME_TOO_LONG", HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG },
142  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
143  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
144  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
146  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
147  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
148  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
149  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
150  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
151  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
152  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
153  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
154  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
155  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
156  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
157  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
158  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
160  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
161  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
162  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
163  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
164 
165  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
166  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
167 
168  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
169 
170  /* suricata warnings/errors */
171  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
172  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
173  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
174 
175  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
176  { "FAILED_PROTOCOL_CHANGE", HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE },
177 
178  { NULL, -1 },
179 };
180 
181 /* app-layer-frame-documentation tag start: HttpFrameTypes */
185 };
186 
188  {
189  "request",
191  },
192  {
193  "response",
195  },
196  { NULL, -1 },
197 };
198 /* app-layer-frame-documentation tag end: HttpFrameTypes */
199 
200 static int HTTPGetFrameIdByName(const char *frame_name)
201 {
202  int id = SCMapEnumNameToValue(frame_name, http_frame_table);
203  if (id < 0) {
204  return -1;
205  }
206  return id;
207 }
208 
209 static const char *HTTPGetFrameNameById(const uint8_t frame_id)
210 {
211  const char *name = SCMapEnumValueToName(frame_id, http_frame_table);
212  return name;
213 }
214 
215 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
216 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
217 static uint64_t HTPStateGetTxCnt(void *alstate);
218 #ifdef UNITTESTS
219 static void HTPParserRegisterTests(void);
220 #endif
221 
222 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
223 {
224  uint64_t id = HTPStateGetTxCnt(s);
225  BUG_ON(id == 0);
226  return id - 1;
227 }
228 
229 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
230 {
231  return s->transaction_cnt;
232 }
233 
234 #ifdef DEBUG
235 /**
236  * \internal
237  *
238  * \brief Lookup the HTP personality string from the numeric personality.
239  *
240  * \todo This needs to be a libhtp function.
241  */
242 static const char *HTPLookupPersonalityString(int p)
243 {
244 #define CASE_HTP_PERSONALITY_STRING(p) \
245  case HTP_SERVER_ ## p: return #p
246 
247  switch (p) {
248  CASE_HTP_PERSONALITY_STRING(MINIMAL);
249  CASE_HTP_PERSONALITY_STRING(GENERIC);
250  CASE_HTP_PERSONALITY_STRING(IDS);
251  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
252  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
253  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
254  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
255  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
256  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
257  CASE_HTP_PERSONALITY_STRING(APACHE_2);
258  }
259 
260  return NULL;
261 }
262 #endif /* DEBUG */
263 
264 /**
265  * \internal
266  *
267  * \brief Lookup the numeric HTP personality from a string.
268  *
269  * \todo This needs to be a libhtp function.
270  */
271 static int HTPLookupPersonality(const char *str)
272 {
273 #define IF_HTP_PERSONALITY_NUM(p) \
274  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
275 
276  IF_HTP_PERSONALITY_NUM(MINIMAL);
277  IF_HTP_PERSONALITY_NUM(GENERIC);
279  IF_HTP_PERSONALITY_NUM(IIS_4_0);
280  IF_HTP_PERSONALITY_NUM(IIS_5_0);
281  IF_HTP_PERSONALITY_NUM(IIS_5_1);
282  IF_HTP_PERSONALITY_NUM(IIS_6_0);
283  IF_HTP_PERSONALITY_NUM(IIS_7_0);
284  IF_HTP_PERSONALITY_NUM(IIS_7_5);
285  IF_HTP_PERSONALITY_NUM(APACHE_2);
286  if (strcasecmp("TOMCAT_6_0", str) == 0) {
287  SCLogError("Personality %s no "
288  "longer supported by libhtp.",
289  str);
290  return -1;
291  } else if ((strcasecmp("APACHE", str) == 0) ||
292  (strcasecmp("APACHE_2_2", str) == 0))
293  {
294  SCLogWarning("Personality %s no "
295  "longer supported by libhtp, failing back to "
296  "Apache2 personality.",
297  str);
298  return HTP_SERVER_APACHE_2;
299  }
300 
301  return -1;
302 }
303 
304 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
305  const uint8_t dir, const uint8_t e)
306 {
307  SCLogDebug("setting event %u", e);
308 
309  if (htud) {
310  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
311  s->events++;
312  return;
313  }
314 
315  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
316  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
317 
318  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
319  if (tx == NULL && tx_id > 0)
320  tx = HTPStateGetTx(s, tx_id - 1);
321  if (tx != NULL) {
322  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
323  if (htud != NULL) {
324  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
325  s->events++;
326  return;
327  }
328  }
329  SCLogDebug("couldn't set event %u", e);
330 }
331 
332 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
333  * connection parser to be used by the HTP library
334  */
335 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
336 {
337  SCEnter();
338 
339  HtpState *s = HTPMalloc(sizeof(HtpState));
340  if (unlikely(s == NULL)) {
341  SCReturnPtr(NULL, "void");
342  }
343 
344  memset(s, 0x00, sizeof(HtpState));
345 
346 #ifdef DEBUG
347  SCMutexLock(&htp_state_mem_lock);
348  htp_state_memcnt++;
349  htp_state_memuse += sizeof(HtpState);
350  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
351  SCMutexUnlock(&htp_state_mem_lock);
352 #endif
353 
354  SCReturnPtr((void *)s, "void");
355 }
356 
357 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
358 {
359  if (likely(htud)) {
360  HtpBodyFree(&state->cfg->request, &htud->request_body);
361  HtpBodyFree(&state->cfg->response, &htud->response_body);
362  bstr_free(htud->request_uri_normalized);
363  if (htud->request_headers_raw)
365  if (htud->response_headers_raw)
368  if (htud->boundary)
369  HTPFree(htud->boundary, htud->boundary_len);
370  if (htud->tx_data.de_state != NULL) {
371  DetectEngineStateFree(htud->tx_data.de_state);
372  }
373  if (htud->file_range) {
374  HTPFileCloseHandleRange(&htp_sbcfg, &htud->files_tc, 0, htud->file_range, NULL, 0);
376  }
379  HTPFree(htud, sizeof(HtpTxUserData));
380  }
381 }
382 
383 /** \brief Function to frees the HTTP state memory and also frees the HTTP
384  * connection parser memory which was used by the HTP library
385  */
386 void HTPStateFree(void *state)
387 {
388  SCEnter();
389 
390  HtpState *s = (HtpState *)state;
391  if (s == NULL) {
392  SCReturn;
393  }
394 
395  /* free the connection parser memory used by HTP library */
396  if (s->connp != NULL) {
397  SCLogDebug("freeing HTP state");
398 
399  uint64_t tx_id;
400  uint64_t total_txs = HTPStateGetTxCnt(state);
401  /* free the list of body chunks */
402  if (s->conn != NULL) {
403  for (tx_id = 0; tx_id < total_txs; tx_id++) {
404  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
405  if (tx != NULL) {
406  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
407  HtpTxUserDataFree(s, htud);
408  htp_tx_set_user_data(tx, NULL);
409  }
410  }
411  }
412  htp_connp_destroy_all(s->connp);
413  }
414 
415  HTPFree(s, sizeof(HtpState));
416 
417 #ifdef DEBUG
418  SCMutexLock(&htp_state_mem_lock);
419  htp_state_memcnt--;
420  htp_state_memuse -= sizeof(HtpState);
421  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
422  SCMutexUnlock(&htp_state_mem_lock);
423 #endif
424 
425  SCReturn;
426 }
427 
428 /**
429  * \brief HTP transaction cleanup callback
430  *
431  * \warning We cannot actually free the transactions here. It seems that
432  * HTP only accepts freeing of transactions in the response callback.
433  */
434 static void HTPStateTransactionFree(void *state, uint64_t id)
435 {
436  SCEnter();
437 
438  HtpState *s = (HtpState *)state;
439 
440  SCLogDebug("state %p, id %"PRIu64, s, id);
441 
442  htp_tx_t *tx = HTPStateGetTx(s, id);
443  if (tx != NULL) {
444  /* This will remove obsolete body chunks */
445  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
446  HtpTxUserDataFree(s, htud);
447  htp_tx_set_user_data(tx, NULL);
448 
449  /* hack: even if libhtp considers the tx incomplete, we want to
450  * free it here. htp_tx_destroy however, will refuse to do this.
451  * As htp_tx_destroy_incomplete isn't available in the public API,
452  * we hack around it here. */
453  if (unlikely(!(
454  tx->request_progress == HTP_REQUEST_COMPLETE &&
455  tx->response_progress == HTP_RESPONSE_COMPLETE)))
456  {
457  tx->request_progress = HTP_REQUEST_COMPLETE;
458  tx->response_progress = HTP_RESPONSE_COMPLETE;
459  }
460  htp_tx_destroy(tx);
461  }
462 }
463 
464 /**
465  * \brief Sets a flag that informs the HTP app layer that some module in the
466  * engine needs the http request body data.
467  * \initonly
468  */
470 {
471  SCEnter();
472 
473  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
474  SCReturn;
475 }
476 
477 /**
478  * \brief Sets a flag that informs the HTP app layer that some module in the
479  * engine needs the http request body data.
480  * \initonly
481  */
483 {
484  SCEnter();
485 
486  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
487  SCReturn;
488 }
489 
490 /**
491  * \brief Sets a flag that informs the HTP app layer that some module in the
492  * engine needs the http request multi part header.
493  *
494  * \initonly
495  */
496 static void AppLayerHtpNeedMultipartHeader(void)
497 {
498  SCEnter();
500 
501  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
502  SCReturn;
503 }
504 
505 /**
506  * \brief Sets a flag that informs the HTP app layer that some module in the
507  * engine needs the http request file.
508  *
509  * \initonly
510  */
512 {
513  SCEnter();
514  AppLayerHtpNeedMultipartHeader();
517 
518  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
519  SCReturn;
520 }
521 
522 static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags)
523 {
524  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
525  if (tx_ud) {
526  SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags);
527  if (flags & STREAM_TOCLIENT) {
528  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
529  } else {
530  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
531  }
532  }
533 }
534 
535 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
536 {
537  SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64,
539  if (flags & HTP_STREAM_DEPTH_SET) {
540  uint32_t stream_depth = FileReassemblyDepth();
541  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
542  SCLogDebug("true");
543  return true;
544  }
545  } else {
546  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
547  return true;
548  }
549  }
550  SCLogDebug("false");
551  return false;
552 }
553 
554 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
555  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
556 {
557  uint32_t chunk_len = 0;
558  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
559  (content_len_so_far < (uint64_t)body_limit) &&
560  (content_len_so_far + (uint64_t)data_len) > body_limit)
561  {
562  chunk_len = body_limit - content_len_so_far;
563  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
564  (content_len_so_far < (uint64_t)stream_depth) &&
565  (content_len_so_far + (uint64_t)data_len) > stream_depth)
566  {
567  chunk_len = stream_depth - content_len_so_far;
568  }
569  SCLogDebug("len %u", chunk_len);
570  return (chunk_len == 0 ? data_len : chunk_len);
571 }
572 
573 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
574 struct {
575  const char *msg;
576  uint8_t de;
577 } htp_errors[] = {
578  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
579  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
580  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
581  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
582  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
583 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
584  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
585 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
586  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
587  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
588  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
589  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
590 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
591  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
592  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
593  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
594  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
595 };
596 
597 struct {
598  const char *msg;
599  uint8_t de;
600 } htp_warnings[] = {
601  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
602  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
603  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
604  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
605  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
606 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
607  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
608  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
609  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
610  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
611  * luckily, "Request server port=" is unique */
612 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
614  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
615  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
616  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
617  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
618  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
619  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
620  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
621  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
622  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
623  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
624  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
625  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
626  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
627  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
628  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
629  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
630  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
631  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
632 };
633 
634 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
635 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
636 
637 /**
638  * \internal
639  *
640  * \brief Get the warning id for the warning msg.
641  *
642  * \param msg warning message
643  *
644  * \retval id the id or 0 in case of not found
645  */
646 static uint8_t HTPHandleWarningGetId(const char *msg)
647 {
648  SCLogDebug("received warning \"%s\"", msg);
649  size_t idx;
650  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
651  if (strncmp(htp_warnings[idx].msg, msg,
652  strlen(htp_warnings[idx].msg)) == 0)
653  {
654  return htp_warnings[idx].de;
655  }
656  }
657 
658  return 0;
659 }
660 
661 /**
662  * \internal
663  *
664  * \brief Get the error id for the error msg.
665  *
666  * \param msg error message
667  *
668  * \retval id the id or 0 in case of not found
669  */
670 static uint8_t HTPHandleErrorGetId(const char *msg)
671 {
672  SCLogDebug("received error \"%s\"", msg);
673 
674  size_t idx;
675  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
676  if (strncmp(htp_errors[idx].msg, msg,
677  strlen(htp_errors[idx].msg)) == 0)
678  {
679  return htp_errors[idx].de;
680  }
681  }
682 
683  return 0;
684 }
685 
686 /**
687  * \internal
688  *
689  * \brief Check state for errors, warnings and add any as events
690  *
691  * \param s state
692  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
693  */
694 static void HTPHandleError(HtpState *s, const uint8_t dir)
695 {
696  if (s == NULL || s->conn == NULL ||
697  s->conn->messages == NULL) {
698  return;
699  }
700 
701  size_t size = htp_list_size(s->conn->messages);
702  size_t msg;
703  if(size >= HTP_MAX_MESSAGES) {
705  //only once per HtpState
706  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
708  //too noisy in fuzzing
709  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
710  }
711  // ignore further messages
712  return;
713  }
714 
715  for (msg = s->htp_messages_offset; msg < size; msg++) {
716  htp_log_t *log = htp_list_get(s->conn->messages, msg);
717  if (log == NULL)
718  continue;
719 
720  HtpTxUserData *htud = NULL;
721  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
722  if (tx != NULL)
723  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
724 
725  SCLogDebug("message %s", log->msg);
726 
727  uint8_t id = HTPHandleErrorGetId(log->msg);
728  if (id == 0) {
729  id = HTPHandleWarningGetId(log->msg);
730  if (id == 0)
732  }
733 
734  if (id > 0) {
735  HTPSetEvent(s, htud, dir, id);
736  }
737  }
738  s->htp_messages_offset = (uint16_t)msg;
739  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
740 }
741 
742 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
743 {
744 #ifdef DEBUG
745  BUG_ON(s == NULL || tx == NULL);
746 #endif
747  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
748  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
749  HTP_HOSTH_INVALID))
750  {
751  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
752  if (htud == NULL)
753  return;
754 
755  if (tx->flags & HTP_REQUEST_INVALID_T_E)
756  HTPSetEvent(s, htud, STREAM_TOSERVER,
758  if (tx->flags & HTP_REQUEST_INVALID_C_L)
759  HTPSetEvent(s, htud, STREAM_TOSERVER,
761  if (tx->flags & HTP_HOST_MISSING)
762  HTPSetEvent(s, htud, STREAM_TOSERVER,
764  if (tx->flags & HTP_HOST_AMBIGUOUS)
765  HTPSetEvent(s, htud, STREAM_TOSERVER,
767  if (tx->flags & HTP_HOSTU_INVALID)
768  HTPSetEvent(s, htud, STREAM_TOSERVER,
770  if (tx->flags & HTP_HOSTH_INVALID)
771  HTPSetEvent(s, htud, STREAM_TOSERVER,
773  }
774  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
775  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
776  if (htud == NULL)
777  return;
778  HTPSetEvent(s, htud, STREAM_TOSERVER,
780  }
781  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
782  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
783  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
784  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
785  if (htud == NULL)
786  return;
787  HTPSetEvent(s, htud, STREAM_TOSERVER,
789  }
790 }
791 
792 static int Setup(Flow *f, HtpState *hstate)
793 {
794  /* store flow ref in state so callbacks can access it */
795  hstate->f = f;
796 
797  HTPCfgRec *htp_cfg_rec = &cfglist;
798  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
799  void *user_data = NULL;
800 
801  if (FLOW_IS_IPV4(f)) {
802  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
803  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
804  }
805  else if (FLOW_IS_IPV6(f)) {
806  SCLogDebug("Looking up HTP config for ipv6");
807  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
808  }
809  else {
810  SCLogError("unknown address family, bug!");
811  goto error;
812  }
813 
814  if (user_data != NULL) {
815  htp_cfg_rec = user_data;
816  htp = htp_cfg_rec->cfg;
817  SCLogDebug("LIBHTP using config: %p", htp);
818  } else {
819  SCLogDebug("Using default HTP config: %p", htp);
820  }
821 
822  if (NULL == htp) {
823 #ifdef DEBUG_VALIDATION
824  BUG_ON(htp == NULL);
825 #endif
826  /* should never happen if HTPConfigure is properly invoked */
827  goto error;
828  }
829 
830  hstate->connp = htp_connp_create(htp);
831  if (hstate->connp == NULL) {
832  goto error;
833  }
834 
835  hstate->conn = htp_connp_get_connection(hstate->connp);
836 
837  htp_connp_set_user_data(hstate->connp, (void *)hstate);
838  hstate->cfg = htp_cfg_rec;
839 
840  SCLogDebug("New hstate->connp %p", hstate->connp);
841 
842  struct timeval tv = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
843  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &tv);
844 
846  htp_cfg_rec->request.inspect_min_size);
848  htp_cfg_rec->response.inspect_min_size);
849  return 0;
850 error:
851  return -1;
852 }
853 
854 /**
855  * \brief Function to handle the reassembled data from client and feed it to
856  * the HTP library to process it.
857  *
858  * \param flow Pointer to the flow the data belong to
859  * \param htp_state Pointer the state in which the parsed value to be stored
860  * \param pstate Application layer parser state for this session
861  *
862  * \retval On success returns 1 or on failure returns -1.
863  */
864 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state, AppLayerParserState *pstate,
865  StreamSlice stream_slice, void *local_data)
866 {
867  SCEnter();
868  int ret = 0;
869  HtpState *hstate = (HtpState *)htp_state;
870 
871  /* On the first invocation, create the connection parser structure to
872  * be used by HTP library. This is looked up via IP in the radix
873  * tree. Failing that, the default HTP config is used.
874  */
875  if (NULL == hstate->conn) {
876  if (Setup(f, hstate) != 0) {
878  }
879  }
880  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
881  hstate->slice = &stream_slice;
882 
883  const uint8_t *input = StreamSliceGetData(&stream_slice);
884  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
885 
886  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
887  /* pass the new data to the htp parser */
888  if (input_len > 0) {
889  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
890  switch (r) {
891  case HTP_STREAM_ERROR:
892  ret = -1;
893  break;
894  default:
895  break;
896  }
897  HTPHandleError(hstate, STREAM_TOSERVER);
898  }
899 
900  /* if the TCP connection is closed, then close the HTTP connection */
902  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
903  {
904  htp_connp_req_close(hstate->connp, &ts);
905  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
906  SCLogDebug("stream eof encountered, closing htp handle for ts");
907  }
908 
909  SCLogDebug("hstate->connp %p", hstate->connp);
910  hstate->slice = NULL;
911 
912  if (ret < 0) {
914  }
916 }
917 
918 /**
919  * \brief Function to handle the reassembled data from server and feed it to
920  * the HTP library to process it.
921  *
922  * \param flow Pointer to the flow the data belong to
923  * \param htp_state Pointer the state in which the parsed value to be stored
924  * \param pstate Application layer parser state for this session
925  * \param input Pointer the received HTTP server data
926  * \param input_len Length in bytes of the received data
927  * \param output Pointer to the output (not used in this function)
928  *
929  * \retval On success returns 1 or on failure returns -1
930  */
931 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state, AppLayerParserState *pstate,
932  StreamSlice stream_slice, void *local_data)
933 {
934  SCEnter();
935  int ret = 0;
936  HtpState *hstate = (HtpState *)htp_state;
937 
938  const uint8_t *input = StreamSliceGetData(&stream_slice);
939  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
940 
941  /* On the first invocation, create the connection parser structure to
942  * be used by HTP library. This is looked up via IP in the radix
943  * tree. Failing that, the default HTP config is used.
944  */
945  if (NULL == hstate->conn) {
946  if (Setup(f, hstate) != 0) {
948  }
949  }
950  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
951  hstate->slice = &stream_slice;
952 
953  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
954  htp_tx_t *tx = NULL;
955  size_t consumed = 0;
956  if (input_len > 0) {
957  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
958  switch (r) {
959  case HTP_STREAM_ERROR:
960  ret = -1;
961  break;
962  case HTP_STREAM_TUNNEL:
963  tx = htp_connp_get_out_tx(hstate->connp);
964  if (tx != NULL && tx->response_status_number == 101) {
965  htp_header_t *h =
966  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
967  if (h == NULL || bstr_cmp_c(h->value, "h2c") != 0) {
968  break;
969  }
971  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
972  break;
973  }
974  uint16_t dp = 0;
975  if (tx->request_port_number != -1) {
976  dp = (uint16_t)tx->request_port_number;
977  }
978  consumed = htp_connp_res_data_consumed(hstate->connp);
979  hstate->slice = NULL;
980  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_HTTP2)) {
981  HTPSetEvent(hstate, NULL, STREAM_TOCLIENT,
983  }
984  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
985  // and we need to parser the remaining part with HTTP2
986  if (consumed > 0 && consumed < input_len) {
987  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
988  }
990  }
991  break;
992  default:
993  break;
994  }
995  HTPHandleError(hstate, STREAM_TOCLIENT);
996  }
997 
998  /* if we the TCP connection is closed, then close the HTTP connection */
1000  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
1001  {
1002  htp_connp_close(hstate->connp, &ts);
1003  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
1004  }
1005 
1006  SCLogDebug("hstate->connp %p", hstate->connp);
1007  hstate->slice = NULL;
1008 
1009  if (ret < 0) {
1011  }
1013 }
1014 
1015 /**
1016  * \param name /Lowercase/ version of the variable name
1017  */
1018 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1019  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1020 {
1021 #ifdef PRINT
1022  printf("DATA START: \n");
1023  PrintRawDataFp(stdout, data, len);
1024  printf("DATA END: \n");
1025 #endif
1026  size_t x;
1027  int quote = 0;
1028 
1029  for (x = 0; x < len; x++) {
1030  if (!(isspace(data[x])))
1031  break;
1032  }
1033 
1034  if (x >= len)
1035  return 0;
1036 
1037  uint8_t *line = data+x;
1038  size_t line_len = len-x;
1039  size_t offset = 0;
1040 #ifdef PRINT
1041  printf("LINE START: \n");
1042  PrintRawDataFp(stdout, line, line_len);
1043  printf("LINE END: \n");
1044 #endif
1045  for (x = 0 ; x < line_len; x++) {
1046  if (x > 0) {
1047  if (line[x - 1] != '\\' && line[x] == '\"') {
1048  quote++;
1049  }
1050 
1051  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1052  uint8_t *token = line + offset;
1053  size_t token_len = x - offset;
1054 
1055  if ((x + 1) == line_len) {
1056  token_len++;
1057  }
1058 
1059  offset = x + 1;
1060 
1061  while (offset < line_len && isspace(line[offset])) {
1062  x++;
1063  offset++;
1064  }
1065 #ifdef PRINT
1066  printf("TOKEN START: \n");
1067  PrintRawDataFp(stdout, token, token_len);
1068  printf("TOKEN END: \n");
1069 #endif
1070  if (token_len > name_len) {
1071  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1072  uint8_t *value = token + name_len;
1073  size_t value_len = token_len - name_len;
1074 
1075  if (value[0] == '\"') {
1076  value++;
1077  value_len--;
1078  }
1079  if (value[value_len-1] == '\"') {
1080  value_len--;
1081  }
1082 #ifdef PRINT
1083  printf("VALUE START: \n");
1084  PrintRawDataFp(stdout, value, value_len);
1085  printf("VALUE END: \n");
1086 #endif
1087  *retptr = value;
1088  *retlen = value_len;
1089  return 1;
1090  }
1091  }
1092  }
1093  }
1094  }
1095 
1096  return 0;
1097 }
1098 
1099 /**
1100  * \param name /Lowercase/ version of the variable name
1101  */
1102 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1103  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1104 {
1105  SCEnter();
1106 #ifdef PRINT
1107  printf("DATA START: \n");
1108  PrintRawDataFp(stdout, data, len);
1109  printf("DATA END: \n");
1110 #endif
1111  size_t x;
1112  int quote = 0;
1113 
1114  for (x = 0; x < len; x++) {
1115  if (!(isspace(data[x])))
1116  break;
1117  }
1118 
1119  if (x >= len) {
1120  SCReturnInt(0);
1121  }
1122 
1123  uint8_t *line = data+x;
1124  size_t line_len = len-x;
1125  size_t offset = 0;
1126 #ifdef PRINT
1127  printf("LINE START: \n");
1128  PrintRawDataFp(stdout, line, line_len);
1129  printf("LINE END: \n");
1130 #endif
1131  for (x = 0 ; x < line_len; x++) {
1132  if (x > 0) {
1133  if (line[x - 1] != '\\' && line[x] == '\"') {
1134  quote++;
1135  }
1136 
1137  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1138  uint8_t *token = line + offset;
1139  size_t token_len = x - offset;
1140 
1141  if ((x + 1) == line_len) {
1142  token_len++;
1143  }
1144 
1145  offset = x + 1;
1146 
1147  while (offset < line_len && isspace(line[offset])) {
1148  x++;
1149  offset++;
1150  }
1151 #ifdef PRINT
1152  printf("TOKEN START: \n");
1153  PrintRawDataFp(stdout, token, token_len);
1154  printf("TOKEN END: \n");
1155 #endif
1156  if (token_len > name_len) {
1157  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1158  uint8_t *value = token + name_len;
1159  size_t value_len = token_len - name_len;
1160 
1161  if (value[0] == '\"') {
1162  value++;
1163  value_len--;
1164  }
1165  if (value[value_len-1] == '\"') {
1166  value_len--;
1167  }
1168 #ifdef PRINT
1169  printf("VALUE START: \n");
1170  PrintRawDataFp(stdout, value, value_len);
1171  printf("VALUE END: \n");
1172 #endif
1173  *retptr = value;
1174  *retlen = value_len;
1175  SCReturnInt(1);
1176  }
1177  }
1178  }
1179  }
1180  }
1181 
1182  SCReturnInt(0);
1183 }
1184 
1185 /**
1186  * \brief setup multipart parsing: extract boundary and store it
1187  *
1188  * \param d HTTP transaction
1189  * \param htud transaction userdata
1190  *
1191  * \retval 1 ok, multipart set up
1192  * \retval 0 ok, not multipart though
1193  * \retval -1 error: problem with the boundary
1194  *
1195  * If the request contains a multipart message, this function will
1196  * set the HTP_BOUNDARY_SET in the transaction.
1197  */
1198 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1199 {
1200  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1201  "Content-Type");
1202  if (h != NULL && bstr_len(h->value) > 0) {
1203  uint8_t *boundary = NULL;
1204  size_t boundary_len = 0;
1205 
1206  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1207  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1208  &boundary, &boundary_len);
1209  if (r == 1) {
1210 #ifdef PRINT
1211  printf("BOUNDARY START: \n");
1212  PrintRawDataFp(stdout, boundary, boundary_len);
1213  printf("BOUNDARY END: \n");
1214 #endif
1215  if (boundary_len < HTP_BOUNDARY_MAX) {
1216  htud->boundary = HTPMalloc(boundary_len);
1217  if (htud->boundary == NULL) {
1218  return -1;
1219  }
1220  htud->boundary_len = (uint8_t)boundary_len;
1221  memcpy(htud->boundary, boundary, boundary_len);
1222 
1223  htud->tsflags |= HTP_BOUNDARY_SET;
1224  } else {
1225  SCLogDebug("invalid boundary");
1226  return -1;
1227  }
1228  SCReturnInt(1);
1229  }
1230  //SCReturnInt(1);
1231  }
1232  SCReturnInt(0);
1233 }
1234 
1235 #define C_D_HDR "content-disposition:"
1236 #define C_D_HDR_LEN 20
1237 #define C_T_HDR "content-type:"
1238 #define C_T_HDR_LEN 13
1240 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1241  HtpTxUserData *htud,
1242  uint8_t *header, uint32_t header_len,
1243  uint8_t **filename, uint16_t *filename_len,
1244  uint8_t **filetype, uint16_t *filetype_len)
1245 {
1246  uint8_t *fn = NULL;
1247  size_t fn_len = 0;
1248  uint8_t *ft = NULL;
1249  size_t ft_len = 0;
1250 
1251 #ifdef PRINT
1252  printf("HEADER START: \n");
1253  PrintRawDataFp(stdout, header, header_len);
1254  printf("HEADER END: \n");
1255 #endif
1256 
1257  while (header_len > 0) {
1258  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1259  uint8_t *line = header;
1260  uint32_t line_len;
1261 
1262  if (next_line == NULL) {
1263  line_len = header_len;
1264  } else {
1265  line_len = next_line - header;
1266  }
1267  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1268  if (sc == NULL) {
1269  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1271  /* if the : we found is the final char, it means we have
1272  * no value */
1273  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1274  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1276  } else {
1277 #ifdef PRINT
1278  printf("LINE START: \n");
1279  PrintRawDataFp(stdout, line, line_len);
1280  printf("LINE END: \n");
1281 #endif
1282  if (line_len >= C_D_HDR_LEN &&
1283  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1284  uint8_t *value = line + C_D_HDR_LEN;
1285  uint32_t value_len = line_len - C_D_HDR_LEN;
1286 
1287  /* parse content-disposition */
1288  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1289  value, value_len, &fn, &fn_len);
1290  } else if (line_len >= C_T_HDR_LEN &&
1291  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1292  SCLogDebug("content-type line");
1293  uint8_t *value = line + C_T_HDR_LEN;
1294  uint32_t value_len = line_len - C_T_HDR_LEN;
1295 
1296  (void)HTTPParseContentTypeHeader(NULL, 0,
1297  value, value_len, &ft, &ft_len);
1298  }
1299  }
1300 
1301  if (next_line == NULL) {
1302  SCLogDebug("no next_line");
1303  break;
1304  }
1305  header_len -= ((next_line + 2) - header);
1306  header = next_line + 2;
1307  } /* while (header_len > 0) */
1308 
1309  if (fn_len > USHRT_MAX)
1310  fn_len = USHRT_MAX;
1311  if (ft_len > USHRT_MAX)
1312  ft_len = USHRT_MAX;
1313 
1314  *filename = fn;
1315  *filename_len = (uint16_t)fn_len;
1316  *filetype = ft;
1317  *filetype_len = (uint16_t)ft_len;
1318 }
1319 
1320 /**
1321  * \brief Create a single buffer from the HtpBodyChunks in our list
1322  *
1323  * \param htud transaction user data
1324  * \param chunks_buffers pointer to pass back the buffer to the caller
1325  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1326  */
1327 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1328  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1329 {
1331  chunks_buffer, chunks_buffer_len,
1332  htud->request_body.body_parsed);
1333 }
1334 
1335 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1336 {
1337  SCEnter();
1338  if (tx && tx->tx_data.de_state) {
1339  if (dir == STREAM_TOSERVER) {
1340  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1341  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1342  } else if (STREAM_TOCLIENT) {
1343  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1344  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1345  }
1346  }
1347 }
1348 
1349 /**
1350  * \brief Setup boundary buffers
1351  */
1352 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1353  uint8_t *boundary, uint32_t boundary_len)
1354 {
1355  memset(boundary, '-', boundary_len);
1356  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1357 }
1358 
1359 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1360  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1361 {
1362  int result = 0;
1363  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1364  uint16_t expected_boundary_len = htud->boundary_len + 2;
1365  uint16_t expected_boundary_end_len = htud->boundary_len + 4;
1366  int tx_progress = 0;
1367 
1368 #ifdef PRINT
1369  printf("CHUNK START: \n");
1370  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1371  printf("CHUNK END: \n");
1372 #endif
1373 
1374  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1375 
1376  /* search for the header start, header end and form end */
1377  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1378  boundary, expected_boundary_len);
1379  /* end of the multipart form */
1380  const uint8_t *form_end = NULL;
1381  /* end marker belonging to header_start */
1382  const uint8_t *header_end = NULL;
1383  if (header_start != NULL) {
1384  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1385  (uint8_t *)"\r\n\r\n", 4);
1386  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1387  boundary, expected_boundary_end_len);
1388  }
1389 
1390  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1391  header_end, form_end);
1392 
1393  /* we currently only handle multipart for ts. When we support it for tc,
1394  * we will need to supply right direction */
1395  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1396  /* if we're in the file storage process, deal with that now */
1397  if (htud->tsflags & HTP_FILENAME_SET) {
1398  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1399  SCLogDebug("reached the end of the file");
1400 
1401  const uint8_t *filedata = chunks_buffer;
1402  uint32_t filedata_len = 0;
1403  uint8_t flags = 0;
1404 
1405  if (header_start != NULL) {
1406  if (header_start == filedata + 2) {
1407  /* last chunk had all data, but not the boundary */
1408  SCLogDebug("last chunk had all data, but not the boundary");
1409  filedata_len = 0;
1410  } else if (header_start > filedata + 2) {
1411  SCLogDebug("some data from last file before the boundary");
1412  /* some data from last file before the boundary */
1413  filedata_len = header_start - filedata - 2;
1414  }
1415  }
1416  /* body parsing done, we did not get our form end. Use all data
1417  * we still have and signal to files API we have an issue. */
1418  if (tx_progress > HTP_REQUEST_BODY) {
1419  filedata_len = chunks_buffer_len;
1421  }
1422 
1423  if (filedata_len > chunks_buffer_len) {
1424  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1426  goto end;
1427  }
1428 #ifdef PRINT
1429  printf("FILEDATA (final chunk) START: \n");
1430  PrintRawDataFp(stdout, filedata, filedata_len);
1431  printf("FILEDATA (final chunk) END: \n");
1432 #endif
1433  if (!(htud->tsflags & HTP_DONTSTORE)) {
1434  if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
1435  -1) {
1436  goto end;
1437  }
1438  }
1439 
1440  htud->tsflags &=~ HTP_FILENAME_SET;
1441 
1442  /* fall through */
1443  } else {
1444  SCLogDebug("not yet at the end of the file");
1445 
1446  if (chunks_buffer_len > expected_boundary_end_len) {
1447  const uint8_t *filedata = chunks_buffer;
1448  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1449  for (; filedata_len < chunks_buffer_len; filedata_len++) {
1450  // take as much as we can until the beginning of a new line
1451  if (chunks_buffer[filedata_len] == '\r') {
1452  if (filedata_len + 1 == chunks_buffer_len ||
1453  chunks_buffer[filedata_len + 1] == '\n') {
1454  break;
1455  }
1456  }
1457  }
1458 
1459 #ifdef PRINT
1460  printf("FILEDATA (part) START: \n");
1461  PrintRawDataFp(stdout, filedata, filedata_len);
1462  printf("FILEDATA (part) END: \n");
1463 #endif
1464 
1465  if (!(htud->tsflags & HTP_DONTSTORE)) {
1466  result = HTPFileStoreChunk(
1467  hstate, htud, filedata, filedata_len, STREAM_TOSERVER);
1468  if (result == -1) {
1469  goto end;
1470  } else if (result == -2) {
1471  /* we know for sure we're not storing the file */
1472  htud->tsflags |= HTP_DONTSTORE;
1473  }
1474  }
1475 
1476  htud->request_body.body_parsed += filedata_len;
1477  } else {
1478  SCLogDebug("chunk too small to already process in part");
1479  }
1480 
1481  goto end;
1482  }
1483  }
1484 
1485  while (header_start != NULL && header_end != NULL &&
1486  header_end != form_end &&
1487  header_start < (chunks_buffer + chunks_buffer_len) &&
1488  header_end < (chunks_buffer + chunks_buffer_len) &&
1489  header_start < header_end)
1490  {
1491  uint8_t *filename = NULL;
1492  uint16_t filename_len = 0;
1493  uint8_t *filetype = NULL;
1494  uint16_t filetype_len = 0;
1495 
1496  uint32_t header_len = header_end - header_start;
1497  SCLogDebug("header_len %u", header_len);
1498  uint8_t *header = (uint8_t *)header_start;
1499 
1500  /* skip empty records */
1501  if (expected_boundary_len == header_len) {
1502  goto next;
1503  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1504  header_len -= (expected_boundary_len + 2);
1505  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1506  }
1507 
1508  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1509  &filename, &filename_len, &filetype, &filetype_len);
1510 
1511  if (filename != NULL) {
1512  const uint8_t *filedata = NULL;
1513  uint32_t filedata_len = 0;
1514 
1515  SCLogDebug("we have a filename");
1516 
1517  htud->tsflags |= HTP_FILENAME_SET;
1518  htud->tsflags &= ~HTP_DONTSTORE;
1519 
1520  SCLogDebug("header_end %p", header_end);
1521  SCLogDebug("form_end %p", form_end);
1522 
1523  /* everything until the final boundary is the file */
1524  if (form_end != NULL) {
1525  SCLogDebug("have form_end");
1526 
1527  filedata = header_end + 4;
1528  if (form_end == filedata) {
1529  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1531  goto end;
1532  } else if (form_end < filedata) {
1533  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1535  goto end;
1536  }
1537 
1538  filedata_len = form_end - (header_end + 4 + 2);
1539  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1540 
1541  /* or is it? */
1542  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1543  boundary, expected_boundary_len);
1544  if (header_next != NULL) {
1545  filedata_len -= (form_end - header_next);
1546  }
1547 
1548  if (filedata_len > chunks_buffer_len) {
1549  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1551  goto end;
1552  }
1553  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1554 #ifdef PRINT
1555  printf("FILEDATA START: \n");
1556  PrintRawDataFp(stdout, filedata, filedata_len);
1557  printf("FILEDATA END: \n");
1558 #endif
1559 
1560  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata, filedata_len,
1561  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1562  if (result == -1) {
1563  goto end;
1564  } else if (result == -2) {
1565  htud->tsflags |= HTP_DONTSTORE;
1566  } else {
1567  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1568  goto end;
1569  }
1570  }
1571  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1572 
1573  htud->request_body.body_parsed += (header_end - chunks_buffer);
1574  htud->tsflags &= ~HTP_FILENAME_SET;
1575  } else {
1576  SCLogDebug("chunk doesn't contain form end");
1577 
1578  filedata = header_end + 4;
1579  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1580  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1581 
1582  if (filedata_len > chunks_buffer_len) {
1583  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1585  goto end;
1586  }
1587 
1588 #ifdef PRINT
1589  printf("FILEDATA START: \n");
1590  PrintRawDataFp(stdout, filedata, filedata_len);
1591  printf("FILEDATA END: \n");
1592 #endif
1593  /* form doesn't end in this chunk, but the part might. Lets
1594  * see if have another coming up */
1595  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1596  boundary, expected_boundary_len);
1597  SCLogDebug("header_next %p", header_next);
1598  if (header_next == NULL) {
1599  SCLogDebug("more file data to come");
1600 
1601  uint32_t offset = (header_end + 4) - chunks_buffer;
1602  SCLogDebug("offset %u", offset);
1603  htud->request_body.body_parsed += offset;
1604 
1605  if (filedata_len >= (uint32_t)(expected_boundary_len + 2)) {
1606  filedata_len -= (expected_boundary_len + 2 - 1);
1607  SCLogDebug("opening file with partial data");
1608  } else {
1609  filedata = NULL;
1610  filedata_len = 0;
1611  }
1612  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1613  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1614  if (result == -1) {
1615  goto end;
1616  } else if (result == -2) {
1617  htud->tsflags |= HTP_DONTSTORE;
1618  }
1619  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1620  htud->request_body.body_parsed += filedata_len;
1621  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1622 
1623  } else if (header_next - filedata > 2) {
1624  filedata_len = header_next - filedata - 2;
1625  SCLogDebug("filedata_len %u", filedata_len);
1626 
1627  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1628  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1629  if (result == -1) {
1630  goto end;
1631  } else if (result == -2) {
1632  htud->tsflags |= HTP_DONTSTORE;
1633  } else {
1634  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1635  goto end;
1636  }
1637  }
1638  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1639 
1640  htud->tsflags &= ~HTP_FILENAME_SET;
1641  htud->request_body.body_parsed += (header_end - chunks_buffer);
1642  }
1643  }
1644  }
1645 next:
1646  SCLogDebug("header_start %p, header_end %p, form_end %p",
1647  header_start, header_end, form_end);
1648 
1649  /* Search next boundary entry after the start of body */
1650  uint32_t cursizeread = header_end - chunks_buffer;
1651  header_start = Bs2bmSearch(header_end + 4,
1652  chunks_buffer_len - (cursizeread + 4),
1653  boundary, expected_boundary_len);
1654  if (header_start != NULL) {
1655  header_end = Bs2bmSearch(header_end + 4,
1656  chunks_buffer_len - (cursizeread + 4),
1657  (uint8_t *) "\r\n\r\n", 4);
1658  }
1659  }
1660 
1661  /* if we're parsing the multipart and we're not currently processing a
1662  * file, we move the body pointer forward. */
1663  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1664  if (chunks_buffer_len > expected_boundary_end_len) {
1665  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1666 
1667  htud->request_body.body_parsed += move;
1668  SCLogDebug("form not ready, file not set, parsing non-file "
1669  "record: moved %u", move);
1670  }
1671  }
1672 
1673 end:
1674  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1675  return 0;
1676 }
1677 
1678 /** \internal
1679  * \brief Handle POST or PUT, no multipart body data
1680  */
1681 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1682  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1683 {
1684  int result = 0;
1685 
1686  /* see if we need to open the file */
1687  if (!(htud->tsflags & HTP_FILENAME_SET))
1688  {
1689  uint8_t *filename = NULL;
1690  size_t filename_len = 0;
1691 
1692  /* get the name */
1693  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1694  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1695  filename_len = bstr_len(tx->parsed_uri->path);
1696  }
1697 
1698  if (filename != NULL) {
1699  if (filename_len > SC_FILENAME_MAX) {
1700  // explicitly truncate the file name if too long
1701  filename_len = SC_FILENAME_MAX;
1702  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1703  }
1704  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1705  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1706  if (result == -1) {
1707  goto end;
1708  } else if (result == -2) {
1709  htud->tsflags |= HTP_DONTSTORE;
1710  } else {
1711  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1712  htud->tsflags |= HTP_FILENAME_SET;
1713  htud->tsflags &= ~HTP_DONTSTORE;
1714  }
1715  }
1716  }
1717  else
1718  {
1719  /* otherwise, just store the data */
1720 
1721  if (!(htud->tsflags & HTP_DONTSTORE)) {
1722  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOSERVER);
1723  if (result == -1) {
1724  goto end;
1725  } else if (result == -2) {
1726  /* we know for sure we're not storing the file */
1727  htud->tsflags |= HTP_DONTSTORE;
1728  }
1729  }
1730  }
1731 
1732  return 0;
1733 end:
1734  return -1;
1735 }
1736 
1737 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1738  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1739 {
1740  SCEnter();
1741 
1742  int result = 0;
1743 
1744  /* see if we need to open the file
1745  * we check for tx->response_line in case of junk
1746  * interpreted as body before response line
1747  */
1748  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1749  (tx->response_line != NULL || tx->is_protocol_0_9))
1750  {
1751  SCLogDebug("setting up file name");
1752 
1753  uint8_t *filename = NULL;
1754  size_t filename_len = 0;
1755 
1756  /* try Content-Disposition header first */
1757  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1758  "Content-Disposition");
1759  if (h != NULL && bstr_len(h->value) > 0) {
1760  /* parse content-disposition */
1761  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1762  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1763  }
1764 
1765  /* fall back to name from the uri */
1766  if (filename == NULL) {
1767  /* get the name */
1768  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1769  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1770  filename_len = bstr_len(tx->parsed_uri->path);
1771  }
1772  }
1773 
1774  if (filename != NULL) {
1775  // set range if present
1776  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1777  if (filename_len > SC_FILENAME_MAX) {
1778  // explicitly truncate the file name if too long
1779  filename_len = SC_FILENAME_MAX;
1780  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1781  }
1782  if (h_content_range != NULL) {
1783  result = HTPFileOpenWithRange(hstate, htud, filename, (uint16_t)filename_len, data,
1784  data_len, HtpGetActiveResponseTxID(hstate), h_content_range->value, htud);
1785  } else {
1786  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1787  HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1788  }
1789  SCLogDebug("result %d", result);
1790  if (result == -1) {
1791  goto end;
1792  } else if (result == -2) {
1793  htud->tcflags |= HTP_DONTSTORE;
1794  } else {
1795  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1796  htud->tcflags |= HTP_FILENAME_SET;
1797  htud->tcflags &= ~HTP_DONTSTORE;
1798  }
1799  }
1800  }
1801  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1802  {
1803  /* otherwise, just store the data */
1804 
1805  if (!(htud->tcflags & HTP_DONTSTORE)) {
1806  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOCLIENT);
1807  SCLogDebug("result %d", result);
1808  if (result == -1) {
1809  goto end;
1810  } else if (result == -2) {
1811  /* we know for sure we're not storing the file */
1812  htud->tcflags |= HTP_DONTSTORE;
1813  }
1814  }
1815  }
1816 
1817  htud->response_body.body_parsed += data_len;
1818  return 0;
1819 end:
1820  return -1;
1821 }
1822 
1823 /**
1824  * \brief Function callback to append chunks for Requests
1825  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1826  * \retval int HTP_OK if all goes well
1827  */
1828 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1829 {
1830  SCEnter();
1831 
1832  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1833  SCReturnInt(HTP_OK);
1834 
1835  if (d->len == 0)
1836  SCReturnInt(HTP_OK);
1837 
1838 #ifdef PRINT
1839  printf("HTPBODY START: \n");
1840  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1841  printf("HTPBODY END: \n");
1842 #endif
1843 
1844  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1845  if (hstate == NULL) {
1846  SCReturnInt(HTP_ERROR);
1847  }
1848 
1849  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1850  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1851 
1852  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1853  if (tx_ud == NULL) {
1854  SCReturnInt(HTP_OK);
1855  }
1856  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
1857 
1858  if (!tx_ud->response_body_init) {
1859  tx_ud->response_body_init = 1;
1860 
1861  if (d->tx->request_method_number == HTP_M_POST) {
1862  SCLogDebug("POST");
1863  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1864  if (r == 1) {
1866  } else if (r == 0) {
1868  SCLogDebug("not multipart");
1869  }
1870  } else if (d->tx->request_method_number == HTP_M_PUT) {
1872  }
1873  }
1874 
1875  /* see if we can get rid of htp body chunks */
1876  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1877 
1878  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1879  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1880 
1881  /* within limits, add the body chunk to the state. */
1882  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1883  uint32_t stream_depth = FileReassemblyDepth();
1884  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1885  hstate->cfg->request.body_limit,
1886  stream_depth,
1887  tx_ud->tsflags,
1888  (uint32_t)d->len);
1889  BUG_ON(len > (uint32_t)d->len);
1890 
1891  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1892 
1893  const uint8_t *chunks_buffer = NULL;
1894  uint32_t chunks_buffer_len = 0;
1895 
1897  /* multi-part body handling starts here */
1898  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1899  goto end;
1900  }
1901 
1902  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1903  if (chunks_buffer == NULL) {
1904  goto end;
1905  }
1906 #ifdef PRINT
1907  printf("REASSCHUNK START: \n");
1908  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1909  printf("REASSCHUNK END: \n");
1910 #endif
1911 
1912  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1913 
1914  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1916  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1917  }
1918 
1919  } else {
1920  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1921  SCLogDebug("closing file that was being stored");
1922  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1923  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1924  }
1925  }
1926 
1927 end:
1928  if (hstate->conn != NULL) {
1929  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1931  hstate->cfg->request.inspect_min_size,
1932  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1933 
1934  /* if we reach the inspect_min_size we'll trigger inspection,
1935  * so make sure that raw stream is also inspected. Set the
1936  * data to be used to the amount of raw bytes we've seen to
1937  * get here. */
1938  if (tx_ud->request_body.body_inspected == 0 &&
1940  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1941  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1942  {
1943  const uint32_t data_size = (uint32_t)(
1944  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1945  const uint32_t depth = MIN(data_size, hstate->cfg->request.inspect_min_size);
1946 
1947  /* body still in progress, but due to min inspect size we need to inspect now */
1948  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, depth);
1949  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1950  }
1951  /* after the start of the body, disable the depth logic */
1952  } else if (tx_ud->request_body.body_inspected > 0) {
1953  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1954  }
1955  }
1956  SCReturnInt(HTP_OK);
1957 }
1958 
1959 /**
1960  * \brief Function callback to append chunks for Responses
1961  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1962  * \retval int HTP_OK if all goes well
1963  */
1964 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1965 {
1966  SCEnter();
1967 
1968  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1969  SCReturnInt(HTP_OK);
1970 
1971  if (d->len == 0)
1972  SCReturnInt(HTP_OK);
1973 
1974  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1975  if (hstate == NULL) {
1976  SCReturnInt(HTP_ERROR);
1977  }
1978 
1979  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1980  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1981 
1982  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1983  if (tx_ud == NULL) {
1984  SCReturnInt(HTP_OK);
1985  }
1986  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
1987  if (!tx_ud->request_body_init) {
1988  tx_ud->request_body_init = 1;
1989  }
1990 
1991  /* see if we can get rid of htp body chunks */
1992  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1993 
1994  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1995  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1996 
1997  /* within limits, add the body chunk to the state. */
1998  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1999  uint32_t stream_depth = FileReassemblyDepth();
2000  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
2001  hstate->cfg->response.body_limit,
2002  stream_depth,
2003  tx_ud->tcflags,
2004  (uint32_t)d->len);
2005  BUG_ON(len > (uint32_t)d->len);
2006 
2007  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
2008 
2009  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
2010  } else {
2011  if (tx_ud->tcflags & HTP_FILENAME_SET) {
2012  SCLogDebug("closing file that was being stored");
2013  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
2014  tx_ud->tcflags &= ~HTP_FILENAME_SET;
2015  }
2016  }
2017 
2018  if (hstate->conn != NULL) {
2019  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
2021  hstate->cfg->response.inspect_min_size,
2022  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
2023  /* if we reach the inspect_min_size we'll trigger inspection,
2024  * so make sure that raw stream is also inspected. Set the
2025  * data to be used to the amount of raw bytes we've seen to
2026  * get here. */
2027  if (tx_ud->response_body.body_inspected == 0 &&
2029  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2030  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2031  {
2032  const uint32_t data_size = (uint32_t)((uint64_t)hstate->conn->out_data_counter -
2033  hstate->last_response_data_stamp);
2034  const uint32_t depth = MIN(data_size, hstate->cfg->response.inspect_min_size);
2035 
2036  /* body still in progress, but due to min inspect size we need to inspect now */
2037  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, depth);
2038  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2039  }
2040  /* after the start of the body, disable the depth logic */
2041  } else if (tx_ud->response_body.body_inspected > 0) {
2042  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
2043  }
2044  }
2045  SCReturnInt(HTP_OK);
2046 }
2047 
2048 /**
2049  * \brief Print the stats of the HTTP requests
2050  */
2052 {
2053 #ifdef DEBUG
2054  SCEnter();
2055  SCMutexLock(&htp_state_mem_lock);
2056  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2057  htp_state_memcnt, htp_state_memuse);
2058  SCMutexUnlock(&htp_state_mem_lock);
2059  SCReturn;
2060 #endif
2061 }
2062 
2063 /** \brief Clears the HTTP server configuration memory used by HTP library */
2064 void HTPFreeConfig(void)
2065 {
2066  SCEnter();
2067 
2068  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2069  !AppLayerParserConfParserEnabled("tcp", "http"))
2070  {
2071  SCReturn;
2072  }
2073 
2074  HTPCfgRec *nextrec = cfglist.next;
2075  SCRadixReleaseRadixTree(cfgtree);
2076  cfgtree = NULL;
2077  htp_config_destroy(cfglist.cfg);
2078  while (nextrec != NULL) {
2079  HTPCfgRec *htprec = nextrec;
2080  nextrec = nextrec->next;
2081 
2082  htp_config_destroy(htprec->cfg);
2083  SCFree(htprec);
2084  }
2085  SCReturn;
2086 }
2087 
2088 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2089 {
2090  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2091  if (htud != NULL) {
2092  htud->request_has_trailers = 1;
2093  }
2094  return HTP_OK;
2095 }
2096 
2097 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2098 {
2099  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2100  if (htud != NULL) {
2101  htud->response_has_trailers = 1;
2102  }
2103  return HTP_OK;
2104 }
2105 
2106 /**\internal
2107  * \brief called at start of request
2108  * Set min inspect size.
2109  */
2110 static int HTPCallbackRequestStart(htp_tx_t *tx)
2111 {
2112  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2113  if (hstate == NULL) {
2114  SCReturnInt(HTP_ERROR);
2115  }
2116 
2117  uint64_t consumed = hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2118  SCLogDebug("HTTP request start: data offset %" PRIu64 ", in_data_counter %" PRIu64, consumed,
2119  (uint64_t)hstate->conn->in_data_counter);
2120 
2121  /* app-layer-frame-documentation tag start: frame registration http request */
2123  hstate->f, hstate->slice, consumed, -1, 0, HTTP_FRAME_REQUEST);
2124  if (frame) {
2125  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2126  hstate->request_frame_id = frame->id;
2127  AppLayerFrameSetTxId(frame, HtpGetActiveRequestTxID(hstate));
2128  }
2129  /* app-layer-frame-documentation tag end: frame registration http request */
2130 
2131  if (hstate->cfg)
2132  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2133  hstate->cfg->request.inspect_min_size);
2134 
2135  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2136  if (tx_ud == NULL) {
2137  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2138  if (unlikely(tx_ud == NULL)) {
2139  SCReturnInt(HTP_OK);
2140  }
2141  tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files
2142  htp_tx_set_user_data(tx, tx_ud);
2143  }
2144  SCReturnInt(HTP_OK);
2145 }
2146 
2147 /**\internal
2148  * \brief called at start of response
2149  * Set min inspect size.
2150  */
2151 static int HTPCallbackResponseStart(htp_tx_t *tx)
2152 {
2153  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2154  if (hstate == NULL) {
2155  SCReturnInt(HTP_ERROR);
2156  }
2157 
2158  uint64_t consumed = hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2159  SCLogDebug("HTTP response start: data offset %" PRIu64 ", out_data_counter %" PRIu64, consumed,
2160  (uint64_t)hstate->conn->out_data_counter);
2161 
2163  hstate->f, hstate->slice, consumed, -1, 1, HTTP_FRAME_RESPONSE);
2164  if (frame) {
2165  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2166  hstate->response_frame_id = frame->id;
2167  AppLayerFrameSetTxId(frame, HtpGetActiveResponseTxID(hstate));
2168  }
2169 
2170  if (hstate->cfg)
2171  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
2172  hstate->cfg->response.inspect_min_size);
2173 
2174  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2175  if (tx_ud == NULL) {
2176  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2177  if (unlikely(tx_ud == NULL)) {
2178  SCReturnInt(HTP_OK);
2179  }
2180  tx_ud->tx_data.file_tx =
2181  STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed.
2182  htp_tx_set_user_data(tx, tx_ud);
2183  }
2184  SCReturnInt(HTP_OK);
2185 }
2186 
2187 /**
2188  * \brief callback for request to store the recent incoming request
2189  into the recent_in_tx for the given htp state
2190  * \param connp pointer to the current connection parser which has the htp
2191  * state in it as user data
2192  */
2193 static int HTPCallbackRequestComplete(htp_tx_t *tx)
2194 {
2195  SCEnter();
2196 
2197  if (tx == NULL) {
2198  SCReturnInt(HTP_ERROR);
2199  }
2200 
2201  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2202  if (hstate == NULL) {
2203  SCReturnInt(HTP_ERROR);
2204  }
2205 
2206  const uint64_t abs_right_edge =
2207  hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2208 
2209  /* app-layer-frame-documentation tag start: updating frame->len */
2210  if (hstate->request_frame_id > 0) {
2211  Frame *frame = AppLayerFrameGetById(hstate->f, 0, hstate->request_frame_id);
2212  if (frame) {
2213  const uint64_t request_size = abs_right_edge - hstate->last_request_data_stamp;
2214 
2215  SCLogDebug("HTTP request complete: data offset %" PRIu64 ", request_size %" PRIu64,
2216  hstate->last_request_data_stamp, request_size);
2217  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2218  request_size);
2219  frame->len = (int64_t)request_size;
2220  /* app-layer-frame-documentation tag end: updating frame->len */
2221  }
2222  hstate->request_frame_id = 0;
2223  }
2224 
2225  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2226  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2227 
2228  SCLogDebug("HTTP request completed");
2229 
2230  HTPErrorCheckTxRequestFlags(hstate, tx);
2231 
2232  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2233  if (htud != NULL) {
2234  if (htud->tsflags & HTP_FILENAME_SET) {
2235  SCLogDebug("closing file that was being stored");
2236  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
2237  htud->tsflags &= ~HTP_FILENAME_SET;
2238  if (abs_right_edge < (uint64_t)UINT32_MAX) {
2240  hstate->f->protoctx, STREAM_TOSERVER, (uint32_t)abs_right_edge);
2241  }
2242  }
2243  }
2244 
2245  hstate->last_request_data_stamp = abs_right_edge;
2246  /* request done, do raw reassembly now to inspect state and stream
2247  * at the same time. */
2248  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
2249  SCReturnInt(HTP_OK);
2250 }
2251 
2252 /**
2253  * \brief callback for response to remove the recent received requests
2254  from the recent_in_tx for the given htp state
2255  * \param connp pointer to the current connection parser which has the htp
2256  * state in it as user data
2257  */
2258 static int HTPCallbackResponseComplete(htp_tx_t *tx)
2259 {
2260  SCEnter();
2261 
2262  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2263  if (hstate == NULL) {
2264  SCReturnInt(HTP_ERROR);
2265  }
2266 
2267  /* we have one whole transaction now */
2268  hstate->transaction_cnt++;
2269 
2270  const uint64_t abs_right_edge =
2271  hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2272 
2273  if (hstate->response_frame_id > 0) {
2274  Frame *frame = AppLayerFrameGetById(hstate->f, 1, hstate->response_frame_id);
2275  if (frame) {
2276  const uint64_t response_size = abs_right_edge - hstate->last_response_data_stamp;
2277 
2278  SCLogDebug("HTTP response complete: data offset %" PRIu64 ", response_size %" PRIu64,
2279  hstate->last_response_data_stamp, response_size);
2280  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2281  response_size);
2282  frame->len = (int64_t)response_size;
2283  }
2284  hstate->response_frame_id = 0;
2285  }
2286 
2287  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2288  if (htud != NULL) {
2289  if (htud->tcflags & HTP_FILENAME_SET) {
2290  SCLogDebug("closing file that was being stored");
2291  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
2292  htud->tcflags &= ~HTP_FILENAME_SET;
2293  }
2294  }
2295 
2296  /* response done, do raw reassembly now to inspect state and stream
2297  * at the same time. */
2298  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2299 
2300  /* handle HTTP CONNECT */
2301  if (tx->request_method_number == HTP_M_CONNECT) {
2302  /* any 2XX status response implies that the connection will become
2303  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2304  if ((tx->response_status_number >= 200) &&
2305  (tx->response_status_number < 300) &&
2306  (hstate->transaction_cnt == 1)) {
2307  uint16_t dp = 0;
2308  if (tx->request_port_number != -1) {
2309  dp = (uint16_t)tx->request_port_number;
2310  }
2311  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2312  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN)) {
2313  HTPSetEvent(
2314  hstate, htud, STREAM_TOCLIENT, HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE);
2315  }
2316  tx->request_progress = HTP_REQUEST_COMPLETE;
2317  tx->response_progress = HTP_RESPONSE_COMPLETE;
2318  }
2319  }
2320 
2321  hstate->last_response_data_stamp = abs_right_edge;
2322  SCReturnInt(HTP_OK);
2323 }
2324 
2325 static int HTPCallbackRequestLine(htp_tx_t *tx)
2326 {
2327  HtpTxUserData *tx_ud;
2328  bstr *request_uri_normalized;
2329  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2330  const HTPCfgRec *cfg = hstate->cfg;
2331 
2332  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2333  if (request_uri_normalized == NULL)
2334  return HTP_OK;
2335 
2336  tx_ud = htp_tx_get_user_data(tx);
2337  if (unlikely(tx_ud == NULL)) {
2338  bstr_free(request_uri_normalized);
2339  return HTP_OK;
2340  }
2341  if (unlikely(tx_ud->request_uri_normalized != NULL))
2342  bstr_free(tx_ud->request_uri_normalized);
2343  tx_ud->request_uri_normalized = request_uri_normalized;
2344 
2345  if (tx->flags) {
2346  HTPErrorCheckTxRequestFlags(hstate, tx);
2347  }
2348  return HTP_OK;
2349 }
2350 
2351 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2352 {
2353  if (part == NULL)
2354  return HTP_OK;
2355 
2356  uint64_t flags = 0;
2357  size_t prevlen = bstr_len(part);
2358  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2359  // shorter string means that uri was encoded
2360  if (res == HTP_OK && prevlen > bstr_len(part)) {
2361  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2362  if (htud == NULL)
2363  return HTP_OK;
2364  HtpState *s = htp_connp_get_user_data(tx->connp);
2365  if (s == NULL)
2366  return HTP_OK;
2367  HTPSetEvent(s, htud, STREAM_TOSERVER,
2369  }
2370 
2371  return HTP_OK;
2372 }
2373 
2374 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2375 {
2376  if (tx->parsed_uri == NULL)
2377  return HTP_OK;
2378 
2379  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2380 }
2381 
2382 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2383 {
2384  if (tx->parsed_uri == NULL)
2385  return HTP_OK;
2386 
2387  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2388 }
2389 
2390 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2391 {
2392  void *ptmp;
2393  if (tx_data->len == 0 || tx_data->tx == NULL)
2394  return HTP_OK;
2395 
2396  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2397  if (tx_ud == NULL) {
2398  return HTP_OK;
2399  }
2400  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2401  tx_ud->request_headers_raw_len,
2402  tx_ud->request_headers_raw_len + tx_data->len);
2403  if (ptmp == NULL) {
2404  return HTP_OK;
2405  }
2406  tx_ud->request_headers_raw = ptmp;
2407 
2408  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2409  tx_data->data, tx_data->len);
2410  tx_ud->request_headers_raw_len += tx_data->len;
2411 
2412  if (tx_data->tx && tx_data->tx->flags) {
2413  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2414  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2415  }
2416  return HTP_OK;
2417 }
2418 
2419 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2420 {
2421  void *ptmp;
2422  if (tx_data->len == 0 || tx_data->tx == NULL)
2423  return HTP_OK;
2424 
2425  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2426  if (tx_ud == NULL) {
2427  return HTP_OK;
2428  }
2429  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2430  tx_ud->response_headers_raw_len,
2431  tx_ud->response_headers_raw_len + tx_data->len);
2432  if (ptmp == NULL) {
2433  return HTP_OK;
2434  }
2435  tx_ud->response_headers_raw = ptmp;
2436 
2437  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2438  tx_data->data, tx_data->len);
2439  tx_ud->response_headers_raw_len += tx_data->len;
2440 
2441  return HTP_OK;
2442 }
2443 
2444 /*
2445  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2446  */
2447 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2448 {
2449  cfg_prec->uri_include_all = FALSE;
2456 
2457  if (!g_disable_randomness) {
2459  } else {
2460  cfg_prec->randomize = 0;
2461  }
2463 
2464  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2465  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2466  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2467  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2468 
2469  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2470  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2471 
2472  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2473  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2474 
2475  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2476  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequestComplete);
2477 
2478  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2479  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponseComplete);
2480 
2481  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2482 
2483  /* don't convert + to space by default */
2484  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2485  // enables request decompression
2486  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2487 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2488  // disable by default
2489  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2490 #endif
2491 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2492  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2494 #endif
2495 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2496  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2498 #endif
2499 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2500  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2501 #endif
2502  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2503  * only the hard limit. So we set both here to the (current) htp defaults.
2504  * The reason we do this is that if the user sets the hard limit in the
2505  * config, we have to set the soft limit as well. If libhtp starts using
2506  * the soft limit in the future, we at least make sure we control what
2507  * it's value is. */
2508  htp_config_set_field_limits(cfg_prec->cfg,
2511  return;
2512 }
2513 
2514 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2515  * but we can get both <0 and >RAND_MAX values from RandomGet
2516  */
2517 static int RandomGetWrap(void)
2518 {
2519  unsigned long r;
2520 
2521  do {
2522  r = RandomGet();
2523  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2524 
2525  return r % RAND_MAX;
2526 }
2527 
2528 /*
2529  * We have this splitup so that in case double decoding has been enabled
2530  * for query and path, they would be called first on the callback queue,
2531  * before the callback set by Phase2() is called. We need this, since
2532  * the callback in Phase2() generates the normalized uri which utilizes
2533  * the query and path. */
2534 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2535 {
2536  /* randomize inspection size if needed */
2537  if (cfg_prec->randomize) {
2538  int rdrange = cfg_prec->randomize_range;
2539 
2540  long int r = RandomGetWrap();
2541  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2542  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2543 
2544  r = RandomGetWrap();
2545  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2546  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2547  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2548  " %u and 'request-body-inspect-window' set to %u after"
2549  " randomization.",
2550  name, cfg_prec->request.inspect_min_size, cfg_prec->request.inspect_window);
2551 
2552  r = RandomGetWrap();
2553  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2554  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2555 
2556  r = RandomGetWrap();
2557  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2558  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2559 
2560  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2561  " %u and 'response-body-inspect-window' set to %u after"
2562  " randomization.",
2563  name, cfg_prec->response.inspect_min_size, cfg_prec->response.inspect_window);
2564  }
2565 
2566  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2567  return;
2568 }
2569 
2570 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2571  SCRadixTree *tree)
2572 {
2573  if (cfg_prec == NULL || s == NULL || tree == NULL)
2574  return;
2575 
2576  ConfNode *p = NULL;
2577 
2578  /* Default Parameters */
2579  TAILQ_FOREACH(p, &s->head, next) {
2580 
2581  if (strcasecmp("address", p->name) == 0) {
2582  ConfNode *pval;
2583  /* Addresses */
2584  TAILQ_FOREACH(pval, &p->head, next) {
2585  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2586  pval->val);
2587 
2588  /* IPV6 or IPV4? */
2589  if (strchr(pval->val, ':') != NULL) {
2590  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2591  s->name, pval->val, cfg_prec->cfg);
2592  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2593  SCLogWarning("LIBHTP failed to "
2594  "add ipv6 server %s, ignoring",
2595  pval->val);
2596  }
2597  } else {
2598  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2599  s->name, pval->val, cfg_prec->cfg);
2600  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2601  SCLogWarning("LIBHTP failed "
2602  "to add ipv4 server %s, ignoring",
2603  pval->val);
2604  }
2605  } /* else - if (strchr(pval->val, ':') != NULL) */
2606  } /* TAILQ_FOREACH(pval, &p->head, next) */
2607 
2608  } else if (strcasecmp("personality", p->name) == 0) {
2609  /* Personalities */
2610  int personality = HTPLookupPersonality(p->val);
2611  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2612  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2613 
2614  if (personality >= 0) {
2615  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2616  personality);
2617  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2618  SCLogWarning("LIBHTP Failed adding "
2619  "personality \"%s\", ignoring",
2620  p->val);
2621  } else {
2622  SCLogDebug("LIBHTP personality set to %s",
2623  HTPLookupPersonalityString(personality));
2624  }
2625 
2626  /* The IDS personality by default converts the path (and due to
2627  * our query string callback also the query string) to lowercase.
2628  * Signatures do not expect this, so override it. */
2629  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2630  } else {
2631  SCLogWarning("LIBHTP Unknown personality "
2632  "\"%s\", ignoring",
2633  p->val);
2634  continue;
2635  }
2636 
2637  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2638  strcasecmp("request_body_limit", p->name) == 0) {
2639  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2640  SCLogError("Error parsing request-body-limit "
2641  "from conf file - %s. Killing engine",
2642  p->val);
2643  exit(EXIT_FAILURE);
2644  }
2645 
2646  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2647  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2648  SCLogError("Error parsing response-body-limit "
2649  "from conf file - %s. Killing engine",
2650  p->val);
2651  exit(EXIT_FAILURE);
2652  }
2653 
2654  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2655  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2656  SCLogError("Error parsing request-body-minimal-inspect-size "
2657  "from conf file - %s. Killing engine",
2658  p->val);
2659  exit(EXIT_FAILURE);
2660  }
2661 
2662  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2663  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2664  SCLogError("Error parsing request-body-inspect-window "
2665  "from conf file - %s. Killing engine",
2666  p->val);
2667  exit(EXIT_FAILURE);
2668  }
2669 
2670  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2671  if (ConfValIsTrue(p->val)) {
2672  htp_config_register_request_line(cfg_prec->cfg,
2673  HTPCallbackDoubleDecodeQuery);
2674  }
2675 
2676  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2677  if (ConfValIsTrue(p->val)) {
2678  htp_config_register_request_line(cfg_prec->cfg,
2679  HTPCallbackDoubleDecodePath);
2680  }
2681 
2682  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2683  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2684  SCLogError("Error parsing response-body-minimal-inspect-size "
2685  "from conf file - %s. Killing engine",
2686  p->val);
2687  exit(EXIT_FAILURE);
2688  }
2689 
2690  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2691  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2692  SCLogError("Error parsing response-body-inspect-window "
2693  "from conf file - %s. Killing engine",
2694  p->val);
2695  exit(EXIT_FAILURE);
2696  }
2697 
2698  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2699  uint32_t value = 2;
2700  if (ParseSizeStringU32(p->val, &value) < 0) {
2701  SCLogError("Error parsing response-body-inspect-window "
2702  "from conf file - %s. Killing engine",
2703  p->val);
2704  exit(EXIT_FAILURE);
2705  }
2706 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2707  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2708 #else
2709  SCLogWarning("can't set response-body-decompress-layer-limit "
2710  "to %u, libhtp version too old",
2711  value);
2712 #endif
2713  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2714  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2715  HTP_DECODER_URL_PATH,
2716  ConfValIsTrue(p->val));
2717  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2718  if (strlen(p->val) == 1) {
2719  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2720  HTP_DECODER_URL_PATH,
2721  p->val[0]);
2722  } else {
2723  SCLogError("Invalid entry "
2724  "for libhtp param path-bestfit-replacement-char");
2725  }
2726  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2727  htp_config_set_convert_lowercase(cfg_prec->cfg,
2728  HTP_DECODER_URL_PATH,
2729  ConfValIsTrue(p->val));
2730  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2731  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2732  HTP_DECODER_URL_PATH,
2733  ConfValIsTrue(p->val));
2734  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2735  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2736  HTP_DECODER_URL_PATH,
2737  ConfValIsTrue(p->val));
2738  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2739  htp_config_set_path_separators_compress(cfg_prec->cfg,
2740  HTP_DECODER_URL_PATH,
2741  ConfValIsTrue(p->val));
2742  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2743  htp_config_set_path_separators_decode(cfg_prec->cfg,
2744  HTP_DECODER_URL_PATH,
2745  ConfValIsTrue(p->val));
2746  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2747  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2748  HTP_DECODER_URL_PATH,
2749  ConfValIsTrue(p->val));
2750  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2751  enum htp_url_encoding_handling_t handling;
2752  if (strcasecmp(p->val, "preserve_percent") == 0) {
2753  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2754  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2755  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2756  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2757  handling = HTP_URL_DECODE_PROCESS_INVALID;
2758  } else {
2759  SCLogError("Invalid entry "
2760  "for libhtp param path-url-encoding-invalid-handling");
2761  return;
2762  }
2763  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2764  HTP_DECODER_URL_PATH,
2765  handling);
2766  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2767  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2768  HTP_DECODER_URL_PATH,
2769  ConfValIsTrue(p->val));
2770  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2771  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2772  SCLogDebug("uri-include-all %s",
2773  cfg_prec->uri_include_all ? "enabled" : "disabled");
2774  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2775  htp_config_set_plusspace_decode(cfg_prec->cfg,
2776  HTP_DECODER_URLENCODED,
2777  ConfValIsTrue(p->val));
2778  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2779  uint32_t limit = 0;
2780  if (ParseSizeStringU32(p->val, &limit) < 0) {
2781  SCLogError("Error meta-field-limit "
2782  "from conf file - %s. Killing engine",
2783  p->val);
2784  exit(EXIT_FAILURE);
2785  }
2786  if (limit == 0) {
2787  FatalError("Error meta-field-limit "
2788  "from conf file cannot be 0. Killing engine");
2789  }
2790  /* set default soft-limit with our new hard limit */
2791  htp_config_set_field_limits(cfg_prec->cfg,
2793  (size_t)limit);
2794 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2795  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2796  uint32_t limit = 0;
2797  if (ParseSizeStringU32(p->val, &limit) < 0) {
2798  FatalError("failed to parse 'lzma-memlimit' "
2799  "from conf file - %s.",
2800  p->val);
2801  }
2802  if (limit == 0) {
2803  FatalError("'lzma-memlimit' "
2804  "from conf file cannot be 0.");
2805  }
2806  /* set default soft-limit with our new hard limit */
2807  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2808  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2809 #endif
2810 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2811  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2812  if (ConfValIsTrue(p->val)) {
2813  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2814  } else if (!ConfValIsFalse(p->val)) {
2815  int8_t limit;
2816  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2817  FatalError("failed to parse 'lzma-enabled' "
2818  "from conf file - %s.",
2819  p->val);
2820  }
2821  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2822  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2823  }
2824 #endif
2825 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2826  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2827  uint32_t limit = 0;
2828  if (ParseSizeStringU32(p->val, &limit) < 0) {
2829  FatalError("failed to parse 'compression-bomb-limit' "
2830  "from conf file - %s.",
2831  p->val);
2832  }
2833  if (limit == 0) {
2834  FatalError("'compression-bomb-limit' "
2835  "from conf file cannot be 0.");
2836  }
2837  /* set default soft-limit with our new hard limit */
2838  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2839  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2840 #endif
2841 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2842  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2843  uint32_t limit = 0;
2844  // between 1 usec and 1 second
2845  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2846  FatalError("failed to parse 'decompression-time-limit' "
2847  "from conf file - %s.",
2848  p->val);
2849  }
2850  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2851  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2852 #endif
2853  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2854  if (!g_disable_randomness) {
2855  cfg_prec->randomize = ConfValIsTrue(p->val);
2856  }
2857  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2858  uint32_t range;
2859  if (StringParseU32RangeCheck(&range, 10, 0,
2860  (const char *)p->val, 0, 100) < 0) {
2861  SCLogError("Invalid value for randomize"
2862  "-inspection-range setting from conf file - \"%s\"."
2863  " It should be a valid integer less than or equal to 100."
2864  " Killing engine",
2865  p->val);
2866  exit(EXIT_FAILURE);
2867  }
2868  cfg_prec->randomize_range = range;
2869  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2870  if (ConfValIsTrue(p->val)) {
2871  cfg_prec->http_body_inline = 1;
2872  } else if (ConfValIsFalse(p->val)) {
2873  cfg_prec->http_body_inline = 0;
2874  } else {
2875  if (strcmp("auto", p->val) != 0) {
2876  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2877  }
2878  if (EngineModeIsIPS()) {
2879  cfg_prec->http_body_inline = 1;
2880  } else {
2881  cfg_prec->http_body_inline = 0;
2882  }
2883  }
2884  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2885  ConfNode *pval;
2886 
2887  TAILQ_FOREACH(pval, &p->head, next) {
2888  if (strcasecmp("enabled", pval->name) == 0) {
2889  if (ConfValIsTrue(pval->val)) {
2890  cfg_prec->swf_decompression_enabled = 1;
2891  } else if (ConfValIsFalse(pval->val)) {
2892  cfg_prec->swf_decompression_enabled = 0;
2893  } else {
2894  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2895  }
2896  } else if (strcasecmp("type", pval->name) == 0) {
2897  if (strcasecmp("no", pval->val) == 0) {
2899  } else if (strcasecmp("deflate", pval->val) == 0) {
2901  } else if (strcasecmp("lzma", pval->val) == 0) {
2903  } else if (strcasecmp("both", pval->val) == 0) {
2905  } else {
2906  SCLogError("Invalid entry for "
2907  "swf-decompression.type: %s - "
2908  "Killing engine",
2909  pval->val);
2910  exit(EXIT_FAILURE);
2911  }
2912  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2913  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2914  SCLogError("Error parsing swf-decompression.compression-depth "
2915  "from conf file - %s. Killing engine",
2916  p->val);
2917  exit(EXIT_FAILURE);
2918  }
2919  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2920  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2921  SCLogError("Error parsing swf-decompression.decompression-depth "
2922  "from conf file - %s. Killing engine",
2923  p->val);
2924  exit(EXIT_FAILURE);
2925  }
2926  } else {
2927  SCLogWarning("Ignoring unknown param %s", pval->name);
2928  }
2929  }
2930  } else {
2931  SCLogWarning("LIBHTP Ignoring unknown "
2932  "default config: %s",
2933  p->name);
2934  }
2935  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2936 
2937  return;
2938 }
2939 
2940 void HTPConfigure(void)
2941 {
2942  SCEnter();
2943 
2944  cfglist.next = NULL;
2945 
2949 
2950  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2951  if (NULL == cfgtree)
2952  exit(EXIT_FAILURE);
2953 
2954  /* Default Config */
2955  cfglist.cfg = htp_config_create();
2956  if (NULL == cfglist.cfg) {
2957  FatalError("Failed to create HTP default config");
2958  }
2959  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2960  HTPConfigSetDefaultsPhase1(&cfglist);
2961  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2962  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2963  cfgtree);
2964  } else {
2965  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2966  }
2967  HTPConfigSetDefaultsPhase2("default", &cfglist);
2968 
2969  HTPParseMemcap();
2970 
2971  /* Read server config and create a parser for each IP in radix tree */
2972  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2973  if (server_config == NULL) {
2974  server_config = ConfGetNode("libhtp.server-config");
2975  if (server_config == NULL) {
2976  SCLogDebug("LIBHTP Configuring %p", server_config);
2977  SCReturn;
2978  }
2979  }
2980  SCLogDebug("LIBHTP Configuring %p", server_config);
2981 
2982  ConfNode *si;
2983  /* Server Nodes */
2984  TAILQ_FOREACH(si, &server_config->head, next) {
2985  /* Need the named node, not the index */
2986  ConfNode *s = TAILQ_FIRST(&si->head);
2987  if (NULL == s) {
2988  SCLogDebug("LIBHTP s NULL");
2989  continue;
2990  }
2991 
2992  SCLogDebug("LIBHTP server %s", s->name);
2993 
2994  HTPCfgRec *nextrec = cfglist.next;
2995  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2996  if (NULL == htprec)
2997  exit(EXIT_FAILURE);
2998  memset(htprec, 0x00, sizeof(*htprec));
2999 
3000  cfglist.next = htprec;
3001 
3002  cfglist.next->next = nextrec;
3003  cfglist.next->cfg = htp_config_create();
3004  if (NULL == cfglist.next->cfg) {
3005  FatalError("Failed to create HTP server config");
3006  }
3007 
3008  HTPConfigSetDefaultsPhase1(htprec);
3009  HTPConfigParseParameters(htprec, s, cfgtree);
3010  HTPConfigSetDefaultsPhase2(s->name, htprec);
3011  }
3012 
3013  SCReturn;
3014 }
3015 
3017 {
3018 #ifdef DEBUG
3019  SCMutexLock(&htp_state_mem_lock);
3020  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
3021  SCMutexUnlock(&htp_state_mem_lock);
3022 #endif
3023 }
3024 
3025 /** \internal
3026  * \brief get files callback
3027  * \param state state ptr
3028  * \param direction flow direction
3029  * \retval files files ptr
3030  */
3031 static AppLayerGetFileState HTPGetTxFiles(void *state, void *txv, uint8_t direction)
3032 {
3033  AppLayerGetFileState files = { .fc = NULL, .cfg = &htp_sbcfg };
3034  htp_tx_t *tx = (htp_tx_t *)txv;
3035  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3036  if (tx_ud) {
3037  if (direction & STREAM_TOCLIENT) {
3038  files.fc = &tx_ud->files_tc;
3039  } else {
3040  files.fc = &tx_ud->files_ts;
3041  }
3042  }
3043  return files;
3044 }
3045 
3046 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
3047 {
3048  if (direction & STREAM_TOSERVER)
3049  return ((htp_tx_t *)tx)->request_progress;
3050  else
3051  return ((htp_tx_t *)tx)->response_progress;
3052 }
3053 
3054 static uint64_t HTPStateGetTxCnt(void *alstate)
3055 {
3056  HtpState *http_state = (HtpState *)alstate;
3057 
3058  if (http_state != NULL && http_state->conn != NULL) {
3059  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
3060  if (size < 0)
3061  return 0ULL;
3062  SCLogDebug("size %"PRIu64, size);
3063  return (uint64_t)size;
3064  } else {
3065  return 0ULL;
3066  }
3067 }
3068 
3069 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
3070 {
3071  HtpState *http_state = (HtpState *)alstate;
3072 
3073  if (http_state != NULL && http_state->conn != NULL)
3074  return htp_list_get(http_state->conn->transactions, tx_id);
3075  else
3076  return NULL;
3077 }
3078 
3079 void *HtpGetTxForH2(void *alstate)
3080 {
3081  // gets last transaction
3082  HtpState *http_state = (HtpState *)alstate;
3083  if (http_state != NULL && http_state->conn != NULL) {
3084  size_t txid = htp_list_array_size(http_state->conn->transactions);
3085  if (txid > 0) {
3086  return htp_list_get(http_state->conn->transactions, txid - 1);
3087  }
3088  }
3089  return NULL;
3090 }
3091 
3092 static int HTPStateGetEventInfo(const char *event_name,
3093  int *event_id, AppLayerEventType *event_type)
3094 {
3095  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3096  if (*event_id == -1) {
3097  SCLogError("event \"%s\" not present in "
3098  "http's enum map table.",
3099  event_name);
3100  /* this should be treated as fatal */
3101  return -1;
3102  }
3103 
3104  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3105 
3106  return 0;
3107 }
3108 
3109 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3110  AppLayerEventType *event_type)
3111 {
3112  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3113  if (*event_name == NULL) {
3114  SCLogError("event \"%d\" not present in "
3115  "http's enum map table.",
3116  event_id);
3117  /* this should be treated as fatal */
3118  return -1;
3119  }
3120 
3121  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3122 
3123  return 0;
3124 }
3125 
3126 static AppLayerTxData *HTPGetTxData(void *vtx)
3127 {
3128  htp_tx_t *tx = (htp_tx_t *)vtx;
3129  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3130  if (tx_ud) {
3131  return &tx_ud->tx_data;
3132  }
3133  return NULL;
3134 }
3135 
3136 static AppLayerStateData *HTPGetStateData(void *vstate)
3137 {
3138  HtpState *s = vstate;
3139  return &s->state_data;
3140 }
3141 
3142 static int HTPRegisterPatternsForProtocolDetection(void)
3143 {
3144  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3145  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3146  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3147  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3148  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3149  const char *spacings[] = { "|20|", "|09|", NULL };
3150  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3151 
3152  int methods_pos;
3153  int spacings_pos;
3154  int versions_pos;
3155  int register_result;
3156  char method_buffer[32] = "";
3157 
3158  /* Loop through all the methods ands spacings and register the patterns */
3159  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3160  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3161 
3162  /* Combine the method name and the spacing */
3163  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3164 
3165  /* Register the new method+spacing pattern
3166  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3167  * but the pattern matching should only be one char
3168  */
3169  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3170  method_buffer, (uint16_t)strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3171  if (register_result < 0) {
3172  return -1;
3173  }
3174  }
3175  }
3176 
3177  /* Loop through all the http version patterns that are TO_CLIENT */
3178  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3179  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3180  versions[versions_pos], (uint16_t)strlen(versions[versions_pos]), 0,
3181  STREAM_TOCLIENT);
3182  if (register_result < 0) {
3183  return -1;
3184  }
3185  }
3186 
3187  return 0;
3188 }
3189 
3190 /**
3191  * \brief Register the HTTP protocol and state handling functions to APP layer
3192  * of the engine.
3193  */
3195 {
3196  SCEnter();
3197 
3198  const char *proto_name = "http";
3199 
3200  /** HTTP */
3201  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3203  if (HTPRegisterPatternsForProtocolDetection() < 0)
3204  return;
3205  } else {
3206  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3207  proto_name);
3208  return;
3209  }
3210 
3211  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3212  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3213  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3214  AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles);
3216  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3217  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3218  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3219 
3221  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3222  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3224  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3225 
3226  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3227  AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData);
3228 
3230  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3231 
3233  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3235  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3236  SC_ATOMIC_INIT(htp_config_flags);
3237  /* This parser accepts gaps. */
3241  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_TOCLIENT);
3242  /* app-layer-frame-documentation tag start: registering relevant callbacks */
3244  IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById);
3245  /* app-layer-frame-documentation tag end: registering relevant callbacks */
3246  HTPConfigure();
3247  } else {
3248  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3249  "still on.", proto_name);
3250  }
3251 #ifdef UNITTESTS
3252  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3253 #endif
3254 
3255  SCReturn;
3256 }
3257 
3258 #ifdef UNITTESTS
3259 #include "detect-engine-alert.h"
3260 
3261 static HTPCfgRec cfglist_backup;
3262 
3264 {
3265  cfglist_backup = cfglist;
3266 
3267  return;
3268 }
3269 
3271 {
3272  cfglist = cfglist_backup;
3273 
3274  return;
3275 }
3276 
3277 /** \test Test case where chunks are sent in smaller chunks and check the
3278  * response of the parser from HTP library. */
3279 static int HTPParserTest01(void)
3280 {
3281  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3282  " Data is c0oL!";
3283  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3284 
3285  TcpSession ssn;
3286  memset(&ssn, 0, sizeof(ssn));
3287 
3290 
3291  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3292  FAIL_IF_NULL(f);
3293  f->protoctx = &ssn;
3294  f->proto = IPPROTO_TCP;
3295  f->alproto = ALPROTO_HTTP1;
3296 
3297  StreamTcpInitConfig(true);
3298 
3299  uint32_t u;
3300  for (u = 0; u < httplen1; u++) {
3301  uint8_t flags = 0;
3302 
3303  if (u == 0)
3304  flags = STREAM_TOSERVER|STREAM_START;
3305  else if (u == (httplen1 - 1))
3306  flags = STREAM_TOSERVER|STREAM_EOF;
3307  else
3308  flags = STREAM_TOSERVER;
3309 
3310  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3311  FAIL_IF(r != 0);
3312  }
3313 
3314  HtpState *htp_state = f->alstate;
3315  FAIL_IF_NULL(htp_state);
3316 
3317  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3318  FAIL_IF_NULL(tx);
3319 
3320  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3321  FAIL_IF_NULL(h);
3322 
3323  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3324  FAIL_IF(tx->request_method_number != HTP_M_POST);
3325  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3326 
3328  StreamTcpFreeConfig(true);
3329  UTHFreeFlow(f);
3330  PASS;
3331 }
3332 
3333 /** \test Test folding in 1 read case */
3334 static int HTPParserTest01b(void)
3335 {
3336  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3337  " Data is c0oL!";
3338  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3339 
3340  TcpSession ssn;
3341  memset(&ssn, 0, sizeof(ssn));
3342 
3345 
3346  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3347  FAIL_IF_NULL(f);
3348  f->protoctx = &ssn;
3349  f->proto = IPPROTO_TCP;
3350  f->alproto = ALPROTO_HTTP1;
3351 
3352  StreamTcpInitConfig(true);
3353 
3354  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3355  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3356  FAIL_IF(r != 0);
3357 
3358  HtpState *htp_state = f->alstate;
3359  FAIL_IF_NULL(htp_state);
3360 
3361  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3362  FAIL_IF_NULL(tx);
3363 
3364  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3365  FAIL_IF_NULL(h);
3366 
3367  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3368  FAIL_IF(tx->request_method_number != HTP_M_POST);
3369  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3370 
3372  StreamTcpFreeConfig(true);
3373  UTHFreeFlow(f);
3374  PASS;
3375 }
3376 
3377 /** \test Test folding in 1byte per read case */
3378 static int HTPParserTest01c(void)
3379 {
3380  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3381  " Data is c0oL!";
3382  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3383 
3384  TcpSession ssn;
3385  memset(&ssn, 0, sizeof(ssn));
3386 
3389 
3390  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3391  FAIL_IF_NULL(f);
3392  f->protoctx = &ssn;
3393  f->proto = IPPROTO_TCP;
3394  f->alproto = ALPROTO_HTTP1;
3395 
3396  StreamTcpInitConfig(true);
3397 
3398  uint32_t u;
3399  for (u = 0; u < httplen1; u++) {
3400  uint8_t flags = 0;
3401 
3402  if (u == 0)
3403  flags = STREAM_TOSERVER|STREAM_START;
3404  else if (u == (httplen1 - 1))
3405  flags = STREAM_TOSERVER|STREAM_EOF;
3406  else
3407  flags = STREAM_TOSERVER;
3408 
3409  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3410  FAIL_IF(r != 0);
3411  }
3412 
3413  HtpState *htp_state = f->alstate;
3414  FAIL_IF_NULL(htp_state);
3415 
3416  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3417  FAIL_IF_NULL(tx);
3418 
3419  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3420  FAIL_IF_NULL(h);
3421 
3422  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3423  FAIL_IF(tx->request_method_number != HTP_M_POST);
3424  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3425 
3427  StreamTcpFreeConfig(true);
3428  UTHFreeFlow(f);
3429  PASS;
3430 }
3431 
3432 /** \test Test case where chunks are sent in smaller chunks and check the
3433  * response of the parser from HTP library. */
3434 static int HTPParserTest01a(void)
3435 {
3436  int result = 0;
3437  Flow *f = NULL;
3438  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3439  " Data is c0oL!";
3440  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3441  TcpSession ssn;
3442  HtpState *htp_state = NULL;
3443  int r = 0;
3445 
3446  memset(&ssn, 0, sizeof(ssn));
3447 
3448  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3449  if (f == NULL)
3450  goto end;
3451  f->protoctx = &ssn;
3452  f->proto = IPPROTO_TCP;
3453  f->alproto = ALPROTO_HTTP1;
3454 
3455  StreamTcpInitConfig(true);
3456 
3457  uint32_t u;
3458  for (u = 0; u < httplen1; u++) {
3459  uint8_t flags = 0;
3460 
3461  if (u == 0)
3462  flags = STREAM_TOSERVER|STREAM_START;
3463  else if (u == (httplen1 - 1))
3464  flags = STREAM_TOSERVER|STREAM_EOF;
3465  else
3466  flags = STREAM_TOSERVER;
3467 
3468  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3469  if (r != 0) {
3470  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3471  " 0: ", u, r);
3472  goto end;
3473  }
3474  }
3475 
3476  htp_state = f->alstate;
3477  if (htp_state == NULL) {
3478  printf("no http state: ");
3479  goto end;
3480  }
3481 
3482  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3483  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3484  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3485  || tx->request_method_number != HTP_M_POST ||
3486  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3487  {
3488  printf("expected header value: Victor/1.0 and got %s: and expected"
3489  " method: POST and got %s, expected protocol number HTTP/1.0"
3490  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3491  bstr_util_strdup_to_c(tx->request_method),
3492  bstr_util_strdup_to_c(tx->request_protocol));
3493  goto end;
3494  }
3495  result = 1;
3496 end:
3497  if (alp_tctx != NULL)
3499  StreamTcpFreeConfig(true);
3500  UTHFreeFlow(f);
3501  return result;
3502 }
3503 
3504 /** \test See how it deals with an incomplete request. */
3505 static int HTPParserTest02(void)
3506 {
3507  int result = 0;
3508  Flow *f = NULL;
3509  uint8_t httpbuf1[] = "POST";
3510  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3511  TcpSession ssn;
3512  HtpState *http_state = NULL;
3514 
3515  memset(&ssn, 0, sizeof(ssn));
3516 
3517  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3518  if (f == NULL)
3519  goto end;
3520  f->protoctx = &ssn;
3521  f->proto = IPPROTO_TCP;
3522  f->alproto = ALPROTO_HTTP1;
3523 
3524  StreamTcpInitConfig(true);
3525 
3526  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3527  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3528  if (r != 0) {
3529  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3530  goto end;
3531  }
3532 
3533  http_state = f->alstate;
3534  if (http_state == NULL) {
3535  printf("no http state: ");
3536  goto end;
3537  }
3538 
3539  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3540  FAIL_IF_NULL(tx);
3541  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3542  FAIL_IF_NOT_NULL(h);
3543 
3544  FAIL_IF_NULL(tx->request_method);
3545  char *method = bstr_util_strdup_to_c(tx->request_method);
3546  FAIL_IF_NULL(method);
3547 
3548  FAIL_IF(strcmp(method, "POST") != 0);
3549  SCFree(method);
3550 
3551  result = 1;
3552 end:
3553  if (alp_tctx != NULL)
3555  StreamTcpFreeConfig(true);
3556  UTHFreeFlow(f);
3557  return result;
3558 }
3559 
3560 /** \test Test case where method is invalid and data is sent in smaller chunks
3561  * and check the response of the parser from HTP library. */
3562 static int HTPParserTest03(void)
3563 {
3564  int result = 0;
3565  Flow *f = NULL;
3566  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3567  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3568  TcpSession ssn;
3569  HtpState *htp_state = NULL;
3570  int r = 0;
3572 
3573  memset(&ssn, 0, sizeof(ssn));
3574 
3575  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3576  if (f == NULL)
3577  goto end;
3578  f->protoctx = &ssn;
3579  f->proto = IPPROTO_TCP;
3580  f->alproto = ALPROTO_HTTP1;
3581 
3582  StreamTcpInitConfig(true);
3583 
3584  uint32_t u;
3585  for (u = 0; u < httplen1; u++) {
3586  uint8_t flags = 0;
3587 
3588  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3589  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3590  else flags = STREAM_TOSERVER;
3591 
3592  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3593  if (r != 0) {
3594  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3595  " 0: ", u, r);
3596  goto end;
3597  }
3598  }
3599  htp_state = f->alstate;
3600  if (htp_state == NULL) {
3601  printf("no http state: ");
3602  goto end;
3603  }
3604 
3605  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3606 
3607  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3608  if (tx->request_method_number != HTP_M_UNKNOWN ||
3609  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3610  {
3611  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3612  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3613  bstr_util_strdup_to_c(tx->request_protocol));
3614  goto end;
3615  }
3616  result = 1;
3617 end:
3618  if (alp_tctx != NULL)
3620  StreamTcpFreeConfig(true);
3621  UTHFreeFlow(f);
3622  return result;
3623 }
3624 
3625 /** \test Test case where invalid data is sent and check the response of the
3626  * parser from HTP library. */
3627 static int HTPParserTest04(void)
3628 {
3629  int result = 0;
3630  Flow *f = NULL;
3631  HtpState *htp_state = NULL;
3632  uint8_t httpbuf1[] = "World!\r\n";
3633  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3634  TcpSession ssn;
3635  int r = 0;
3637 
3638  memset(&ssn, 0, sizeof(ssn));
3639 
3640  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3641  if (f == NULL)
3642  goto end;
3643  f->protoctx = &ssn;
3644  f->proto = IPPROTO_TCP;
3645  f->alproto = ALPROTO_HTTP1;
3646 
3647  StreamTcpInitConfig(true);
3648 
3650  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3651  if (r != 0) {
3652  goto end;
3653  }
3654 
3655  htp_state = f->alstate;
3656  if (htp_state == NULL) {
3657  printf("no http state: ");
3658  goto end;
3659  }
3660 
3661  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3662  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3663  if (tx->request_method_number != HTP_M_UNKNOWN ||
3664  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3665  {
3666  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3667  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3668  bstr_util_strdup_to_c(tx->request_protocol));
3669  goto end;
3670  }
3671  result = 1;
3672 end:
3673  if (alp_tctx != NULL)
3675  StreamTcpFreeConfig(true);
3676  UTHFreeFlow(f);
3677  return result;
3678 }
3679 
3680 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3681  * properly parsed them and also keeps them separated. */
3682 static int HTPParserTest05(void)
3683 {
3684  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3685  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3686  uint8_t httpbuf2[] = "Post D";
3687  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3688  uint8_t httpbuf3[] = "ata is c0oL!";
3689  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3690 
3691  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3692  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3693  uint8_t httpbuf5[] = "post R";
3694  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3695  uint8_t httpbuf6[] = "esults are tha bomb!";
3696  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3697 
3698  TcpSession ssn;
3699  memset(&ssn, 0, sizeof(ssn));
3700 
3703 
3704  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3705  FAIL_IF_NULL(f);
3706  f->protoctx = &ssn;
3707  f->proto = IPPROTO_TCP;
3708  f->alproto = ALPROTO_HTTP1;
3709 
3710  StreamTcpInitConfig(true);
3711 
3712  int r = AppLayerParserParse(
3713  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3714  FAIL_IF(r != 0);
3715 
3716  r = AppLayerParserParse(
3717  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3718  FAIL_IF(r != 0);
3719 
3720  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3721  FAIL_IF(r != 0);
3722 
3723  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3724  FAIL_IF(r != 0);
3725 
3726  r = AppLayerParserParse(
3727  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3728  FAIL_IF(r != 0);
3729 
3730  r = AppLayerParserParse(
3731  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3732  FAIL_IF(r != 0);
3733 
3734  HtpState *http_state = f->alstate;
3735  FAIL_IF_NULL(http_state);
3736 
3737  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3738  FAIL_IF_NULL(tx);
3739  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3740  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3741 
3742  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3743  FAIL_IF_NULL(h);
3744 
3745  FAIL_IF_NOT(tx->response_status_number == 200);
3746 
3748  StreamTcpFreeConfig(true);
3749  UTHFreeFlow(f);
3750  PASS;
3751 }
3752 
3753 /** \test Test proper chunked encoded response body
3754  */
3755 static int HTPParserTest06(void)
3756 {
3757  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3758  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3759  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3760  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3761  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3762  "GMT\r\n"
3763  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3764  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3765  "FrontPage/5.0.2.2510\r\n"
3766  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3767  "chunked\r\n"
3768  "Content-Type: text/html\r\n\r\n"
3769  "580\r\n"
3770  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3771  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3772  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3773  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3774  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3775  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3776  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3777  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3778  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3779  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3780  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3781  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3782  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3783  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3784  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3785  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3786  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3787  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3788  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3789  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3790  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3791  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3792  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3793  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3794  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3795  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3796  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3797  "aHA=\r\n0\r\n\r\n";
3798  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3799  TcpSession ssn;
3800 
3803 
3804  memset(&ssn, 0, sizeof(ssn));
3805 
3806  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3807  FAIL_IF_NULL(f);
3808  f->protoctx = &ssn;
3809  f->proto = IPPROTO_TCP;
3810  f->alproto = ALPROTO_HTTP1;
3811 
3812  StreamTcpInitConfig(true);
3813 
3814  int r = AppLayerParserParse(
3815  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3816  FAIL_IF(r != 0);
3817  r = AppLayerParserParse(
3818  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3819  FAIL_IF(r != 0);
3820 
3821  HtpState *http_state = f->alstate;
3822  FAIL_IF_NULL(http_state);
3823 
3824  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3825  FAIL_IF_NULL(tx);
3826 
3827  FAIL_IF(tx->request_method_number != HTP_M_GET);
3828  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3829 
3830  FAIL_IF(tx->response_status_number != 200);
3831  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3832 
3833  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3834  FAIL_IF_NULL(h);
3835 
3837  StreamTcpFreeConfig(true);
3838  UTHFreeFlow(f);
3839  PASS;
3840 }
3841 
3842 /** \test
3843  */
3844 static int HTPParserTest07(void)
3845 {
3846  int result = 0;
3847  Flow *f = NULL;
3848  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3849  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3850  TcpSession ssn;
3851  HtpState *htp_state = NULL;
3852  int r = 0;
3854 
3855  memset(&ssn, 0, sizeof(ssn));
3856 
3857  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3858  if (f == NULL)
3859  goto end;
3860  f->protoctx = &ssn;
3861  f->proto = IPPROTO_TCP;
3862  f->alproto = ALPROTO_HTTP1;
3863 
3864  StreamTcpInitConfig(true);
3865 
3866  uint32_t u;
3867  for (u = 0; u < httplen1; u++) {
3868  uint8_t flags = 0;
3869 
3870  if (u == 0)
3871  flags = STREAM_TOSERVER|STREAM_START;
3872  else if (u == (httplen1 - 1))
3873  flags = STREAM_TOSERVER|STREAM_EOF;
3874  else
3875  flags = STREAM_TOSERVER;
3876 
3877  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3878  if (r != 0) {
3879  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3880  " 0: ", u, r);
3881  goto end;
3882  }
3883  }
3884 
3885  htp_state = f->alstate;
3886  if (htp_state == NULL) {
3887  printf("no http state: ");
3888  goto end;
3889  }
3890 
3891  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3892  size_t reflen = sizeof(ref) - 1;
3893 
3894  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3895  if (tx == NULL)
3896  goto end;
3897  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3898  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3899  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3900  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3901  (uintmax_t)reflen,
3902  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3903  goto end;
3904  }
3905 
3906  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3907  bstr_len(tx_ud->request_uri_normalized)) != 0)
3908  {
3909  printf("normalized uri \"");
3910  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3911  printf("\" != \"");
3912  PrintRawUriFp(stdout, ref, reflen);
3913  printf("\": ");
3914  goto end;
3915  }
3916  }
3917 
3918  result = 1;
3919 end:
3920  if (alp_tctx != NULL)
3922  StreamTcpFreeConfig(true);
3923  UTHFreeFlow(f);
3924  return result;
3925 }
3926 
3927 #include "conf-yaml-loader.h"
3928 
3929 /** \test Abort
3930  */
3931 static int HTPParserTest08(void)
3932 {
3933  int result = 0;
3934  Flow *f = NULL;
3935  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3936  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3937  TcpSession ssn;
3939 
3940  char input[] = "\
3941 %YAML 1.1\n\
3942 ---\n\
3943 libhtp:\n\
3944 \n\
3945  default-config:\n\
3946  personality: IDS\n\
3947 ";
3948 
3950  ConfInit();
3952 
3953  ConfYamlLoadString(input, strlen(input));
3954  HTPConfigure();
3955 
3956  HtpState *htp_state = NULL;
3957  int r = 0;
3958  memset(&ssn, 0, sizeof(ssn));
3959 
3960  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3961  if (f == NULL)
3962  goto end;
3963  f->protoctx = &ssn;
3964  f->proto = IPPROTO_TCP;
3965  f->alproto = ALPROTO_HTTP1;
3966 
3967  StreamTcpInitConfig(true);
3968 
3969  uint8_t flags = 0;
3970  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3971 
3972  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3973  if (r != 0) {
3974  printf("toserver chunk returned %" PRId32 ", expected"
3975  " 0: ", r);
3976  result = 0;
3977  goto end;
3978  }
3979 
3980  htp_state = f->alstate;
3981  if (htp_state == NULL) {
3982  printf("no http state: ");
3983  result = 0;
3984  goto end;
3985  }
3986 
3987  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3988  if (tx == NULL)
3989  goto end;
3990  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3991  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3992  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3993  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3994  bstr_len(tx_ud->request_uri_normalized));
3995  }
3996 
3997  result = 1;
3998 end:
3999  if (alp_tctx != NULL)
4001  StreamTcpFreeConfig(true);
4002  HTPFreeConfig();
4003  ConfDeInit();
4006  UTHFreeFlow(f);
4007  return result;
4008 }
4009 
4010 /** \test Abort
4011  */
4012 static int HTPParserTest09(void)
4013 {
4014  int result = 0;
4015  Flow *f = NULL;
4016  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
4017  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4018  TcpSession ssn;
4020 
4021  char input[] = "\
4022 %YAML 1.1\n\
4023 ---\n\
4024 libhtp:\n\
4025 \n\
4026  default-config:\n\
4027  personality: Apache_2_2\n\
4028 ";
4029 
4031  ConfInit();
4033 
4034  ConfYamlLoadString(input, strlen(input));
4035  HTPConfigure();
4036 
4037  HtpState *htp_state = NULL;
4038  int r = 0;
4039 
4040  memset(&ssn, 0, sizeof(ssn));
4041 
4042  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4043  if (f == NULL)
4044  goto end;
4045  f->protoctx = &ssn;
4046  f->proto = IPPROTO_TCP;
4047  f->alproto = ALPROTO_HTTP1;
4048 
4049  StreamTcpInitConfig(true);
4050 
4051  uint8_t flags = 0;
4052  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
4053 
4054  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
4055  if (r != 0) {
4056  printf("toserver chunk returned %" PRId32 ", expected"
4057  " 0: ", r);
4058  goto end;
4059  }
4060 
4061  htp_state = f->alstate;
4062  if (htp_state == NULL) {
4063  printf("no http state: ");
4064  goto end;
4065  }
4066 
4067  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4068  if (tx == NULL)
4069  goto end;
4070  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4071  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4072  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4073  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4074  bstr_len(tx_ud->request_uri_normalized));
4075  }
4076 
4077  result = 1;
4078 end:
4079  if (alp_tctx != NULL)
4081  StreamTcpFreeConfig(true);
4082  HTPFreeConfig();
4083  ConfDeInit();
4086  UTHFreeFlow(f);
4087  return result;
4088 }
4089 
4090 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4091  */
4092 static int HTPParserTest10(void)
4093 {
4094  int result = 0;
4095  Flow *f = NULL;
4096  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4097  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4098  TcpSession ssn;
4099  HtpState *htp_state = NULL;
4100  int r = 0;
4102 
4103  memset(&ssn, 0, sizeof(ssn));
4104 
4105  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4106  if (f == NULL)
4107  goto end;
4108  f->protoctx = &ssn;
4109  f->proto = IPPROTO_TCP;
4110  f->alproto = ALPROTO_HTTP1;
4111 
4112  StreamTcpInitConfig(true);
4113 
4114  uint32_t u;
4115  for (u = 0; u < httplen1; u++) {
4116  uint8_t flags = 0;
4117 
4118  if (u == 0)
4119  flags = STREAM_TOSERVER|STREAM_START;
4120  else if (u == (httplen1 - 1))
4121  flags = STREAM_TOSERVER|STREAM_EOF;
4122  else
4123  flags = STREAM_TOSERVER;
4124 
4125  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4126  if (r != 0) {
4127  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4128  " 0: ", u, r);
4129  goto end;
4130  }
4131  }
4132 
4133  htp_state = f->alstate;
4134  if (htp_state == NULL) {
4135  printf("no http state: ");
4136  goto end;
4137  }
4138 
4139  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4140  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4141  if (h == NULL) {
4142  goto end;
4143  }
4144 
4145  char *name = bstr_util_strdup_to_c(h->name);
4146  if (name == NULL) {
4147  goto end;
4148  }
4149 
4150  if (strcmp(name, "Host") != 0) {
4151  printf("header name not \"Host\", instead \"%s\": ", name);
4152  free(name);
4153  goto end;
4154  }
4155  free(name);
4156 
4157  char *value = bstr_util_strdup_to_c(h->value);
4158  if (value == NULL) {
4159  goto end;
4160  }
4161 
4162  if (strcmp(value, "www.google.com") != 0) {
4163  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4164  free(value);
4165  goto end;
4166  }
4167  free(value);
4168 
4169  result = 1;
4170 end:
4171  if (alp_tctx != NULL)
4173  StreamTcpFreeConfig(true);
4174  UTHFreeFlow(f);
4175  return result;
4176 }
4177 
4178 /** \test double encoding in path
4179  */
4180 static int HTPParserTest11(void)
4181 {
4182  int result = 0;
4183  Flow *f = NULL;
4184  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4185  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4186  TcpSession ssn;
4187  HtpState *htp_state = NULL;
4188  int r = 0;
4190 
4191  memset(&ssn, 0, sizeof(ssn));
4192 
4193  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4194  if (f == NULL)
4195  goto end;
4196  f->protoctx = &ssn;
4197  f->proto = IPPROTO_TCP;
4198  f->alproto = ALPROTO_HTTP1;
4199 
4200  StreamTcpInitConfig(true);
4201 
4202  uint32_t u;
4203  for (u = 0; u < httplen1; u++) {
4204  uint8_t flags = 0;
4205 
4206  if (u == 0)
4207  flags = STREAM_TOSERVER|STREAM_START;
4208  else if (u == (httplen1 - 1))
4209  flags = STREAM_TOSERVER|STREAM_EOF;
4210  else
4211  flags = STREAM_TOSERVER;
4212 
4213  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4214  if (r != 0) {
4215  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4216  " 0: ", u, r);
4217  goto end;
4218  }
4219  }
4220 
4221  htp_state = f->alstate;
4222  if (htp_state == NULL) {
4223  printf("no http state: ");
4224  goto end;
4225  }
4226 
4227  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4228  if (tx == NULL)
4229  goto end;
4230  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4231  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4232  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4233  printf("normalized uri len should be 2, is %"PRIuMAX,
4234  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4235  goto end;
4236  }
4237 
4238  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4239  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4240  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4241  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4242  {
4243  printf("normalized uri \"");
4244  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4245  printf("\": ");
4246  goto end;
4247  }
4248  }
4249 
4250  result = 1;
4251 end:
4252  if (alp_tctx != NULL)
4254  StreamTcpFreeConfig(true);
4255  UTHFreeFlow(f);
4256  return result;
4257 }
4258 
4259 /** \test double encoding in query
4260  */
4261 static int HTPParserTest12(void)
4262 {
4263  int result = 0;
4264  Flow *f = NULL;
4265  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4266  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4267  TcpSession ssn;
4268  HtpState *htp_state = NULL;
4269  int r = 0;
4271 
4272  memset(&ssn, 0, sizeof(ssn));
4273 
4274  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4275  if (f == NULL)
4276  goto end;
4277  f->protoctx = &ssn;
4278  f->proto = IPPROTO_TCP;
4279  f->alproto = ALPROTO_HTTP1;
4280 
4281  StreamTcpInitConfig(true);
4282 
4283  uint32_t u;
4284  for (u = 0; u < httplen1; u++) {
4285  uint8_t flags = 0;
4286 
4287  if (u == 0)
4288  flags = STREAM_TOSERVER|STREAM_START;
4289  else if (u == (httplen1 - 1))
4290  flags = STREAM_TOSERVER|STREAM_EOF;
4291  else
4292  flags = STREAM_TOSERVER;
4293 
4294  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4295  if (r != 0) {
4296  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4297  " 0: ", u, r);
4298  goto end;
4299  }
4300  }
4301 
4302  htp_state = f->alstate;
4303  if (htp_state == NULL) {
4304  printf("no http state: ");
4305  goto end;
4306  }
4307 
4308  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4309  if (tx == NULL)
4310  goto end;
4311  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4312  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4313  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4314  printf("normalized uri len should be 5, is %"PRIuMAX,
4315  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4316  goto end;
4317  }
4318 
4319  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4320  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4321  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4322  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4323  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4324  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4325  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4326  {
4327  printf("normalized uri \"");
4328  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4329  printf("\": ");
4330  goto end;
4331  }
4332  }
4333 
4334  result = 1;
4335  end:
4336  if (alp_tctx != NULL)
4338  StreamTcpFreeConfig(true);
4339  UTHFreeFlow(f);
4340  return result;
4341 }
4342 
4343 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4344  */
4345 static int HTPParserTest13(void)
4346 {
4347  int result = 0;
4348  Flow *f = NULL;
4349  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4350  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4351  TcpSession ssn;
4352  HtpState *htp_state = NULL;
4353  int r = 0;
4355 
4356  memset(&ssn, 0, sizeof(ssn));
4357 
4358  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4359  if (f == NULL)
4360  goto end;
4361  f->protoctx = &ssn;
4362  f->proto = IPPROTO_TCP;
4363  f->alproto = ALPROTO_HTTP1;
4364 
4365  StreamTcpInitConfig(true);
4366 
4367  uint32_t u;
4368  for (u = 0; u < httplen1; u++) {
4369  uint8_t flags = 0;
4370 
4371  if (u == 0)
4372  flags = STREAM_TOSERVER|STREAM_START;
4373  else if (u == (httplen1 - 1))
4374  flags = STREAM_TOSERVER|STREAM_EOF;
4375  else
4376  flags = STREAM_TOSERVER;
4377 
4378  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4379  if (r != 0) {
4380  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4381  " 0: ", u, r);
4382  goto end;
4383  }
4384  }
4385 
4386  htp_state = f->alstate;
4387  if (htp_state == NULL) {
4388  printf("no http state: ");
4389  goto end;
4390  }
4391 
4392  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4393  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4394  if (h == NULL) {
4395  goto end;
4396  }
4397 
4398  char *name = bstr_util_strdup_to_c(h->name);
4399  if (name == NULL) {
4400  goto end;
4401  }
4402 
4403  if (strcmp(name, "Host") != 0) {
4404  printf("header name not \"Host\", instead \"%s\": ", name);
4405  free(name);
4406  goto end;
4407  }
4408  free(name);
4409 
4410  char *value = bstr_util_strdup_to_c(h->value);
4411  if (value == NULL) {
4412  goto end;
4413  }
4414 
4415  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4416  printf("header value not \"www.google.com\", instead \"");
4417  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4418  printf("\": ");
4419  free(value);
4420  goto end;
4421  }
4422  free(value);
4423 
4424  result = 1;
4425 end:
4426  if (alp_tctx != NULL)
4428  StreamTcpFreeConfig(true);
4429  UTHFreeFlow(f);
4430  return result;
4431 }
4432 
4433 /** \test Test basic config */
4434 static int HTPParserConfigTest01(void)
4435 {
4436  int ret = 0;
4437  char input[] = "\
4438 %YAML 1.1\n\
4439 ---\n\
4440 libhtp:\n\
4441 \n\
4442  default-config:\n\
4443  personality: IDS\n\
4444 \n\
4445  server-config:\n\
4446 \n\
4447  - apache-tomcat:\n\
4448  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4449  personality: Tomcat_6_0\n\
4450 \n\
4451  - iis7:\n\
4452  address: \n\
4453  - 192.168.0.0/24\n\
4454  - 192.168.10.0/24\n\
4455  personality: IIS_7_0\n\
4456 ";
4457 
4459  ConfInit();
4460 
4461  ConfYamlLoadString(input, strlen(input));
4462 
4463  ConfNode *outputs;
4464  outputs = ConfGetNode("libhtp.default-config.personality");
4465  if (outputs == NULL) {
4466  goto end;
4467  }
4468 
4469  outputs = ConfGetNode("libhtp.server-config");
4470  if (outputs == NULL) {
4471  goto end;
4472  }
4473 
4474  ConfNode *node = TAILQ_FIRST(&outputs->head);
4475  if (node == NULL) {
4476  goto end;
4477  }
4478  if (strcmp(node->name, "0") != 0) {
4479  goto end;
4480  }
4481  node = TAILQ_FIRST(&node->head);
4482  if (node == NULL) {
4483  goto end;
4484  }
4485  if (strcmp(node->name, "apache-tomcat") != 0) {
4486  goto end;
4487  }
4488 
4489  int i = 0;
4490  ConfNode *n;
4491 
4492  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4493  if (node2 == NULL) {
4494  goto end;
4495  }
4496  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4497  goto end;
4498  }
4499 
4500  node = ConfNodeLookupChild(node, "address");
4501  if (node == NULL) {
4502  goto end;
4503  }
4504  TAILQ_FOREACH(n, &node->head, next) {
4505  if (n == NULL) {
4506  goto end;
4507  }
4508 
4509  switch(i) {
4510  case 0:
4511  if (strcmp(n->name, "0") != 0) {
4512  goto end;
4513  }
4514  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4515  goto end;
4516  }
4517  break;
4518  case 1:
4519  if (strcmp(n->name, "1") != 0) {
4520  goto end;
4521  }
4522  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4523  goto end;
4524  }
4525  break;
4526  case 2:
4527  if (strcmp(n->name, "2") != 0) {
4528  goto end;
4529  }
4530  if (strcmp(n->val, "::1") != 0) {
4531  goto end;
4532  }
4533  break;
4534  default:
4535  goto end;
4536  }
4537  i++;
4538  }
4539 
4540  outputs = ConfGetNode("libhtp.server-config");
4541  if (outputs == NULL) {
4542  goto end;
4543  }
4544 
4545  node = TAILQ_FIRST(&outputs->head);
4546  node = TAILQ_NEXT(node, next);
4547  if (node == NULL) {
4548  goto end;
4549  }
4550  if (strcmp(node->name, "1") != 0) {
4551  goto end;
4552  }
4553  node = TAILQ_FIRST(&node->head);
4554  if (node == NULL) {
4555  goto end;
4556  }
4557  if (strcmp(node->name, "iis7") != 0) {
4558  goto end;
4559  }
4560 
4561  node2 = ConfNodeLookupChild(node, "personality");
4562  if (node2 == NULL) {
4563  goto end;
4564  }
4565  if (strcmp(node2->val, "IIS_7_0") != 0) {
4566  goto end;
4567  }
4568 
4569  node = ConfNodeLookupChild(node, "address");
4570  if (node == NULL) {
4571  goto end;
4572  }
4573 
4574  i = 0;
4575  TAILQ_FOREACH(n, &node->head, next) {
4576  if (n == NULL) {
4577  goto end;
4578  }
4579 
4580  switch(i) {
4581  case 0:
4582  if (strcmp(n->name, "0") != 0) {
4583  goto end;
4584  }
4585  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4586  goto end;
4587  }
4588  break;
4589  case 1:
4590  if (strcmp(n->name, "1") != 0) {
4591  goto end;
4592  }
4593  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4594  goto end;
4595  }
4596  break;
4597  default:
4598  goto end;
4599  }
4600  i++;
4601  }
4602 
4603  ret = 1;
4604 
4605 end:
4606  ConfDeInit();
4608 
4609  return ret;
4610 }
4611 
4612 /** \test Test config builds radix correctly */
4613 static int HTPParserConfigTest02(void)
4614 {
4615  int ret = 0;
4616  char input[] = "\
4617 %YAML 1.1\n\
4618 ---\n\
4619 libhtp:\n\
4620 \n\
4621  default-config:\n\
4622  personality: IDS\n\
4623 \n\
4624  server-config:\n\
4625 \n\
4626  - apache-tomcat:\n\
4627  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4628  personality: Tomcat_6_0\n\
4629 \n\
4630  - iis7:\n\
4631  address: \n\
4632  - 192.168.0.0/24\n\
4633  - 192.168.10.0/24\n\
4634  personality: IIS_7_0\n\
4635 ";
4636 
4638  ConfInit();
4640 
4641  ConfYamlLoadString(input, strlen(input));
4642 
4643  HTPConfigure();
4644 
4645  if (cfglist.cfg == NULL) {
4646  printf("No default config created.\n");
4647  goto end;
4648  }
4649 
4650  if (cfgtree == NULL) {
4651  printf("No config tree created.\n");
4652  goto end;
4653  }
4654 
4655  htp_cfg_t *htp = cfglist.cfg;
4656  uint8_t buf[128];
4657  const char *addr;
4658  void *user_data = NULL;
4659 
4660  addr = "192.168.10.42";
4661  if (inet_pton(AF_INET, addr, buf) == 1) {
4662  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4663  if (user_data != NULL) {
4664  HTPCfgRec *htp_cfg_rec = user_data;
4665  htp = htp_cfg_rec->cfg;
4666  SCLogDebug("LIBHTP using config: %p", htp);
4667  }
4668  if (htp == NULL) {
4669  printf("Could not get config for: %s\n", addr);
4670  goto end;
4671  }
4672  }
4673  else {
4674  printf("Failed to parse address: %s\n", addr);
4675  goto end;
4676  }
4677 
4678  user_data = NULL;
4679  addr = "::1";
4680  if (inet_pton(AF_INET6, addr, buf) == 1) {
4681  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4682  if (user_data != NULL) {
4683  HTPCfgRec *htp_cfg_rec = user_data;
4684  htp = htp_cfg_rec->cfg;
4685  SCLogDebug("LIBHTP using config: %p", htp);
4686  }
4687  if (htp == NULL) {
4688  printf("Could not get config for: %s\n", addr);
4689  goto end;
4690  }
4691  }
4692  else {
4693  printf("Failed to parse address: %s\n", addr);
4694  goto end;
4695  }
4696 
4697  ret = 1;
4698 
4699 end:
4700  HTPFreeConfig();
4701  ConfDeInit();
4704 
4705  return ret;
4706 }
4707 
4708 /** \test Test traffic is handled by the correct htp config */
4709 static int HTPParserConfigTest03(void)
4710 {
4711  int result = 1;
4712  Flow *f = NULL;
4713  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4714  " Data is c0oL!";
4715  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4716  TcpSession ssn;
4718 
4719  HtpState *htp_state = NULL;
4720  int r = 0;
4721  char input[] = "\
4722 %YAML 1.1\n\
4723 ---\n\
4724 libhtp:\n\
4725 \n\
4726  default-config:\n\
4727  personality: IDS\n\
4728 \n\
4729  server-config:\n\
4730 \n\
4731  - apache-tomcat:\n\
4732  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4733  personality: Tomcat_6_0\n\
4734 \n\
4735  - iis7:\n\
4736  address: \n\
4737  - 192.168.0.0/24\n\
4738  - 192.168.10.0/24\n\
4739  personality: IIS_7_0\n\
4740 ";
4741 
4743  ConfInit();
4745 
4746  ConfYamlLoadString(input, strlen(input));
4747 
4748  HTPConfigure();
4749 
4750  const char *addr = "192.168.10.42";
4751 
4752  memset(&ssn, 0, sizeof(ssn));
4753 
4754  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4755  if (f == NULL)
4756  goto end;
4757  f->protoctx = &ssn;
4758  f->proto = IPPROTO_TCP;
4759  f->alproto = ALPROTO_HTTP1;
4760 
4761  htp_cfg_t *htp = cfglist.cfg;
4762 
4763  void *user_data = NULL;
4764  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4765  if (user_data != NULL) {
4766  HTPCfgRec *htp_cfg_rec = user_data;
4767  htp = htp_cfg_rec->cfg;
4768  SCLogDebug("LIBHTP using config: %p", htp);
4769  }
4770  if (htp == NULL) {
4771  printf("Could not get config for: %s\n", addr);
4772  goto end;
4773  }
4774 
4775  StreamTcpInitConfig(true);
4776 
4777  uint32_t u;
4778  for (u = 0; u < httplen1; u++) {
4779  uint8_t flags = 0;
4780 
4781  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4782  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4783  else flags = STREAM_TOSERVER;
4784 
4785  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4786  if (r != 0) {
4787  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4788  " 0: ", u, r);
4789  result = 0;
4790  goto end;
4791  }
4792  }
4793 
4794  htp_state = f->alstate;
4795  if (htp_state == NULL) {
4796  printf("no http state: ");
4797  result = 0;
4798  goto end;
4799  }
4800 
4801  if (HTPStateGetTxCnt(htp_state) != 2) {
4802  printf("HTPStateGetTxCnt(htp_state) failure\n");
4803  goto end;
4804  }
4805 
4806  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4807  if (tx == NULL)
4808  goto end;
4809  if (tx->cfg != htp) {
4810  printf("wrong HTP config (%p instead of %p - default=%p): ",
4811  tx->cfg, htp, cfglist.cfg);
4812  goto end;
4813  }
4814  tx = HTPStateGetTx(htp_state, 1);
4815  if (tx == NULL)
4816  goto end;
4817  if (tx->cfg != htp) {
4818  printf("wrong HTP config (%p instead of %p - default=%p): ",
4819  tx->cfg, htp, cfglist.cfg);
4820  goto end;
4821  }
4822 
4823 end:
4824  if (alp_tctx != NULL)
4826  HTPFreeConfig();
4827  ConfDeInit();
4830 
4831  StreamTcpFreeConfig(true);
4832  UTHFreeFlow(f);
4833  return result;
4834 }
4835 
4836 /* disabled when we upgraded to libhtp 0.5.x */
4837 #if 0
4838 static int HTPParserConfigTest04(void)
4839 {
4840  int result = 0;
4841 
4842  char input[] = "\
4843 %YAML 1.1\n\
4844 ---\n\
4845 libhtp:\n\
4846 \n\
4847  default-config:\n\
4848  personality: IDS\n\
4849  path-control-char-handling: status_400\n\
4850  path-convert-utf8: yes\n\
4851  path-invalid-encoding-handling: remove_percent\n\
4852 \n\
4853  server-config:\n\
4854 \n\
4855  - apache-tomcat:\n\
4856  personality: Tomcat_6_0\n\
4857  path-invalid-utf8-handling: none\n\
4858  path-nul-encoded-handling: status_404\n\
4859  path-nul-raw-handling: status_400\n\
4860 \n\
4861  - iis7:\n\
4862  personality: IIS_7_0\n\
4863  path-replacement-char: o\n\
4864  path-unicode-mapping: status_400\n\
4865 ";
4866 
4868  ConfInit();
4870 
4871  ConfYamlLoadString(input, strlen(input));
4872 
4873  HTPConfigure();
4874 
4875  HTPCfgRec *cfg_rec = &cfglist;
4876  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4877  cfg_rec->cfg->path_convert_utf8 != 1 ||
4878  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4879  printf("failed 1\n");
4880  goto end;
4881  }
4882 
4883  cfg_rec = cfg_rec->next;
4884  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4885  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4886  printf("failed 2\n");
4887  goto end;
4888  }
4889 
4890  cfg_rec = cfg_rec->next;
4891  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4892  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4893  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4894  printf("failed 3\n");
4895  goto end;
4896  }
4897 
4898  result = 1;
4899 
4900 end:
4901  HTPFreeConfig();
4902  ConfDeInit();
4905 
4906  return result;
4907 }
4908 #endif
4909 
4910 /** \test Test %2f decoding in profile Apache_2_2
4911  *
4912  * %2f in path is left untouched
4913  * %2f in query string is normalized to %2F
4914  * %252f in query string is decoded/normalized to %2F
4915  */
4916 static int HTPParserDecodingTest01(void)
4917 {
4918  uint8_t httpbuf1[] =
4919  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4920  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4921  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4922  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4923  TcpSession ssn;
4926 
4927  char input[] = "\
4928 %YAML 1.1\n\
4929 ---\n\
4930 libhtp:\n\
4931 \n\
4932  default-config:\n\
4933  personality: Apache_2\n\
4934 ";
4935 
4937  ConfInit();
4939  ConfYamlLoadString(input, strlen(input));
4940  HTPConfigure();
4941  const char *addr = "4.3.2.1";
4942  memset(&ssn, 0, sizeof(ssn));
4943 
4944  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4945  FAIL_IF_NULL(f);
4946  f->protoctx = &ssn;
4947  f->proto = IPPROTO_TCP;
4948  f->alproto = ALPROTO_HTTP1;
4949 
4950  StreamTcpInitConfig(true);
4951 
4952  for (uint32_t u = 0; u < httplen1; u++) {
4953  uint8_t flags = 0;
4954  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4955  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4956  else flags = STREAM_TOSERVER;
4957 
4958  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4959  FAIL_IF(r != 0);
4960  }
4961 
4962  HtpState *htp_state = f->alstate;
4963  FAIL_IF_NULL(htp_state);
4964 
4965  uint8_t ref1[] = "/abc%2fdef";
4966  size_t reflen = sizeof(ref1) - 1;
4967 
4968  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4969  FAIL_IF_NULL(tx);
4970 
4971  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4972  FAIL_IF_NULL(tx_ud);
4974  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4975  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4976  bstr_len(tx_ud->request_uri_normalized)) != 0);
4977 
4978  uint8_t ref2[] = "/abc/def?ghi/jkl";
4979  reflen = sizeof(ref2) - 1;
4980 
4981  tx = HTPStateGetTx(htp_state, 1);
4982  FAIL_IF_NULL(tx);
4983  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4984  FAIL_IF_NULL(tx_ud);
4986  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4987 
4988  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4989  bstr_len(tx_ud->request_uri_normalized)) != 0);
4990 
4991  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4992  reflen = sizeof(ref3) - 1;
4993  tx = HTPStateGetTx(htp_state, 2);
4994  FAIL_IF_NULL(tx);
4995  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4996  FAIL_IF_NULL(tx_ud);
4998  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4999 
5000  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5001  bstr_len(tx_ud->request_uri_normalized)) != 0);
5002 
5004  HTPFreeConfig();
5005  ConfDeInit();
5008 
5009  StreamTcpFreeConfig(true);
5010  UTHFreeFlow(f);
5011  PASS;
5012 }
5013 
5014 static int HTPParserDecodingTest01a(void)
5015 {
5016  uint8_t httpbuf1[] = "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5017  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5018  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5019  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5020  TcpSession ssn;
5023 
5024  char input[] = "\
5025 %YAML 1.1\n\
5026 ---\n\
5027 libhtp:\n\
5028 \n\
5029  default-config:\n\
5030  personality: Apache_2\n\
5031 ";
5032 
5034  ConfInit();
5036  ConfYamlLoadString(input, strlen(input));
5037  HTPConfigure();
5038  const char *addr = "4.3.2.1";
5039  memset(&ssn, 0, sizeof(ssn));
5040 
5041  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5042  FAIL_IF_NULL(f);
5043  f->protoctx = &ssn;
5044  f->proto = IPPROTO_TCP;
5045  f->alproto = ALPROTO_HTTP1;
5046 
5047  StreamTcpInitConfig(true);
5048 
5049  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
5050  (STREAM_TOSERVER | STREAM_START | STREAM_EOF), httpbuf1, httplen1);
5051  FAIL_IF(r != 0);
5052 
5053  HtpState *htp_state = f->alstate;
5054  FAIL_IF_NULL(htp_state);
5055 
5056  uint8_t ref1[] = "/abc%2fdef";
5057  size_t reflen = sizeof(ref1) - 1;
5058 
5059  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5060  FAIL_IF_NULL(tx);
5061 
5062  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5063  FAIL_IF_NULL(tx_ud);
5065  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5066  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5067  bstr_len(tx_ud->request_uri_normalized)) != 0);
5068 
5069  uint8_t ref2[] = "/abc/def?ghi/jkl";
5070  reflen = sizeof(ref2) - 1;
5071 
5072  tx = HTPStateGetTx(htp_state, 1);
5073  FAIL_IF_NULL(tx);
5074  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5075  FAIL_IF_NULL(tx_ud);
5077  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5078 
5079  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5080  bstr_len(tx_ud->request_uri_normalized)) != 0);
5081 
5082  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5083  reflen = sizeof(ref3) - 1;
5084  tx = HTPStateGetTx(htp_state, 2);
5085  FAIL_IF_NULL(tx);
5086  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5087  FAIL_IF_NULL(tx_ud);
5089  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5090 
5091  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5092  bstr_len(tx_ud->request_uri_normalized)) != 0);
5093 
5095  HTPFreeConfig();
5096  ConfDeInit();
5099 
5100  StreamTcpFreeConfig(true);
5101  UTHFreeFlow(f);
5102  PASS;
5103 }
5104 
5105 /** \test Test %2f decoding in profile IDS
5106  *
5107  * %2f in path decoded to /
5108  * %2f in query string is decoded to /
5109  * %252f in query string is decoded to %2F
5110  */
5111 static int HTPParserDecodingTest02(void)
5112 {
5113  int result = 0;
5114  Flow *f = NULL;
5115  uint8_t httpbuf1[] =
5116  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5117  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5118  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5119  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5120  TcpSession ssn;
5122 
5123  HtpState *htp_state = NULL;
5124  int r = 0;
5125  char input[] = "\
5126 %YAML 1.1\n\
5127 ---\n\
5128 libhtp:\n\
5129 \n\
5130  default-config:\n\
5131  personality: IDS\n\
5132  double-decode-path: no\n\
5133  double-decode-query: no\n\
5134 ";
5135 
5137  ConfInit();
5139  ConfYamlLoadString(input, strlen(input));
5140  HTPConfigure();
5141  const char *addr = "4.3.2.1";
5142  memset(&ssn, 0, sizeof(ssn));
5143 
5144  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5145  if (f == NULL)
5146  goto end;
5147  f->protoctx = &ssn;
5148  f->proto = IPPROTO_TCP;
5149  f->alproto = ALPROTO_HTTP1;
5150 
5151  StreamTcpInitConfig(true);
5152 
5153  uint32_t u;
5154  for (u = 0; u < httplen1; u++) {
5155  uint8_t flags = 0;
5156 
5157  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5158  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5159  else flags = STREAM_TOSERVER;
5160 
5161  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5162  if (r != 0) {
5163  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5164  " 0: ", u, r);
5165  goto end;
5166  }
5167  }
5168 
5169  htp_state = f->alstate;
5170  if (htp_state == NULL) {
5171  printf("no http state: ");
5172  goto end;
5173  }
5174 
5175  uint8_t ref1[] = "/abc/def";
5176  size_t reflen = sizeof(ref1) - 1;
5177 
5178  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5179  if (tx == NULL)
5180  goto end;
5181  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5182  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5183  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5184  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5185  (uintmax_t)reflen,
5186  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5187  goto end;
5188  }
5189 
5190  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5191  bstr_len(tx_ud->request_uri_normalized)) != 0)
5192  {
5193  printf("normalized uri \"");
5194  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5195  printf("\" != \"");
5196  PrintRawUriFp(stdout, ref1, reflen);
5197  printf("\": ");
5198  goto end;
5199  }
5200  }
5201 
5202  uint8_t ref2[] = "/abc/def?ghi/jkl";
5203  reflen = sizeof(ref2) - 1;
5204 
5205  tx = HTPStateGetTx(htp_state, 1);
5206  if (tx == NULL)
5207  goto end;
5208  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5209  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5210  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5211  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5212  (uintmax_t)reflen,
5213  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5214  goto end;
5215  }
5216 
5217  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5218  bstr_len(tx_ud->request_uri_normalized)) != 0)
5219  {
5220  printf("normalized uri \"");
5221  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5222  printf("\" != \"");
5223  PrintRawUriFp(stdout, ref2, reflen);
5224  printf("\": ");
5225  goto end;
5226  }
5227  }
5228 
5229  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5230  reflen = sizeof(ref3) - 1;
5231  tx = HTPStateGetTx(htp_state, 2);
5232  if (tx == NULL)
5233  goto end;
5234  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5235  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5236  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5237  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5238  (uintmax_t)reflen,
5239  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5240  goto end;
5241  }
5242 
5243  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5244  bstr_len(tx_ud->request_uri_normalized)) != 0)
5245  {
5246  printf("normalized uri \"");
5247  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5248  printf("\" != \"");
5249  PrintRawUriFp(stdout, ref3, reflen);
5250  printf("\": ");
5251  goto end;
5252  }
5253  }
5254 
5255  result = 1;
5256 
5257 end:
5258  if (alp_tctx != NULL)
5260  HTPFreeConfig();
5261  ConfDeInit();
5264 
5265  StreamTcpFreeConfig(true);
5266  UTHFreeFlow(f);
5267  return result;
5268 }
5269 
5270 /** \test Test %2f decoding in profile IDS with double-decode-* options
5271  *
5272  * %252f in path decoded to /
5273  * %252f in query string is decoded to /
5274  */
5275 static int HTPParserDecodingTest03(void)
5276 {
5277  int result = 0;
5278  Flow *f = NULL;
5279  uint8_t httpbuf1[] =
5280  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5281  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5282  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5283  TcpSession ssn;
5285 
5286  HtpState *htp_state = NULL;
5287  int r = 0;
5288  char input[] = "\
5289 %YAML 1.1\n\
5290 ---\n\
5291 libhtp:\n\
5292 \n\
5293  default-config:\n\
5294  personality: IDS\n\
5295  double-decode-path: yes\n\
5296  double-decode-query: yes\n\
5297 ";
5298 
5300  ConfInit();
5302  ConfYamlLoadString(input, strlen(input));
5303  HTPConfigure();
5304  const char *addr = "4.3.2.1";
5305  memset(&ssn, 0, sizeof(ssn));
5306 
5307  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5308  if (f == NULL)
5309  goto end;
5310  f->protoctx = &ssn;
5311  f->proto = IPPROTO_TCP;
5312  f->alproto = ALPROTO_HTTP1;
5313 
5314  StreamTcpInitConfig(true);
5315 
5316  uint32_t u;
5317  for (u = 0; u < httplen1; u++) {
5318  uint8_t flags = 0;
5319 
5320  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5321  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5322  else flags = STREAM_TOSERVER;
5323 
5324  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5325  if (r != 0) {
5326  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5327  " 0: ", u, r);
5328  goto end;
5329  }
5330  }
5331 
5332  htp_state = f->alstate;
5333  if (htp_state == NULL) {
5334  printf("no http state: ");
5335  goto end;
5336  }
5337 
5338  uint8_t ref1[] = "/abc/def";
5339  size_t reflen = sizeof(ref1) - 1;
5340 
5341  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5342  if (tx == NULL)
5343  goto end;
5344  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5345  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5346  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5347  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5348  (uintmax_t)reflen,
5349  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5350  goto end;
5351  }
5352 
5353  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5354  bstr_len(tx_ud->request_uri_normalized)) != 0)
5355  {
5356  printf("normalized uri \"");
5357  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5358  printf("\" != \"");
5359  PrintRawUriFp(stdout, ref1, reflen);
5360  printf("\": ");
5361  goto end;
5362  }
5363  }
5364 
5365  uint8_t ref2[] = "/abc/def?ghi/jkl";
5366  reflen = sizeof(ref2) - 1;
5367 
5368  tx = HTPStateGetTx(htp_state, 1);
5369  if (tx == NULL)
5370  goto end;
5371  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5372  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5373  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5374  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5375  (uintmax_t)reflen,
5376  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5377  goto end;
5378  }
5379 
5380  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5381  bstr_len(tx_ud->request_uri_normalized)) != 0)
5382  {
5383  printf("normalized uri \"");
5384  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5385  printf("\" != \"");
5386  PrintRawUriFp(stdout, ref2, reflen);
5387  printf("\": ");
5388  goto end;
5389  }
5390  }
5391 
5392  result = 1;
5393 
5394 end:
5395  if (alp_tctx != NULL)
5397  HTPFreeConfig();
5398  ConfDeInit();
5401 
5402  StreamTcpFreeConfig(true);
5403  UTHFreeFlow(f);
5404  return result;
5405 }
5406 
5407 /** \test Test http:// in query profile IDS
5408  */
5409 static int HTPParserDecodingTest04(void)
5410 {
5411  int result = 0;
5412  Flow *f = NULL;
5413  uint8_t httpbuf1[] =
5414  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5415  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5416  TcpSession ssn;
5418 
5419  HtpState *htp_state = NULL;
5420  int r = 0;
5421  char input[] = "\
5422 %YAML 1.1\n\
5423 ---\n\
5424 libhtp:\n\
5425 \n\
5426  default-config:\n\
5427  personality: IDS\n\
5428  double-decode-path: yes\n\
5429  double-decode-query: yes\n\
5430 ";
5431 
5433  ConfInit();
5435  ConfYamlLoadString(input, strlen(input));
5436  HTPConfigure();
5437  const char *addr = "4.3.2.1";
5438  memset(&ssn, 0, sizeof(ssn));
5439 
5440  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5441  if (f == NULL)
5442  goto end;
5443  f->protoctx = &ssn;
5444  f->proto = IPPROTO_TCP;
5445  f->alproto = ALPROTO_HTTP1;
5446 
5447  StreamTcpInitConfig(true);
5448 
5449  uint32_t u;
5450  for (u = 0; u < httplen1; u++) {
5451  uint8_t flags = 0;
5452 
5453  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5454  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5455  else flags = STREAM_TOSERVER;
5456 
5457  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5458  if (r != 0) {
5459  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5460  " 0: ", u, r);
5461  goto end;
5462  }
5463  }
5464 
5465  htp_state = f->alstate;
5466  if (htp_state == NULL) {
5467  printf("no http state: ");
5468  goto end;
5469  }
5470 
5471  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5472  size_t reflen = sizeof(ref1) - 1;
5473 
5474  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5475  if (tx == NULL)
5476  goto end;
5477  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5478  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5479  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5480  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5481  (uintmax_t)reflen,
5482  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5483  goto end;
5484  }
5485 
5486  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5487  bstr_len(tx_ud->request_uri_normalized)) != 0)
5488  {
5489  printf("normalized uri \"");
5490  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5491  printf("\" != \"");
5492  PrintRawUriFp(stdout, ref1, reflen);
5493  printf("\": ");
5494  goto end;
5495  }
5496  }
5497 
5498  result = 1;
5499 
5500 end:
5501  if (alp_tctx != NULL)
5503  HTPFreeConfig();
5504  ConfDeInit();
5507 
5508  StreamTcpFreeConfig(true);
5509  UTHFreeFlow(f);
5510  return result;
5511 }
5512 
5513 /** \test Test \ char in query profile IDS. Bug 739
5514  */
5515 static int HTPParserDecodingTest05(void)
5516 {
5517  int result = 0;
5518  Flow *f = NULL;
5519  uint8_t httpbuf1[] =
5520  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5521  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5522  TcpSession ssn;
5524 
5525  HtpState *htp_state = NULL;
5526  int r = 0;
5527  char input[] = "\
5528 %YAML 1.1\n\
5529 ---\n\
5530 libhtp:\n\
5531 \n\
5532  default-config:\n\
5533  personality: IDS\n\
5534  double-decode-path: yes\n\
5535  double-decode-query: yes\n\
5536 ";
5537 
5539  ConfInit();
5541  ConfYamlLoadString(input, strlen(input));
5542  HTPConfigure();
5543  const char *addr = "4.3.2.1";
5544  memset(&ssn, 0, sizeof(ssn));
5545 
5546  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5547  if (f == NULL)
5548  goto end;
5549  f->protoctx = &ssn;
5550  f->proto = IPPROTO_TCP;
5551  f->alproto = ALPROTO_HTTP1;
5552 
5553  StreamTcpInitConfig(true);
5554 
5555  uint32_t u;
5556  for (u = 0; u < httplen1; u++) {
5557  uint8_t flags = 0;
5558 
5559  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5560  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5561  else flags = STREAM_TOSERVER;
5562 
5563  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5564  if (r != 0) {
5565  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5566  " 0: ", u, r);
5567  goto end;
5568  }
5569  }
5570 
5571  htp_state = f->alstate;
5572  if (htp_state == NULL) {
5573  printf("no http state: ");
5574  goto end;
5575  }
5576 
5577  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5578  size_t reflen = sizeof(ref1) - 1;
5579 
5580  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5581  if (tx == NULL)
5582  goto end;
5583  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5584  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5585  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5586  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5587  (uintmax_t)reflen,
5588  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5589  goto end;
5590  }
5591 
5592  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5593  bstr_len(tx_ud->request_uri_normalized)) != 0)
5594  {
5595  printf("normalized uri \"");
5596  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5597  printf("\" != \"");
5598  PrintRawUriFp(stdout, ref1, reflen);
5599  printf("\": ");
5600  goto end;
5601  }
5602  }
5603 
5604  result = 1;
5605 
5606 end:
5607  if (alp_tctx != NULL)
5609  HTPFreeConfig();
5610  ConfDeInit();
5613 
5614  StreamTcpFreeConfig(true);
5615  UTHFreeFlow(f);
5616  return result;
5617 }
5618 
5619 /** \test Test + char in query. Bug 1035
5620  */
5621 static int HTPParserDecodingTest06(void)
5622 {
5623  int result = 0;
5624  Flow *f = NULL;
5625  uint8_t httpbuf1[] =
5626  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5627  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5628  TcpSession ssn;
5630 
5631  HtpState *htp_state = NULL;
5632  int r = 0;
5633  char input[] = "\
5634 %YAML 1.1\n\
5635 ---\n\
5636 libhtp:\n\
5637 \n\
5638  default-config:\n\
5639  personality: IDS\n\
5640  double-decode-path: yes\n\
5641  double-decode-query: yes\n\
5642 ";
5643 
5645  ConfInit();
5647  ConfYamlLoadString(input, strlen(input));
5648  HTPConfigure();
5649  const char *addr = "4.3.2.1";
5650  memset(&ssn, 0, sizeof(ssn));
5651 
5652  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5653  if (f == NULL)
5654  goto end;
5655  f->protoctx = &ssn;
5656  f->proto = IPPROTO_TCP;
5657  f->alproto = ALPROTO_HTTP1;
5658 
5659  StreamTcpInitConfig(true);
5660 
5661  uint32_t u;
5662  for (u = 0; u < httplen1; u++) {
5663  uint8_t flags = 0;
5664 
5665  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5666  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5667  else flags = STREAM_TOSERVER;
5668 
5669  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5670  if (r != 0) {
5671  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5672  " 0: ", u, r);
5673  goto end;
5674  }
5675  }
5676 
5677  htp_state = f->alstate;
5678  if (htp_state == NULL) {
5679  printf("no http state: ");
5680  goto end;
5681  }
5682 
5683  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5684  size_t reflen = sizeof(ref1) - 1;
5685 
5686  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5687  if (tx == NULL)
5688  goto end;
5689  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5690  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5691  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5692  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5693  (uintmax_t)reflen,
5694  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5695  goto end;
5696  }
5697 
5698  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5699  bstr_len(tx_ud->request_uri_normalized)) != 0)
5700  {
5701  printf("normalized uri \"");
5702  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5703  printf("\" != \"");
5704  PrintRawUriFp(stdout, ref1, reflen);
5705  printf("\": ");
5706  goto end;
5707  }
5708  }
5709 
5710  result = 1;
5711 
5712 end:
5713  if (alp_tctx != NULL)
5715  HTPFreeConfig();
5716  ConfDeInit();
5719 
5720  StreamTcpFreeConfig(true);
5721  UTHFreeFlow(f);
5722  return result;
5723 }
5724 
5725 /** \test Test + char in query. Bug 1035
5726  */
5727 static int HTPParserDecodingTest07(void)
5728 {
5729  int result = 0;
5730  Flow *f = NULL;
5731  uint8_t httpbuf1[] =
5732  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5733  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5734  TcpSession ssn;
5736 
5737  HtpState *htp_state = NULL;
5738  int r = 0;
5739  char input[] = "\
5740 %YAML 1.1\n\
5741 ---\n\
5742 libhtp:\n\
5743 \n\
5744  default-config:\n\
5745  personality: IDS\n\
5746  double-decode-path: yes\n\
5747  double-decode-query: yes\n\
5748  query-plusspace-decode: yes\n\
5749 ";
5750 
5752  ConfInit();
5754  ConfYamlLoadString(input, strlen(input));
5755  HTPConfigure();
5756  const char *addr = "4.3.2.1";
5757  memset(&ssn, 0, sizeof(ssn));
5758 
5759  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5760  if (f == NULL)
5761  goto end;
5762  f->protoctx = &ssn;
5763  f->proto = IPPROTO_TCP;
5764  f->alproto = ALPROTO_HTTP1;
5765 
5766  StreamTcpInitConfig(true);
5767 
5768  uint32_t u;
5769  for (u = 0; u < httplen1; u++) {
5770  uint8_t flags = 0;
5771 
5772  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5773  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5774  else flags = STREAM_TOSERVER;
5775 
5776  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5777  if (r != 0) {
5778  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5779  " 0: ", u, r);
5780  goto end;
5781  }
5782  }
5783 
5784  htp_state = f->alstate;
5785  if (htp_state == NULL) {
5786  printf("no http state: ");
5787  goto end;
5788  }
5789 
5790  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5791  size_t reflen = sizeof(ref1) - 1;
5792 
5793  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5794  if (tx == NULL)
5795  goto end;
5796  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5797  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5798  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5799  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5800  (uintmax_t)reflen,
5801  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5802  goto end;
5803  }
5804 
5805  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5806  bstr_len(tx_ud->request_uri_normalized)) != 0)
5807  {
5808  printf("normalized uri \"");
5809  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5810  printf("\" != \"");
5811  PrintRawUriFp(stdout, ref1, reflen);
5812  printf("\": ");
5813  goto end;
5814  }
5815  }
5816 
5817  result = 1;
5818 
5819 end:
5820  if (alp_tctx != NULL)
5822  HTPFreeConfig();
5823  ConfDeInit();
5826 
5827  StreamTcpFreeConfig(true);
5828  UTHFreeFlow(f);
5829  return result;
5830 }
5831 
5832 /** \test Test 'proxy' URI normalization. Ticket 1008
5833  */
5834 static int HTPParserDecodingTest08(void)