suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "decode.h"
40 #include "threads.h"
41 #include "counters.h"
42 
43 #include "util-print.h"
44 #include "util-pool.h"
45 #include "util-radix-tree.h"
46 #include "util-file.h"
47 #include "util-byte.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-detect-proto.h"
59 #include "app-layer-frames.h"
60 #include "app-layer-htp.h"
61 #include "app-layer-htp-body.h"
62 #include "app-layer-htp-file.h"
63 #include "app-layer-htp-libhtp.h"
64 #include "app-layer-htp-xff.h"
65 #include "app-layer-htp-range.h"
66 #include "app-layer-htp-mem.h"
67 
68 #include "util-spm.h"
69 #include "util-debug.h"
70 #include "util-time.h"
71 #include "util-misc.h"
72 #include "util-enum.h"
73 
74 #include "util-unittest.h"
75 #include "util-unittest-helper.h"
76 #include "flow-util.h"
77 
78 #include "detect-engine.h"
79 #include "detect-engine-build.h"
80 #include "detect-engine-state.h"
81 #include "detect-parse.h"
82 
83 #include "decode-events.h"
84 
85 #include "util-memcmp.h"
86 #include "util-random.h"
87 #include "util-validate.h"
88 
89 //#define PRINT
90 
91 /** Fast lookup tree (radix) for the various HTP configurations */
92 static SCRadixTree *cfgtree;
93 /** List of HTP configurations. */
94 static HTPCfgRec cfglist;
95 
97 
98 /** Limit to the number of libhtp messages that can be handled */
99 #define HTP_MAX_MESSAGES 512
100 
101 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
102 
103 #ifdef DEBUG
104 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
105 static uint64_t htp_state_memuse = 0;
106 static uint64_t htp_state_memcnt = 0;
107 #endif
108 
110  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
111  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
112  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
113  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
114  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
115  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
116  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
118  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
120  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
122  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
124  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
126  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
128  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
129  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
131  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
132  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
133  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
134  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
135  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
136  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
137  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
138  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
139  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
140  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
141  { "FILE_NAME_TOO_LONG", HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG },
142  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
143  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
144  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
146  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
147  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
148  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
149  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
150  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
151  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
152  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
153  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
154  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
155  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
156  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
157  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
158  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
160  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
161  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
162  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
163  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
164 
165  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
166  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
167 
168  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
169  { "REQUEST_CHUNK_EXTENSION", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
170 
171  /* suricata warnings/errors */
172  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
173  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
174  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
175 
176  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
177  { "FAILED_PROTOCOL_CHANGE", HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE },
178 
179  { NULL, -1 },
180 };
181 
182 /* app-layer-frame-documentation tag start: HttpFrameTypes */
186 };
187 
189  {
190  "request",
192  },
193  {
194  "response",
196  },
197  { NULL, -1 },
198 };
199 /* app-layer-frame-documentation tag end: HttpFrameTypes */
200 
201 static int HTTPGetFrameIdByName(const char *frame_name)
202 {
203  int id = SCMapEnumNameToValue(frame_name, http_frame_table);
204  if (id < 0) {
205  return -1;
206  }
207  return id;
208 }
209 
210 static const char *HTTPGetFrameNameById(const uint8_t frame_id)
211 {
212  const char *name = SCMapEnumValueToName(frame_id, http_frame_table);
213  return name;
214 }
215 
216 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
217 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
218 static uint64_t HTPStateGetTxCnt(void *alstate);
219 #ifdef UNITTESTS
220 static void HTPParserRegisterTests(void);
221 #endif
222 
223 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
224 {
225  uint64_t id = HTPStateGetTxCnt(s);
226  BUG_ON(id == 0);
227  return id - 1;
228 }
229 
230 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
231 {
232  return s->transaction_cnt;
233 }
234 
235 #ifdef DEBUG
236 /**
237  * \internal
238  *
239  * \brief Lookup the HTP personality string from the numeric personality.
240  *
241  * \todo This needs to be a libhtp function.
242  */
243 static const char *HTPLookupPersonalityString(int p)
244 {
245 #define CASE_HTP_PERSONALITY_STRING(p) \
246  case HTP_SERVER_ ## p: return #p
247 
248  switch (p) {
249  CASE_HTP_PERSONALITY_STRING(MINIMAL);
250  CASE_HTP_PERSONALITY_STRING(GENERIC);
251  CASE_HTP_PERSONALITY_STRING(IDS);
252  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
253  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
254  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
255  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
256  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
257  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
258  CASE_HTP_PERSONALITY_STRING(APACHE_2);
259  }
260 
261  return NULL;
262 }
263 #endif /* DEBUG */
264 
265 /**
266  * \internal
267  *
268  * \brief Lookup the numeric HTP personality from a string.
269  *
270  * \todo This needs to be a libhtp function.
271  */
272 static int HTPLookupPersonality(const char *str)
273 {
274 #define IF_HTP_PERSONALITY_NUM(p) \
275  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
276 
277  IF_HTP_PERSONALITY_NUM(MINIMAL);
278  IF_HTP_PERSONALITY_NUM(GENERIC);
280  IF_HTP_PERSONALITY_NUM(IIS_4_0);
281  IF_HTP_PERSONALITY_NUM(IIS_5_0);
282  IF_HTP_PERSONALITY_NUM(IIS_5_1);
283  IF_HTP_PERSONALITY_NUM(IIS_6_0);
284  IF_HTP_PERSONALITY_NUM(IIS_7_0);
285  IF_HTP_PERSONALITY_NUM(IIS_7_5);
286  IF_HTP_PERSONALITY_NUM(APACHE_2);
287  if (strcasecmp("TOMCAT_6_0", str) == 0) {
288  SCLogError("Personality %s no "
289  "longer supported by libhtp.",
290  str);
291  return -1;
292  } else if ((strcasecmp("APACHE", str) == 0) ||
293  (strcasecmp("APACHE_2_2", str) == 0))
294  {
295  SCLogWarning("Personality %s no "
296  "longer supported by libhtp, failing back to "
297  "Apache2 personality.",
298  str);
299  return HTP_SERVER_APACHE_2;
300  }
301 
302  return -1;
303 }
304 
305 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
306  const uint8_t dir, const uint8_t e)
307 {
308  SCLogDebug("setting event %u", e);
309 
310  if (htud) {
311  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
312  s->events++;
313  return;
314  }
315 
316  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
317  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
318 
319  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
320  if (tx == NULL && tx_id > 0)
321  tx = HTPStateGetTx(s, tx_id - 1);
322  if (tx != NULL) {
323  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
324  if (htud != NULL) {
325  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
326  s->events++;
327  return;
328  }
329  }
330  SCLogDebug("couldn't set event %u", e);
331 }
332 
333 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
334  * connection parser to be used by the HTP library
335  */
336 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
337 {
338  SCEnter();
339 
340  HtpState *s = HTPMalloc(sizeof(HtpState));
341  if (unlikely(s == NULL)) {
342  SCReturnPtr(NULL, "void");
343  }
344 
345  memset(s, 0x00, sizeof(HtpState));
346 
347 #ifdef DEBUG
348  SCMutexLock(&htp_state_mem_lock);
349  htp_state_memcnt++;
350  htp_state_memuse += sizeof(HtpState);
351  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
352  SCMutexUnlock(&htp_state_mem_lock);
353 #endif
354 
355  SCReturnPtr((void *)s, "void");
356 }
357 
358 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
359 {
360  if (likely(htud)) {
361  HtpBodyFree(&state->cfg->request, &htud->request_body);
362  HtpBodyFree(&state->cfg->response, &htud->response_body);
363  bstr_free(htud->request_uri_normalized);
364  if (htud->request_headers_raw)
366  if (htud->response_headers_raw)
369  if (htud->boundary)
370  HTPFree(htud->boundary, htud->boundary_len);
371  if (htud->tx_data.de_state != NULL) {
372  DetectEngineStateFree(htud->tx_data.de_state);
373  }
374  if (htud->file_range) {
375  HTPFileCloseHandleRange(&htp_sbcfg, &htud->files_tc, 0, htud->file_range, NULL, 0);
377  }
380  HTPFree(htud, sizeof(HtpTxUserData));
381  }
382 }
383 
384 /** \brief Function to frees the HTTP state memory and also frees the HTTP
385  * connection parser memory which was used by the HTP library
386  */
387 void HTPStateFree(void *state)
388 {
389  SCEnter();
390 
391  HtpState *s = (HtpState *)state;
392  if (s == NULL) {
393  SCReturn;
394  }
395 
396  /* free the connection parser memory used by HTP library */
397  if (s->connp != NULL) {
398  SCLogDebug("freeing HTP state");
399 
400  uint64_t tx_id;
401  uint64_t total_txs = HTPStateGetTxCnt(state);
402  /* free the list of body chunks */
403  if (s->conn != NULL) {
404  for (tx_id = s->tx_freed; tx_id < total_txs; tx_id++) {
405  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
406  if (tx != NULL) {
407  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
408  HtpTxUserDataFree(s, htud);
409  htp_tx_set_user_data(tx, NULL);
410  }
411  }
412  }
413  htp_connp_destroy_all(s->connp);
414  }
415 
416  HTPFree(s, sizeof(HtpState));
417 
418 #ifdef DEBUG
419  SCMutexLock(&htp_state_mem_lock);
420  htp_state_memcnt--;
421  htp_state_memuse -= sizeof(HtpState);
422  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
423  SCMutexUnlock(&htp_state_mem_lock);
424 #endif
425 
426  SCReturn;
427 }
428 
429 /**
430  * \brief HTP transaction cleanup callback
431  *
432  * \warning We cannot actually free the transactions here. It seems that
433  * HTP only accepts freeing of transactions in the response callback.
434  */
435 static void HTPStateTransactionFree(void *state, uint64_t id)
436 {
437  SCEnter();
438 
439  HtpState *s = (HtpState *)state;
440 
441  SCLogDebug("state %p, id %"PRIu64, s, id);
442 
443  htp_tx_t *tx = HTPStateGetTx(s, id);
444  if (tx != NULL) {
445  /* This will remove obsolete body chunks */
446  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
447  HtpTxUserDataFree(s, htud);
448  htp_tx_set_user_data(tx, NULL);
449 
450  /* hack: even if libhtp considers the tx incomplete, we want to
451  * free it here. htp_tx_destroy however, will refuse to do this.
452  * As htp_tx_destroy_incomplete isn't available in the public API,
453  * we hack around it here. */
454  if (unlikely(!(
455  tx->request_progress == HTP_REQUEST_COMPLETE &&
456  tx->response_progress == HTP_RESPONSE_COMPLETE)))
457  {
458  tx->request_progress = HTP_REQUEST_COMPLETE;
459  tx->response_progress = HTP_RESPONSE_COMPLETE;
460  }
461  // replaces tx in the s->conn->transactions list by NULL
462  htp_tx_destroy(tx);
463  }
464  s->tx_freed += htp_connp_tx_freed(s->connp);
465 }
466 
467 /**
468  * \brief Sets a flag that informs the HTP app layer that some module in the
469  * engine needs the http request body data.
470  * \initonly
471  */
473 {
474  SCEnter();
475 
476  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
477  SCReturn;
478 }
479 
480 /**
481  * \brief Sets a flag that informs the HTP app layer that some module in the
482  * engine needs the http request body data.
483  * \initonly
484  */
486 {
487  SCEnter();
488 
489  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
490  SCReturn;
491 }
492 
493 /**
494  * \brief Sets a flag that informs the HTP app layer that some module in the
495  * engine needs the http request multi part header.
496  *
497  * \initonly
498  */
499 static void AppLayerHtpNeedMultipartHeader(void)
500 {
501  SCEnter();
503 
504  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
505  SCReturn;
506 }
507 
508 /**
509  * \brief Sets a flag that informs the HTP app layer that some module in the
510  * engine needs the http request file.
511  *
512  * \initonly
513  */
515 {
516  SCEnter();
517  AppLayerHtpNeedMultipartHeader();
520 
521  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
522  SCReturn;
523 }
524 
525 static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags)
526 {
527  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
528  if (tx_ud) {
529  SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags);
530  if (flags & STREAM_TOCLIENT) {
531  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
532  } else {
533  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
534  }
535  }
536 }
537 
538 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
539 {
540  SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64,
542  if (flags & HTP_STREAM_DEPTH_SET) {
543  uint32_t stream_depth = FileReassemblyDepth();
544  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
545  SCLogDebug("true");
546  return true;
547  }
548  } else {
549  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
550  return true;
551  }
552  }
553  SCLogDebug("false");
554  return false;
555 }
556 
557 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
558  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
559 {
560  uint32_t chunk_len = 0;
561  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
562  (content_len_so_far < (uint64_t)body_limit) &&
563  (content_len_so_far + (uint64_t)data_len) > body_limit)
564  {
565  chunk_len = body_limit - content_len_so_far;
566  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
567  (content_len_so_far < (uint64_t)stream_depth) &&
568  (content_len_so_far + (uint64_t)data_len) > stream_depth)
569  {
570  chunk_len = stream_depth - content_len_so_far;
571  }
572  SCLogDebug("len %u", chunk_len);
573  return (chunk_len == 0 ? data_len : chunk_len);
574 }
575 
576 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
577 struct {
578  const char *msg;
579  uint8_t de;
580 } htp_errors[] = {
581  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
582  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
583  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
584  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
585  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
586 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
587  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
588 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
589  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
590  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
591  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
592  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
593 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
594  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
595  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
596  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
597  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
598 };
599 
600 struct {
601  const char *msg;
602  uint8_t de;
603 } htp_warnings[] = {
604  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
605  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
606  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
607  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
608  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
609  /* { "Host information in request headers required by HTTP/1.1",
610  HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING { "Host information
611  ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
612  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
613  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
614  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number
615  * differs from the actual TCP port=%d", port, connp->conn->server_port); luckily, "Request
616  * server port=" is unique */
617  /* { "Request server port number differs from the actual TCP port",
618  HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
620  { "Request line: URI contains non-compliant delimiter",
622  { "Request line: non-compliant delimiter between Method and URI",
624  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
625  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
626  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
627  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
628  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
629  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
630  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
631  { "Transfer-encoding has abnormal chunked value",
633  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0",
635  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
636  { "Invalid response line: invalid response status",
638  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
639  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
640  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
641  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST },
642  { "Ambiguous response C-L value",
644  { "Request chunk extension", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
645 };
646 
647 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
648 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
649 
650 /**
651  * \internal
652  *
653  * \brief Get the warning id for the warning msg.
654  *
655  * \param msg warning message
656  *
657  * \retval id the id or 0 in case of not found
658  */
659 static uint8_t HTPHandleWarningGetId(const char *msg)
660 {
661  SCLogDebug("received warning \"%s\"", msg);
662  size_t idx;
663  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
664  if (strncmp(htp_warnings[idx].msg, msg,
665  strlen(htp_warnings[idx].msg)) == 0)
666  {
667  return htp_warnings[idx].de;
668  }
669  }
670 
671  return 0;
672 }
673 
674 /**
675  * \internal
676  *
677  * \brief Get the error id for the error msg.
678  *
679  * \param msg error message
680  *
681  * \retval id the id or 0 in case of not found
682  */
683 static uint8_t HTPHandleErrorGetId(const char *msg)
684 {
685  SCLogDebug("received error \"%s\"", msg);
686 
687  size_t idx;
688  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
689  if (strncmp(htp_errors[idx].msg, msg,
690  strlen(htp_errors[idx].msg)) == 0)
691  {
692  return htp_errors[idx].de;
693  }
694  }
695 
696  return 0;
697 }
698 
699 /**
700  * \internal
701  *
702  * \brief Check state for errors, warnings and add any as events
703  *
704  * \param s state
705  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
706  */
707 static void HTPHandleError(HtpState *s, const uint8_t dir)
708 {
709  if (s == NULL || s->conn == NULL ||
710  s->conn->messages == NULL) {
711  return;
712  }
713 
714  size_t size = htp_list_size(s->conn->messages);
715  size_t msg;
716  if(size >= HTP_MAX_MESSAGES) {
718  //only once per HtpState
719  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
721  //too noisy in fuzzing
722  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
723  }
724  // ignore further messages
725  return;
726  }
727 
728  for (msg = s->htp_messages_offset; msg < size; msg++) {
729  htp_log_t *log = htp_list_get(s->conn->messages, msg);
730  if (log == NULL)
731  continue;
732 
733  HtpTxUserData *htud = NULL;
734  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
735  if (tx != NULL)
736  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
737 
738  SCLogDebug("message %s", log->msg);
739 
740  uint8_t id = HTPHandleErrorGetId(log->msg);
741  if (id == 0) {
742  id = HTPHandleWarningGetId(log->msg);
743  if (id == 0)
745  }
746 
747  if (id > 0) {
748  HTPSetEvent(s, htud, dir, id);
749  }
750  }
751  s->htp_messages_offset = (uint16_t)msg;
752  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
753 }
754 
755 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
756 {
757 #ifdef DEBUG
758  BUG_ON(s == NULL || tx == NULL);
759 #endif
760  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
761  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
762  HTP_HOSTH_INVALID))
763  {
764  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
765  if (htud == NULL)
766  return;
767 
768  if (tx->flags & HTP_REQUEST_INVALID_T_E)
769  HTPSetEvent(s, htud, STREAM_TOSERVER,
771  if (tx->flags & HTP_REQUEST_INVALID_C_L)
772  HTPSetEvent(s, htud, STREAM_TOSERVER,
774  if (tx->flags & HTP_HOST_MISSING)
775  HTPSetEvent(s, htud, STREAM_TOSERVER,
777  if (tx->flags & HTP_HOST_AMBIGUOUS)
778  HTPSetEvent(s, htud, STREAM_TOSERVER,
780  if (tx->flags & HTP_HOSTU_INVALID)
781  HTPSetEvent(s, htud, STREAM_TOSERVER,
783  if (tx->flags & HTP_HOSTH_INVALID)
784  HTPSetEvent(s, htud, STREAM_TOSERVER,
786  }
787  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
788  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
789  if (htud == NULL)
790  return;
791  HTPSetEvent(s, htud, STREAM_TOSERVER,
793  }
794  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
795  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
796  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
797  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
798  if (htud == NULL)
799  return;
800  HTPSetEvent(s, htud, STREAM_TOSERVER,
802  }
803 }
804 
805 static int Setup(Flow *f, HtpState *hstate)
806 {
807  /* store flow ref in state so callbacks can access it */
808  hstate->f = f;
809 
810  HTPCfgRec *htp_cfg_rec = &cfglist;
811  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
812  void *user_data = NULL;
813 
814  if (FLOW_IS_IPV4(f)) {
815  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
816  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
817  }
818  else if (FLOW_IS_IPV6(f)) {
819  SCLogDebug("Looking up HTP config for ipv6");
820  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
821  }
822  else {
823  SCLogError("unknown address family, bug!");
824  goto error;
825  }
826 
827  if (user_data != NULL) {
828  htp_cfg_rec = user_data;
829  htp = htp_cfg_rec->cfg;
830  SCLogDebug("LIBHTP using config: %p", htp);
831  } else {
832  SCLogDebug("Using default HTP config: %p", htp);
833  }
834 
835  if (NULL == htp) {
836 #ifdef DEBUG_VALIDATION
837  BUG_ON(htp == NULL);
838 #endif
839  /* should never happen if HTPConfigure is properly invoked */
840  goto error;
841  }
842 
843  hstate->connp = htp_connp_create(htp);
844  if (hstate->connp == NULL) {
845  goto error;
846  }
847 
848  hstate->conn = htp_connp_get_connection(hstate->connp);
849 
850  htp_connp_set_user_data(hstate->connp, (void *)hstate);
851  hstate->cfg = htp_cfg_rec;
852 
853  SCLogDebug("New hstate->connp %p", hstate->connp);
854 
855  struct timeval tv = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
856  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &tv);
857 
859  htp_cfg_rec->request.inspect_min_size);
861  htp_cfg_rec->response.inspect_min_size);
862  return 0;
863 error:
864  return -1;
865 }
866 
867 /**
868  * \brief Function to handle the reassembled data from client and feed it to
869  * the HTP library to process it.
870  *
871  * \param flow Pointer to the flow the data belong to
872  * \param htp_state Pointer the state in which the parsed value to be stored
873  * \param pstate Application layer parser state for this session
874  *
875  * \retval On success returns 1 or on failure returns -1.
876  */
877 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state, AppLayerParserState *pstate,
878  StreamSlice stream_slice, void *local_data)
879 {
880  SCEnter();
881  int ret = 0;
882  HtpState *hstate = (HtpState *)htp_state;
883 
884  /* On the first invocation, create the connection parser structure to
885  * be used by HTP library. This is looked up via IP in the radix
886  * tree. Failing that, the default HTP config is used.
887  */
888  if (NULL == hstate->conn) {
889  if (Setup(f, hstate) != 0) {
891  }
892  }
893  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
894  hstate->slice = &stream_slice;
895 
896  const uint8_t *input = StreamSliceGetData(&stream_slice);
897  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
898 
899  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
900  /* pass the new data to the htp parser */
901  if (input_len > 0) {
902  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
903  switch (r) {
904  case HTP_STREAM_ERROR:
905  ret = -1;
906  break;
907  default:
908  break;
909  }
910  HTPHandleError(hstate, STREAM_TOSERVER);
911  }
912 
913  /* if the TCP connection is closed, then close the HTTP connection */
915  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
916  {
917  htp_connp_req_close(hstate->connp, &ts);
918  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
919  SCLogDebug("stream eof encountered, closing htp handle for ts");
920  }
921 
922  SCLogDebug("hstate->connp %p", hstate->connp);
923  hstate->slice = NULL;
924 
925  if (ret < 0) {
927  }
929 }
930 
931 /**
932  * \brief Function to handle the reassembled data from server and feed it to
933  * the HTP library to process it.
934  *
935  * \param flow Pointer to the flow the data belong to
936  * \param htp_state Pointer the state in which the parsed value to be stored
937  * \param pstate Application layer parser state for this session
938  * \param input Pointer the received HTTP server data
939  * \param input_len Length in bytes of the received data
940  * \param output Pointer to the output (not used in this function)
941  *
942  * \retval On success returns 1 or on failure returns -1
943  */
944 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state, AppLayerParserState *pstate,
945  StreamSlice stream_slice, void *local_data)
946 {
947  SCEnter();
948  int ret = 0;
949  HtpState *hstate = (HtpState *)htp_state;
950 
951  const uint8_t *input = StreamSliceGetData(&stream_slice);
952  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
953 
954  /* On the first invocation, create the connection parser structure to
955  * be used by HTP library. This is looked up via IP in the radix
956  * tree. Failing that, the default HTP config is used.
957  */
958  if (NULL == hstate->conn) {
959  if (Setup(f, hstate) != 0) {
961  }
962  }
963  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
964  hstate->slice = &stream_slice;
965 
966  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
967  htp_tx_t *tx = NULL;
968  size_t consumed = 0;
969  if (input_len > 0) {
970  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
971  switch (r) {
972  case HTP_STREAM_ERROR:
973  ret = -1;
974  break;
975  case HTP_STREAM_TUNNEL:
976  tx = htp_connp_get_out_tx(hstate->connp);
977  if (tx != NULL && tx->response_status_number == 101) {
978  htp_header_t *h =
979  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
980  if (h == NULL || bstr_cmp_c(h->value, "h2c") != 0) {
981  break;
982  }
984  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
985  break;
986  }
987  uint16_t dp = 0;
988  if (tx->request_port_number != -1) {
989  dp = (uint16_t)tx->request_port_number;
990  }
991  consumed = htp_connp_res_data_consumed(hstate->connp);
992  hstate->slice = NULL;
993  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_HTTP2)) {
994  HTPSetEvent(hstate, NULL, STREAM_TOCLIENT,
996  }
997  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
998  // and we need to parser the remaining part with HTTP2
999  if (consumed > 0 && consumed < input_len) {
1000  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
1001  }
1003  }
1004  break;
1005  default:
1006  break;
1007  }
1008  HTPHandleError(hstate, STREAM_TOCLIENT);
1009  }
1010 
1011  /* if we the TCP connection is closed, then close the HTTP connection */
1013  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
1014  {
1015  htp_connp_close(hstate->connp, &ts);
1016  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
1017  }
1018 
1019  SCLogDebug("hstate->connp %p", hstate->connp);
1020  hstate->slice = NULL;
1021 
1022  if (ret < 0) {
1024  }
1026 }
1027 
1028 /**
1029  * \param name /Lowercase/ version of the variable name
1030  */
1031 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1032  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1033 {
1034 #ifdef PRINT
1035  printf("DATA START: \n");
1036  PrintRawDataFp(stdout, data, len);
1037  printf("DATA END: \n");
1038 #endif
1039  size_t x;
1040  int quote = 0;
1041 
1042  for (x = 0; x < len; x++) {
1043  if (!(isspace(data[x])))
1044  break;
1045  }
1046 
1047  if (x >= len)
1048  return 0;
1049 
1050  uint8_t *line = data+x;
1051  size_t line_len = len-x;
1052  size_t offset = 0;
1053 #ifdef PRINT
1054  printf("LINE START: \n");
1055  PrintRawDataFp(stdout, line, line_len);
1056  printf("LINE END: \n");
1057 #endif
1058  for (x = 0 ; x < line_len; x++) {
1059  if (x > 0) {
1060  if (line[x - 1] != '\\' && line[x] == '\"') {
1061  quote++;
1062  }
1063 
1064  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1065  uint8_t *token = line + offset;
1066  size_t token_len = x - offset;
1067 
1068  if ((x + 1) == line_len) {
1069  token_len++;
1070  }
1071 
1072  offset = x + 1;
1073 
1074  while (offset < line_len && isspace(line[offset])) {
1075  x++;
1076  offset++;
1077  }
1078 #ifdef PRINT
1079  printf("TOKEN START: \n");
1080  PrintRawDataFp(stdout, token, token_len);
1081  printf("TOKEN END: \n");
1082 #endif
1083  if (token_len > name_len) {
1084  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1085  uint8_t *value = token + name_len;
1086  size_t value_len = token_len - name_len;
1087 
1088  if (value[0] == '\"') {
1089  value++;
1090  value_len--;
1091  }
1092  if (value[value_len-1] == '\"') {
1093  value_len--;
1094  }
1095 #ifdef PRINT
1096  printf("VALUE START: \n");
1097  PrintRawDataFp(stdout, value, value_len);
1098  printf("VALUE END: \n");
1099 #endif
1100  *retptr = value;
1101  *retlen = value_len;
1102  return 1;
1103  }
1104  }
1105  }
1106  }
1107  }
1108 
1109  return 0;
1110 }
1111 
1112 /**
1113  * \param name /Lowercase/ version of the variable name
1114  */
1115 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1116  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1117 {
1118  SCEnter();
1119 #ifdef PRINT
1120  printf("DATA START: \n");
1121  PrintRawDataFp(stdout, data, len);
1122  printf("DATA END: \n");
1123 #endif
1124  size_t x;
1125  int quote = 0;
1126 
1127  for (x = 0; x < len; x++) {
1128  if (!(isspace(data[x])))
1129  break;
1130  }
1131 
1132  if (x >= len) {
1133  SCReturnInt(0);
1134  }
1135 
1136  uint8_t *line = data+x;
1137  size_t line_len = len-x;
1138  size_t offset = 0;
1139 #ifdef PRINT
1140  printf("LINE START: \n");
1141  PrintRawDataFp(stdout, line, line_len);
1142  printf("LINE END: \n");
1143 #endif
1144  for (x = 0 ; x < line_len; x++) {
1145  if (x > 0) {
1146  if (line[x - 1] != '\\' && line[x] == '\"') {
1147  quote++;
1148  }
1149 
1150  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1151  uint8_t *token = line + offset;
1152  size_t token_len = x - offset;
1153 
1154  if ((x + 1) == line_len) {
1155  token_len++;
1156  }
1157 
1158  offset = x + 1;
1159 
1160  while (offset < line_len && isspace(line[offset])) {
1161  x++;
1162  offset++;
1163  }
1164 #ifdef PRINT
1165  printf("TOKEN START: \n");
1166  PrintRawDataFp(stdout, token, token_len);
1167  printf("TOKEN END: \n");
1168 #endif
1169  if (token_len > name_len) {
1170  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1171  uint8_t *value = token + name_len;
1172  size_t value_len = token_len - name_len;
1173 
1174  if (value[0] == '\"') {
1175  value++;
1176  value_len--;
1177  }
1178  if (value[value_len-1] == '\"') {
1179  value_len--;
1180  }
1181 #ifdef PRINT
1182  printf("VALUE START: \n");
1183  PrintRawDataFp(stdout, value, value_len);
1184  printf("VALUE END: \n");
1185 #endif
1186  *retptr = value;
1187  *retlen = value_len;
1188  SCReturnInt(1);
1189  }
1190  }
1191  }
1192  }
1193  }
1194 
1195  SCReturnInt(0);
1196 }
1197 
1198 /**
1199  * \brief setup multipart parsing: extract boundary and store it
1200  *
1201  * \param d HTTP transaction
1202  * \param htud transaction userdata
1203  *
1204  * \retval 1 ok, multipart set up
1205  * \retval 0 ok, not multipart though
1206  * \retval -1 error: problem with the boundary
1207  *
1208  * If the request contains a multipart message, this function will
1209  * set the HTP_BOUNDARY_SET in the transaction.
1210  */
1211 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1212 {
1213  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1214  "Content-Type");
1215  if (h != NULL && bstr_len(h->value) > 0) {
1216  uint8_t *boundary = NULL;
1217  size_t boundary_len = 0;
1218 
1219  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1220  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1221  &boundary, &boundary_len);
1222  if (r == 1) {
1223 #ifdef PRINT
1224  printf("BOUNDARY START: \n");
1225  PrintRawDataFp(stdout, boundary, boundary_len);
1226  printf("BOUNDARY END: \n");
1227 #endif
1228  if (boundary_len < HTP_BOUNDARY_MAX) {
1229  htud->boundary = HTPMalloc(boundary_len);
1230  if (htud->boundary == NULL) {
1231  return -1;
1232  }
1233  htud->boundary_len = (uint8_t)boundary_len;
1234  memcpy(htud->boundary, boundary, boundary_len);
1235 
1236  htud->tsflags |= HTP_BOUNDARY_SET;
1237  } else {
1238  SCLogDebug("invalid boundary");
1239  return -1;
1240  }
1241  SCReturnInt(1);
1242  }
1243  //SCReturnInt(1);
1244  }
1245  SCReturnInt(0);
1246 }
1247 
1248 #define C_D_HDR "content-disposition:"
1249 #define C_D_HDR_LEN 20
1250 #define C_T_HDR "content-type:"
1251 #define C_T_HDR_LEN 13
1253 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1254  HtpTxUserData *htud,
1255  uint8_t *header, uint32_t header_len,
1256  uint8_t **filename, uint16_t *filename_len,
1257  uint8_t **filetype, uint16_t *filetype_len)
1258 {
1259  uint8_t *fn = NULL;
1260  size_t fn_len = 0;
1261  uint8_t *ft = NULL;
1262  size_t ft_len = 0;
1263 
1264 #ifdef PRINT
1265  printf("HEADER START: \n");
1266  PrintRawDataFp(stdout, header, header_len);
1267  printf("HEADER END: \n");
1268 #endif
1269 
1270  while (header_len > 0) {
1271  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1272  uint8_t *line = header;
1273  uint32_t line_len;
1274 
1275  if (next_line == NULL) {
1276  line_len = header_len;
1277  } else {
1278  line_len = next_line - header;
1279  }
1280  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1281  if (sc == NULL) {
1282  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1284  /* if the : we found is the final char, it means we have
1285  * no value */
1286  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1287  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1289  } else {
1290 #ifdef PRINT
1291  printf("LINE START: \n");
1292  PrintRawDataFp(stdout, line, line_len);
1293  printf("LINE END: \n");
1294 #endif
1295  if (line_len >= C_D_HDR_LEN &&
1296  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1297  uint8_t *value = line + C_D_HDR_LEN;
1298  uint32_t value_len = line_len - C_D_HDR_LEN;
1299 
1300  /* parse content-disposition */
1301  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1302  value, value_len, &fn, &fn_len);
1303  } else if (line_len >= C_T_HDR_LEN &&
1304  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1305  SCLogDebug("content-type line");
1306  uint8_t *value = line + C_T_HDR_LEN;
1307  uint32_t value_len = line_len - C_T_HDR_LEN;
1308 
1309  (void)HTTPParseContentTypeHeader(NULL, 0,
1310  value, value_len, &ft, &ft_len);
1311  }
1312  }
1313 
1314  if (next_line == NULL) {
1315  SCLogDebug("no next_line");
1316  break;
1317  }
1318  header_len -= ((next_line + 2) - header);
1319  header = next_line + 2;
1320  } /* while (header_len > 0) */
1321 
1322  if (fn_len > USHRT_MAX)
1323  fn_len = USHRT_MAX;
1324  if (ft_len > USHRT_MAX)
1325  ft_len = USHRT_MAX;
1326 
1327  *filename = fn;
1328  *filename_len = (uint16_t)fn_len;
1329  *filetype = ft;
1330  *filetype_len = (uint16_t)ft_len;
1331 }
1332 
1333 /**
1334  * \brief Create a single buffer from the HtpBodyChunks in our list
1335  *
1336  * \param htud transaction user data
1337  * \param chunks_buffers pointer to pass back the buffer to the caller
1338  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1339  */
1340 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1341  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1342 {
1344  chunks_buffer, chunks_buffer_len,
1345  htud->request_body.body_parsed);
1346 }
1347 
1348 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1349 {
1350  SCEnter();
1351  if (tx && tx->tx_data.de_state) {
1352  if (dir == STREAM_TOSERVER) {
1353  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1354  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1355  } else if (STREAM_TOCLIENT) {
1356  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1357  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1358  }
1359  }
1360 }
1361 
1362 /**
1363  * \brief Setup boundary buffers
1364  */
1365 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1366  uint8_t *boundary, uint32_t boundary_len)
1367 {
1368  memset(boundary, '-', boundary_len);
1369  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1370 }
1371 
1372 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1373  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1374 {
1375  int result = 0;
1376  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1377  uint16_t expected_boundary_len = htud->boundary_len + 2;
1378  uint16_t expected_boundary_end_len = htud->boundary_len + 4;
1379  int tx_progress = 0;
1380 
1381 #ifdef PRINT
1382  printf("CHUNK START: \n");
1383  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1384  printf("CHUNK END: \n");
1385 #endif
1386 
1387  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1388 
1389  /* search for the header start, header end and form end */
1390  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1391  boundary, expected_boundary_len);
1392  /* end of the multipart form */
1393  const uint8_t *form_end = NULL;
1394  /* end marker belonging to header_start */
1395  const uint8_t *header_end = NULL;
1396  if (header_start != NULL) {
1397  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1398  (uint8_t *)"\r\n\r\n", 4);
1399  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1400  boundary, expected_boundary_end_len);
1401  }
1402 
1403  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1404  header_end, form_end);
1405 
1406  /* we currently only handle multipart for ts. When we support it for tc,
1407  * we will need to supply right direction */
1408  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1409  /* if we're in the file storage process, deal with that now */
1410  if (htud->tsflags & HTP_FILENAME_SET) {
1411  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1412  SCLogDebug("reached the end of the file");
1413 
1414  const uint8_t *filedata = chunks_buffer;
1415  uint32_t filedata_len = 0;
1416  uint8_t flags = 0;
1417 
1418  if (header_start != NULL) {
1419  if (header_start == filedata + 2) {
1420  /* last chunk had all data, but not the boundary */
1421  SCLogDebug("last chunk had all data, but not the boundary");
1422  filedata_len = 0;
1423  } else if (header_start > filedata + 2) {
1424  SCLogDebug("some data from last file before the boundary");
1425  /* some data from last file before the boundary */
1426  filedata_len = header_start - filedata - 2;
1427  }
1428  }
1429  /* body parsing done, we did not get our form end. Use all data
1430  * we still have and signal to files API we have an issue. */
1431  if (tx_progress > HTP_REQUEST_BODY) {
1432  filedata_len = chunks_buffer_len;
1434  }
1435 
1436  if (filedata_len > chunks_buffer_len) {
1437  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1439  goto end;
1440  }
1441 #ifdef PRINT
1442  printf("FILEDATA (final chunk) START: \n");
1443  PrintRawDataFp(stdout, filedata, filedata_len);
1444  printf("FILEDATA (final chunk) END: \n");
1445 #endif
1446  if (!(htud->tsflags & HTP_DONTSTORE)) {
1447  if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
1448  -1) {
1449  goto end;
1450  }
1451  }
1452 
1453  htud->tsflags &=~ HTP_FILENAME_SET;
1454 
1455  /* fall through */
1456  } else {
1457  SCLogDebug("not yet at the end of the file");
1458 
1459  if (chunks_buffer_len > expected_boundary_end_len) {
1460  const uint8_t *filedata = chunks_buffer;
1461  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1462  for (; filedata_len < chunks_buffer_len; filedata_len++) {
1463  // take as much as we can until the beginning of a new line
1464  if (chunks_buffer[filedata_len] == '\r') {
1465  if (filedata_len + 1 == chunks_buffer_len ||
1466  chunks_buffer[filedata_len + 1] == '\n') {
1467  break;
1468  }
1469  }
1470  }
1471 
1472 #ifdef PRINT
1473  printf("FILEDATA (part) START: \n");
1474  PrintRawDataFp(stdout, filedata, filedata_len);
1475  printf("FILEDATA (part) END: \n");
1476 #endif
1477 
1478  if (!(htud->tsflags & HTP_DONTSTORE)) {
1479  result = HTPFileStoreChunk(
1480  hstate, htud, filedata, filedata_len, STREAM_TOSERVER);
1481  if (result == -1) {
1482  goto end;
1483  } else if (result == -2) {
1484  /* we know for sure we're not storing the file */
1485  htud->tsflags |= HTP_DONTSTORE;
1486  }
1487  }
1488 
1489  htud->request_body.body_parsed += filedata_len;
1490  } else {
1491  SCLogDebug("chunk too small to already process in part");
1492  }
1493 
1494  goto end;
1495  }
1496  }
1497 
1498  while (header_start != NULL && header_end != NULL &&
1499  header_end != form_end &&
1500  header_start < (chunks_buffer + chunks_buffer_len) &&
1501  header_end < (chunks_buffer + chunks_buffer_len) &&
1502  header_start < header_end)
1503  {
1504  uint8_t *filename = NULL;
1505  uint16_t filename_len = 0;
1506  uint8_t *filetype = NULL;
1507  uint16_t filetype_len = 0;
1508 
1509  uint32_t header_len = header_end - header_start;
1510  SCLogDebug("header_len %u", header_len);
1511  uint8_t *header = (uint8_t *)header_start;
1512 
1513  /* skip empty records */
1514  if (expected_boundary_len == header_len) {
1515  goto next;
1516  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1517  header_len -= (expected_boundary_len + 2);
1518  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1519  }
1520 
1521  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1522  &filename, &filename_len, &filetype, &filetype_len);
1523 
1524  if (filename != NULL) {
1525  const uint8_t *filedata = NULL;
1526  uint32_t filedata_len = 0;
1527 
1528  SCLogDebug("we have a filename");
1529 
1530  htud->tsflags |= HTP_FILENAME_SET;
1531  htud->tsflags &= ~HTP_DONTSTORE;
1532 
1533  SCLogDebug("header_end %p", header_end);
1534  SCLogDebug("form_end %p", form_end);
1535 
1536  /* everything until the final boundary is the file */
1537  if (form_end != NULL) {
1538  SCLogDebug("have form_end");
1539 
1540  filedata = header_end + 4;
1541  if (form_end == filedata) {
1542  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1544  goto end;
1545  } else if (form_end < filedata) {
1546  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1548  goto end;
1549  }
1550 
1551  filedata_len = form_end - (header_end + 4 + 2);
1552  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1553 
1554  /* or is it? */
1555  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1556  boundary, expected_boundary_len);
1557  if (header_next != NULL) {
1558  filedata_len -= (form_end - header_next);
1559  }
1560 
1561  if (filedata_len > chunks_buffer_len) {
1562  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1564  goto end;
1565  }
1566  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1567 #ifdef PRINT
1568  printf("FILEDATA START: \n");
1569  PrintRawDataFp(stdout, filedata, filedata_len);
1570  printf("FILEDATA END: \n");
1571 #endif
1572 
1573  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata, filedata_len,
1574  STREAM_TOSERVER);
1575  if (result == -1) {
1576  goto end;
1577  } else if (result == -2) {
1578  htud->tsflags |= HTP_DONTSTORE;
1579  } else {
1580  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1581  goto end;
1582  }
1583  }
1584  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1585 
1586  htud->request_body.body_parsed += (header_end - chunks_buffer);
1587  htud->tsflags &= ~HTP_FILENAME_SET;
1588  } else {
1589  SCLogDebug("chunk doesn't contain form end");
1590 
1591  filedata = header_end + 4;
1592  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1593  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1594 
1595  if (filedata_len > chunks_buffer_len) {
1596  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1598  goto end;
1599  }
1600 
1601 #ifdef PRINT
1602  printf("FILEDATA START: \n");
1603  PrintRawDataFp(stdout, filedata, filedata_len);
1604  printf("FILEDATA END: \n");
1605 #endif
1606  /* form doesn't end in this chunk, but the part might. Lets
1607  * see if have another coming up */
1608  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1609  boundary, expected_boundary_len);
1610  SCLogDebug("header_next %p", header_next);
1611  if (header_next == NULL) {
1612  SCLogDebug("more file data to come");
1613 
1614  uint32_t offset = (header_end + 4) - chunks_buffer;
1615  SCLogDebug("offset %u", offset);
1616  htud->request_body.body_parsed += offset;
1617 
1618  if (filedata_len >= (uint32_t)(expected_boundary_len + 2)) {
1619  filedata_len -= (expected_boundary_len + 2 - 1);
1620  // take as much as we can until start of boundary
1621  for (size_t nb = 0; nb < (size_t)expected_boundary_len + 1; nb++) {
1622  if (filedata[filedata_len] == '\r') {
1623  if (nb == expected_boundary_len ||
1624  filedata[filedata_len + 1] == '\n') {
1625  break;
1626  }
1627  }
1628  filedata_len++;
1629  }
1630  SCLogDebug("opening file with partial data");
1631  } else {
1632  filedata = NULL;
1633  filedata_len = 0;
1634  }
1635  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1636  filedata_len, STREAM_TOSERVER);
1637  if (result == -1) {
1638  goto end;
1639  } else if (result == -2) {
1640  htud->tsflags |= HTP_DONTSTORE;
1641  }
1642  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1643  htud->request_body.body_parsed += filedata_len;
1644  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1645 
1646  } else if (header_next - filedata > 2) {
1647  filedata_len = header_next - filedata - 2;
1648  SCLogDebug("filedata_len %u", filedata_len);
1649 
1650  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1651  filedata_len, STREAM_TOSERVER);
1652  if (result == -1) {
1653  goto end;
1654  } else if (result == -2) {
1655  htud->tsflags |= HTP_DONTSTORE;
1656  } else {
1657  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1658  goto end;
1659  }
1660  }
1661  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1662 
1663  htud->tsflags &= ~HTP_FILENAME_SET;
1664  htud->request_body.body_parsed += (header_end - chunks_buffer);
1665  }
1666  }
1667  }
1668 next:
1669  SCLogDebug("header_start %p, header_end %p, form_end %p",
1670  header_start, header_end, form_end);
1671 
1672  /* Search next boundary entry after the start of body */
1673  uint32_t cursizeread = header_end - chunks_buffer;
1674  header_start = Bs2bmSearch(header_end + 4,
1675  chunks_buffer_len - (cursizeread + 4),
1676  boundary, expected_boundary_len);
1677  if (header_start != NULL) {
1678  header_end = Bs2bmSearch(header_end + 4,
1679  chunks_buffer_len - (cursizeread + 4),
1680  (uint8_t *) "\r\n\r\n", 4);
1681  }
1682  }
1683 
1684  /* if we're parsing the multipart and we're not currently processing a
1685  * file, we move the body pointer forward. */
1686  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1687  if (chunks_buffer_len > expected_boundary_end_len) {
1688  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1689 
1690  htud->request_body.body_parsed += move;
1691  SCLogDebug("form not ready, file not set, parsing non-file "
1692  "record: moved %u", move);
1693  }
1694  }
1695 
1696 end:
1697  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1698  return 0;
1699 }
1700 
1701 /** \internal
1702  * \brief Handle POST or PUT, no multipart body data
1703  */
1704 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1705  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1706 {
1707  int result = 0;
1708 
1709  /* see if we need to open the file */
1710  if (!(htud->tsflags & HTP_FILENAME_SET))
1711  {
1712  uint8_t *filename = NULL;
1713  size_t filename_len = 0;
1714 
1715  /* get the name */
1716  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1717  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1718  filename_len = bstr_len(tx->parsed_uri->path);
1719  }
1720 
1721  if (filename != NULL) {
1722  if (filename_len > SC_FILENAME_MAX) {
1723  // explicitly truncate the file name if too long
1724  filename_len = SC_FILENAME_MAX;
1725  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1726  }
1727  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1728  STREAM_TOSERVER);
1729  if (result == -1) {
1730  goto end;
1731  } else if (result == -2) {
1732  htud->tsflags |= HTP_DONTSTORE;
1733  } else {
1734  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1735  htud->tsflags |= HTP_FILENAME_SET;
1736  htud->tsflags &= ~HTP_DONTSTORE;
1737  }
1738  }
1739  }
1740  else
1741  {
1742  /* otherwise, just store the data */
1743 
1744  if (!(htud->tsflags & HTP_DONTSTORE)) {
1745  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOSERVER);
1746  if (result == -1) {
1747  goto end;
1748  } else if (result == -2) {
1749  /* we know for sure we're not storing the file */
1750  htud->tsflags |= HTP_DONTSTORE;
1751  }
1752  }
1753  }
1754 
1755  return 0;
1756 end:
1757  return -1;
1758 }
1759 
1760 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1761  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1762 {
1763  SCEnter();
1764 
1765  int result = 0;
1766 
1767  /* see if we need to open the file
1768  * we check for tx->response_line in case of junk
1769  * interpreted as body before response line
1770  */
1771  if (!(htud->tcflags & HTP_FILENAME_SET)) {
1772  SCLogDebug("setting up file name");
1773 
1774  uint8_t *filename = NULL;
1775  size_t filename_len = 0;
1776 
1777  /* try Content-Disposition header first */
1778  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1779  "Content-Disposition");
1780  if (h != NULL && bstr_len(h->value) > 0) {
1781  /* parse content-disposition */
1782  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1783  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1784  }
1785 
1786  /* fall back to name from the uri */
1787  if (filename == NULL) {
1788  /* get the name */
1789  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1790  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1791  filename_len = bstr_len(tx->parsed_uri->path);
1792  }
1793  }
1794 
1795  if (filename != NULL) {
1796  // set range if present
1797  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1798  if (filename_len > SC_FILENAME_MAX) {
1799  // explicitly truncate the file name if too long
1800  filename_len = SC_FILENAME_MAX;
1801  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1802  }
1803  if (h_content_range != NULL) {
1804  result = HTPFileOpenWithRange(hstate, htud, filename, (uint16_t)filename_len, data,
1805  data_len, tx, h_content_range->value, htud);
1806  } else {
1807  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1808  STREAM_TOCLIENT);
1809  }
1810  SCLogDebug("result %d", result);
1811  if (result == -1) {
1812  goto end;
1813  } else if (result == -2) {
1814  htud->tcflags |= HTP_DONTSTORE;
1815  } else {
1816  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1817  htud->tcflags |= HTP_FILENAME_SET;
1818  htud->tcflags &= ~HTP_DONTSTORE;
1819  }
1820  }
1821  } else {
1822  /* otherwise, just store the data */
1823 
1824  if (!(htud->tcflags & HTP_DONTSTORE)) {
1825  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOCLIENT);
1826  SCLogDebug("result %d", result);
1827  if (result == -1) {
1828  goto end;
1829  } else if (result == -2) {
1830  /* we know for sure we're not storing the file */
1831  htud->tcflags |= HTP_DONTSTORE;
1832  }
1833  }
1834  }
1835 
1836  htud->response_body.body_parsed += data_len;
1837  return 0;
1838 end:
1839  return -1;
1840 }
1841 
1842 /**
1843  * \brief Function callback to append chunks for Requests
1844  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1845  * \retval int HTP_OK if all goes well
1846  */
1847 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1848 {
1849  SCEnter();
1850 
1851  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1852  SCReturnInt(HTP_OK);
1853 
1854  if (d->len == 0)
1855  SCReturnInt(HTP_OK);
1856 
1857 #ifdef PRINT
1858  printf("HTPBODY START: \n");
1859  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1860  printf("HTPBODY END: \n");
1861 #endif
1862 
1863  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1864  if (hstate == NULL) {
1865  SCReturnInt(HTP_ERROR);
1866  }
1867 
1868  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1869  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1870 
1871  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1872  if (tx_ud == NULL) {
1873  SCReturnInt(HTP_OK);
1874  }
1875  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
1876 
1877  if (!tx_ud->response_body_init) {
1878  tx_ud->response_body_init = 1;
1879 
1880  if (d->tx->request_method_number == HTP_M_POST) {
1881  SCLogDebug("POST");
1882  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1883  if (r == 1) {
1885  } else if (r == 0) {
1887  SCLogDebug("not multipart");
1888  }
1889  } else if (d->tx->request_method_number == HTP_M_PUT) {
1891  }
1892  }
1893 
1894  /* see if we can get rid of htp body chunks */
1895  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1896 
1897  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1898  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1899 
1900  /* within limits, add the body chunk to the state. */
1901  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1902  uint32_t stream_depth = FileReassemblyDepth();
1903  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1904  hstate->cfg->request.body_limit,
1905  stream_depth,
1906  tx_ud->tsflags,
1907  (uint32_t)d->len);
1908  BUG_ON(len > (uint32_t)d->len);
1909 
1910  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1911 
1912  const uint8_t *chunks_buffer = NULL;
1913  uint32_t chunks_buffer_len = 0;
1914 
1916  /* multi-part body handling starts here */
1917  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1918  goto end;
1919  }
1920 
1921  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1922  if (chunks_buffer == NULL) {
1923  goto end;
1924  }
1925 #ifdef PRINT
1926  printf("REASSCHUNK START: \n");
1927  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1928  printf("REASSCHUNK END: \n");
1929 #endif
1930 
1931  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1932 
1933  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1935  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1936  }
1937 
1938  } else {
1939  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1940  SCLogDebug("closing file that was being stored");
1941  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1942  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1943  }
1944  }
1945 
1946 end:
1947  if (hstate->conn != NULL) {
1948  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1950  hstate->cfg->request.inspect_min_size,
1951  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1952 
1953  /* if we reach the inspect_min_size we'll trigger inspection,
1954  * so make sure that raw stream is also inspected. Set the
1955  * data to be used to the amount of raw bytes we've seen to
1956  * get here. */
1957  if (tx_ud->request_body.body_inspected == 0 &&
1959  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1960  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1961  {
1962  const uint32_t data_size = (uint32_t)(
1963  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1964  const uint32_t depth = MIN(data_size, hstate->cfg->request.inspect_min_size);
1965 
1966  /* body still in progress, but due to min inspect size we need to inspect now */
1967  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, depth);
1968  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1969  }
1970  /* after the start of the body, disable the depth logic */
1971  } else if (tx_ud->request_body.body_inspected > 0) {
1972  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1973  }
1974  }
1975  SCReturnInt(HTP_OK);
1976 }
1977 
1978 /**
1979  * \brief Function callback to append chunks for Responses
1980  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1981  * \retval int HTP_OK if all goes well
1982  */
1983 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1984 {
1985  SCEnter();
1986 
1987  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1988  SCReturnInt(HTP_OK);
1989 
1990  if (d->len == 0)
1991  SCReturnInt(HTP_OK);
1992 
1993  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1994  if (hstate == NULL) {
1995  SCReturnInt(HTP_ERROR);
1996  }
1997 
1998  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1999  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
2000 
2001  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
2002  if (tx_ud == NULL) {
2003  SCReturnInt(HTP_OK);
2004  }
2005  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
2006  if (!tx_ud->request_body_init) {
2007  tx_ud->request_body_init = 1;
2008  }
2009 
2010  /* see if we can get rid of htp body chunks */
2011  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
2012 
2013  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
2014  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
2015 
2016  /* within limits, add the body chunk to the state. */
2017  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
2018  uint32_t stream_depth = FileReassemblyDepth();
2019  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
2020  hstate->cfg->response.body_limit,
2021  stream_depth,
2022  tx_ud->tcflags,
2023  (uint32_t)d->len);
2024  BUG_ON(len > (uint32_t)d->len);
2025 
2026  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
2027 
2028  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
2029  } else {
2030  if (tx_ud->tcflags & HTP_FILENAME_SET) {
2031  SCLogDebug("closing file that was being stored");
2032  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
2033  tx_ud->tcflags &= ~HTP_FILENAME_SET;
2034  }
2035  }
2036 
2037  if (hstate->conn != NULL) {
2038  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
2040  hstate->cfg->response.inspect_min_size,
2041  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
2042  /* if we reach the inspect_min_size we'll trigger inspection,
2043  * so make sure that raw stream is also inspected. Set the
2044  * data to be used to the amount of raw bytes we've seen to
2045  * get here. */
2046  if (tx_ud->response_body.body_inspected == 0 &&
2048  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2049  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2050  {
2051  const uint32_t data_size = (uint32_t)((uint64_t)hstate->conn->out_data_counter -
2052  hstate->last_response_data_stamp);
2053  const uint32_t depth = MIN(data_size, hstate->cfg->response.inspect_min_size);
2054 
2055  /* body still in progress, but due to min inspect size we need to inspect now */
2056  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, depth);
2057  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2058  }
2059  /* after the start of the body, disable the depth logic */
2060  } else if (tx_ud->response_body.body_inspected > 0) {
2061  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
2062  }
2063  }
2064  SCReturnInt(HTP_OK);
2065 }
2066 
2067 /**
2068  * \brief Print the stats of the HTTP requests
2069  */
2071 {
2072 #ifdef DEBUG
2073  SCEnter();
2074  SCMutexLock(&htp_state_mem_lock);
2075  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2076  htp_state_memcnt, htp_state_memuse);
2077  SCMutexUnlock(&htp_state_mem_lock);
2078  SCReturn;
2079 #endif
2080 }
2081 
2082 /** \brief Clears the HTTP server configuration memory used by HTP library */
2083 void HTPFreeConfig(void)
2084 {
2085  SCEnter();
2086 
2087  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2088  !AppLayerParserConfParserEnabled("tcp", "http"))
2089  {
2090  SCReturn;
2091  }
2092 
2093  HTPCfgRec *nextrec = cfglist.next;
2094  SCRadixReleaseRadixTree(cfgtree);
2095  cfgtree = NULL;
2096  htp_config_destroy(cfglist.cfg);
2097  while (nextrec != NULL) {
2098  HTPCfgRec *htprec = nextrec;
2099  nextrec = nextrec->next;
2100 
2101  htp_config_destroy(htprec->cfg);
2102  SCFree(htprec);
2103  }
2104  SCReturn;
2105 }
2106 
2107 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2108 {
2109  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2110  if (htud != NULL) {
2111  htud->request_has_trailers = 1;
2112  }
2113  return HTP_OK;
2114 }
2115 
2116 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2117 {
2118  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2119  if (htud != NULL) {
2120  htud->response_has_trailers = 1;
2121  }
2122  return HTP_OK;
2123 }
2124 
2125 /**\internal
2126  * \brief called at start of request
2127  * Set min inspect size.
2128  */
2129 static int HTPCallbackRequestStart(htp_tx_t *tx)
2130 {
2131  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2132  if (hstate == NULL) {
2133  SCReturnInt(HTP_ERROR);
2134  }
2135 
2136  uint64_t consumed = hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2137  SCLogDebug("HTTP request start: data offset %" PRIu64 ", in_data_counter %" PRIu64, consumed,
2138  (uint64_t)hstate->conn->in_data_counter);
2139 
2140  /* app-layer-frame-documentation tag start: frame registration http request */
2142  hstate->f, hstate->slice, consumed, -1, 0, HTTP_FRAME_REQUEST);
2143  if (frame) {
2144  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2145  hstate->request_frame_id = frame->id;
2146  AppLayerFrameSetTxId(frame, HtpGetActiveRequestTxID(hstate));
2147  }
2148  /* app-layer-frame-documentation tag end: frame registration http request */
2149 
2150  if (hstate->cfg)
2151  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2152  hstate->cfg->request.inspect_min_size);
2153 
2154  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2155  if (tx_ud == NULL) {
2156  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2157  if (unlikely(tx_ud == NULL)) {
2158  SCReturnInt(HTP_OK);
2159  }
2160  tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files
2161  htp_tx_set_user_data(tx, tx_ud);
2162  }
2163  SCReturnInt(HTP_OK);
2164 }
2165 
2166 /**\internal
2167  * \brief called at start of response
2168  * Set min inspect size.
2169  */
2170 static int HTPCallbackResponseStart(htp_tx_t *tx)
2171 {
2172  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2173  if (hstate == NULL) {
2174  SCReturnInt(HTP_ERROR);
2175  }
2176 
2177  uint64_t consumed = hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2178  SCLogDebug("HTTP response start: data offset %" PRIu64 ", out_data_counter %" PRIu64, consumed,
2179  (uint64_t)hstate->conn->out_data_counter);
2180 
2182  hstate->f, hstate->slice, consumed, -1, 1, HTTP_FRAME_RESPONSE);
2183  if (frame) {
2184  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2185  hstate->response_frame_id = frame->id;
2186  AppLayerFrameSetTxId(frame, HtpGetActiveResponseTxID(hstate));
2187  }
2188 
2189  if (hstate->cfg)
2190  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
2191  hstate->cfg->response.inspect_min_size);
2192 
2193  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2194  if (tx_ud == NULL) {
2195  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2196  if (unlikely(tx_ud == NULL)) {
2197  SCReturnInt(HTP_OK);
2198  }
2199  tx_ud->tx_data.file_tx =
2200  STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed.
2201  htp_tx_set_user_data(tx, tx_ud);
2202  }
2203  SCReturnInt(HTP_OK);
2204 }
2205 
2206 /**
2207  * \brief callback for request to store the recent incoming request
2208  into the recent_in_tx for the given htp state
2209  * \param connp pointer to the current connection parser which has the htp
2210  * state in it as user data
2211  */
2212 static int HTPCallbackRequestComplete(htp_tx_t *tx)
2213 {
2214  SCEnter();
2215 
2216  if (tx == NULL) {
2217  SCReturnInt(HTP_ERROR);
2218  }
2219 
2220  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2221  if (hstate == NULL) {
2222  SCReturnInt(HTP_ERROR);
2223  }
2224 
2225  const uint64_t abs_right_edge =
2226  hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2227 
2228  /* app-layer-frame-documentation tag start: updating frame->len */
2229  if (hstate->request_frame_id > 0) {
2230  Frame *frame = AppLayerFrameGetById(hstate->f, 0, hstate->request_frame_id);
2231  if (frame) {
2232  const uint64_t request_size = abs_right_edge - hstate->last_request_data_stamp;
2233 
2234  SCLogDebug("HTTP request complete: data offset %" PRIu64 ", request_size %" PRIu64,
2235  hstate->last_request_data_stamp, request_size);
2236  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2237  request_size);
2238  frame->len = (int64_t)request_size;
2239  /* app-layer-frame-documentation tag end: updating frame->len */
2240  }
2241  hstate->request_frame_id = 0;
2242  }
2243 
2244  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2245  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2246 
2247  SCLogDebug("HTTP request completed");
2248 
2249  HTPErrorCheckTxRequestFlags(hstate, tx);
2250 
2251  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2252  if (htud != NULL) {
2253  if (htud->tsflags & HTP_FILENAME_SET) {
2254  SCLogDebug("closing file that was being stored");
2255  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
2256  htud->tsflags &= ~HTP_FILENAME_SET;
2257  if (abs_right_edge < (uint64_t)UINT32_MAX) {
2259  hstate->f->protoctx, STREAM_TOSERVER, (uint32_t)abs_right_edge);
2260  }
2261  }
2262  }
2263 
2264  hstate->last_request_data_stamp = abs_right_edge;
2265  /* request done, do raw reassembly now to inspect state and stream
2266  * at the same time. */
2267  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
2268  SCReturnInt(HTP_OK);
2269 }
2270 
2271 /**
2272  * \brief callback for response to remove the recent received requests
2273  from the recent_in_tx for the given htp state
2274  * \param connp pointer to the current connection parser which has the htp
2275  * state in it as user data
2276  */
2277 static int HTPCallbackResponseComplete(htp_tx_t *tx)
2278 {
2279  SCEnter();
2280 
2281  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2282  if (hstate == NULL) {
2283  SCReturnInt(HTP_ERROR);
2284  }
2285 
2286  /* we have one whole transaction now */
2287  hstate->transaction_cnt++;
2288 
2289  const uint64_t abs_right_edge =
2290  hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2291 
2292  if (hstate->response_frame_id > 0) {
2293  Frame *frame = AppLayerFrameGetById(hstate->f, 1, hstate->response_frame_id);
2294  if (frame) {
2295  const uint64_t response_size = abs_right_edge - hstate->last_response_data_stamp;
2296 
2297  SCLogDebug("HTTP response complete: data offset %" PRIu64 ", response_size %" PRIu64,
2298  hstate->last_response_data_stamp, response_size);
2299  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2300  response_size);
2301  frame->len = (int64_t)response_size;
2302  }
2303  hstate->response_frame_id = 0;
2304  }
2305 
2306  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2307  if (htud != NULL) {
2308  if (htud->tcflags & HTP_FILENAME_SET) {
2309  SCLogDebug("closing file that was being stored");
2310  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
2311  htud->tcflags &= ~HTP_FILENAME_SET;
2312  }
2313  }
2314 
2315  /* response done, do raw reassembly now to inspect state and stream
2316  * at the same time. */
2317  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2318 
2319  /* handle HTTP CONNECT */
2320  if (tx->request_method_number == HTP_M_CONNECT) {
2321  /* any 2XX status response implies that the connection will become
2322  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2323  if ((tx->response_status_number >= 200) &&
2324  (tx->response_status_number < 300) &&
2325  (hstate->transaction_cnt == 1)) {
2326  uint16_t dp = 0;
2327  if (tx->request_port_number != -1) {
2328  dp = (uint16_t)tx->request_port_number;
2329  }
2330  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2331  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN)) {
2332  HTPSetEvent(
2333  hstate, htud, STREAM_TOCLIENT, HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE);
2334  }
2335  tx->request_progress = HTP_REQUEST_COMPLETE;
2336  tx->response_progress = HTP_RESPONSE_COMPLETE;
2337  }
2338  }
2339 
2340  hstate->last_response_data_stamp = abs_right_edge;
2341  SCReturnInt(HTP_OK);
2342 }
2343 
2344 static int HTPCallbackRequestLine(htp_tx_t *tx)
2345 {
2346  HtpTxUserData *tx_ud;
2347  bstr *request_uri_normalized;
2348  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2349  const HTPCfgRec *cfg = hstate->cfg;
2350 
2351  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2352  if (request_uri_normalized == NULL)
2353  return HTP_OK;
2354 
2355  tx_ud = htp_tx_get_user_data(tx);
2356  if (unlikely(tx_ud == NULL)) {
2357  bstr_free(request_uri_normalized);
2358  return HTP_OK;
2359  }
2360  if (unlikely(tx_ud->request_uri_normalized != NULL))
2361  bstr_free(tx_ud->request_uri_normalized);
2362  tx_ud->request_uri_normalized = request_uri_normalized;
2363 
2364  if (tx->flags) {
2365  HTPErrorCheckTxRequestFlags(hstate, tx);
2366  }
2367  return HTP_OK;
2368 }
2369 
2370 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2371 {
2372  if (part == NULL)
2373  return HTP_OK;
2374 
2375  uint64_t flags = 0;
2376  size_t prevlen = bstr_len(part);
2377  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2378  // shorter string means that uri was encoded
2379  if (res == HTP_OK && prevlen > bstr_len(part)) {
2380  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2381  if (htud == NULL)
2382  return HTP_OK;
2383  HtpState *s = htp_connp_get_user_data(tx->connp);
2384  if (s == NULL)
2385  return HTP_OK;
2386  HTPSetEvent(s, htud, STREAM_TOSERVER,
2388  }
2389 
2390  return HTP_OK;
2391 }
2392 
2393 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2394 {
2395  if (tx->parsed_uri == NULL)
2396  return HTP_OK;
2397 
2398  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2399 }
2400 
2401 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2402 {
2403  if (tx->parsed_uri == NULL)
2404  return HTP_OK;
2405 
2406  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2407 }
2408 
2409 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2410 {
2411  void *ptmp;
2412  if (tx_data->len == 0 || tx_data->tx == NULL)
2413  return HTP_OK;
2414 
2415  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2416  if (tx_ud == NULL) {
2417  return HTP_OK;
2418  }
2419  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2420  tx_ud->request_headers_raw_len,
2421  tx_ud->request_headers_raw_len + tx_data->len);
2422  if (ptmp == NULL) {
2423  return HTP_OK;
2424  }
2425  tx_ud->request_headers_raw = ptmp;
2426 
2427  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2428  tx_data->data, tx_data->len);
2429  tx_ud->request_headers_raw_len += tx_data->len;
2430 
2431  if (tx_data->tx && tx_data->tx->flags) {
2432  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2433  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2434  }
2435  return HTP_OK;
2436 }
2437 
2438 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2439 {
2440  void *ptmp;
2441  if (tx_data->len == 0 || tx_data->tx == NULL)
2442  return HTP_OK;
2443 
2444  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2445  if (tx_ud == NULL) {
2446  return HTP_OK;
2447  }
2448  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2449  tx_ud->response_headers_raw_len,
2450  tx_ud->response_headers_raw_len + tx_data->len);
2451  if (ptmp == NULL) {
2452  return HTP_OK;
2453  }
2454  tx_ud->response_headers_raw = ptmp;
2455 
2456  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2457  tx_data->data, tx_data->len);
2458  tx_ud->response_headers_raw_len += tx_data->len;
2459 
2460  return HTP_OK;
2461 }
2462 
2463 /*
2464  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2465  */
2466 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2467 {
2468  cfg_prec->uri_include_all = false;
2475 
2476  if (!g_disable_randomness) {
2478  } else {
2479  cfg_prec->randomize = 0;
2480  }
2482 
2483  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2484  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2485  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2486  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2487 
2488  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2489  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2490 
2491  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2492  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2493 
2494  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2495  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequestComplete);
2496 
2497  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2498  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponseComplete);
2499 
2500  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2501 
2502  /* don't convert + to space by default */
2503  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2504  // enables request decompression
2505  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2506 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2507  // disable by default
2508  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2509 #endif
2510 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2511  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2513 #endif
2514 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2515  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2517 #endif
2518 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2519  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2520 #endif
2521 #ifdef HAVE_HTP_CONFIG_SET_MAX_TX
2522 #define HTP_CONFIG_DEFAULT_MAX_TX_LIMIT 512
2523  htp_config_set_max_tx(cfg_prec->cfg, HTP_CONFIG_DEFAULT_MAX_TX_LIMIT);
2524 #endif
2525  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2526  * only the hard limit. So we set both here to the (current) htp defaults.
2527  * The reason we do this is that if the user sets the hard limit in the
2528  * config, we have to set the soft limit as well. If libhtp starts using
2529  * the soft limit in the future, we at least make sure we control what
2530  * it's value is. */
2531  htp_config_set_field_limits(cfg_prec->cfg,
2534  return;
2535 }
2536 
2537 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2538  * but we can get both <0 and >RAND_MAX values from RandomGet
2539  */
2540 static int RandomGetWrap(void)
2541 {
2542  unsigned long r;
2543 
2544  do {
2545  r = RandomGet();
2546  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2547 
2548  return r % RAND_MAX;
2549 }
2550 
2551 /*
2552  * We have this splitup so that in case double decoding has been enabled
2553  * for query and path, they would be called first on the callback queue,
2554  * before the callback set by Phase2() is called. We need this, since
2555  * the callback in Phase2() generates the normalized uri which utilizes
2556  * the query and path. */
2557 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2558 {
2559  /* randomize inspection size if needed */
2560  if (cfg_prec->randomize) {
2561  int rdrange = cfg_prec->randomize_range;
2562 
2563  long int r = RandomGetWrap();
2564  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2565  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2566 
2567  r = RandomGetWrap();
2568  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2569  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2570  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2571  " %u and 'request-body-inspect-window' set to %u after"
2572  " randomization.",
2573  name, cfg_prec->request.inspect_min_size, cfg_prec->request.inspect_window);
2574 
2575  r = RandomGetWrap();
2576  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2577  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2578 
2579  r = RandomGetWrap();
2580  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2581  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2582 
2583  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2584  " %u and 'response-body-inspect-window' set to %u after"
2585  " randomization.",
2586  name, cfg_prec->response.inspect_min_size, cfg_prec->response.inspect_window);
2587  }
2588 
2589  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2590  return;
2591 }
2592 
2593 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2594  SCRadixTree *tree)
2595 {
2596  if (cfg_prec == NULL || s == NULL || tree == NULL)
2597  return;
2598 
2599  ConfNode *p = NULL;
2600 
2601  /* Default Parameters */
2602  TAILQ_FOREACH(p, &s->head, next) {
2603 
2604  if (strcasecmp("address", p->name) == 0) {
2605  ConfNode *pval;
2606  /* Addresses */
2607  TAILQ_FOREACH(pval, &p->head, next) {
2608  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2609  pval->val);
2610 
2611  /* IPV6 or IPV4? */
2612  if (strchr(pval->val, ':') != NULL) {
2613  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2614  s->name, pval->val, cfg_prec->cfg);
2615  if (!SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec)) {
2616  SCLogWarning("LIBHTP failed to "
2617  "add ipv6 server %s, ignoring",
2618  pval->val);
2619  }
2620  } else {
2621  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2622  s->name, pval->val, cfg_prec->cfg);
2623  if (!SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec)) {
2624  SCLogWarning("LIBHTP failed "
2625  "to add ipv4 server %s, ignoring",
2626  pval->val);
2627  }
2628  } /* else - if (strchr(pval->val, ':') != NULL) */
2629  } /* TAILQ_FOREACH(pval, &p->head, next) */
2630 
2631  } else if (strcasecmp("personality", p->name) == 0) {
2632  /* Personalities */
2633  int personality = HTPLookupPersonality(p->val);
2634  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2635  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2636 
2637  if (personality >= 0) {
2638  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2639  personality);
2640  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2641  SCLogWarning("LIBHTP Failed adding "
2642  "personality \"%s\", ignoring",
2643  p->val);
2644  } else {
2645  SCLogDebug("LIBHTP personality set to %s",
2646  HTPLookupPersonalityString(personality));
2647  }
2648 
2649  /* The IDS personality by default converts the path (and due to
2650  * our query string callback also the query string) to lowercase.
2651  * Signatures do not expect this, so override it. */
2652  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2653  } else {
2654  SCLogWarning("LIBHTP Unknown personality "
2655  "\"%s\", ignoring",
2656  p->val);
2657  continue;
2658  }
2659 
2660  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2661  strcasecmp("request_body_limit", p->name) == 0) {
2662  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2663  SCLogError("Error parsing request-body-limit "
2664  "from conf file - %s. Killing engine",
2665  p->val);
2666  exit(EXIT_FAILURE);
2667  }
2668 
2669  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2670  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2671  SCLogError("Error parsing response-body-limit "
2672  "from conf file - %s. Killing engine",
2673  p->val);
2674  exit(EXIT_FAILURE);
2675  }
2676 
2677  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2678  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2679  SCLogError("Error parsing request-body-minimal-inspect-size "
2680  "from conf file - %s. Killing engine",
2681  p->val);
2682  exit(EXIT_FAILURE);
2683  }
2684 
2685  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2686  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2687  SCLogError("Error parsing request-body-inspect-window "
2688  "from conf file - %s. Killing engine",
2689  p->val);
2690  exit(EXIT_FAILURE);
2691  }
2692 
2693  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2694  if (ConfValIsTrue(p->val)) {
2695  htp_config_register_request_line(cfg_prec->cfg,
2696  HTPCallbackDoubleDecodeQuery);
2697  }
2698 
2699  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2700  if (ConfValIsTrue(p->val)) {
2701  htp_config_register_request_line(cfg_prec->cfg,
2702  HTPCallbackDoubleDecodePath);
2703  }
2704 
2705  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2706  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2707  SCLogError("Error parsing response-body-minimal-inspect-size "
2708  "from conf file - %s. Killing engine",
2709  p->val);
2710  exit(EXIT_FAILURE);
2711  }
2712 
2713  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2714  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2715  SCLogError("Error parsing response-body-inspect-window "
2716  "from conf file - %s. Killing engine",
2717  p->val);
2718  exit(EXIT_FAILURE);
2719  }
2720 
2721  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2722  uint32_t value = 2;
2723  if (ParseSizeStringU32(p->val, &value) < 0) {
2724  SCLogError("Error parsing response-body-inspect-window "
2725  "from conf file - %s. Killing engine",
2726  p->val);
2727  exit(EXIT_FAILURE);
2728  }
2729 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2730  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2731 #else
2732  SCLogWarning("can't set response-body-decompress-layer-limit "
2733  "to %u, libhtp version too old",
2734  value);
2735 #endif
2736  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2737  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2738  HTP_DECODER_URL_PATH,
2739  ConfValIsTrue(p->val));
2740  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2741  if (strlen(p->val) == 1) {
2742  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2743  HTP_DECODER_URL_PATH,
2744  p->val[0]);
2745  } else {
2746  SCLogError("Invalid entry "
2747  "for libhtp param path-bestfit-replacement-char");
2748  }
2749  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2750  htp_config_set_convert_lowercase(cfg_prec->cfg,
2751  HTP_DECODER_URL_PATH,
2752  ConfValIsTrue(p->val));
2753  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2754  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2755  HTP_DECODER_URL_PATH,
2756  ConfValIsTrue(p->val));
2757  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2758  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2759  HTP_DECODER_URL_PATH,
2760  ConfValIsTrue(p->val));
2761  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2762  htp_config_set_path_separators_compress(cfg_prec->cfg,
2763  HTP_DECODER_URL_PATH,
2764  ConfValIsTrue(p->val));
2765  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2766  htp_config_set_path_separators_decode(cfg_prec->cfg,
2767  HTP_DECODER_URL_PATH,
2768  ConfValIsTrue(p->val));
2769  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2770  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2771  HTP_DECODER_URL_PATH,
2772  ConfValIsTrue(p->val));
2773  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2774  enum htp_url_encoding_handling_t handling;
2775  if (strcasecmp(p->val, "preserve_percent") == 0) {
2776  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2777  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2778  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2779  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2780  handling = HTP_URL_DECODE_PROCESS_INVALID;
2781  } else {
2782  SCLogError("Invalid entry "
2783  "for libhtp param path-url-encoding-invalid-handling");
2784  return;
2785  }
2786  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2787  HTP_DECODER_URL_PATH,
2788  handling);
2789  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2790  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2791  HTP_DECODER_URL_PATH,
2792  ConfValIsTrue(p->val));
2793  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2794  cfg_prec->uri_include_all = (1 == ConfValIsTrue(p->val));
2795  SCLogDebug("uri-include-all %s",
2796  cfg_prec->uri_include_all ? "enabled" : "disabled");
2797  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2798  htp_config_set_plusspace_decode(cfg_prec->cfg,
2799  HTP_DECODER_URLENCODED,
2800  ConfValIsTrue(p->val));
2801  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2802  uint32_t limit = 0;
2803  if (ParseSizeStringU32(p->val, &limit) < 0) {
2804  SCLogError("Error meta-field-limit "
2805  "from conf file - %s. Killing engine",
2806  p->val);
2807  exit(EXIT_FAILURE);
2808  }
2809  if (limit == 0) {
2810  FatalError("Error meta-field-limit "
2811  "from conf file cannot be 0. Killing engine");
2812  }
2813  /* set default soft-limit with our new hard limit */
2814  htp_config_set_field_limits(cfg_prec->cfg,
2816  (size_t)limit);
2817 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2818  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2819  uint32_t limit = 0;
2820  if (ParseSizeStringU32(p->val, &limit) < 0) {
2821  FatalError("failed to parse 'lzma-memlimit' "
2822  "from conf file - %s.",
2823  p->val);
2824  }
2825  if (limit == 0) {
2826  FatalError("'lzma-memlimit' "
2827  "from conf file cannot be 0.");
2828  }
2829  /* set default soft-limit with our new hard limit */
2830  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2831  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2832 #endif
2833 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2834  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2835  if (ConfValIsTrue(p->val)) {
2836  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2837  } else if (!ConfValIsFalse(p->val)) {
2838  int8_t limit;
2839  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2840  FatalError("failed to parse 'lzma-enabled' "
2841  "from conf file - %s.",
2842  p->val);
2843  }
2844  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2845  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2846  }
2847 #endif
2848 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2849  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2850  uint32_t limit = 0;
2851  if (ParseSizeStringU32(p->val, &limit) < 0) {
2852  FatalError("failed to parse 'compression-bomb-limit' "
2853  "from conf file - %s.",
2854  p->val);
2855  }
2856  if (limit == 0) {
2857  FatalError("'compression-bomb-limit' "
2858  "from conf file cannot be 0.");
2859  }
2860  /* set default soft-limit with our new hard limit */
2861  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2862  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2863 #endif
2864 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2865  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2866  uint32_t limit = 0;
2867  // between 1 usec and 1 second
2868  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2869  FatalError("failed to parse 'decompression-time-limit' "
2870  "from conf file - %s.",
2871  p->val);
2872  }
2873  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2874  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2875 #endif
2876 #ifdef HAVE_HTP_CONFIG_SET_MAX_TX
2877  } else if (strcasecmp("max-tx", p->name) == 0) {
2878  uint32_t limit = 0;
2879  if (ParseSizeStringU32(p->val, &limit) < 0) {
2880  FatalError("failed to parse 'max-tx' "
2881  "from conf file - %s.",
2882  p->val);
2883  }
2884  /* set default soft-limit with our new hard limit */
2885  SCLogConfig("Setting HTTP max-tx limit to %" PRIu32 " bytes", limit);
2886  htp_config_set_max_tx(cfg_prec->cfg, (size_t)limit);
2887 #endif
2888  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2889  if (!g_disable_randomness) {
2890  cfg_prec->randomize = ConfValIsTrue(p->val);
2891  }
2892  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2893  uint32_t range;
2894  if (StringParseU32RangeCheck(&range, 10, 0,
2895  (const char *)p->val, 0, 100) < 0) {
2896  SCLogError("Invalid value for randomize"
2897  "-inspection-range setting from conf file - \"%s\"."
2898  " It should be a valid integer less than or equal to 100."
2899  " Killing engine",
2900  p->val);
2901  exit(EXIT_FAILURE);
2902  }
2903  cfg_prec->randomize_range = range;
2904  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2905  if (ConfValIsTrue(p->val)) {
2906  cfg_prec->http_body_inline = 1;
2907  } else if (ConfValIsFalse(p->val)) {
2908  cfg_prec->http_body_inline = 0;
2909  } else {
2910  if (strcmp("auto", p->val) != 0) {
2911  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2912  }
2913  if (EngineModeIsIPS()) {
2914  cfg_prec->http_body_inline = 1;
2915  } else {
2916  cfg_prec->http_body_inline = 0;
2917  }
2918  }
2919  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2920  ConfNode *pval;
2921 
2922  TAILQ_FOREACH(pval, &p->head, next) {
2923  if (strcasecmp("enabled", pval->name) == 0) {
2924  if (ConfValIsTrue(pval->val)) {
2925  cfg_prec->swf_decompression_enabled = 1;
2926  } else if (ConfValIsFalse(pval->val)) {
2927  cfg_prec->swf_decompression_enabled = 0;
2928  } else {
2929  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2930  }
2931  } else if (strcasecmp("type", pval->name) == 0) {
2932  if (strcasecmp("no", pval->val) == 0) {
2934  } else if (strcasecmp("deflate", pval->val) == 0) {
2936  } else if (strcasecmp("lzma", pval->val) == 0) {
2938  } else if (strcasecmp("both", pval->val) == 0) {
2940  } else {
2941  SCLogError("Invalid entry for "
2942  "swf-decompression.type: %s - "
2943  "Killing engine",
2944  pval->val);
2945  exit(EXIT_FAILURE);
2946  }
2947  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2948  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2949  SCLogError("Error parsing swf-decompression.compression-depth "
2950  "from conf file - %s. Killing engine",
2951  p->val);
2952  exit(EXIT_FAILURE);
2953  }
2954  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2955  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2956  SCLogError("Error parsing swf-decompression.decompression-depth "
2957  "from conf file - %s. Killing engine",
2958  p->val);
2959  exit(EXIT_FAILURE);
2960  }
2961  } else {
2962  SCLogWarning("Ignoring unknown param %s", pval->name);
2963  }
2964  }
2965  } else {
2966  SCLogWarning("LIBHTP Ignoring unknown "
2967  "default config: %s",
2968  p->name);
2969  }
2970  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2971 
2972  return;
2973 }
2974 
2975 void HTPConfigure(void)
2976 {
2977  SCEnter();
2978 
2979  cfglist.next = NULL;
2980 
2984 
2985  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2986  if (NULL == cfgtree)
2987  exit(EXIT_FAILURE);
2988 
2989  /* Default Config */
2990  cfglist.cfg = htp_config_create();
2991  if (NULL == cfglist.cfg) {
2992  FatalError("Failed to create HTP default config");
2993  }
2994  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2995  HTPConfigSetDefaultsPhase1(&cfglist);
2996  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2997  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2998  cfgtree);
2999  } else {
3000  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
3001  }
3002  HTPConfigSetDefaultsPhase2("default", &cfglist);
3003 
3004  HTPParseMemcap();
3005 
3006  /* Read server config and create a parser for each IP in radix tree */
3007  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
3008  if (server_config == NULL) {
3009  server_config = ConfGetNode("libhtp.server-config");
3010  if (server_config == NULL) {
3011  SCLogDebug("LIBHTP Configuring %p", server_config);
3012  SCReturn;
3013  }
3014  }
3015  SCLogDebug("LIBHTP Configuring %p", server_config);
3016 
3017  ConfNode *si;
3018  /* Server Nodes */
3019  TAILQ_FOREACH(si, &server_config->head, next) {
3020  /* Need the named node, not the index */
3021  ConfNode *s = TAILQ_FIRST(&si->head);
3022  if (NULL == s) {
3023  SCLogDebug("LIBHTP s NULL");
3024  continue;
3025  }
3026 
3027  SCLogDebug("LIBHTP server %s", s->name);
3028 
3029  HTPCfgRec *nextrec = cfglist.next;
3030  HTPCfgRec *htprec = SCCalloc(1, sizeof(HTPCfgRec));
3031  if (NULL == htprec)
3032  exit(EXIT_FAILURE);
3033 
3034  cfglist.next = htprec;
3035 
3036  cfglist.next->next = nextrec;
3037  cfglist.next->cfg = htp_config_create();
3038  if (NULL == cfglist.next->cfg) {
3039  FatalError("Failed to create HTP server config");
3040  }
3041 
3042  HTPConfigSetDefaultsPhase1(htprec);
3043  HTPConfigParseParameters(htprec, s, cfgtree);
3044  HTPConfigSetDefaultsPhase2(s->name, htprec);
3045  }
3046 
3047  SCReturn;
3048 }
3049 
3051 {
3052 #ifdef DEBUG
3053  SCMutexLock(&htp_state_mem_lock);
3054  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
3055  SCMutexUnlock(&htp_state_mem_lock);
3056 #endif
3057 }
3058 
3059 /** \internal
3060  * \brief get files callback
3061  * \param state state ptr
3062  * \param direction flow direction
3063  * \retval files files ptr
3064  */
3065 static AppLayerGetFileState HTPGetTxFiles(void *state, void *txv, uint8_t direction)
3066 {
3067  AppLayerGetFileState files = { .fc = NULL, .cfg = &htp_sbcfg };
3068  htp_tx_t *tx = (htp_tx_t *)txv;
3069  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3070  if (tx_ud) {
3071  if (direction & STREAM_TOCLIENT) {
3072  files.fc = &tx_ud->files_tc;
3073  } else {
3074  files.fc = &tx_ud->files_ts;
3075  }
3076  }
3077  return files;
3078 }
3079 
3080 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
3081 {
3082  if (direction & STREAM_TOSERVER)
3083  return ((htp_tx_t *)tx)->request_progress;
3084  else
3085  return ((htp_tx_t *)tx)->response_progress;
3086 }
3087 
3088 static uint64_t HTPStateGetTxCnt(void *alstate)
3089 {
3090  HtpState *http_state = (HtpState *)alstate;
3091 
3092  if (http_state != NULL && http_state->conn != NULL) {
3093  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
3094  if (size < 0)
3095  return 0ULL;
3096  SCLogDebug("size %"PRIu64, size);
3097  return (uint64_t)size + http_state->tx_freed;
3098  } else {
3099  return 0ULL;
3100  }
3101 }
3102 
3103 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
3104 {
3105  HtpState *http_state = (HtpState *)alstate;
3106 
3107  if (http_state != NULL && http_state->conn != NULL && tx_id >= http_state->tx_freed)
3108  return htp_list_get(http_state->conn->transactions, tx_id - http_state->tx_freed);
3109  else
3110  return NULL;
3111 }
3112 
3113 void *HtpGetTxForH2(void *alstate)
3114 {
3115  // gets last transaction
3116  HtpState *http_state = (HtpState *)alstate;
3117  if (http_state != NULL && http_state->conn != NULL) {
3118  size_t txid = HTPStateGetTxCnt(http_state);
3119  if (txid > http_state->tx_freed) {
3120  return htp_list_get(http_state->conn->transactions, txid - http_state->tx_freed - 1);
3121  }
3122  }
3123  return NULL;
3124 }
3125 
3126 static int HTPStateGetEventInfo(const char *event_name,
3127  int *event_id, AppLayerEventType *event_type)
3128 {
3129  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3130  if (*event_id == -1) {
3131  SCLogError("event \"%s\" not present in "
3132  "http's enum map table.",
3133  event_name);
3134  /* this should be treated as fatal */
3135  return -1;
3136  }
3137 
3138  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3139 
3140  return 0;
3141 }
3142 
3143 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3144  AppLayerEventType *event_type)
3145 {
3146  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3147  if (*event_name == NULL) {
3148  SCLogError("event \"%d\" not present in "
3149  "http's enum map table.",
3150  event_id);
3151  /* this should be treated as fatal */
3152  return -1;
3153  }
3154 
3155  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3156 
3157  return 0;
3158 }
3159 
3160 static AppLayerTxData *HTPGetTxData(void *vtx)
3161 {
3162  htp_tx_t *tx = (htp_tx_t *)vtx;
3163  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3164  if (tx_ud) {
3165  return &tx_ud->tx_data;
3166  }
3167  return NULL;
3168 }
3169 
3170 static AppLayerStateData *HTPGetStateData(void *vstate)
3171 {
3172  HtpState *s = vstate;
3173  return &s->state_data;
3174 }
3175 
3176 static int HTPRegisterPatternsForProtocolDetection(void)
3177 {
3178  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3179  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3180  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3181  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3182  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3183  const char *spacings[] = { "|20|", "|09|", NULL };
3184  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3185 
3186  int methods_pos;
3187  int spacings_pos;
3188  int versions_pos;
3189  int register_result;
3190  char method_buffer[32] = "";
3191 
3192  /* Loop through all the methods ands spacings and register the patterns */
3193  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3194  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3195 
3196  /* Combine the method name and the spacing */
3197  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3198 
3199  /* Register the new method+spacing pattern
3200  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3201  * but the pattern matching should only be one char
3202  */
3203  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3204  method_buffer, (uint16_t)strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3205  if (register_result < 0) {
3206  return -1;
3207  }
3208  }
3209  }
3210 
3211  /* Loop through all the http version patterns that are TO_CLIENT */
3212  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3213  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3214  versions[versions_pos], (uint16_t)strlen(versions[versions_pos]), 0,
3215  STREAM_TOCLIENT);
3216  if (register_result < 0) {
3217  return -1;
3218  }
3219  }
3220 
3221  return 0;
3222 }
3223 
3224 /**
3225  * \brief Register the HTTP protocol and state handling functions to APP layer
3226  * of the engine.
3227  */
3229 {
3230  SCEnter();
3231 
3232  const char *proto_name = "http";
3233 
3234  /** HTTP */
3235  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3237  if (HTPRegisterPatternsForProtocolDetection() < 0)
3238  return;
3239  } else {
3240  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3241  proto_name);
3242  return;
3243  }
3244 
3245  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3246  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3247  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3248  AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles);
3250  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3251  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3252  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3253 
3255  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3256  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3258  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3259 
3260  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3261  AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData);
3262 
3264  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3265 
3267  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3269  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3270  SC_ATOMIC_INIT(htp_config_flags);
3271  /* This parser accepts gaps. */
3275  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_TOCLIENT);
3276  /* app-layer-frame-documentation tag start: registering relevant callbacks */
3278  IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById);
3279  /* app-layer-frame-documentation tag end: registering relevant callbacks */
3280  HTPConfigure();
3281  } else {
3282  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3283  "still on.", proto_name);
3284  }
3285 #ifdef UNITTESTS
3286  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3287 #endif
3288 
3289  SCReturn;
3290 }
3291 
3292 #ifdef UNITTESTS
3293 #include "detect-engine-alert.h"
3294 
3295 static HTPCfgRec cfglist_backup;
3296 
3298 {
3299  cfglist_backup = cfglist;
3300 
3301  return;
3302 }
3303 
3305 {
3306  cfglist = cfglist_backup;
3307 
3308  return;
3309 }
3310 
3311 /** \test Test case where chunks are sent in smaller chunks and check the
3312  * response of the parser from HTP library. */
3313 static int HTPParserTest01(void)
3314 {
3315  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3316  " Data is c0oL!";
3317  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3318 
3319  TcpSession ssn;
3320  memset(&ssn, 0, sizeof(ssn));
3321 
3324 
3325  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3326  FAIL_IF_NULL(f);
3327  f->protoctx = &ssn;
3328  f->proto = IPPROTO_TCP;
3329  f->alproto = ALPROTO_HTTP1;
3330 
3331  StreamTcpInitConfig(true);
3332 
3333  uint32_t u;
3334  for (u = 0; u < httplen1; u++) {
3335  uint8_t flags = 0;
3336 
3337  if (u == 0)
3338  flags = STREAM_TOSERVER|STREAM_START;
3339  else if (u == (httplen1 - 1))
3340  flags = STREAM_TOSERVER|STREAM_EOF;
3341  else
3342  flags = STREAM_TOSERVER;
3343 
3344  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3345  FAIL_IF(r != 0);
3346  }
3347 
3348  HtpState *htp_state = f->alstate;
3349  FAIL_IF_NULL(htp_state);
3350 
3351  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3352  FAIL_IF_NULL(tx);
3353 
3354  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3355  FAIL_IF_NULL(h);
3356 
3357  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3358  FAIL_IF(tx->request_method_number != HTP_M_POST);
3359  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3360 
3362  StreamTcpFreeConfig(true);
3363  UTHFreeFlow(f);
3364  PASS;
3365 }
3366 
3367 /** \test Test folding in 1 read case */
3368 static int HTPParserTest01b(void)
3369 {
3370  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3371  " Data is c0oL!";
3372  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3373 
3374  TcpSession ssn;
3375  memset(&ssn, 0, sizeof(ssn));
3376 
3379 
3380  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3381  FAIL_IF_NULL(f);
3382  f->protoctx = &ssn;
3383  f->proto = IPPROTO_TCP;
3384  f->alproto = ALPROTO_HTTP1;
3385 
3386  StreamTcpInitConfig(true);
3387 
3388  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3389  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3390  FAIL_IF(r != 0);
3391 
3392  HtpState *htp_state = f->alstate;
3393  FAIL_IF_NULL(htp_state);
3394 
3395  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3396  FAIL_IF_NULL(tx);
3397 
3398  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3399  FAIL_IF_NULL(h);
3400 
3401  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3402  FAIL_IF(tx->request_method_number != HTP_M_POST);
3403  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3404 
3406  StreamTcpFreeConfig(true);
3407  UTHFreeFlow(f);
3408  PASS;
3409 }
3410 
3411 /** \test Test folding in 1byte per read case */
3412 static int HTPParserTest01c(void)
3413 {
3414  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3415  " Data is c0oL!";
3416  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3417 
3418  TcpSession ssn;
3419  memset(&ssn, 0, sizeof(ssn));
3420 
3423 
3424  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3425  FAIL_IF_NULL(f);
3426  f->protoctx = &ssn;
3427  f->proto = IPPROTO_TCP;
3428  f->alproto = ALPROTO_HTTP1;
3429 
3430  StreamTcpInitConfig(true);
3431 
3432  uint32_t u;
3433  for (u = 0; u < httplen1; u++) {
3434  uint8_t flags = 0;
3435 
3436  if (u == 0)
3437  flags = STREAM_TOSERVER|STREAM_START;
3438  else if (u == (httplen1 - 1))
3439  flags = STREAM_TOSERVER|STREAM_EOF;
3440  else
3441  flags = STREAM_TOSERVER;
3442 
3443  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3444  FAIL_IF(r != 0);
3445  }
3446 
3447  HtpState *htp_state = f->alstate;
3448  FAIL_IF_NULL(htp_state);
3449 
3450  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3451  FAIL_IF_NULL(tx);
3452 
3453  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3454  FAIL_IF_NULL(h);
3455 
3456  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3457  FAIL_IF(tx->request_method_number != HTP_M_POST);
3458  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3459 
3461  StreamTcpFreeConfig(true);
3462  UTHFreeFlow(f);
3463  PASS;
3464 }
3465 
3466 /** \test Test case where chunks are sent in smaller chunks and check the
3467  * response of the parser from HTP library. */
3468 static int HTPParserTest01a(void)
3469 {
3470  int result = 0;
3471  Flow *f = NULL;
3472  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3473  " Data is c0oL!";
3474  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3475  TcpSession ssn;
3476  HtpState *htp_state = NULL;
3477  int r = 0;
3479 
3480  memset(&ssn, 0, sizeof(ssn));
3481 
3482  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3483  if (f == NULL)
3484  goto end;
3485  f->protoctx = &ssn;
3486  f->proto = IPPROTO_TCP;
3487  f->alproto = ALPROTO_HTTP1;
3488 
3489  StreamTcpInitConfig(true);
3490 
3491  uint32_t u;
3492  for (u = 0; u < httplen1; u++) {
3493  uint8_t flags = 0;
3494 
3495  if (u == 0)
3496  flags = STREAM_TOSERVER|STREAM_START;
3497  else if (u == (httplen1 - 1))
3498  flags = STREAM_TOSERVER|STREAM_EOF;
3499  else
3500  flags = STREAM_TOSERVER;
3501 
3502  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3503  if (r != 0) {
3504  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3505  " 0: ", u, r);
3506  goto end;
3507  }
3508  }
3509 
3510  htp_state = f->alstate;
3511  if (htp_state == NULL) {
3512  printf("no http state: ");
3513  goto end;
3514  }
3515 
3516  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3517  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3518  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3519  || tx->request_method_number != HTP_M_POST ||
3520  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3521  {
3522  printf("expected header value: Victor/1.0 and got %s: and expected"
3523  " method: POST and got %s, expected protocol number HTTP/1.0"
3524  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3525  bstr_util_strdup_to_c(tx->request_method),
3526  bstr_util_strdup_to_c(tx->request_protocol));
3527  goto end;
3528  }
3529  result = 1;
3530 end:
3531  if (alp_tctx != NULL)
3533  StreamTcpFreeConfig(true);
3534  UTHFreeFlow(f);
3535  return result;
3536 }
3537 
3538 /** \test See how it deals with an incomplete request. */
3539 static int HTPParserTest02(void)
3540 {
3541  int result = 0;
3542  Flow *f = NULL;
3543  uint8_t httpbuf1[] = "POST";
3544  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3545  TcpSession ssn;
3546  HtpState *http_state = NULL;
3548 
3549  memset(&ssn, 0, sizeof(ssn));
3550 
3551  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3552  if (f == NULL)
3553  goto end;
3554  f->protoctx = &ssn;
3555  f->proto = IPPROTO_TCP;
3556  f->alproto = ALPROTO_HTTP1;
3557 
3558  StreamTcpInitConfig(true);
3559 
3560  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3561  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3562  if (r != 0) {
3563  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3564  goto end;
3565  }
3566 
3567  http_state = f->alstate;
3568  if (http_state == NULL) {
3569  printf("no http state: ");
3570  goto end;
3571  }
3572 
3573  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3574  FAIL_IF_NULL(tx);
3575  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3576  FAIL_IF_NOT_NULL(h);
3577 
3578  FAIL_IF_NULL(tx->request_method);
3579  char *method = bstr_util_strdup_to_c(tx->request_method);
3580  FAIL_IF_NULL(method);
3581 
3582  FAIL_IF(strcmp(method, "POST") != 0);
3583  SCFree(method);
3584 
3585  result = 1;
3586 end:
3587  if (alp_tctx != NULL)
3589  StreamTcpFreeConfig(true);
3590  UTHFreeFlow(f);
3591  return result;
3592 }
3593 
3594 /** \test Test case where method is invalid and data is sent in smaller chunks
3595  * and check the response of the parser from HTP library. */
3596 static int HTPParserTest03(void)
3597 {
3598  int result = 0;
3599  Flow *f = NULL;
3600  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3601  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3602  TcpSession ssn;
3603  HtpState *htp_state = NULL;
3604  int r = 0;
3606 
3607  memset(&ssn, 0, sizeof(ssn));
3608 
3609  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3610  if (f == NULL)
3611  goto end;
3612  f->protoctx = &ssn;
3613  f->proto = IPPROTO_TCP;
3614  f->alproto = ALPROTO_HTTP1;
3615 
3616  StreamTcpInitConfig(true);
3617 
3618  uint32_t u;
3619  for (u = 0; u < httplen1; u++) {
3620  uint8_t flags = 0;
3621 
3622  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3623  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3624  else flags = STREAM_TOSERVER;
3625 
3626  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3627  if (r != 0) {
3628  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3629  " 0: ", u, r);
3630  goto end;
3631  }
3632  }
3633  htp_state = f->alstate;
3634  if (htp_state == NULL) {
3635  printf("no http state: ");
3636  goto end;
3637  }
3638 
3639  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3640 
3641  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3642  if (tx->request_method_number != HTP_M_UNKNOWN ||
3643  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3644  {
3645  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3646  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3647  bstr_util_strdup_to_c(tx->request_protocol));
3648  goto end;
3649  }
3650  result = 1;
3651 end:
3652  if (alp_tctx != NULL)
3654  StreamTcpFreeConfig(true);
3655  UTHFreeFlow(f);
3656  return result;
3657 }
3658 
3659 /** \test Test case where invalid data is sent and check the response of the
3660  * parser from HTP library. */
3661 static int HTPParserTest04(void)
3662 {
3663  int result = 0;
3664  Flow *f = NULL;
3665  HtpState *htp_state = NULL;
3666  uint8_t httpbuf1[] = "World!\r\n";
3667  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3668  TcpSession ssn;
3669  int r = 0;
3671 
3672  memset(&ssn, 0, sizeof(ssn));
3673 
3674  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3675  if (f == NULL)
3676  goto end;
3677  f->protoctx = &ssn;
3678  f->proto = IPPROTO_TCP;
3679  f->alproto = ALPROTO_HTTP1;
3680 
3681  StreamTcpInitConfig(true);
3682 
3684  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3685  if (r != 0) {
3686  goto end;
3687  }
3688 
3689  htp_state = f->alstate;
3690  if (htp_state == NULL) {
3691  printf("no http state: ");
3692  goto end;
3693  }
3694 
3695  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3696  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3697  if (tx->request_method_number != HTP_M_UNKNOWN ||
3698  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3699  {
3700  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3701  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3702  bstr_util_strdup_to_c(tx->request_protocol));
3703  goto end;
3704  }
3705  result = 1;
3706 end:
3707  if (alp_tctx != NULL)
3709  StreamTcpFreeConfig(true);
3710  UTHFreeFlow(f);
3711  return result;
3712 }
3713 
3714 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3715  * properly parsed them and also keeps them separated. */
3716 static int HTPParserTest05(void)
3717 {
3718  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3719  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3720  uint8_t httpbuf2[] = "Post D";
3721  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3722  uint8_t httpbuf3[] = "ata is c0oL!";
3723  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3724 
3725  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3726  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3727  uint8_t httpbuf5[] = "post R";
3728  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3729  uint8_t httpbuf6[] = "esults are tha bomb!";
3730  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3731 
3732  TcpSession ssn;
3733  memset(&ssn, 0, sizeof(ssn));
3734 
3737 
3738  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3739  FAIL_IF_NULL(f);
3740  f->protoctx = &ssn;
3741  f->proto = IPPROTO_TCP;
3742  f->alproto = ALPROTO_HTTP1;
3743 
3744  StreamTcpInitConfig(true);
3745 
3746  int r = AppLayerParserParse(
3747  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3748  FAIL_IF(r != 0);
3749 
3750  r = AppLayerParserParse(
3751  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3752  FAIL_IF(r != 0);
3753 
3754  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3755  FAIL_IF(r != 0);
3756 
3757  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3758  FAIL_IF(r != 0);
3759 
3760  r = AppLayerParserParse(
3761  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3762  FAIL_IF(r != 0);
3763 
3764  r = AppLayerParserParse(
3765  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3766  FAIL_IF(r != 0);
3767 
3768  HtpState *http_state = f->alstate;
3769  FAIL_IF_NULL(http_state);
3770 
3771  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3772  FAIL_IF_NULL(tx);
3773  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3774  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3775 
3776  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3777  FAIL_IF_NULL(h);
3778 
3779  FAIL_IF_NOT(tx->response_status_number == 200);
3780 
3782  StreamTcpFreeConfig(true);
3783  UTHFreeFlow(f);
3784  PASS;
3785 }
3786 
3787 /** \test Test proper chunked encoded response body
3788  */
3789 static int HTPParserTest06(void)
3790 {
3791  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3792  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3793  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3794  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3795  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3796  "GMT\r\n"
3797  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3798  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3799  "FrontPage/5.0.2.2510\r\n"
3800  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3801  "chunked\r\n"
3802  "Content-Type: text/html\r\n\r\n"
3803  "580\r\n"
3804  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3805  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3806  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3807  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3808  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3809  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3810  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3811  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3812  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3813  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3814  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3815  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3816  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3817  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3818  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3819  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3820  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3821  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3822  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3823  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3824  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3825  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3826  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3827  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3828  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3829  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3830  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3831  "aHA=\r\n0\r\n\r\n";
3832  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3833  TcpSession ssn;
3834 
3837 
3838  memset(&ssn, 0, sizeof(ssn));
3839 
3840  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3841  FAIL_IF_NULL(f);
3842  f->protoctx = &ssn;
3843  f->proto = IPPROTO_TCP;
3844  f->alproto = ALPROTO_HTTP1;
3845 
3846  StreamTcpInitConfig(true);
3847 
3848  int r = AppLayerParserParse(
3849  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3850  FAIL_IF(r != 0);
3851  r = AppLayerParserParse(
3852  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3853  FAIL_IF(r != 0);
3854 
3855  HtpState *http_state = f->alstate;
3856  FAIL_IF_NULL(http_state);
3857 
3858  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3859  FAIL_IF_NULL(tx);
3860 
3861  FAIL_IF(tx->request_method_number != HTP_M_GET);
3862  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3863 
3864  FAIL_IF(tx->response_status_number != 200);
3865  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3866 
3867  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3868  FAIL_IF_NULL(h);
3869 
3871  StreamTcpFreeConfig(true);
3872  UTHFreeFlow(f);
3873  PASS;
3874 }
3875 
3876 /** \test
3877  */
3878 static int HTPParserTest07(void)
3879 {
3880  int result = 0;
3881  Flow *f = NULL;
3882  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3883  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3884  TcpSession ssn;
3885  HtpState *htp_state = NULL;
3886  int r = 0;
3888 
3889  memset(&ssn, 0, sizeof(ssn));
3890 
3891  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3892  if (f == NULL)
3893  goto end;
3894  f->protoctx = &ssn;
3895  f->proto = IPPROTO_TCP;
3896  f->alproto = ALPROTO_HTTP1;
3897 
3898  StreamTcpInitConfig(true);
3899 
3900  uint32_t u;
3901  for (u = 0; u < httplen1; u++) {
3902  uint8_t flags = 0;
3903 
3904  if (u == 0)
3905  flags = STREAM_TOSERVER|STREAM_START;
3906  else if (u == (httplen1 - 1))
3907  flags = STREAM_TOSERVER|STREAM_EOF;
3908  else
3909  flags = STREAM_TOSERVER;
3910 
3911  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3912  if (r != 0) {
3913  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3914  " 0: ", u, r);
3915  goto end;
3916  }
3917  }
3918 
3919  htp_state = f->alstate;
3920  if (htp_state == NULL) {
3921  printf("no http state: ");
3922  goto end;
3923  }
3924 
3925  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3926  size_t reflen = sizeof(ref) - 1;
3927 
3928  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3929  if (tx == NULL)
3930  goto end;
3931  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3932  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3933  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3934  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3935  (uintmax_t)reflen,
3936  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3937  goto end;
3938  }
3939 
3940  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3941  bstr_len(tx_ud->request_uri_normalized)) != 0)
3942  {
3943  printf("normalized uri \"");
3944  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3945  printf("\" != \"");
3946  PrintRawUriFp(stdout, ref, reflen);
3947  printf("\": ");
3948  goto end;
3949  }
3950  }
3951 
3952  result = 1;
3953 end:
3954  if (alp_tctx != NULL)
3956  StreamTcpFreeConfig(true);
3957  UTHFreeFlow(f);
3958  return result;
3959 }
3960 
3961 #include "conf-yaml-loader.h"
3962 
3963 /** \test Abort
3964  */
3965 static int HTPParserTest08(void)
3966 {
3967  int result = 0;
3968  Flow *f = NULL;
3969  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3970  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3971  TcpSession ssn;
3973 
3974  char input[] = "\
3975 %YAML 1.1\n\
3976 ---\n\
3977 libhtp:\n\
3978 \n\
3979  default-config:\n\
3980  personality: IDS\n\
3981 ";
3982 
3984  ConfInit();
3986 
3987  ConfYamlLoadString(input, strlen(input));
3988  HTPConfigure();
3989 
3990  HtpState *htp_state = NULL;
3991  int r = 0;
3992  memset(&ssn, 0, sizeof(ssn));
3993 
3994  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3995  if (f == NULL)
3996  goto end;
3997  f->protoctx = &ssn;
3998  f->proto = IPPROTO_TCP;
3999  f->alproto = ALPROTO_HTTP1;
4000 
4001  StreamTcpInitConfig(true);
4002 
4003  uint8_t flags = 0;
4004  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
4005 
4006  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
4007  if (r != 0) {
4008  printf("toserver chunk returned %" PRId32 ", expected"
4009  " 0: ", r);
4010  result = 0;
4011  goto end;
4012  }
4013 
4014  htp_state = f->alstate;
4015  if (htp_state == NULL) {
4016  printf("no http state: ");
4017  result = 0;
4018  goto end;
4019  }
4020 
4021  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4022  if (tx == NULL)
4023  goto end;
4024  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4025  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4026  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4027  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4028  bstr_len(tx_ud->request_uri_normalized));
4029  }
4030 
4031  result = 1;
4032 end:
4033  if (alp_tctx != NULL)
4035  StreamTcpFreeConfig(true);
4036  HTPFreeConfig();
4037  ConfDeInit();
4040  UTHFreeFlow(f);
4041  return result;
4042 }
4043 
4044 /** \test Abort
4045  */
4046 static int HTPParserTest09(void)
4047 {
4048  int result = 0;
4049  Flow *f = NULL;
4050  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
4051  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4052  TcpSession ssn;
4054 
4055  char input[] = "\
4056 %YAML 1.1\n\
4057 ---\n\
4058 libhtp:\n\
4059 \n\
4060  default-config:\n\
4061  personality: Apache_2_2\n\
4062 ";
4063 
4065  ConfInit();
4067 
4068  ConfYamlLoadString(input, strlen(input));
4069  HTPConfigure();
4070 
4071  HtpState *htp_state = NULL;
4072  int r = 0;
4073 
4074  memset(&ssn, 0, sizeof(ssn));
4075 
4076  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4077  if (f == NULL)
4078  goto end;
4079  f->protoctx = &ssn;
4080  f->proto = IPPROTO_TCP;
4081  f->alproto = ALPROTO_HTTP1;
4082 
4083  StreamTcpInitConfig(true);
4084 
4085  uint8_t flags = 0;
4086  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
4087 
4088  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
4089  if (r != 0) {
4090  printf("toserver chunk returned %" PRId32 ", expected"
4091  " 0: ", r);
4092  goto end;
4093  }
4094 
4095  htp_state = f->alstate;
4096  if (htp_state == NULL) {
4097  printf("no http state: ");
4098  goto end;
4099  }
4100 
4101  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4102  if (tx == NULL)
4103  goto end;
4104  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4105  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4106  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4107  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4108  bstr_len(tx_ud->request_uri_normalized));
4109  }
4110 
4111  result = 1;
4112 end:
4113  if (alp_tctx != NULL)
4115  StreamTcpFreeConfig(true);
4116  HTPFreeConfig();
4117  ConfDeInit();
4120  UTHFreeFlow(f);
4121  return result;
4122 }
4123 
4124 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4125  */
4126 static int HTPParserTest10(void)
4127 {
4128  int result = 0;
4129  Flow *f = NULL;
4130  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4131  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4132  TcpSession ssn;
4133  HtpState *htp_state = NULL;
4134  int r = 0;
4136 
4137  memset(&ssn, 0, sizeof(ssn));
4138 
4139  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4140  if (f == NULL)
4141  goto end;
4142  f->protoctx = &ssn;
4143  f->proto = IPPROTO_TCP;
4144  f->alproto = ALPROTO_HTTP1;
4145 
4146  StreamTcpInitConfig(true);
4147 
4148  uint32_t u;
4149  for (u = 0; u < httplen1; u++) {
4150  uint8_t flags = 0;
4151 
4152  if (u == 0)
4153  flags = STREAM_TOSERVER|STREAM_START;
4154  else if (u == (httplen1 - 1))
4155  flags = STREAM_TOSERVER|STREAM_EOF;
4156  else
4157  flags = STREAM_TOSERVER;
4158 
4159  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4160  if (r != 0) {
4161  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4162  " 0: ", u, r);
4163  goto end;
4164  }
4165  }
4166 
4167  htp_state = f->alstate;
4168  if (htp_state == NULL) {
4169  printf("no http state: ");
4170  goto end;
4171  }
4172 
4173  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4174  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4175  if (h == NULL) {
4176  goto end;
4177  }
4178 
4179  char *name = bstr_util_strdup_to_c(h->name);
4180  if (name == NULL) {
4181  goto end;
4182  }
4183 
4184  if (strcmp(name, "Host") != 0) {
4185  printf("header name not \"Host\", instead \"%s\": ", name);
4186  free(name);
4187  goto end;
4188  }
4189  free(name);
4190 
4191  char *value = bstr_util_strdup_to_c(h->value);
4192  if (value == NULL) {
4193  goto end;
4194  }
4195 
4196  if (strcmp(value, "www.google.com") != 0) {
4197  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4198  free(value);
4199  goto end;
4200  }
4201  free(value);
4202 
4203  result = 1;
4204 end:
4205  if (alp_tctx != NULL)
4207  StreamTcpFreeConfig(true);
4208  UTHFreeFlow(f);
4209  return result;
4210 }
4211 
4212 /** \test double encoding in path
4213  */
4214 static int HTPParserTest11(void)
4215 {
4216  int result = 0;
4217  Flow *f = NULL;
4218  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4219  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4220  TcpSession ssn;
4221  HtpState *htp_state = NULL;
4222  int r = 0;
4224 
4225  memset(&ssn, 0, sizeof(ssn));
4226 
4227  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4228  if (f == NULL)
4229  goto end;
4230  f->protoctx = &ssn;
4231  f->proto = IPPROTO_TCP;
4232  f->alproto = ALPROTO_HTTP1;
4233 
4234  StreamTcpInitConfig(true);
4235 
4236  uint32_t u;
4237  for (u = 0; u < httplen1; u++) {
4238  uint8_t flags = 0;
4239 
4240  if (u == 0)
4241  flags = STREAM_TOSERVER|STREAM_START;
4242  else if (u == (httplen1 - 1))
4243  flags = STREAM_TOSERVER|STREAM_EOF;
4244  else
4245  flags = STREAM_TOSERVER;
4246 
4247  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4248  if (r != 0) {
4249  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4250  " 0: ", u, r);
4251  goto end;
4252  }
4253  }
4254 
4255  htp_state = f->alstate;
4256  if (htp_state == NULL) {
4257  printf("no http state: ");
4258  goto end;
4259  }
4260 
4261  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4262  if (tx == NULL)
4263  goto end;
4264  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4265  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4266  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4267  printf("normalized uri len should be 2, is %"PRIuMAX,
4268  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4269  goto end;
4270  }
4271 
4272  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4273  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4274  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4275  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4276  {
4277  printf("normalized uri \"");
4278  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4279  printf("\": ");
4280  goto end;
4281  }
4282  }
4283 
4284  result = 1;
4285 end:
4286  if (alp_tctx != NULL)
4288  StreamTcpFreeConfig(true);
4289  UTHFreeFlow(f);
4290  return result;
4291 }
4292 
4293 /** \test double encoding in query
4294  */
4295 static int HTPParserTest12(void)
4296 {
4297  int result = 0;
4298  Flow *f = NULL;
4299  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4300  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4301  TcpSession ssn;
4302  HtpState *htp_state = NULL;
4303  int r = 0;
4305 
4306  memset(&ssn, 0, sizeof(ssn));
4307 
4308  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4309  if (f == NULL)
4310  goto end;
4311  f->protoctx = &ssn;
4312  f->proto = IPPROTO_TCP;
4313  f->alproto = ALPROTO_HTTP1;
4314 
4315  StreamTcpInitConfig(true);
4316 
4317  uint32_t u;
4318  for (u = 0; u < httplen1; u++) {
4319  uint8_t flags = 0;
4320 
4321  if (u == 0)
4322  flags = STREAM_TOSERVER|STREAM_START;
4323  else if (u == (httplen1 - 1))
4324  flags = STREAM_TOSERVER|STREAM_EOF;
4325  else
4326  flags = STREAM_TOSERVER;
4327 
4328  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4329  if (r != 0) {
4330  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4331  " 0: ", u, r);
4332  goto end;
4333  }
4334  }
4335 
4336  htp_state = f->alstate;
4337  if (htp_state == NULL) {
4338  printf("no http state: ");
4339  goto end;
4340  }
4341 
4342  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4343  if (tx == NULL)
4344  goto end;
4345  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4346  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4347  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4348  printf("normalized uri len should be 5, is %"PRIuMAX,
4349  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4350  goto end;
4351  }
4352 
4353  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4354  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4355  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4356  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4357  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4358  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4359  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4360  {
4361  printf("normalized uri \"");
4362  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4363  printf("\": ");
4364  goto end;
4365  }
4366  }
4367 
4368  result = 1;
4369  end:
4370  if (alp_tctx != NULL)
4372  StreamTcpFreeConfig(true);
4373  UTHFreeFlow(f);
4374  return result;
4375 }
4376 
4377 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4378  */
4379 static int HTPParserTest13(void)
4380 {
4381  int result = 0;
4382  Flow *f = NULL;
4383  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4384  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4385  TcpSession ssn;
4386  HtpState *htp_state = NULL;
4387  int r = 0;
4389 
4390  memset(&ssn, 0, sizeof(ssn));
4391 
4392  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4393  if (f == NULL)
4394  goto end;
4395  f->protoctx = &ssn;
4396  f->proto = IPPROTO_TCP;
4397  f->alproto = ALPROTO_HTTP1;
4398 
4399  StreamTcpInitConfig(true);
4400 
4401  uint32_t u;
4402  for (u = 0; u < httplen1; u++) {
4403  uint8_t flags = 0;
4404 
4405  if (u == 0)
4406  flags = STREAM_TOSERVER|STREAM_START;
4407  else if (u == (httplen1 - 1))
4408  flags = STREAM_TOSERVER|STREAM_EOF;
4409  else
4410  flags = STREAM_TOSERVER;
4411 
4412  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4413  if (r != 0) {
4414  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4415  " 0: ", u, r);
4416  goto end;
4417  }
4418  }
4419 
4420  htp_state = f->alstate;
4421  if (htp_state == NULL) {
4422  printf("no http state: ");
4423  goto end;
4424  }
4425 
4426  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4427  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4428  if (h == NULL) {
4429  goto end;
4430  }
4431 
4432  char *name = bstr_util_strdup_to_c(h->name);
4433  if (name == NULL) {
4434  goto end;
4435  }
4436 
4437  if (strcmp(name, "Host") != 0) {
4438  printf("header name not \"Host\", instead \"%s\": ", name);
4439  free(name);
4440  goto end;
4441  }
4442  free(name);
4443 
4444  char *value = bstr_util_strdup_to_c(h->value);
4445  if (value == NULL) {
4446  goto end;
4447  }
4448 
4449  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4450  printf("header value not \"www.google.com\", instead \"");
4451  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4452  printf("\": ");
4453  free(value);
4454  goto end;
4455  }
4456  free(value);
4457 
4458  result = 1;
4459 end:
4460  if (alp_tctx != NULL)
4462  StreamTcpFreeConfig(true);
4463  UTHFreeFlow(f);
4464  return result;
4465 }
4466 
4467 /** \test Test basic config */
4468 static int HTPParserConfigTest01(void)
4469 {
4470  int ret = 0;
4471  char input[] = "\
4472 %YAML 1.1\n\
4473 ---\n\
4474 libhtp:\n\
4475 \n\
4476  default-config:\n\
4477  personality: IDS\n\
4478 \n\
4479  server-config:\n\
4480 \n\
4481  - apache-tomcat:\n\
4482  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4483  personality: Tomcat_6_0\n\
4484 \n\
4485  - iis7:\n\
4486  address: \n\
4487  - 192.168.0.0/24\n\
4488  - 192.168.10.0/24\n\
4489  personality: IIS_7_0\n\
4490 ";
4491 
4493  ConfInit();
4494 
4495  ConfYamlLoadString(input, strlen(input));
4496 
4497  ConfNode *outputs;
4498  outputs = ConfGetNode("libhtp.default-config.personality");
4499  if (outputs == NULL) {
4500  goto end;
4501  }
4502 
4503  outputs = ConfGetNode("libhtp.server-config");
4504  if (outputs == NULL) {
4505  goto end;
4506  }
4507 
4508  ConfNode *node = TAILQ_FIRST(&outputs->head);
4509  if (node == NULL) {
4510  goto end;
4511  }
4512  if (strcmp(node->name, "0") != 0) {
4513  goto end;
4514  }
4515  node = TAILQ_FIRST(&node->head);
4516  if (node == NULL) {
4517  goto end;
4518  }
4519  if (strcmp(node->name, "apache-tomcat") != 0) {
4520  goto end;
4521  }
4522 
4523  int i = 0;
4524  ConfNode *n;
4525 
4526  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4527  if (node2 == NULL) {
4528  goto end;
4529  }
4530  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4531  goto end;
4532  }
4533 
4534  node = ConfNodeLookupChild(node, "address");
4535  if (node == NULL) {
4536  goto end;
4537  }
4538  TAILQ_FOREACH(n, &node->head, next) {
4539  if (n == NULL) {
4540  goto end;
4541  }
4542 
4543  switch(i) {
4544  case 0:
4545  if (strcmp(n->name, "0") != 0) {
4546  goto end;
4547  }
4548  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4549  goto end;
4550  }
4551  break;
4552  case 1:
4553  if (strcmp(n->name, "1") != 0) {
4554  goto end;
4555  }
4556  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4557  goto end;
4558  }
4559  break;
4560  case 2:
4561  if (strcmp(n->name, "2") != 0) {
4562  goto end;
4563  }
4564  if (strcmp(n->val, "::1") != 0) {
4565  goto end;
4566  }
4567  break;
4568  default:
4569  goto end;
4570  }
4571  i++;
4572  }
4573 
4574  outputs = ConfGetNode("libhtp.server-config");
4575  if (outputs == NULL) {
4576  goto end;
4577  }
4578 
4579  node = TAILQ_FIRST(&outputs->head);
4580  node = TAILQ_NEXT(node, next);
4581  if (node == NULL) {
4582  goto end;
4583  }
4584  if (strcmp(node->name, "1") != 0) {
4585  goto end;
4586  }
4587  node = TAILQ_FIRST(&node->head);
4588  if (node == NULL) {
4589  goto end;
4590  }
4591  if (strcmp(node->name, "iis7") != 0) {
4592  goto end;
4593  }
4594 
4595  node2 = ConfNodeLookupChild(node, "personality");
4596  if (node2 == NULL) {
4597  goto end;
4598  }
4599  if (strcmp(node2->val, "IIS_7_0") != 0) {
4600  goto end;
4601  }
4602 
4603  node = ConfNodeLookupChild(node, "address");
4604  if (node == NULL) {
4605  goto end;
4606  }
4607 
4608  i = 0;
4609  TAILQ_FOREACH(n, &node->head, next) {
4610  if (n == NULL) {
4611  goto end;
4612  }
4613 
4614  switch(i) {
4615  case 0:
4616  if (strcmp(n->name, "0") != 0) {
4617  goto end;
4618  }
4619  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4620  goto end;
4621  }
4622  break;
4623  case 1:
4624  if (strcmp(n->name, "1") != 0) {
4625  goto end;
4626  }
4627  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4628  goto end;
4629  }
4630  break;
4631  default:
4632  goto end;
4633  }
4634  i++;
4635  }
4636 
4637  ret = 1;
4638 
4639 end:
4640  ConfDeInit();
4642 
4643  return ret;
4644 }
4645 
4646 /** \test Test config builds radix correctly */
4647 static int HTPParserConfigTest02(void)
4648 {
4649  int ret = 0;
4650  char input[] = "\
4651 %YAML 1.1\n\
4652 ---\n\
4653 libhtp:\n\
4654 \n\
4655  default-config:\n\
4656  personality: IDS\n\
4657 \n\
4658  server-config:\n\
4659 \n\
4660  - apache-tomcat:\n\
4661  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4662  personality: Tomcat_6_0\n\
4663 \n\
4664  - iis7:\n\
4665  address: \n\
4666  - 192.168.0.0/24\n\
4667  - 192.168.10.0/24\n\
4668  personality: IIS_7_0\n\
4669 ";
4670 
4672  ConfInit();
4674 
4675  ConfYamlLoadString(input, strlen(input));
4676 
4677  HTPConfigure();
4678 
4679  if (cfglist.cfg == NULL) {
4680  printf("No default config created.\n");
4681  goto end;
4682  }
4683 
4684  if (cfgtree == NULL) {
4685  printf("No config tree created.\n");
4686  goto end;
4687  }
4688 
4689  htp_cfg_t *htp = cfglist.cfg;
4690  uint8_t buf[128];
4691  const char *addr;
4692  void *user_data = NULL;
4693 
4694  addr = "192.168.10.42";
4695  if (inet_pton(AF_INET, addr, buf) == 1) {
4696  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4697  if (user_data != NULL) {
4698  HTPCfgRec *htp_cfg_rec = user_data;
4699  htp = htp_cfg_rec->cfg;
4700  SCLogDebug("LIBHTP using config: %p", htp);
4701  }
4702  if (htp == NULL) {
4703  printf("Could not get config for: %s\n", addr);
4704  goto end;
4705  }
4706  }
4707  else {
4708  printf("Failed to parse address: %s\n", addr);
4709  goto end;
4710  }
4711 
4712  user_data = NULL;
4713  addr = "::1";
4714  if (inet_pton(AF_INET6, addr, buf) == 1) {
4715  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4716  if (user_data != NULL) {
4717  HTPCfgRec *htp_cfg_rec = user_data;
4718  htp = htp_cfg_rec->cfg;
4719  SCLogDebug("LIBHTP using config: %p", htp);
4720  }
4721  if (htp == NULL) {
4722  printf("Could not get config for: %s\n", addr);
4723  goto end;
4724  }
4725  }
4726  else {
4727  printf("Failed to parse address: %s\n", addr);
4728  goto end;
4729  }
4730 
4731  ret = 1;
4732 
4733 end:
4734  HTPFreeConfig();
4735  ConfDeInit();
4738 
4739  return ret;
4740 }
4741 
4742 /** \test Test traffic is handled by the correct htp config */
4743 static int HTPParserConfigTest03(void)
4744 {
4745  int result = 1;
4746  Flow *f = NULL;
4747  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4748  " Data is c0oL!";
4749  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4750  TcpSession ssn;
4752 
4753  HtpState *htp_state = NULL;
4754  int r = 0;
4755  char input[] = "\
4756 %YAML 1.1\n\
4757 ---\n\
4758 libhtp:\n\
4759 \n\
4760  default-config:\n\
4761  personality: IDS\n\
4762 \n\
4763  server-config:\n\
4764 \n\
4765  - apache-tomcat:\n\
4766  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4767  personality: Tomcat_6_0\n\
4768 \n\
4769  - iis7:\n\
4770  address: \n\
4771  - 192.168.0.0/24\n\
4772  - 192.168.10.0/24\n\
4773  personality: IIS_7_0\n\
4774 ";
4775 
4777  ConfInit();
4779 
4780  ConfYamlLoadString(input, strlen(input));
4781 
4782  HTPConfigure();
4783 
4784  const char *addr = "192.168.10.42";
4785 
4786  memset(&ssn, 0, sizeof(ssn));
4787 
4788  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4789  if (f == NULL)
4790  goto end;
4791  f->protoctx = &ssn;
4792  f->proto = IPPROTO_TCP;
4793  f->alproto = ALPROTO_HTTP1;
4794 
4795  htp_cfg_t *htp = cfglist.cfg;
4796 
4797  void *user_data = NULL;
4798  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4799  if (user_data != NULL) {
4800  HTPCfgRec *htp_cfg_rec = user_data;
4801  htp = htp_cfg_rec->cfg;
4802  SCLogDebug("LIBHTP using config: %p", htp);
4803  }
4804  if (htp == NULL) {
4805  printf("Could not get config for: %s\n", addr);
4806  goto end;
4807  }
4808 
4809  StreamTcpInitConfig(true);
4810 
4811  uint32_t u;
4812  for (u = 0; u < httplen1; u++) {
4813  uint8_t flags = 0;
4814 
4815  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4816  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4817  else flags = STREAM_TOSERVER;
4818 
4819  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4820  if (r != 0) {
4821  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4822  " 0: ", u, r);
4823  result = 0;
4824  goto end;
4825  }
4826  }
4827 
4828  htp_state = f->alstate;
4829  if (htp_state == NULL) {
4830  printf("no http state: ");
4831  result = 0;
4832  goto end;
4833  }
4834 
4835  if (HTPStateGetTxCnt(htp_state) != 2) {
4836  printf("HTPStateGetTxCnt(htp_state) failure\n");
4837  goto end;
4838  }
4839 
4840  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4841  if (tx == NULL)
4842  goto end;
4843  if (tx->cfg != htp) {
4844  printf("wrong HTP config (%p instead of %p - default=%p): ",
4845  tx->cfg, htp, cfglist.cfg);
4846  goto end;
4847  }
4848  tx = HTPStateGetTx(htp_state, 1);
4849  if (tx == NULL)
4850  goto end;
4851  if (tx->cfg != htp) {
4852  printf("wrong HTP config (%p instead of %p - default=%p): ",
4853  tx->cfg, htp, cfglist.cfg);
4854  goto end;
4855  }
4856 
4857 end:
4858  if (alp_tctx != NULL)
4860  HTPFreeConfig();
4861  ConfDeInit();
4864 
4865  StreamTcpFreeConfig(true);
4866  UTHFreeFlow(f);
4867  return result;
4868 }
4869 
4870 /* disabled when we upgraded to libhtp 0.5.x */
4871 #if 0
4872 static int HTPParserConfigTest04(void)
4873 {
4874  int result = 0;
4875 
4876  char input[] = "\
4877 %YAML 1.1\n\
4878 ---\n\
4879 libhtp:\n\
4880 \n\
4881  default-config:\n\
4882  personality: IDS\n\
4883  path-control-char-handling: status_400\n\
4884  path-convert-utf8: yes\n\
4885  path-invalid-encoding-handling: remove_percent\n\
4886 \n\
4887  server-config:\n\
4888 \n\
4889  - apache-tomcat:\n\
4890  personality: Tomcat_6_0\n\
4891  path-invalid-utf8-handling: none\n\
4892  path-nul-encoded-handling: status_404\n\
4893  path-nul-raw-handling: status_400\n\
4894 \n\
4895  - iis7:\n\
4896  personality: IIS_7_0\n\
4897  path-replacement-char: o\n\
4898  path-unicode-mapping: status_400\n\
4899 ";
4900 
4902  ConfInit();
4904 
4905  ConfYamlLoadString(input, strlen(input));
4906 
4907  HTPConfigure();
4908 
4909  HTPCfgRec *cfg_rec = &cfglist;
4910  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4911  cfg_rec->cfg->path_convert_utf8 != 1 ||
4912  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4913  printf("failed 1\n");
4914  goto end;
4915  }
4916 
4917  cfg_rec = cfg_rec->next;
4918  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4919  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4920  printf("failed 2\n");
4921  goto end;
4922  }
4923 
4924  cfg_rec = cfg_rec->next;
4925  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4926  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4927  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4928  printf("failed 3\n");
4929  goto end;
4930  }
4931 
4932  result = 1;
4933 
4934 end:
4935  HTPFreeConfig();
4936  ConfDeInit();
4939 
4940  return result;
4941 }
4942 #endif
4943 
4944 /** \test Test %2f decoding in profile Apache_2_2
4945  *
4946  * %2f in path is left untouched
4947  * %2f in query string is normalized to %2F
4948  * %252f in query string is decoded/normalized to %2F
4949  */
4950 static int HTPParserDecodingTest01(void)
4951 {
4952  uint8_t httpbuf1[] =
4953  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4954  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4955  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4956  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4957  TcpSession ssn;
4960 
4961  char input[] = "\
4962 %YAML 1.1\n\
4963 ---\n\
4964 libhtp:\n\
4965 \n\
4966  default-config:\n\
4967  personality: Apache_2\n\
4968 ";
4969 
4971  ConfInit();
4973  ConfYamlLoadString(input, strlen(input));
4974  HTPConfigure();
4975  const char *addr = "4.3.2.1";
4976  memset(&ssn, 0, sizeof(ssn));
4977 
4978  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4979  FAIL_IF_NULL(f);
4980  f->protoctx = &ssn;
4981  f->proto = IPPROTO_TCP;
4982  f->alproto = ALPROTO_HTTP1;
4983 
4984  StreamTcpInitConfig(true);
4985 
4986  for (uint32_t u = 0; u < httplen1; u++) {
4987  uint8_t flags = 0;
4988  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4989  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4990  else flags = STREAM_TOSERVER;
4991 
4992  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4993  FAIL_IF(r != 0);
4994  }
4995 
4996  HtpState *htp_state = f->alstate;
4997  FAIL_IF_NULL(htp_state);
4998 
4999  uint8_t ref1[] = "/abc%2fdef";
5000  size_t reflen = sizeof(ref1) - 1;
5001 
5002  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5003  FAIL_IF_NULL(tx);
5004 
5005  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5006  FAIL_IF_NULL(tx_ud);
5008  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5009  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5010  bstr_len(tx_ud->request_uri_normalized)) != 0);
5011 
5012  uint8_t ref2[] = "/abc/def?ghi/jkl";
5013  reflen = sizeof(ref2) - 1;
5014 
5015  tx = HTPStateGetTx(htp_state, 1);
5016  FAIL_IF_NULL(tx);
5017  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5018  FAIL_IF_NULL(tx_ud);
5020  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5021 
5022  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5023  bstr_len(tx_ud->request_uri_normalized)) != 0);
5024 
5025  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5026  reflen = sizeof(ref3) - 1;
5027  tx = HTPStateGetTx(htp_state, 2);
5028  FAIL_IF_NULL(tx);
5029  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5030  FAIL_IF_NULL(tx_ud);
5032  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5033 
5034  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5035  bstr_len(tx_ud->request_uri_normalized)) != 0);
5036 
5038  HTPFreeConfig();
5039  ConfDeInit();
5042 
5043  StreamTcpFreeConfig(true);
5044  UTHFreeFlow(f);
5045  PASS;
5046 }
5047 
5048 static int HTPParserDecodingTest01a(void)
5049 {
5050  uint8_t httpbuf1[] = "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5051  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5052  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5053  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5054  TcpSession ssn;
5057 
5058  char input[] = "\
5059 %YAML 1.1\n\
5060 ---\n\
5061 libhtp:\n\
5062 \n\
5063  default-config:\n\
5064  personality: Apache_2\n\
5065 ";
5066 
5068  ConfInit();
5070  ConfYamlLoadString(input, strlen(input));
5071  HTPConfigure();
5072  const char *addr = "4.3.2.1";
5073  memset(&ssn, 0, sizeof(ssn));
5074 
5075  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5076  FAIL_IF_NULL(f);
5077  f->protoctx = &ssn;
5078  f->proto = IPPROTO_TCP;
5079  f->alproto = ALPROTO_HTTP1;
5080 
5081  StreamTcpInitConfig(true);
5082 
5083  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
5084  (STREAM_TOSERVER | STREAM_START | STREAM_EOF), httpbuf1, httplen1);
5085  FAIL_IF(r != 0);
5086 
5087  HtpState *htp_state = f->alstate;
5088  FAIL_IF_NULL(htp_state);
5089 
5090  uint8_t ref1[] = "/abc%2fdef";
5091  size_t reflen = sizeof(ref1) - 1;
5092 
5093  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5094  FAIL_IF_NULL(tx);
5095 
5096  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5097  FAIL_IF_NULL(tx_ud);
5099  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5100  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5101  bstr_len(tx_ud->request_uri_normalized)) != 0);
5102 
5103  uint8_t ref2[] = "/abc/def?ghi/jkl";
5104  reflen = sizeof(ref2) - 1;
5105 
5106  tx = HTPStateGetTx(htp_state, 1);
5107  FAIL_IF_NULL(tx);
5108  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5109  FAIL_IF_NULL(tx_ud);
5111  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5112 
5113  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5114  bstr_len(tx_ud->request_uri_normalized)) != 0);
5115 
5116  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5117  reflen = sizeof(ref3) - 1;
5118  tx = HTPStateGetTx(htp_state, 2);
5119  FAIL_IF_NULL(tx);
5120  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5121  FAIL_IF_NULL(tx_ud);
5123  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5124 
5125  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5126  bstr_len(tx_ud->request_uri_normalized)) != 0);
5127 
5129  HTPFreeConfig();
5130  ConfDeInit();
5133 
5134  StreamTcpFreeConfig(true);
5135  UTHFreeFlow(f);
5136  PASS;
5137 }
5138 
5139 /** \test Test %2f decoding in profile IDS
5140  *
5141  * %2f in path decoded to /
5142  * %2f in query string is decoded to /
5143  * %252f in query string is decoded to %2F
5144  */
5145 static int HTPParserDecodingTest02(void)
5146 {
5147  int result = 0;
5148  Flow *f = NULL;
5149  uint8_t httpbuf1[] =
5150  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5151  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5152  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5153  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5154  TcpSession ssn;
5156 
5157  HtpState *htp_state = NULL;
5158  int r = 0;
5159  char input[] = "\
5160 %YAML 1.1\n\
5161 ---\n\
5162 libhtp:\n\
5163 \n\
5164  default-config:\n\
5165  personality: IDS\n\
5166  double-decode-path: no\n\
5167  double-decode-query: no\n\
5168 ";
5169 
5171  ConfInit();
5173  ConfYamlLoadString(input, strlen(input));
5174  HTPConfigure();
5175  const char *addr = "4.3.2.1";
5176  memset(&ssn, 0, sizeof(ssn));
5177 
5178  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5179  if (f == NULL)
5180  goto end;
5181  f->protoctx = &ssn;
5182  f->proto = IPPROTO_TCP;
5183  f->alproto = ALPROTO_HTTP1;
5184 
5185  StreamTcpInitConfig(true);
5186 
5187  uint32_t u;
5188  for (u = 0; u < httplen1; u++) {
5189  uint8_t flags = 0;
5190 
5191  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5192  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5193  else flags = STREAM_TOSERVER;
5194 
5195  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5196  if (r != 0) {
5197  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5198  " 0: ", u, r);
5199  goto end;
5200  }
5201  }
5202 
5203  htp_state = f->alstate;
5204  if (htp_state == NULL) {
5205  printf("no http state: ");
5206  goto end;
5207  }
5208 
5209  uint8_t ref1[] = "/abc/def";
5210  size_t reflen = sizeof(ref1) - 1;
5211 
5212  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5213  if (tx == NULL)
5214  goto end;
5215  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5216  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5217  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5218  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5219  (uintmax_t)reflen,
5220  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5221  goto end;
5222  }
5223 
5224  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5225  bstr_len(tx_ud->request_uri_normalized)) != 0)
5226  {
5227  printf("normalized uri \"");
5228  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5229  printf("\" != \"");
5230  PrintRawUriFp(stdout, ref1, reflen);
5231  printf("\": ");
5232  goto end;
5233  }
5234  }
5235 
5236  uint8_t ref2[] = "/abc/def?ghi/jkl";
5237  reflen = sizeof(ref2) - 1;
5238 
5239  tx = HTPStateGetTx(htp_state, 1);
5240  if (tx == NULL)
5241  goto end;
5242  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5243  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5244  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5245  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5246  (uintmax_t)reflen,
5247  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5248  goto end;
5249  }
5250 
5251  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5252  bstr_len(tx_ud->request_uri_normalized)) != 0)
5253  {
5254  printf("normalized uri \"");
5255  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5256  printf("\" != \"");
5257  PrintRawUriFp(stdout, ref2, reflen);
5258  printf("\": ");
5259  goto end;
5260  }
5261  }
5262 
5263  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5264  reflen = sizeof(ref3) - 1;
5265  tx = HTPStateGetTx(htp_state, 2);
5266  if (tx == NULL)
5267  goto end;
5268  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5269  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5270  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5271  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5272  (uintmax_t)reflen,
5273  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5274  goto end;
5275  }
5276 
5277  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5278  bstr_len(tx_ud->request_uri_normalized)) != 0)
5279  {
5280  printf("normalized uri \"");
5281  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5282  printf("\" != \"");
5283  PrintRawUriFp(stdout, ref3, reflen);
5284  printf("\": ");
5285  goto end;
5286  }
5287  }
5288 
5289  result = 1;
5290 
5291 end:
5292  if (alp_tctx != NULL)
5294  HTPFreeConfig();
5295  ConfDeInit();
5298 
5299  StreamTcpFreeConfig(true);
5300  UTHFreeFlow(f);
5301  return result;
5302 }
5303 
5304 /** \test Test %2f decoding in profile IDS with double-decode-* options
5305  *
5306  * %252f in path decoded to /
5307  * %252f in query string is decoded to /
5308  */
5309 static int HTPParserDecodingTest03(void)
5310 {
5311  int result = 0;
5312  Flow *f = NULL;
5313  uint8_t httpbuf1[] =
5314  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5315  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5316  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5317  TcpSession ssn;
5319 
5320  HtpState *htp_state = NULL;
5321  int r = 0;
5322  char input[] = "\
5323 %YAML 1.1\n\
5324 ---\n\
5325 libhtp:\n\
5326 \n\
5327  default-config:\n\
5328  personality: IDS\n\
5329  double-decode-path: yes\n\
5330  double-decode-query: yes\n\
5331 ";
5332 
5334  ConfInit();
5336  ConfYamlLoadString(input, strlen(input));
5337  HTPConfigure();
5338  const char *addr = "4.3.2.1";
5339  memset(&ssn, 0, sizeof(ssn));
5340 
5341  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5342  if (f == NULL)
5343  goto end;
5344  f->protoctx = &ssn;
5345  f->proto = IPPROTO_TCP;
5346  f->alproto = ALPROTO_HTTP1;
5347 
5348  StreamTcpInitConfig(true);
5349 
5350  uint32_t u;
5351  for (u = 0; u < httplen1; u++) {
5352  uint8_t flags = 0;
5353 
5354  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5355  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5356  else flags = STREAM_TOSERVER;
5357 
5358  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5359  if (r != 0) {
5360  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5361  " 0: ", u, r);
5362  goto end;
5363  }
5364  }
5365 
5366  htp_state = f->alstate;
5367  if (htp_state == NULL) {
5368  printf("no http state: ");
5369  goto end;
5370  }
5371 
5372  uint8_t ref1[] = "/abc/def";
5373  size_t reflen = sizeof(ref1) - 1;
5374 
5375  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5376  if (tx == NULL)
5377  goto end;
5378  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5379  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5380  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5381  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5382  (uintmax_t)reflen,
5383  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5384  goto end;
5385  }
5386 
5387  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5388  bstr_len(tx_ud->request_uri_normalized)) != 0)
5389  {
5390  printf("normalized uri \"");
5391  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5392  printf("\" != \"");
5393  PrintRawUriFp(stdout, ref1, reflen);
5394  printf("\": ");
5395  goto end;
5396  }
5397  }
5398 
5399  uint8_t ref2[] = "/abc/def?ghi/jkl";
5400  reflen = sizeof(ref2) - 1;
5401 
5402  tx = HTPStateGetTx(htp_state, 1);
5403  if (tx == NULL)
5404  goto end;
5405  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5406  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5407  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5408  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5409  (uintmax_t)reflen,
5410  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5411  goto end;
5412  }
5413 
5414  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5415  bstr_len(tx_ud->request_uri_normalized)) != 0)
5416  {
5417  printf("normalized uri \"");
5418  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5419  printf("\" != \"");
5420  PrintRawUriFp(stdout, ref2, reflen);
5421  printf("\": ");
5422  goto end;
5423  }
5424  }
5425 
5426  result = 1;
5427 
5428 end:
5429  if (alp_tctx != NULL)
5431  HTPFreeConfig();
5432  ConfDeInit();
5435 
5436  StreamTcpFreeConfig(true);
5437  UTHFreeFlow(f);
5438  return result;
5439 }
5440 
5441 /** \test Test http:// in query profile IDS
5442  */
5443 static int HTPParserDecodingTest04(void)
5444 {
5445  int result = 0;
5446  Flow *f = NULL;
5447  uint8_t httpbuf1[] =
5448  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5449  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5450  TcpSession ssn;
5452 
5453  HtpState *htp_state = NULL;
5454  int r = 0;
5455  char input[] = "\
5456 %YAML 1.1\n\
5457 ---\n\
5458 libhtp:\n\
5459 \n\
5460  default-config:\n\
5461  personality: IDS\n\
5462  double-decode-path: yes\n\
5463  double-decode-query: yes\n\
5464 ";
5465 
5467  ConfInit();
5469  ConfYamlLoadString(input, strlen(input));
5470  HTPConfigure();
5471  const char *addr = "4.3.2.1";
5472  memset(&ssn, 0, sizeof(ssn));
5473 
5474  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5475  if (f == NULL)
5476  goto end;
5477  f->protoctx = &ssn;
5478  f->proto = IPPROTO_TCP;
5479  f->alproto = ALPROTO_HTTP1;
5480 
5481  StreamTcpInitConfig(true);
5482 
5483  uint32_t u;
5484  for (u = 0; u < httplen1; u++) {
5485  uint8_t flags = 0;
5486 
5487  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5488  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5489  else flags = STREAM_TOSERVER;
5490 
5491  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5492  if (r != 0) {
5493  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5494  " 0: ", u, r);
5495  goto end;
5496  }
5497  }
5498 
5499  htp_state = f->alstate;
5500  if (htp_state == NULL) {
5501  printf("no http state: ");
5502  goto end;
5503  }
5504 
5505  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5506  size_t reflen = sizeof(ref1) - 1;
5507 
5508  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5509  if (tx == NULL)
5510  goto end;
5511  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5512  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5513  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5514  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5515  (uintmax_t)reflen,
5516  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5517  goto end;
5518  }
5519 
5520  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5521  bstr_len(tx_ud->request_uri_normalized)) != 0)
5522  {
5523  printf("normalized uri \"");
5524  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5525  printf("\" != \"");
5526  PrintRawUriFp(stdout, ref1, reflen);
5527  printf("\": ");
5528  goto end;
5529  }
5530  }
5531 
5532  result = 1;
5533 
5534 end:
5535  if (alp_tctx != NULL)
5537  HTPFreeConfig();
5538  ConfDeInit();
5541 
5542  StreamTcpFreeConfig(true);
5543  UTHFreeFlow(f);
5544  return result;
5545 }
5546 
5547 /** \test Test \ char in query profile IDS. Bug 739
5548  */
5549 static int HTPParserDecodingTest05(void)
5550 {
5551  int result = 0;
5552  Flow *f = NULL;
5553  uint8_t httpbuf1[] =
5554  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5555  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5556  TcpSession ssn;
5558 
5559  HtpState *htp_state = NULL;
5560  int r = 0;
5561  char input[] = "\
5562 %YAML 1.1\n\
5563 ---\n\
5564 libhtp:\n\
5565 \n\
5566  default-config:\n\
5567  personality: IDS\n\
5568  double-decode-path: yes\n\
5569  double-decode-query: yes\n\
5570 ";
5571 
5573  ConfInit();
5575  ConfYamlLoadString(input, strlen(input));
5576  HTPConfigure();
5577  const char *addr = "4.3.2.1";
5578  memset(&ssn, 0, sizeof(ssn));
5579 
5580  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5581  if (f == NULL)
5582  goto end;
5583  f->protoctx = &ssn;
5584  f->proto = IPPROTO_TCP;
5585  f->alproto = ALPROTO_HTTP1;
5586 
5587  StreamTcpInitConfig(true);
5588 
5589  uint32_t u;
5590  for (u = 0; u < httplen1; u++) {
5591  uint8_t flags = 0;
5592 
5593  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5594  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5595  else flags = STREAM_TOSERVER;
5596 
5597  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5598  if (r != 0) {
5599  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5600  " 0: ", u, r);
5601  goto end;
5602  }
5603  }
5604 
5605  htp_state = f->alstate;
5606  if (htp_state == NULL) {
5607  printf("no http state: ");
5608  goto end;
5609  }
5610 
5611  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5612  size_t reflen = sizeof(ref1) - 1;
5613 
5614  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5615  if (tx == NULL)
5616  goto end;
5617  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5618  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5619  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5620  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5621  (uintmax_t)reflen,
5622  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5623  goto end;
5624  }
5625 
5626  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5627  bstr_len(tx_ud->request_uri_normalized)) != 0)
5628  {
5629  printf("normalized uri \"");
5630  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5631  printf("\" != \"");
5632  PrintRawUriFp(stdout, ref1, reflen);
5633  printf("\": ");
5634  goto end;
5635  }
5636  }
5637 
5638  result = 1;
5639 
5640 end:
5641  if (alp_tctx != NULL)
5643  HTPFreeConfig();
5644  ConfDeInit();
5647 
5648  StreamTcpFreeConfig(true);
5649  UTHFreeFlow(f);
5650  return result;
5651 }
5652 
5653 /** \test Test + char in query. Bug 1035
5654  */
5655 static int HTPParserDecodingTest06(void)
5656 {
5657  int result = 0;
5658  Flow *f = NULL;
5659  uint8_t httpbuf1[] =
5660  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5661  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5662  TcpSession ssn;
5664 
5665  HtpState *htp_state = NULL;
5666  int r = 0;
5667  char input[] = "\
5668 %YAML 1.1\n\
5669 ---\n\
5670 libhtp:\n\
5671 \n\
5672  default-config:\n\
5673  personality: IDS\n\
5674  double-decode-path: yes\n\
5675  double-decode-query: yes\n\
5676 ";
5677 
5679  ConfInit();
5681  ConfYamlLoadString(input, strlen(input));
5682  HTPConfigure();
5683  const char *addr = "4.3.2.1";
5684  memset(&ssn, 0, sizeof(ssn));
5685 
5686  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5687  if (f == NULL)
5688  goto end;
5689  f->protoctx = &ssn;
5690  f->proto = IPPROTO_TCP;
5691  f->alproto = ALPROTO_HTTP1;
5692 
5693  StreamTcpInitConfig(true);
5694 
5695  uint32_t u;
5696  for (u = 0; u < httplen1; u++) {
5697  uint8_t flags = 0;
5698 
5699  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5700  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5701  else flags = STREAM_TOSERVER;
5702 
5703  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5704  if (r != 0) {
5705  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5706  " 0: ", u, r);
5707  goto end;
5708  }
5709  }
5710 
5711  htp_state = f->alstate;
5712  if (htp_state == NULL) {
5713  printf("no http state: ");
5714  goto end;
5715  }
5716 
5717  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5718  size_t reflen = sizeof(ref1) - 1;
5719 
5720  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5721  if (tx == NULL)
5722  goto end;
5723  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5724  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5725  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5726  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5727  (uintmax_t)reflen,
5728  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5729  goto end;
5730  }
5731 
5732  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5733  bstr_len(tx_ud->request_uri_normalized)) != 0)
5734  {
5735  printf("normalized uri \"");
5736  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5737  printf("\" != \"");
5738  PrintRawUriFp(stdout, ref1, reflen);
5739  printf("\": ");
5740  goto end;
5741  }
5742  }
5743 
5744  result = 1;
5745 
5746 end:
5747  if (alp_tctx != NULL)
5749  HTPFreeConfig();
5750  ConfDeInit();
5753 
5754  StreamTcpFreeConfig(true);
5755  UTHFreeFlow(f);
5756  return result;
5757 }
5758 
5759 /** \test Test + char in query. Bug 1035
5760  */
5761 static int HTPParserDecodingTest07(void)
5762 {
5763  int result = 0;
5764  Flow *f = NULL;
5765  uint8_t httpbuf1[] =
5766  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5767  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5768  TcpSession ssn;
5770 
5771  HtpState *htp_state = NULL;
5772  int r = 0;
5773  char input[] = "\
5774 %YAML 1.1\n\
5775 ---\n\
5776 libhtp:\n\
5777 \n\
5778  default-config:\n\
5779  personality: IDS\n\
5780  double-decode-path: yes\n\
5781  double-decode-query: yes\n\
5782  query-plusspace-decode: yes\n\
5783 ";
5784 
5786  ConfInit();
5788  ConfYamlLoadString(input, strlen(input));
5789  HTPConfigure();
5790  const char *addr = "4.3.2.1";
5791  memset(&ssn, 0, sizeof(ssn));
5792 
5793  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5794  if (f == NULL)
5795  goto end;
5796  f->protoctx = &ssn;
5797  f->proto = IPPROTO_TCP;
5798  f->alproto = ALPROTO_HTTP1;
5799 
5800  StreamTcpInitConfig(true);
5801 
5802  uint32_t u;
5803  for (u = 0; u < httplen1; u++) {
5804  uint8_t flags = 0;
5805 
5806  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5807  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5808  else flags = STREAM_TOSERVER;
5809 
5810  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5811  if (r != 0) {
5812  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5813  " 0: ", u, r);
5814  goto end;
5815  }
5816  }
5817 
5818  htp_state = f->alstate;
5819  if (htp_state == NULL) {
5820  printf("no http state: ");
5821  goto end;
5822  }
5823 
5824  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5825  size_t reflen = sizeof(ref1) - 1;
5826 
5827  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5828  if (tx == NULL)
5829  goto end;
5830  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5831  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {