suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "decode.h"
40 #include "threads.h"
41 #include "counters.h"
42 
43 #include "util-print.h"
44 #include "util-pool.h"
45 #include "util-radix-tree.h"
46 #include "util-file.h"
47 #include "util-byte.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-detect-proto.h"
59 #include "app-layer-frames.h"
60 #include "app-layer-htp.h"
61 #include "app-layer-htp-body.h"
62 #include "app-layer-htp-file.h"
63 #include "app-layer-htp-libhtp.h"
64 #include "app-layer-htp-xff.h"
65 #include "app-layer-htp-range.h"
66 #include "app-layer-htp-mem.h"
67 
68 #include "util-spm.h"
69 #include "util-debug.h"
70 #include "util-time.h"
71 #include "util-misc.h"
72 #include "util-enum.h"
73 
74 #include "util-unittest.h"
75 #include "util-unittest-helper.h"
76 #include "flow-util.h"
77 
78 #include "detect-engine.h"
79 #include "detect-engine-build.h"
80 #include "detect-engine-state.h"
81 #include "detect-parse.h"
82 
83 #include "decode-events.h"
84 
85 #include "util-memcmp.h"
86 #include "util-random.h"
87 #include "util-validate.h"
88 
89 //#define PRINT
90 
91 /** Fast lookup tree (radix) for the various HTP configurations */
92 static SCRadixTree *cfgtree;
93 /** List of HTP configurations. */
94 static HTPCfgRec cfglist;
95 
97 
98 /** Limit to the number of libhtp messages that can be handled */
99 #define HTP_MAX_MESSAGES 512
100 
101 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
102 
103 #ifdef DEBUG
104 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
105 static uint64_t htp_state_memuse = 0;
106 static uint64_t htp_state_memcnt = 0;
107 #endif
108 
110  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
111  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
112  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
113  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
114  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
115  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
116  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
118  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
120  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
122  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
124  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
126  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
128  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
129  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
131  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
132  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
133  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
134  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
135  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
136  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
137  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
138  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
139  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
140  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
141  { "FILE_NAME_TOO_LONG", HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG },
142  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
143  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
144  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
146  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
147  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
148  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
149  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
150  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
151  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
152  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
153  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
154  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
155  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
156  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
157  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
158  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
160  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
161  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
162  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
163  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
164 
165  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
166  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
167 
168  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
169 
170  /* suricata warnings/errors */
171  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
172  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
173  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
174 
175  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
176  { "FAILED_PROTOCOL_CHANGE", HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE },
177 
178  { NULL, -1 },
179 };
180 
181 /* app-layer-frame-documentation tag start: HttpFrameTypes */
185 };
186 
188  {
189  "request",
191  },
192  {
193  "response",
195  },
196  { NULL, -1 },
197 };
198 /* app-layer-frame-documentation tag end: HttpFrameTypes */
199 
200 static int HTTPGetFrameIdByName(const char *frame_name)
201 {
202  int id = SCMapEnumNameToValue(frame_name, http_frame_table);
203  if (id < 0) {
204  return -1;
205  }
206  return id;
207 }
208 
209 static const char *HTTPGetFrameNameById(const uint8_t frame_id)
210 {
211  const char *name = SCMapEnumValueToName(frame_id, http_frame_table);
212  return name;
213 }
214 
215 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
216 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
217 static uint64_t HTPStateGetTxCnt(void *alstate);
218 #ifdef UNITTESTS
219 static void HTPParserRegisterTests(void);
220 #endif
221 
222 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
223 {
224  uint64_t id = HTPStateGetTxCnt(s);
225  BUG_ON(id == 0);
226  return id - 1;
227 }
228 
229 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
230 {
231  return s->transaction_cnt;
232 }
233 
234 #ifdef DEBUG
235 /**
236  * \internal
237  *
238  * \brief Lookup the HTP personality string from the numeric personality.
239  *
240  * \todo This needs to be a libhtp function.
241  */
242 static const char *HTPLookupPersonalityString(int p)
243 {
244 #define CASE_HTP_PERSONALITY_STRING(p) \
245  case HTP_SERVER_ ## p: return #p
246 
247  switch (p) {
248  CASE_HTP_PERSONALITY_STRING(MINIMAL);
249  CASE_HTP_PERSONALITY_STRING(GENERIC);
250  CASE_HTP_PERSONALITY_STRING(IDS);
251  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
252  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
253  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
254  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
255  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
256  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
257  CASE_HTP_PERSONALITY_STRING(APACHE_2);
258  }
259 
260  return NULL;
261 }
262 #endif /* DEBUG */
263 
264 /**
265  * \internal
266  *
267  * \brief Lookup the numeric HTP personality from a string.
268  *
269  * \todo This needs to be a libhtp function.
270  */
271 static int HTPLookupPersonality(const char *str)
272 {
273 #define IF_HTP_PERSONALITY_NUM(p) \
274  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
275 
276  IF_HTP_PERSONALITY_NUM(MINIMAL);
277  IF_HTP_PERSONALITY_NUM(GENERIC);
279  IF_HTP_PERSONALITY_NUM(IIS_4_0);
280  IF_HTP_PERSONALITY_NUM(IIS_5_0);
281  IF_HTP_PERSONALITY_NUM(IIS_5_1);
282  IF_HTP_PERSONALITY_NUM(IIS_6_0);
283  IF_HTP_PERSONALITY_NUM(IIS_7_0);
284  IF_HTP_PERSONALITY_NUM(IIS_7_5);
285  IF_HTP_PERSONALITY_NUM(APACHE_2);
286  if (strcasecmp("TOMCAT_6_0", str) == 0) {
287  SCLogError("Personality %s no "
288  "longer supported by libhtp.",
289  str);
290  return -1;
291  } else if ((strcasecmp("APACHE", str) == 0) ||
292  (strcasecmp("APACHE_2_2", str) == 0))
293  {
294  SCLogWarning("Personality %s no "
295  "longer supported by libhtp, failing back to "
296  "Apache2 personality.",
297  str);
298  return HTP_SERVER_APACHE_2;
299  }
300 
301  return -1;
302 }
303 
304 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
305  const uint8_t dir, const uint8_t e)
306 {
307  SCLogDebug("setting event %u", e);
308 
309  if (htud) {
310  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
311  s->events++;
312  return;
313  }
314 
315  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
316  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
317 
318  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
319  if (tx == NULL && tx_id > 0)
320  tx = HTPStateGetTx(s, tx_id - 1);
321  if (tx != NULL) {
322  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
323  if (htud != NULL) {
324  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
325  s->events++;
326  return;
327  }
328  }
329  SCLogDebug("couldn't set event %u", e);
330 }
331 
332 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
333  * connection parser to be used by the HTP library
334  */
335 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
336 {
337  SCEnter();
338 
339  HtpState *s = HTPMalloc(sizeof(HtpState));
340  if (unlikely(s == NULL)) {
341  SCReturnPtr(NULL, "void");
342  }
343 
344  memset(s, 0x00, sizeof(HtpState));
345 
346 #ifdef DEBUG
347  SCMutexLock(&htp_state_mem_lock);
348  htp_state_memcnt++;
349  htp_state_memuse += sizeof(HtpState);
350  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
351  SCMutexUnlock(&htp_state_mem_lock);
352 #endif
353 
354  SCReturnPtr((void *)s, "void");
355 }
356 
357 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
358 {
359  if (likely(htud)) {
360  HtpBodyFree(&state->cfg->request, &htud->request_body);
361  HtpBodyFree(&state->cfg->response, &htud->response_body);
362  bstr_free(htud->request_uri_normalized);
363  if (htud->request_headers_raw)
365  if (htud->response_headers_raw)
368  if (htud->boundary)
369  HTPFree(htud->boundary, htud->boundary_len);
370  if (htud->tx_data.de_state != NULL) {
371  DetectEngineStateFree(htud->tx_data.de_state);
372  }
373  if (htud->file_range) {
374  HTPFileCloseHandleRange(&htp_sbcfg, &htud->files_tc, 0, htud->file_range, NULL, 0);
376  }
379  HTPFree(htud, sizeof(HtpTxUserData));
380  }
381 }
382 
383 /** \brief Function to frees the HTTP state memory and also frees the HTTP
384  * connection parser memory which was used by the HTP library
385  */
386 void HTPStateFree(void *state)
387 {
388  SCEnter();
389 
390  HtpState *s = (HtpState *)state;
391  if (s == NULL) {
392  SCReturn;
393  }
394 
395  /* free the connection parser memory used by HTP library */
396  if (s->connp != NULL) {
397  SCLogDebug("freeing HTP state");
398 
399  uint64_t tx_id;
400  uint64_t total_txs = HTPStateGetTxCnt(state);
401  /* free the list of body chunks */
402  if (s->conn != NULL) {
403  for (tx_id = 0; tx_id < total_txs; tx_id++) {
404  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
405  if (tx != NULL) {
406  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
407  HtpTxUserDataFree(s, htud);
408  htp_tx_set_user_data(tx, NULL);
409  }
410  }
411  }
412  htp_connp_destroy_all(s->connp);
413  }
414 
415  HTPFree(s, sizeof(HtpState));
416 
417 #ifdef DEBUG
418  SCMutexLock(&htp_state_mem_lock);
419  htp_state_memcnt--;
420  htp_state_memuse -= sizeof(HtpState);
421  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
422  SCMutexUnlock(&htp_state_mem_lock);
423 #endif
424 
425  SCReturn;
426 }
427 
428 /**
429  * \brief HTP transaction cleanup callback
430  *
431  * \warning We cannot actually free the transactions here. It seems that
432  * HTP only accepts freeing of transactions in the response callback.
433  */
434 static void HTPStateTransactionFree(void *state, uint64_t id)
435 {
436  SCEnter();
437 
438  HtpState *s = (HtpState *)state;
439 
440  SCLogDebug("state %p, id %"PRIu64, s, id);
441 
442  htp_tx_t *tx = HTPStateGetTx(s, id);
443  if (tx != NULL) {
444  /* This will remove obsolete body chunks */
445  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
446  HtpTxUserDataFree(s, htud);
447  htp_tx_set_user_data(tx, NULL);
448 
449  /* hack: even if libhtp considers the tx incomplete, we want to
450  * free it here. htp_tx_destroy however, will refuse to do this.
451  * As htp_tx_destroy_incomplete isn't available in the public API,
452  * we hack around it here. */
453  if (unlikely(!(
454  tx->request_progress == HTP_REQUEST_COMPLETE &&
455  tx->response_progress == HTP_RESPONSE_COMPLETE)))
456  {
457  tx->request_progress = HTP_REQUEST_COMPLETE;
458  tx->response_progress = HTP_RESPONSE_COMPLETE;
459  }
460  htp_tx_destroy(tx);
461  }
462 }
463 
464 /**
465  * \brief Sets a flag that informs the HTP app layer that some module in the
466  * engine needs the http request body data.
467  * \initonly
468  */
470 {
471  SCEnter();
472 
473  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
474  SCReturn;
475 }
476 
477 /**
478  * \brief Sets a flag that informs the HTP app layer that some module in the
479  * engine needs the http request body data.
480  * \initonly
481  */
483 {
484  SCEnter();
485 
486  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
487  SCReturn;
488 }
489 
490 /**
491  * \brief Sets a flag that informs the HTP app layer that some module in the
492  * engine needs the http request multi part header.
493  *
494  * \initonly
495  */
496 static void AppLayerHtpNeedMultipartHeader(void)
497 {
498  SCEnter();
500 
501  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
502  SCReturn;
503 }
504 
505 /**
506  * \brief Sets a flag that informs the HTP app layer that some module in the
507  * engine needs the http request file.
508  *
509  * \initonly
510  */
512 {
513  SCEnter();
514  AppLayerHtpNeedMultipartHeader();
517 
518  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
519  SCReturn;
520 }
521 
522 static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags)
523 {
524  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
525  if (tx_ud) {
526  SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags);
527  if (flags & STREAM_TOCLIENT) {
528  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
529  } else {
530  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
531  }
532  }
533 }
534 
535 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
536 {
537  SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64,
539  if (flags & HTP_STREAM_DEPTH_SET) {
540  uint32_t stream_depth = FileReassemblyDepth();
541  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
542  SCLogDebug("true");
543  return true;
544  }
545  } else {
546  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
547  return true;
548  }
549  }
550  SCLogDebug("false");
551  return false;
552 }
553 
554 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
555  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
556 {
557  uint32_t chunk_len = 0;
558  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
559  (content_len_so_far < (uint64_t)body_limit) &&
560  (content_len_so_far + (uint64_t)data_len) > body_limit)
561  {
562  chunk_len = body_limit - content_len_so_far;
563  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
564  (content_len_so_far < (uint64_t)stream_depth) &&
565  (content_len_so_far + (uint64_t)data_len) > stream_depth)
566  {
567  chunk_len = stream_depth - content_len_so_far;
568  }
569  SCLogDebug("len %u", chunk_len);
570  return (chunk_len == 0 ? data_len : chunk_len);
571 }
572 
573 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
574 struct {
575  const char *msg;
576  uint8_t de;
577 } htp_errors[] = {
578  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
579  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
580  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
581  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
582  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
583 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
584  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
585 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
586  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
587  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
588  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
589  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
590 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
591  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
592  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
593  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
594  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
595 };
596 
597 struct {
598  const char *msg;
599  uint8_t de;
600 } htp_warnings[] = {
601  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
602  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
603  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
604  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
605  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
606 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
607  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
608  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
609  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
610  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
611  * luckily, "Request server port=" is unique */
612 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
614  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
615  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
616  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
617  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
618  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
619  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
620  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
621  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
622  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
623  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
624  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
625  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
626  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
627  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
628  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
629  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
630  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
631  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
632 };
633 
634 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
635 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
636 
637 /**
638  * \internal
639  *
640  * \brief Get the warning id for the warning msg.
641  *
642  * \param msg warning message
643  *
644  * \retval id the id or 0 in case of not found
645  */
646 static uint8_t HTPHandleWarningGetId(const char *msg)
647 {
648  SCLogDebug("received warning \"%s\"", msg);
649  size_t idx;
650  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
651  if (strncmp(htp_warnings[idx].msg, msg,
652  strlen(htp_warnings[idx].msg)) == 0)
653  {
654  return htp_warnings[idx].de;
655  }
656  }
657 
658  return 0;
659 }
660 
661 /**
662  * \internal
663  *
664  * \brief Get the error id for the error msg.
665  *
666  * \param msg error message
667  *
668  * \retval id the id or 0 in case of not found
669  */
670 static uint8_t HTPHandleErrorGetId(const char *msg)
671 {
672  SCLogDebug("received error \"%s\"", msg);
673 
674  size_t idx;
675  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
676  if (strncmp(htp_errors[idx].msg, msg,
677  strlen(htp_errors[idx].msg)) == 0)
678  {
679  return htp_errors[idx].de;
680  }
681  }
682 
683  return 0;
684 }
685 
686 /**
687  * \internal
688  *
689  * \brief Check state for errors, warnings and add any as events
690  *
691  * \param s state
692  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
693  */
694 static void HTPHandleError(HtpState *s, const uint8_t dir)
695 {
696  if (s == NULL || s->conn == NULL ||
697  s->conn->messages == NULL) {
698  return;
699  }
700 
701  size_t size = htp_list_size(s->conn->messages);
702  size_t msg;
703  if(size >= HTP_MAX_MESSAGES) {
705  //only once per HtpState
706  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
708  //too noisy in fuzzing
709  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
710  }
711  // ignore further messages
712  return;
713  }
714 
715  for (msg = s->htp_messages_offset; msg < size; msg++) {
716  htp_log_t *log = htp_list_get(s->conn->messages, msg);
717  if (log == NULL)
718  continue;
719 
720  HtpTxUserData *htud = NULL;
721  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
722  if (tx != NULL)
723  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
724 
725  SCLogDebug("message %s", log->msg);
726 
727  uint8_t id = HTPHandleErrorGetId(log->msg);
728  if (id == 0) {
729  id = HTPHandleWarningGetId(log->msg);
730  if (id == 0)
732  }
733 
734  if (id > 0) {
735  HTPSetEvent(s, htud, dir, id);
736  }
737  }
738  s->htp_messages_offset = (uint16_t)msg;
739  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
740 }
741 
742 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
743 {
744 #ifdef DEBUG
745  BUG_ON(s == NULL || tx == NULL);
746 #endif
747  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
748  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
749  HTP_HOSTH_INVALID))
750  {
751  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
752  if (htud == NULL)
753  return;
754 
755  if (tx->flags & HTP_REQUEST_INVALID_T_E)
756  HTPSetEvent(s, htud, STREAM_TOSERVER,
758  if (tx->flags & HTP_REQUEST_INVALID_C_L)
759  HTPSetEvent(s, htud, STREAM_TOSERVER,
761  if (tx->flags & HTP_HOST_MISSING)
762  HTPSetEvent(s, htud, STREAM_TOSERVER,
764  if (tx->flags & HTP_HOST_AMBIGUOUS)
765  HTPSetEvent(s, htud, STREAM_TOSERVER,
767  if (tx->flags & HTP_HOSTU_INVALID)
768  HTPSetEvent(s, htud, STREAM_TOSERVER,
770  if (tx->flags & HTP_HOSTH_INVALID)
771  HTPSetEvent(s, htud, STREAM_TOSERVER,
773  }
774  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
775  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
776  if (htud == NULL)
777  return;
778  HTPSetEvent(s, htud, STREAM_TOSERVER,
780  }
781  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
782  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
783  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
784  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
785  if (htud == NULL)
786  return;
787  HTPSetEvent(s, htud, STREAM_TOSERVER,
789  }
790 }
791 
792 static int Setup(Flow *f, HtpState *hstate)
793 {
794  /* store flow ref in state so callbacks can access it */
795  hstate->f = f;
796 
797  HTPCfgRec *htp_cfg_rec = &cfglist;
798  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
799  void *user_data = NULL;
800 
801  if (FLOW_IS_IPV4(f)) {
802  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
803  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
804  }
805  else if (FLOW_IS_IPV6(f)) {
806  SCLogDebug("Looking up HTP config for ipv6");
807  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
808  }
809  else {
810  SCLogError("unknown address family, bug!");
811  goto error;
812  }
813 
814  if (user_data != NULL) {
815  htp_cfg_rec = user_data;
816  htp = htp_cfg_rec->cfg;
817  SCLogDebug("LIBHTP using config: %p", htp);
818  } else {
819  SCLogDebug("Using default HTP config: %p", htp);
820  }
821 
822  if (NULL == htp) {
823 #ifdef DEBUG_VALIDATION
824  BUG_ON(htp == NULL);
825 #endif
826  /* should never happen if HTPConfigure is properly invoked */
827  goto error;
828  }
829 
830  hstate->connp = htp_connp_create(htp);
831  if (hstate->connp == NULL) {
832  goto error;
833  }
834 
835  hstate->conn = htp_connp_get_connection(hstate->connp);
836 
837  htp_connp_set_user_data(hstate->connp, (void *)hstate);
838  hstate->cfg = htp_cfg_rec;
839 
840  SCLogDebug("New hstate->connp %p", hstate->connp);
841 
842  struct timeval tv = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
843  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &tv);
844 
846  htp_cfg_rec->request.inspect_min_size);
848  htp_cfg_rec->response.inspect_min_size);
849  return 0;
850 error:
851  return -1;
852 }
853 
854 /**
855  * \brief Function to handle the reassembled data from client and feed it to
856  * the HTP library to process it.
857  *
858  * \param flow Pointer to the flow the data belong to
859  * \param htp_state Pointer the state in which the parsed value to be stored
860  * \param pstate Application layer parser state for this session
861  *
862  * \retval On success returns 1 or on failure returns -1.
863  */
864 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state, AppLayerParserState *pstate,
865  StreamSlice stream_slice, void *local_data)
866 {
867  SCEnter();
868  int ret = 0;
869  HtpState *hstate = (HtpState *)htp_state;
870 
871  /* On the first invocation, create the connection parser structure to
872  * be used by HTP library. This is looked up via IP in the radix
873  * tree. Failing that, the default HTP config is used.
874  */
875  if (NULL == hstate->conn) {
876  if (Setup(f, hstate) != 0) {
878  }
879  }
880  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
881  hstate->slice = &stream_slice;
882 
883  const uint8_t *input = StreamSliceGetData(&stream_slice);
884  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
885 
886  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
887  /* pass the new data to the htp parser */
888  if (input_len > 0) {
889  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
890  switch (r) {
891  case HTP_STREAM_ERROR:
892  ret = -1;
893  break;
894  default:
895  break;
896  }
897  HTPHandleError(hstate, STREAM_TOSERVER);
898  }
899 
900  /* if the TCP connection is closed, then close the HTTP connection */
902  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
903  {
904  htp_connp_req_close(hstate->connp, &ts);
905  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
906  SCLogDebug("stream eof encountered, closing htp handle for ts");
907  }
908 
909  SCLogDebug("hstate->connp %p", hstate->connp);
910  hstate->slice = NULL;
911 
912  if (ret < 0) {
914  }
916 }
917 
918 /**
919  * \brief Function to handle the reassembled data from server and feed it to
920  * the HTP library to process it.
921  *
922  * \param flow Pointer to the flow the data belong to
923  * \param htp_state Pointer the state in which the parsed value to be stored
924  * \param pstate Application layer parser state for this session
925  * \param input Pointer the received HTTP server data
926  * \param input_len Length in bytes of the received data
927  * \param output Pointer to the output (not used in this function)
928  *
929  * \retval On success returns 1 or on failure returns -1
930  */
931 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state, AppLayerParserState *pstate,
932  StreamSlice stream_slice, void *local_data)
933 {
934  SCEnter();
935  int ret = 0;
936  HtpState *hstate = (HtpState *)htp_state;
937 
938  const uint8_t *input = StreamSliceGetData(&stream_slice);
939  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
940 
941  /* On the first invocation, create the connection parser structure to
942  * be used by HTP library. This is looked up via IP in the radix
943  * tree. Failing that, the default HTP config is used.
944  */
945  if (NULL == hstate->conn) {
946  if (Setup(f, hstate) != 0) {
948  }
949  }
950  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
951  hstate->slice = &stream_slice;
952 
953  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
954  htp_tx_t *tx = NULL;
955  size_t consumed = 0;
956  if (input_len > 0) {
957  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
958  switch (r) {
959  case HTP_STREAM_ERROR:
960  ret = -1;
961  break;
962  case HTP_STREAM_TUNNEL:
963  tx = htp_connp_get_out_tx(hstate->connp);
964  if (tx != NULL && tx->response_status_number == 101) {
965  htp_header_t *h =
966  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
967  if (h == NULL || bstr_cmp_c(h->value, "h2c") != 0) {
968  break;
969  }
971  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
972  break;
973  }
974  uint16_t dp = 0;
975  if (tx->request_port_number != -1) {
976  dp = (uint16_t)tx->request_port_number;
977  }
978  consumed = htp_connp_res_data_consumed(hstate->connp);
979  hstate->slice = NULL;
980  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_HTTP2)) {
981  HTPSetEvent(hstate, NULL, STREAM_TOCLIENT,
983  }
984  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
985  // and we need to parser the remaining part with HTTP2
986  if (consumed > 0 && consumed < input_len) {
987  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
988  }
990  }
991  break;
992  default:
993  break;
994  }
995  HTPHandleError(hstate, STREAM_TOCLIENT);
996  }
997 
998  /* if we the TCP connection is closed, then close the HTTP connection */
1000  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
1001  {
1002  htp_connp_close(hstate->connp, &ts);
1003  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
1004  }
1005 
1006  SCLogDebug("hstate->connp %p", hstate->connp);
1007  hstate->slice = NULL;
1008 
1009  if (ret < 0) {
1011  }
1013 }
1014 
1015 /**
1016  * \param name /Lowercase/ version of the variable name
1017  */
1018 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1019  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1020 {
1021 #ifdef PRINT
1022  printf("DATA START: \n");
1023  PrintRawDataFp(stdout, data, len);
1024  printf("DATA END: \n");
1025 #endif
1026  size_t x;
1027  int quote = 0;
1028 
1029  for (x = 0; x < len; x++) {
1030  if (!(isspace(data[x])))
1031  break;
1032  }
1033 
1034  if (x >= len)
1035  return 0;
1036 
1037  uint8_t *line = data+x;
1038  size_t line_len = len-x;
1039  size_t offset = 0;
1040 #ifdef PRINT
1041  printf("LINE START: \n");
1042  PrintRawDataFp(stdout, line, line_len);
1043  printf("LINE END: \n");
1044 #endif
1045  for (x = 0 ; x < line_len; x++) {
1046  if (x > 0) {
1047  if (line[x - 1] != '\\' && line[x] == '\"') {
1048  quote++;
1049  }
1050 
1051  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1052  uint8_t *token = line + offset;
1053  size_t token_len = x - offset;
1054 
1055  if ((x + 1) == line_len) {
1056  token_len++;
1057  }
1058 
1059  offset = x + 1;
1060 
1061  while (offset < line_len && isspace(line[offset])) {
1062  x++;
1063  offset++;
1064  }
1065 #ifdef PRINT
1066  printf("TOKEN START: \n");
1067  PrintRawDataFp(stdout, token, token_len);
1068  printf("TOKEN END: \n");
1069 #endif
1070  if (token_len > name_len) {
1071  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1072  uint8_t *value = token + name_len;
1073  size_t value_len = token_len - name_len;
1074 
1075  if (value[0] == '\"') {
1076  value++;
1077  value_len--;
1078  }
1079  if (value[value_len-1] == '\"') {
1080  value_len--;
1081  }
1082 #ifdef PRINT
1083  printf("VALUE START: \n");
1084  PrintRawDataFp(stdout, value, value_len);
1085  printf("VALUE END: \n");
1086 #endif
1087  *retptr = value;
1088  *retlen = value_len;
1089  return 1;
1090  }
1091  }
1092  }
1093  }
1094  }
1095 
1096  return 0;
1097 }
1098 
1099 /**
1100  * \param name /Lowercase/ version of the variable name
1101  */
1102 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1103  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1104 {
1105  SCEnter();
1106 #ifdef PRINT
1107  printf("DATA START: \n");
1108  PrintRawDataFp(stdout, data, len);
1109  printf("DATA END: \n");
1110 #endif
1111  size_t x;
1112  int quote = 0;
1113 
1114  for (x = 0; x < len; x++) {
1115  if (!(isspace(data[x])))
1116  break;
1117  }
1118 
1119  if (x >= len) {
1120  SCReturnInt(0);
1121  }
1122 
1123  uint8_t *line = data+x;
1124  size_t line_len = len-x;
1125  size_t offset = 0;
1126 #ifdef PRINT
1127  printf("LINE START: \n");
1128  PrintRawDataFp(stdout, line, line_len);
1129  printf("LINE END: \n");
1130 #endif
1131  for (x = 0 ; x < line_len; x++) {
1132  if (x > 0) {
1133  if (line[x - 1] != '\\' && line[x] == '\"') {
1134  quote++;
1135  }
1136 
1137  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1138  uint8_t *token = line + offset;
1139  size_t token_len = x - offset;
1140 
1141  if ((x + 1) == line_len) {
1142  token_len++;
1143  }
1144 
1145  offset = x + 1;
1146 
1147  while (offset < line_len && isspace(line[offset])) {
1148  x++;
1149  offset++;
1150  }
1151 #ifdef PRINT
1152  printf("TOKEN START: \n");
1153  PrintRawDataFp(stdout, token, token_len);
1154  printf("TOKEN END: \n");
1155 #endif
1156  if (token_len > name_len) {
1157  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1158  uint8_t *value = token + name_len;
1159  size_t value_len = token_len - name_len;
1160 
1161  if (value[0] == '\"') {
1162  value++;
1163  value_len--;
1164  }
1165  if (value[value_len-1] == '\"') {
1166  value_len--;
1167  }
1168 #ifdef PRINT
1169  printf("VALUE START: \n");
1170  PrintRawDataFp(stdout, value, value_len);
1171  printf("VALUE END: \n");
1172 #endif
1173  *retptr = value;
1174  *retlen = value_len;
1175  SCReturnInt(1);
1176  }
1177  }
1178  }
1179  }
1180  }
1181 
1182  SCReturnInt(0);
1183 }
1184 
1185 /**
1186  * \brief setup multipart parsing: extract boundary and store it
1187  *
1188  * \param d HTTP transaction
1189  * \param htud transaction userdata
1190  *
1191  * \retval 1 ok, multipart set up
1192  * \retval 0 ok, not multipart though
1193  * \retval -1 error: problem with the boundary
1194  *
1195  * If the request contains a multipart message, this function will
1196  * set the HTP_BOUNDARY_SET in the transaction.
1197  */
1198 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1199 {
1200  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1201  "Content-Type");
1202  if (h != NULL && bstr_len(h->value) > 0) {
1203  uint8_t *boundary = NULL;
1204  size_t boundary_len = 0;
1205 
1206  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1207  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1208  &boundary, &boundary_len);
1209  if (r == 1) {
1210 #ifdef PRINT
1211  printf("BOUNDARY START: \n");
1212  PrintRawDataFp(stdout, boundary, boundary_len);
1213  printf("BOUNDARY END: \n");
1214 #endif
1215  if (boundary_len < HTP_BOUNDARY_MAX) {
1216  htud->boundary = HTPMalloc(boundary_len);
1217  if (htud->boundary == NULL) {
1218  return -1;
1219  }
1220  htud->boundary_len = (uint8_t)boundary_len;
1221  memcpy(htud->boundary, boundary, boundary_len);
1222 
1223  htud->tsflags |= HTP_BOUNDARY_SET;
1224  } else {
1225  SCLogDebug("invalid boundary");
1226  return -1;
1227  }
1228  SCReturnInt(1);
1229  }
1230  //SCReturnInt(1);
1231  }
1232  SCReturnInt(0);
1233 }
1234 
1235 #define C_D_HDR "content-disposition:"
1236 #define C_D_HDR_LEN 20
1237 #define C_T_HDR "content-type:"
1238 #define C_T_HDR_LEN 13
1240 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1241  HtpTxUserData *htud,
1242  uint8_t *header, uint32_t header_len,
1243  uint8_t **filename, uint16_t *filename_len,
1244  uint8_t **filetype, uint16_t *filetype_len)
1245 {
1246  uint8_t *fn = NULL;
1247  size_t fn_len = 0;
1248  uint8_t *ft = NULL;
1249  size_t ft_len = 0;
1250 
1251 #ifdef PRINT
1252  printf("HEADER START: \n");
1253  PrintRawDataFp(stdout, header, header_len);
1254  printf("HEADER END: \n");
1255 #endif
1256 
1257  while (header_len > 0) {
1258  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1259  uint8_t *line = header;
1260  uint32_t line_len;
1261 
1262  if (next_line == NULL) {
1263  line_len = header_len;
1264  } else {
1265  line_len = next_line - header;
1266  }
1267  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1268  if (sc == NULL) {
1269  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1271  /* if the : we found is the final char, it means we have
1272  * no value */
1273  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1274  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1276  } else {
1277 #ifdef PRINT
1278  printf("LINE START: \n");
1279  PrintRawDataFp(stdout, line, line_len);
1280  printf("LINE END: \n");
1281 #endif
1282  if (line_len >= C_D_HDR_LEN &&
1283  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1284  uint8_t *value = line + C_D_HDR_LEN;
1285  uint32_t value_len = line_len - C_D_HDR_LEN;
1286 
1287  /* parse content-disposition */
1288  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1289  value, value_len, &fn, &fn_len);
1290  } else if (line_len >= C_T_HDR_LEN &&
1291  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1292  SCLogDebug("content-type line");
1293  uint8_t *value = line + C_T_HDR_LEN;
1294  uint32_t value_len = line_len - C_T_HDR_LEN;
1295 
1296  (void)HTTPParseContentTypeHeader(NULL, 0,
1297  value, value_len, &ft, &ft_len);
1298  }
1299  }
1300 
1301  if (next_line == NULL) {
1302  SCLogDebug("no next_line");
1303  break;
1304  }
1305  header_len -= ((next_line + 2) - header);
1306  header = next_line + 2;
1307  } /* while (header_len > 0) */
1308 
1309  if (fn_len > USHRT_MAX)
1310  fn_len = USHRT_MAX;
1311  if (ft_len > USHRT_MAX)
1312  ft_len = USHRT_MAX;
1313 
1314  *filename = fn;
1315  *filename_len = (uint16_t)fn_len;
1316  *filetype = ft;
1317  *filetype_len = (uint16_t)ft_len;
1318 }
1319 
1320 /**
1321  * \brief Create a single buffer from the HtpBodyChunks in our list
1322  *
1323  * \param htud transaction user data
1324  * \param chunks_buffers pointer to pass back the buffer to the caller
1325  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1326  */
1327 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1328  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1329 {
1331  chunks_buffer, chunks_buffer_len,
1332  htud->request_body.body_parsed);
1333 }
1334 
1335 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1336 {
1337  SCEnter();
1338  if (tx && tx->tx_data.de_state) {
1339  if (dir == STREAM_TOSERVER) {
1340  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1341  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1342  } else if (STREAM_TOCLIENT) {
1343  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1344  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1345  }
1346  }
1347 }
1348 
1349 /**
1350  * \brief Setup boundary buffers
1351  */
1352 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1353  uint8_t *boundary, uint32_t boundary_len)
1354 {
1355  memset(boundary, '-', boundary_len);
1356  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1357 }
1358 
1359 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1360  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1361 {
1362  int result = 0;
1363  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1364  uint16_t expected_boundary_len = htud->boundary_len + 2;
1365  uint16_t expected_boundary_end_len = htud->boundary_len + 4;
1366  int tx_progress = 0;
1367 
1368 #ifdef PRINT
1369  printf("CHUNK START: \n");
1370  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1371  printf("CHUNK END: \n");
1372 #endif
1373 
1374  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1375 
1376  /* search for the header start, header end and form end */
1377  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1378  boundary, expected_boundary_len);
1379  /* end of the multipart form */
1380  const uint8_t *form_end = NULL;
1381  /* end marker belonging to header_start */
1382  const uint8_t *header_end = NULL;
1383  if (header_start != NULL) {
1384  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1385  (uint8_t *)"\r\n\r\n", 4);
1386  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1387  boundary, expected_boundary_end_len);
1388  }
1389 
1390  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1391  header_end, form_end);
1392 
1393  /* we currently only handle multipart for ts. When we support it for tc,
1394  * we will need to supply right direction */
1395  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1396  /* if we're in the file storage process, deal with that now */
1397  if (htud->tsflags & HTP_FILENAME_SET) {
1398  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1399  SCLogDebug("reached the end of the file");
1400 
1401  const uint8_t *filedata = chunks_buffer;
1402  uint32_t filedata_len = 0;
1403  uint8_t flags = 0;
1404 
1405  if (header_start != NULL) {
1406  if (header_start == filedata + 2) {
1407  /* last chunk had all data, but not the boundary */
1408  SCLogDebug("last chunk had all data, but not the boundary");
1409  filedata_len = 0;
1410  } else if (header_start > filedata + 2) {
1411  SCLogDebug("some data from last file before the boundary");
1412  /* some data from last file before the boundary */
1413  filedata_len = header_start - filedata - 2;
1414  }
1415  }
1416  /* body parsing done, we did not get our form end. Use all data
1417  * we still have and signal to files API we have an issue. */
1418  if (tx_progress > HTP_REQUEST_BODY) {
1419  filedata_len = chunks_buffer_len;
1421  }
1422 
1423  if (filedata_len > chunks_buffer_len) {
1424  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1426  goto end;
1427  }
1428 #ifdef PRINT
1429  printf("FILEDATA (final chunk) START: \n");
1430  PrintRawDataFp(stdout, filedata, filedata_len);
1431  printf("FILEDATA (final chunk) END: \n");
1432 #endif
1433  if (!(htud->tsflags & HTP_DONTSTORE)) {
1434  if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
1435  -1) {
1436  goto end;
1437  }
1438  }
1439 
1440  htud->tsflags &=~ HTP_FILENAME_SET;
1441 
1442  /* fall through */
1443  } else {
1444  SCLogDebug("not yet at the end of the file");
1445 
1446  if (chunks_buffer_len > expected_boundary_end_len) {
1447  const uint8_t *filedata = chunks_buffer;
1448  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1449 #ifdef PRINT
1450  printf("FILEDATA (part) START: \n");
1451  PrintRawDataFp(stdout, filedata, filedata_len);
1452  printf("FILEDATA (part) END: \n");
1453 #endif
1454 
1455  if (!(htud->tsflags & HTP_DONTSTORE)) {
1456  result = HTPFileStoreChunk(
1457  hstate, htud, filedata, filedata_len, STREAM_TOSERVER);
1458  if (result == -1) {
1459  goto end;
1460  } else if (result == -2) {
1461  /* we know for sure we're not storing the file */
1462  htud->tsflags |= HTP_DONTSTORE;
1463  }
1464  }
1465 
1466  htud->request_body.body_parsed += filedata_len;
1467  } else {
1468  SCLogDebug("chunk too small to already process in part");
1469  }
1470 
1471  goto end;
1472  }
1473  }
1474 
1475  while (header_start != NULL && header_end != NULL &&
1476  header_end != form_end &&
1477  header_start < (chunks_buffer + chunks_buffer_len) &&
1478  header_end < (chunks_buffer + chunks_buffer_len) &&
1479  header_start < header_end)
1480  {
1481  uint8_t *filename = NULL;
1482  uint16_t filename_len = 0;
1483  uint8_t *filetype = NULL;
1484  uint16_t filetype_len = 0;
1485 
1486  uint32_t header_len = header_end - header_start;
1487  SCLogDebug("header_len %u", header_len);
1488  uint8_t *header = (uint8_t *)header_start;
1489 
1490  /* skip empty records */
1491  if (expected_boundary_len == header_len) {
1492  goto next;
1493  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1494  header_len -= (expected_boundary_len + 2);
1495  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1496  }
1497 
1498  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1499  &filename, &filename_len, &filetype, &filetype_len);
1500 
1501  if (filename != NULL) {
1502  const uint8_t *filedata = NULL;
1503  uint32_t filedata_len = 0;
1504 
1505  SCLogDebug("we have a filename");
1506 
1507  htud->tsflags |= HTP_FILENAME_SET;
1508  htud->tsflags &= ~HTP_DONTSTORE;
1509 
1510  SCLogDebug("header_end %p", header_end);
1511  SCLogDebug("form_end %p", form_end);
1512 
1513  /* everything until the final boundary is the file */
1514  if (form_end != NULL) {
1515  SCLogDebug("have form_end");
1516 
1517  filedata = header_end + 4;
1518  if (form_end == filedata) {
1519  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1521  goto end;
1522  } else if (form_end < filedata) {
1523  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1525  goto end;
1526  }
1527 
1528  filedata_len = form_end - (header_end + 4 + 2);
1529  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1530 
1531  /* or is it? */
1532  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1533  boundary, expected_boundary_len);
1534  if (header_next != NULL) {
1535  filedata_len -= (form_end - header_next);
1536  }
1537 
1538  if (filedata_len > chunks_buffer_len) {
1539  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1541  goto end;
1542  }
1543  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1544 #ifdef PRINT
1545  printf("FILEDATA START: \n");
1546  PrintRawDataFp(stdout, filedata, filedata_len);
1547  printf("FILEDATA END: \n");
1548 #endif
1549 
1550  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata, filedata_len,
1551  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1552  if (result == -1) {
1553  goto end;
1554  } else if (result == -2) {
1555  htud->tsflags |= HTP_DONTSTORE;
1556  } else {
1557  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1558  goto end;
1559  }
1560  }
1561  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1562 
1563  htud->request_body.body_parsed += (header_end - chunks_buffer);
1564  htud->tsflags &= ~HTP_FILENAME_SET;
1565  } else {
1566  SCLogDebug("chunk doesn't contain form end");
1567 
1568  filedata = header_end + 4;
1569  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1570  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1571 
1572  if (filedata_len > chunks_buffer_len) {
1573  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1575  goto end;
1576  }
1577 
1578 #ifdef PRINT
1579  printf("FILEDATA START: \n");
1580  PrintRawDataFp(stdout, filedata, filedata_len);
1581  printf("FILEDATA END: \n");
1582 #endif
1583  /* form doesn't end in this chunk, but the part might. Lets
1584  * see if have another coming up */
1585  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1586  boundary, expected_boundary_len);
1587  SCLogDebug("header_next %p", header_next);
1588  if (header_next == NULL) {
1589  SCLogDebug("more file data to come");
1590 
1591  uint32_t offset = (header_end + 4) - chunks_buffer;
1592  SCLogDebug("offset %u", offset);
1593  htud->request_body.body_parsed += offset;
1594 
1595  if (filedata_len >= (uint32_t)(expected_boundary_len + 2)) {
1596  filedata_len -= (expected_boundary_len + 2 - 1);
1597  SCLogDebug("opening file with partial data");
1598  } else {
1599  filedata = NULL;
1600  filedata_len = 0;
1601  }
1602  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1603  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1604  if (result == -1) {
1605  goto end;
1606  } else if (result == -2) {
1607  htud->tsflags |= HTP_DONTSTORE;
1608  }
1609  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1610  htud->request_body.body_parsed += filedata_len;
1611  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1612 
1613  } else if (header_next - filedata > 2) {
1614  filedata_len = header_next - filedata - 2;
1615  SCLogDebug("filedata_len %u", filedata_len);
1616 
1617  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1618  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1619  if (result == -1) {
1620  goto end;
1621  } else if (result == -2) {
1622  htud->tsflags |= HTP_DONTSTORE;
1623  } else {
1624  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1625  goto end;
1626  }
1627  }
1628  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1629 
1630  htud->tsflags &= ~HTP_FILENAME_SET;
1631  htud->request_body.body_parsed += (header_end - chunks_buffer);
1632  }
1633  }
1634  }
1635 next:
1636  SCLogDebug("header_start %p, header_end %p, form_end %p",
1637  header_start, header_end, form_end);
1638 
1639  /* Search next boundary entry after the start of body */
1640  uint32_t cursizeread = header_end - chunks_buffer;
1641  header_start = Bs2bmSearch(header_end + 4,
1642  chunks_buffer_len - (cursizeread + 4),
1643  boundary, expected_boundary_len);
1644  if (header_start != NULL) {
1645  header_end = Bs2bmSearch(header_end + 4,
1646  chunks_buffer_len - (cursizeread + 4),
1647  (uint8_t *) "\r\n\r\n", 4);
1648  }
1649  }
1650 
1651  /* if we're parsing the multipart and we're not currently processing a
1652  * file, we move the body pointer forward. */
1653  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1654  if (chunks_buffer_len > expected_boundary_end_len) {
1655  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1656 
1657  htud->request_body.body_parsed += move;
1658  SCLogDebug("form not ready, file not set, parsing non-file "
1659  "record: moved %u", move);
1660  }
1661  }
1662 
1663 end:
1664  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1665  return 0;
1666 }
1667 
1668 /** \internal
1669  * \brief Handle POST or PUT, no multipart body data
1670  */
1671 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1672  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1673 {
1674  int result = 0;
1675 
1676  /* see if we need to open the file */
1677  if (!(htud->tsflags & HTP_FILENAME_SET))
1678  {
1679  uint8_t *filename = NULL;
1680  size_t filename_len = 0;
1681 
1682  /* get the name */
1683  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1684  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1685  filename_len = bstr_len(tx->parsed_uri->path);
1686  }
1687 
1688  if (filename != NULL) {
1689  if (filename_len > SC_FILENAME_MAX) {
1690  // explicitly truncate the file name if too long
1691  filename_len = SC_FILENAME_MAX;
1692  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1693  }
1694  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1695  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1696  if (result == -1) {
1697  goto end;
1698  } else if (result == -2) {
1699  htud->tsflags |= HTP_DONTSTORE;
1700  } else {
1701  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1702  htud->tsflags |= HTP_FILENAME_SET;
1703  htud->tsflags &= ~HTP_DONTSTORE;
1704  }
1705  }
1706  }
1707  else
1708  {
1709  /* otherwise, just store the data */
1710 
1711  if (!(htud->tsflags & HTP_DONTSTORE)) {
1712  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOSERVER);
1713  if (result == -1) {
1714  goto end;
1715  } else if (result == -2) {
1716  /* we know for sure we're not storing the file */
1717  htud->tsflags |= HTP_DONTSTORE;
1718  }
1719  }
1720  }
1721 
1722  return 0;
1723 end:
1724  return -1;
1725 }
1726 
1727 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1728  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1729 {
1730  SCEnter();
1731 
1732  int result = 0;
1733 
1734  /* see if we need to open the file
1735  * we check for tx->response_line in case of junk
1736  * interpreted as body before response line
1737  */
1738  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1739  (tx->response_line != NULL || tx->is_protocol_0_9))
1740  {
1741  SCLogDebug("setting up file name");
1742 
1743  uint8_t *filename = NULL;
1744  size_t filename_len = 0;
1745 
1746  /* try Content-Disposition header first */
1747  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1748  "Content-Disposition");
1749  if (h != NULL && bstr_len(h->value) > 0) {
1750  /* parse content-disposition */
1751  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1752  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1753  }
1754 
1755  /* fall back to name from the uri */
1756  if (filename == NULL) {
1757  /* get the name */
1758  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1759  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1760  filename_len = bstr_len(tx->parsed_uri->path);
1761  }
1762  }
1763 
1764  if (filename != NULL) {
1765  // set range if present
1766  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1767  if (filename_len > SC_FILENAME_MAX) {
1768  // explicitly truncate the file name if too long
1769  filename_len = SC_FILENAME_MAX;
1770  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1771  }
1772  if (h_content_range != NULL) {
1773  result = HTPFileOpenWithRange(hstate, htud, filename, (uint16_t)filename_len, data,
1774  data_len, HtpGetActiveResponseTxID(hstate), h_content_range->value, htud);
1775  } else {
1776  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1777  HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1778  }
1779  SCLogDebug("result %d", result);
1780  if (result == -1) {
1781  goto end;
1782  } else if (result == -2) {
1783  htud->tcflags |= HTP_DONTSTORE;
1784  } else {
1785  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1786  htud->tcflags |= HTP_FILENAME_SET;
1787  htud->tcflags &= ~HTP_DONTSTORE;
1788  }
1789  }
1790  }
1791  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1792  {
1793  /* otherwise, just store the data */
1794 
1795  if (!(htud->tcflags & HTP_DONTSTORE)) {
1796  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOCLIENT);
1797  SCLogDebug("result %d", result);
1798  if (result == -1) {
1799  goto end;
1800  } else if (result == -2) {
1801  /* we know for sure we're not storing the file */
1802  htud->tcflags |= HTP_DONTSTORE;
1803  }
1804  }
1805  }
1806 
1807  htud->response_body.body_parsed += data_len;
1808  return 0;
1809 end:
1810  return -1;
1811 }
1812 
1813 /**
1814  * \brief Function callback to append chunks for Requests
1815  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1816  * \retval int HTP_OK if all goes well
1817  */
1818 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1819 {
1820  SCEnter();
1821 
1822  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1823  SCReturnInt(HTP_OK);
1824 
1825  if (d->len == 0)
1826  SCReturnInt(HTP_OK);
1827 
1828 #ifdef PRINT
1829  printf("HTPBODY START: \n");
1830  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1831  printf("HTPBODY END: \n");
1832 #endif
1833 
1834  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1835  if (hstate == NULL) {
1836  SCReturnInt(HTP_ERROR);
1837  }
1838 
1839  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1840  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1841 
1842  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1843  if (tx_ud == NULL) {
1844  SCReturnInt(HTP_OK);
1845  }
1846  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
1847 
1848  if (!tx_ud->response_body_init) {
1849  tx_ud->response_body_init = 1;
1850 
1851  if (d->tx->request_method_number == HTP_M_POST) {
1852  SCLogDebug("POST");
1853  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1854  if (r == 1) {
1856  } else if (r == 0) {
1858  SCLogDebug("not multipart");
1859  }
1860  } else if (d->tx->request_method_number == HTP_M_PUT) {
1862  }
1863  }
1864 
1865  /* see if we can get rid of htp body chunks */
1866  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1867 
1868  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1869  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1870 
1871  /* within limits, add the body chunk to the state. */
1872  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1873  uint32_t stream_depth = FileReassemblyDepth();
1874  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1875  hstate->cfg->request.body_limit,
1876  stream_depth,
1877  tx_ud->tsflags,
1878  (uint32_t)d->len);
1879  BUG_ON(len > (uint32_t)d->len);
1880 
1881  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1882 
1883  const uint8_t *chunks_buffer = NULL;
1884  uint32_t chunks_buffer_len = 0;
1885 
1887  /* multi-part body handling starts here */
1888  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1889  goto end;
1890  }
1891 
1892  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1893  if (chunks_buffer == NULL) {
1894  goto end;
1895  }
1896 #ifdef PRINT
1897  printf("REASSCHUNK START: \n");
1898  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1899  printf("REASSCHUNK END: \n");
1900 #endif
1901 
1902  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1903 
1904  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1906  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1907  }
1908 
1909  } else {
1910  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1911  SCLogDebug("closing file that was being stored");
1912  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1913  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1914  }
1915  }
1916 
1917 end:
1918  if (hstate->conn != NULL) {
1919  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1921  hstate->cfg->request.inspect_min_size,
1922  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1923 
1924  /* if we reach the inspect_min_size we'll trigger inspection,
1925  * so make sure that raw stream is also inspected. Set the
1926  * data to be used to the amount of raw bytes we've seen to
1927  * get here. */
1928  if (tx_ud->request_body.body_inspected == 0 &&
1930  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1931  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1932  {
1933  const uint32_t data_size = (uint32_t)(
1934  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1935  const uint32_t depth = MIN(data_size, hstate->cfg->request.inspect_min_size);
1936 
1937  /* body still in progress, but due to min inspect size we need to inspect now */
1938  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, depth);
1939  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1940  }
1941  /* after the start of the body, disable the depth logic */
1942  } else if (tx_ud->request_body.body_inspected > 0) {
1943  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1944  }
1945  }
1946  SCReturnInt(HTP_OK);
1947 }
1948 
1949 /**
1950  * \brief Function callback to append chunks for Responses
1951  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1952  * \retval int HTP_OK if all goes well
1953  */
1954 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1955 {
1956  SCEnter();
1957 
1958  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1959  SCReturnInt(HTP_OK);
1960 
1961  if (d->len == 0)
1962  SCReturnInt(HTP_OK);
1963 
1964  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1965  if (hstate == NULL) {
1966  SCReturnInt(HTP_ERROR);
1967  }
1968 
1969  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1970  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1971 
1972  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1973  if (tx_ud == NULL) {
1974  SCReturnInt(HTP_OK);
1975  }
1976  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
1977  if (!tx_ud->request_body_init) {
1978  tx_ud->request_body_init = 1;
1979  }
1980 
1981  /* see if we can get rid of htp body chunks */
1982  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1983 
1984  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1985  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1986 
1987  /* within limits, add the body chunk to the state. */
1988  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1989  uint32_t stream_depth = FileReassemblyDepth();
1990  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1991  hstate->cfg->response.body_limit,
1992  stream_depth,
1993  tx_ud->tcflags,
1994  (uint32_t)d->len);
1995  BUG_ON(len > (uint32_t)d->len);
1996 
1997  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1998 
1999  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
2000  } else {
2001  if (tx_ud->tcflags & HTP_FILENAME_SET) {
2002  SCLogDebug("closing file that was being stored");
2003  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
2004  tx_ud->tcflags &= ~HTP_FILENAME_SET;
2005  }
2006  }
2007 
2008  if (hstate->conn != NULL) {
2009  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
2011  hstate->cfg->response.inspect_min_size,
2012  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
2013  /* if we reach the inspect_min_size we'll trigger inspection,
2014  * so make sure that raw stream is also inspected. Set the
2015  * data to be used to the amount of raw bytes we've seen to
2016  * get here. */
2017  if (tx_ud->response_body.body_inspected == 0 &&
2019  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2020  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2021  {
2022  const uint32_t data_size = (uint32_t)((uint64_t)hstate->conn->out_data_counter -
2023  hstate->last_response_data_stamp);
2024  const uint32_t depth = MIN(data_size, hstate->cfg->response.inspect_min_size);
2025 
2026  /* body still in progress, but due to min inspect size we need to inspect now */
2027  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, depth);
2028  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2029  }
2030  /* after the start of the body, disable the depth logic */
2031  } else if (tx_ud->response_body.body_inspected > 0) {
2032  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
2033  }
2034  }
2035  SCReturnInt(HTP_OK);
2036 }
2037 
2038 /**
2039  * \brief Print the stats of the HTTP requests
2040  */
2042 {
2043 #ifdef DEBUG
2044  SCEnter();
2045  SCMutexLock(&htp_state_mem_lock);
2046  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2047  htp_state_memcnt, htp_state_memuse);
2048  SCMutexUnlock(&htp_state_mem_lock);
2049  SCReturn;
2050 #endif
2051 }
2052 
2053 /** \brief Clears the HTTP server configuration memory used by HTP library */
2054 void HTPFreeConfig(void)
2055 {
2056  SCEnter();
2057 
2058  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2059  !AppLayerParserConfParserEnabled("tcp", "http"))
2060  {
2061  SCReturn;
2062  }
2063 
2064  HTPCfgRec *nextrec = cfglist.next;
2065  SCRadixReleaseRadixTree(cfgtree);
2066  cfgtree = NULL;
2067  htp_config_destroy(cfglist.cfg);
2068  while (nextrec != NULL) {
2069  HTPCfgRec *htprec = nextrec;
2070  nextrec = nextrec->next;
2071 
2072  htp_config_destroy(htprec->cfg);
2073  SCFree(htprec);
2074  }
2075  SCReturn;
2076 }
2077 
2078 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2079 {
2080  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2081  if (htud != NULL) {
2082  htud->request_has_trailers = 1;
2083  }
2084  return HTP_OK;
2085 }
2086 
2087 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2088 {
2089  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2090  if (htud != NULL) {
2091  htud->response_has_trailers = 1;
2092  }
2093  return HTP_OK;
2094 }
2095 
2096 /**\internal
2097  * \brief called at start of request
2098  * Set min inspect size.
2099  */
2100 static int HTPCallbackRequestStart(htp_tx_t *tx)
2101 {
2102  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2103  if (hstate == NULL) {
2104  SCReturnInt(HTP_ERROR);
2105  }
2106 
2107  uint64_t consumed = hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2108  SCLogDebug("HTTP request start: data offset %" PRIu64 ", in_data_counter %" PRIu64, consumed,
2109  (uint64_t)hstate->conn->in_data_counter);
2110 
2111  /* app-layer-frame-documentation tag start: frame registration http request */
2113  hstate->f, hstate->slice, consumed, -1, 0, HTTP_FRAME_REQUEST);
2114  if (frame) {
2115  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2116  hstate->request_frame_id = frame->id;
2117  AppLayerFrameSetTxId(frame, HtpGetActiveRequestTxID(hstate));
2118  }
2119  /* app-layer-frame-documentation tag end: frame registration http request */
2120 
2121  if (hstate->cfg)
2122  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2123  hstate->cfg->request.inspect_min_size);
2124 
2125  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2126  if (tx_ud == NULL) {
2127  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2128  if (unlikely(tx_ud == NULL)) {
2129  SCReturnInt(HTP_OK);
2130  }
2131  tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files
2132  htp_tx_set_user_data(tx, tx_ud);
2133  }
2134  SCReturnInt(HTP_OK);
2135 }
2136 
2137 /**\internal
2138  * \brief called at start of response
2139  * Set min inspect size.
2140  */
2141 static int HTPCallbackResponseStart(htp_tx_t *tx)
2142 {
2143  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2144  if (hstate == NULL) {
2145  SCReturnInt(HTP_ERROR);
2146  }
2147 
2148  uint64_t consumed = hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2149  SCLogDebug("HTTP response start: data offset %" PRIu64 ", out_data_counter %" PRIu64, consumed,
2150  (uint64_t)hstate->conn->out_data_counter);
2151 
2153  hstate->f, hstate->slice, consumed, -1, 1, HTTP_FRAME_RESPONSE);
2154  if (frame) {
2155  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2156  hstate->response_frame_id = frame->id;
2157  AppLayerFrameSetTxId(frame, HtpGetActiveResponseTxID(hstate));
2158  }
2159 
2160  if (hstate->cfg)
2161  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
2162  hstate->cfg->response.inspect_min_size);
2163 
2164  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2165  if (tx_ud == NULL) {
2166  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2167  if (unlikely(tx_ud == NULL)) {
2168  SCReturnInt(HTP_OK);
2169  }
2170  tx_ud->tx_data.file_tx =
2171  STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed.
2172  htp_tx_set_user_data(tx, tx_ud);
2173  }
2174  SCReturnInt(HTP_OK);
2175 }
2176 
2177 
2178 /**
2179  * \brief callback for request to store the recent incoming request
2180  in to the recent_in_tx for the given htp state
2181  * \param connp pointer to the current connection parser which has the htp
2182  * state in it as user data
2183  */
2184 static int HTPCallbackRequestComplete(htp_tx_t *tx)
2185 {
2186  SCEnter();
2187 
2188  if (tx == NULL) {
2189  SCReturnInt(HTP_ERROR);
2190  }
2191 
2192  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2193  if (hstate == NULL) {
2194  SCReturnInt(HTP_ERROR);
2195  }
2196 
2197  const uint64_t abs_right_edge =
2198  hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2199 
2200  /* app-layer-frame-documentation tag start: updating frame->len */
2201  if (hstate->request_frame_id > 0) {
2202  Frame *frame = AppLayerFrameGetById(hstate->f, 0, hstate->request_frame_id);
2203  if (frame) {
2204  const uint64_t request_size = abs_right_edge - hstate->last_request_data_stamp;
2205 
2206  SCLogDebug("HTTP request complete: data offset %" PRIu64 ", request_size %" PRIu64,
2207  hstate->last_request_data_stamp, request_size);
2208  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2209  request_size);
2210  frame->len = (int64_t)request_size;
2211  /* app-layer-frame-documentation tag end: updating frame->len */
2212  }
2213  hstate->request_frame_id = 0;
2214  }
2215 
2216  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2217  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2218 
2219  SCLogDebug("HTTP request completed");
2220 
2221  HTPErrorCheckTxRequestFlags(hstate, tx);
2222 
2223  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2224  if (htud != NULL) {
2225  if (htud->tsflags & HTP_FILENAME_SET) {
2226  SCLogDebug("closing file that was being stored");
2227  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
2228  htud->tsflags &= ~HTP_FILENAME_SET;
2229  if (abs_right_edge < (uint64_t)UINT32_MAX) {
2231  hstate->f->protoctx, STREAM_TOSERVER, (uint32_t)abs_right_edge);
2232  }
2233  }
2234  }
2235 
2236  hstate->last_request_data_stamp = abs_right_edge;
2237  /* request done, do raw reassembly now to inspect state and stream
2238  * at the same time. */
2239  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
2240  SCReturnInt(HTP_OK);
2241 }
2242 
2243 /**
2244  * \brief callback for response to remove the recent received requests
2245  from the recent_in_tx for the given htp state
2246  * \param connp pointer to the current connection parser which has the htp
2247  * state in it as user data
2248  */
2249 static int HTPCallbackResponseComplete(htp_tx_t *tx)
2250 {
2251  SCEnter();
2252 
2253  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2254  if (hstate == NULL) {
2255  SCReturnInt(HTP_ERROR);
2256  }
2257 
2258  /* we have one whole transaction now */
2259  hstate->transaction_cnt++;
2260 
2261  const uint64_t abs_right_edge =
2262  hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2263 
2264  if (hstate->response_frame_id > 0) {
2265  Frame *frame = AppLayerFrameGetById(hstate->f, 1, hstate->response_frame_id);
2266  if (frame) {
2267  const uint64_t response_size = abs_right_edge - hstate->last_response_data_stamp;
2268 
2269  SCLogDebug("HTTP response complete: data offset %" PRIu64 ", response_size %" PRIu64,
2270  hstate->last_response_data_stamp, response_size);
2271  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2272  response_size);
2273  frame->len = (int64_t)response_size;
2274  }
2275  hstate->response_frame_id = 0;
2276  }
2277 
2278  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2279  if (htud != NULL) {
2280  if (htud->tcflags & HTP_FILENAME_SET) {
2281  SCLogDebug("closing file that was being stored");
2282  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
2283  htud->tcflags &= ~HTP_FILENAME_SET;
2284  }
2285  }
2286 
2287  /* response done, do raw reassembly now to inspect state and stream
2288  * at the same time. */
2289  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2290 
2291  /* handle HTTP CONNECT */
2292  if (tx->request_method_number == HTP_M_CONNECT) {
2293  /* any 2XX status response implies that the connection will become
2294  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2295  if ((tx->response_status_number >= 200) &&
2296  (tx->response_status_number < 300) &&
2297  (hstate->transaction_cnt == 1)) {
2298  uint16_t dp = 0;
2299  if (tx->request_port_number != -1) {
2300  dp = (uint16_t)tx->request_port_number;
2301  }
2302  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2303  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN)) {
2304  HTPSetEvent(
2305  hstate, htud, STREAM_TOCLIENT, HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE);
2306  }
2307  tx->request_progress = HTP_REQUEST_COMPLETE;
2308  tx->response_progress = HTP_RESPONSE_COMPLETE;
2309  }
2310  }
2311 
2312  hstate->last_response_data_stamp = abs_right_edge;
2313  SCReturnInt(HTP_OK);
2314 }
2315 
2316 static int HTPCallbackRequestLine(htp_tx_t *tx)
2317 {
2318  HtpTxUserData *tx_ud;
2319  bstr *request_uri_normalized;
2320  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2321  const HTPCfgRec *cfg = hstate->cfg;
2322 
2323  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2324  if (request_uri_normalized == NULL)
2325  return HTP_OK;
2326 
2327  tx_ud = htp_tx_get_user_data(tx);
2328  if (unlikely(tx_ud == NULL)) {
2329  return HTP_OK;
2330  }
2331  if (unlikely(tx_ud->request_uri_normalized != NULL))
2332  bstr_free(tx_ud->request_uri_normalized);
2333  tx_ud->request_uri_normalized = request_uri_normalized;
2334 
2335  if (tx->flags) {
2336  HTPErrorCheckTxRequestFlags(hstate, tx);
2337  }
2338  return HTP_OK;
2339 }
2340 
2341 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2342 {
2343  if (part == NULL)
2344  return HTP_OK;
2345 
2346  uint64_t flags = 0;
2347  size_t prevlen = bstr_len(part);
2348  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2349  // shorter string means that uri was encoded
2350  if (res == HTP_OK && prevlen > bstr_len(part)) {
2351  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2352  if (htud == NULL)
2353  return HTP_OK;
2354  HtpState *s = htp_connp_get_user_data(tx->connp);
2355  if (s == NULL)
2356  return HTP_OK;
2357  HTPSetEvent(s, htud, STREAM_TOSERVER,
2359  }
2360 
2361  return HTP_OK;
2362 }
2363 
2364 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2365 {
2366  if (tx->parsed_uri == NULL)
2367  return HTP_OK;
2368 
2369  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2370 }
2371 
2372 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2373 {
2374  if (tx->parsed_uri == NULL)
2375  return HTP_OK;
2376 
2377  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2378 }
2379 
2380 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2381 {
2382  void *ptmp;
2383  if (tx_data->len == 0 || tx_data->tx == NULL)
2384  return HTP_OK;
2385 
2386  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2387  if (tx_ud == NULL) {
2388  return HTP_OK;
2389  }
2390  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2391  tx_ud->request_headers_raw_len,
2392  tx_ud->request_headers_raw_len + tx_data->len);
2393  if (ptmp == NULL) {
2394  return HTP_OK;
2395  }
2396  tx_ud->request_headers_raw = ptmp;
2397 
2398  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2399  tx_data->data, tx_data->len);
2400  tx_ud->request_headers_raw_len += tx_data->len;
2401 
2402  if (tx_data->tx && tx_data->tx->flags) {
2403  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2404  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2405  }
2406  return HTP_OK;
2407 }
2408 
2409 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2410 {
2411  void *ptmp;
2412  if (tx_data->len == 0 || tx_data->tx == NULL)
2413  return HTP_OK;
2414 
2415  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2416  if (tx_ud == NULL) {
2417  return HTP_OK;
2418  }
2419  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2420  tx_ud->response_headers_raw_len,
2421  tx_ud->response_headers_raw_len + tx_data->len);
2422  if (ptmp == NULL) {
2423  return HTP_OK;
2424  }
2425  tx_ud->response_headers_raw = ptmp;
2426 
2427  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2428  tx_data->data, tx_data->len);
2429  tx_ud->response_headers_raw_len += tx_data->len;
2430 
2431  return HTP_OK;
2432 }
2433 
2434 /*
2435  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2436  */
2437 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2438 {
2439  cfg_prec->uri_include_all = FALSE;
2446 
2447  if (!g_disable_randomness) {
2449  } else {
2450  cfg_prec->randomize = 0;
2451  }
2453 
2454  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2455  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2456  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2457  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2458 
2459  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2460  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2461 
2462  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2463  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2464 
2465  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2466  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequestComplete);
2467 
2468  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2469  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponseComplete);
2470 
2471  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2472 
2473  /* don't convert + to space by default */
2474  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2475  // enables request decompression
2476  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2477 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2478  // disable by default
2479  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2480 #endif
2481 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2482  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2484 #endif
2485 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2486  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2488 #endif
2489 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2490  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2491 #endif
2492  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2493  * only the hard limit. So we set both here to the (current) htp defaults.
2494  * The reason we do this is that if the user sets the hard limit in the
2495  * config, we have to set the soft limit as well. If libhtp starts using
2496  * the soft limit in the future, we at least make sure we control what
2497  * it's value is. */
2498  htp_config_set_field_limits(cfg_prec->cfg,
2501  return;
2502 }
2503 
2504 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2505  * but we can get both <0 and >RAND_MAX values from RandomGet
2506  */
2507 static int RandomGetWrap(void)
2508 {
2509  unsigned long r;
2510 
2511  do {
2512  r = RandomGet();
2513  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2514 
2515  return r % RAND_MAX;
2516 }
2517 
2518 /*
2519  * We have this splitup so that in case double decoding has been enabled
2520  * for query and path, they would be called first on the callback queue,
2521  * before the callback set by Phase2() is called. We need this, since
2522  * the callback in Phase2() generates the normalized uri which utilizes
2523  * the query and path. */
2524 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2525 {
2526  /* randomize inspection size if needed */
2527  if (cfg_prec->randomize) {
2528  int rdrange = cfg_prec->randomize_range;
2529 
2530  long int r = RandomGetWrap();
2531  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2532  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2533 
2534  r = RandomGetWrap();
2535  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2536  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2537  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2538  " %u and 'request-body-inspect-window' set to %u after"
2539  " randomization.",
2540  name, cfg_prec->request.inspect_min_size, cfg_prec->request.inspect_window);
2541 
2542  r = RandomGetWrap();
2543  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2544  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2545 
2546  r = RandomGetWrap();
2547  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2548  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2549 
2550  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2551  " %u and 'response-body-inspect-window' set to %u after"
2552  " randomization.",
2553  name, cfg_prec->response.inspect_min_size, cfg_prec->response.inspect_window);
2554  }
2555 
2556  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2557  return;
2558 }
2559 
2560 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2561  SCRadixTree *tree)
2562 {
2563  if (cfg_prec == NULL || s == NULL || tree == NULL)
2564  return;
2565 
2566  ConfNode *p = NULL;
2567 
2568  /* Default Parameters */
2569  TAILQ_FOREACH(p, &s->head, next) {
2570 
2571  if (strcasecmp("address", p->name) == 0) {
2572  ConfNode *pval;
2573  /* Addresses */
2574  TAILQ_FOREACH(pval, &p->head, next) {
2575  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2576  pval->val);
2577 
2578  /* IPV6 or IPV4? */
2579  if (strchr(pval->val, ':') != NULL) {
2580  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2581  s->name, pval->val, cfg_prec->cfg);
2582  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2583  SCLogWarning("LIBHTP failed to "
2584  "add ipv6 server %s, ignoring",
2585  pval->val);
2586  }
2587  } else {
2588  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2589  s->name, pval->val, cfg_prec->cfg);
2590  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2591  SCLogWarning("LIBHTP failed "
2592  "to add ipv4 server %s, ignoring",
2593  pval->val);
2594  }
2595  } /* else - if (strchr(pval->val, ':') != NULL) */
2596  } /* TAILQ_FOREACH(pval, &p->head, next) */
2597 
2598  } else if (strcasecmp("personality", p->name) == 0) {
2599  /* Personalities */
2600  int personality = HTPLookupPersonality(p->val);
2601  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2602  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2603 
2604  if (personality >= 0) {
2605  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2606  personality);
2607  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2608  SCLogWarning("LIBHTP Failed adding "
2609  "personality \"%s\", ignoring",
2610  p->val);
2611  } else {
2612  SCLogDebug("LIBHTP personality set to %s",
2613  HTPLookupPersonalityString(personality));
2614  }
2615 
2616  /* The IDS personality by default converts the path (and due to
2617  * our query string callback also the query string) to lowercase.
2618  * Signatures do not expect this, so override it. */
2619  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2620  } else {
2621  SCLogWarning("LIBHTP Unknown personality "
2622  "\"%s\", ignoring",
2623  p->val);
2624  continue;
2625  }
2626 
2627  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2628  strcasecmp("request_body_limit", p->name) == 0) {
2629  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2630  SCLogError("Error parsing request-body-limit "
2631  "from conf file - %s. Killing engine",
2632  p->val);
2633  exit(EXIT_FAILURE);
2634  }
2635 
2636  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2637  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2638  SCLogError("Error parsing response-body-limit "
2639  "from conf file - %s. Killing engine",
2640  p->val);
2641  exit(EXIT_FAILURE);
2642  }
2643 
2644  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2645  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2646  SCLogError("Error parsing request-body-minimal-inspect-size "
2647  "from conf file - %s. Killing engine",
2648  p->val);
2649  exit(EXIT_FAILURE);
2650  }
2651 
2652  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2653  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2654  SCLogError("Error parsing request-body-inspect-window "
2655  "from conf file - %s. Killing engine",
2656  p->val);
2657  exit(EXIT_FAILURE);
2658  }
2659 
2660  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2661  if (ConfValIsTrue(p->val)) {
2662  htp_config_register_request_line(cfg_prec->cfg,
2663  HTPCallbackDoubleDecodeQuery);
2664  }
2665 
2666  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2667  if (ConfValIsTrue(p->val)) {
2668  htp_config_register_request_line(cfg_prec->cfg,
2669  HTPCallbackDoubleDecodePath);
2670  }
2671 
2672  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2673  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2674  SCLogError("Error parsing response-body-minimal-inspect-size "
2675  "from conf file - %s. Killing engine",
2676  p->val);
2677  exit(EXIT_FAILURE);
2678  }
2679 
2680  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2681  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2682  SCLogError("Error parsing response-body-inspect-window "
2683  "from conf file - %s. Killing engine",
2684  p->val);
2685  exit(EXIT_FAILURE);
2686  }
2687 
2688  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2689  uint32_t value = 2;
2690  if (ParseSizeStringU32(p->val, &value) < 0) {
2691  SCLogError("Error parsing response-body-inspect-window "
2692  "from conf file - %s. Killing engine",
2693  p->val);
2694  exit(EXIT_FAILURE);
2695  }
2696 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2697  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2698 #else
2699  SCLogWarning("can't set response-body-decompress-layer-limit "
2700  "to %u, libhtp version too old",
2701  value);
2702 #endif
2703  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2704  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2705  HTP_DECODER_URL_PATH,
2706  ConfValIsTrue(p->val));
2707  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2708  if (strlen(p->val) == 1) {
2709  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2710  HTP_DECODER_URL_PATH,
2711  p->val[0]);
2712  } else {
2713  SCLogError("Invalid entry "
2714  "for libhtp param path-bestfit-replacement-char");
2715  }
2716  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2717  htp_config_set_convert_lowercase(cfg_prec->cfg,
2718  HTP_DECODER_URL_PATH,
2719  ConfValIsTrue(p->val));
2720  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2721  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2722  HTP_DECODER_URL_PATH,
2723  ConfValIsTrue(p->val));
2724  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2725  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2726  HTP_DECODER_URL_PATH,
2727  ConfValIsTrue(p->val));
2728  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2729  htp_config_set_path_separators_compress(cfg_prec->cfg,
2730  HTP_DECODER_URL_PATH,
2731  ConfValIsTrue(p->val));
2732  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2733  htp_config_set_path_separators_decode(cfg_prec->cfg,
2734  HTP_DECODER_URL_PATH,
2735  ConfValIsTrue(p->val));
2736  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2737  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2738  HTP_DECODER_URL_PATH,
2739  ConfValIsTrue(p->val));
2740  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2741  enum htp_url_encoding_handling_t handling;
2742  if (strcasecmp(p->val, "preserve_percent") == 0) {
2743  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2744  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2745  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2746  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2747  handling = HTP_URL_DECODE_PROCESS_INVALID;
2748  } else {
2749  SCLogError("Invalid entry "
2750  "for libhtp param path-url-encoding-invalid-handling");
2751  return;
2752  }
2753  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2754  HTP_DECODER_URL_PATH,
2755  handling);
2756  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2757  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2758  HTP_DECODER_URL_PATH,
2759  ConfValIsTrue(p->val));
2760  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2761  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2762  SCLogDebug("uri-include-all %s",
2763  cfg_prec->uri_include_all ? "enabled" : "disabled");
2764  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2765  htp_config_set_plusspace_decode(cfg_prec->cfg,
2766  HTP_DECODER_URLENCODED,
2767  ConfValIsTrue(p->val));
2768  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2769  uint32_t limit = 0;
2770  if (ParseSizeStringU32(p->val, &limit) < 0) {
2771  SCLogError("Error meta-field-limit "
2772  "from conf file - %s. Killing engine",
2773  p->val);
2774  exit(EXIT_FAILURE);
2775  }
2776  if (limit == 0) {
2777  FatalError("Error meta-field-limit "
2778  "from conf file cannot be 0. Killing engine");
2779  }
2780  /* set default soft-limit with our new hard limit */
2781  htp_config_set_field_limits(cfg_prec->cfg,
2783  (size_t)limit);
2784 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2785  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2786  uint32_t limit = 0;
2787  if (ParseSizeStringU32(p->val, &limit) < 0) {
2788  FatalError("failed to parse 'lzma-memlimit' "
2789  "from conf file - %s.",
2790  p->val);
2791  }
2792  if (limit == 0) {
2793  FatalError("'lzma-memlimit' "
2794  "from conf file cannot be 0.");
2795  }
2796  /* set default soft-limit with our new hard limit */
2797  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2798  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2799 #endif
2800 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2801  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2802  if (ConfValIsTrue(p->val)) {
2803  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2804  } else if (!ConfValIsFalse(p->val)) {
2805  int8_t limit;
2806  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2807  FatalError("failed to parse 'lzma-enabled' "
2808  "from conf file - %s.",
2809  p->val);
2810  }
2811  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2812  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2813  }
2814 #endif
2815 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2816  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2817  uint32_t limit = 0;
2818  if (ParseSizeStringU32(p->val, &limit) < 0) {
2819  FatalError("failed to parse 'compression-bomb-limit' "
2820  "from conf file - %s.",
2821  p->val);
2822  }
2823  if (limit == 0) {
2824  FatalError("'compression-bomb-limit' "
2825  "from conf file cannot be 0.");
2826  }
2827  /* set default soft-limit with our new hard limit */
2828  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2829  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2830 #endif
2831 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2832  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2833  uint32_t limit = 0;
2834  // between 1 usec and 1 second
2835  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2836  FatalError("failed to parse 'decompression-time-limit' "
2837  "from conf file - %s.",
2838  p->val);
2839  }
2840  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2841  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2842 #endif
2843  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2844  if (!g_disable_randomness) {
2845  cfg_prec->randomize = ConfValIsTrue(p->val);
2846  }
2847  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2848  uint32_t range;
2849  if (StringParseU32RangeCheck(&range, 10, 0,
2850  (const char *)p->val, 0, 100) < 0) {
2851  SCLogError("Invalid value for randomize"
2852  "-inspection-range setting from conf file - \"%s\"."
2853  " It should be a valid integer less than or equal to 100."
2854  " Killing engine",
2855  p->val);
2856  exit(EXIT_FAILURE);
2857  }
2858  cfg_prec->randomize_range = range;
2859  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2860  if (ConfValIsTrue(p->val)) {
2861  cfg_prec->http_body_inline = 1;
2862  } else if (ConfValIsFalse(p->val)) {
2863  cfg_prec->http_body_inline = 0;
2864  } else {
2865  if (strcmp("auto", p->val) != 0) {
2866  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2867  }
2868  if (EngineModeIsIPS()) {
2869  cfg_prec->http_body_inline = 1;
2870  } else {
2871  cfg_prec->http_body_inline = 0;
2872  }
2873  }
2874  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2875  ConfNode *pval;
2876 
2877  TAILQ_FOREACH(pval, &p->head, next) {
2878  if (strcasecmp("enabled", pval->name) == 0) {
2879  if (ConfValIsTrue(pval->val)) {
2880  cfg_prec->swf_decompression_enabled = 1;
2881  } else if (ConfValIsFalse(pval->val)) {
2882  cfg_prec->swf_decompression_enabled = 0;
2883  } else {
2884  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2885  }
2886  } else if (strcasecmp("type", pval->name) == 0) {
2887  if (strcasecmp("no", pval->val) == 0) {
2889  } else if (strcasecmp("deflate", pval->val) == 0) {
2891  } else if (strcasecmp("lzma", pval->val) == 0) {
2893  } else if (strcasecmp("both", pval->val) == 0) {
2895  } else {
2896  SCLogError("Invalid entry for "
2897  "swf-decompression.type: %s - "
2898  "Killing engine",
2899  pval->val);
2900  exit(EXIT_FAILURE);
2901  }
2902  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2903  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2904  SCLogError("Error parsing swf-decompression.compression-depth "
2905  "from conf file - %s. Killing engine",
2906  p->val);
2907  exit(EXIT_FAILURE);
2908  }
2909  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2910  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2911  SCLogError("Error parsing swf-decompression.decompression-depth "
2912  "from conf file - %s. Killing engine",
2913  p->val);
2914  exit(EXIT_FAILURE);
2915  }
2916  } else {
2917  SCLogWarning("Ignoring unknown param %s", pval->name);
2918  }
2919  }
2920  } else {
2921  SCLogWarning("LIBHTP Ignoring unknown "
2922  "default config: %s",
2923  p->name);
2924  }
2925  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2926 
2927  return;
2928 }
2929 
2930 void HTPConfigure(void)
2931 {
2932  SCEnter();
2933 
2934  cfglist.next = NULL;
2935 
2939 
2940  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2941  if (NULL == cfgtree)
2942  exit(EXIT_FAILURE);
2943 
2944  /* Default Config */
2945  cfglist.cfg = htp_config_create();
2946  if (NULL == cfglist.cfg) {
2947  FatalError("Failed to create HTP default config");
2948  }
2949  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2950  HTPConfigSetDefaultsPhase1(&cfglist);
2951  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2952  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2953  cfgtree);
2954  } else {
2955  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2956  }
2957  HTPConfigSetDefaultsPhase2("default", &cfglist);
2958 
2959  HTPParseMemcap();
2960 
2961  /* Read server config and create a parser for each IP in radix tree */
2962  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2963  if (server_config == NULL) {
2964  server_config = ConfGetNode("libhtp.server-config");
2965  if (server_config == NULL) {
2966  SCLogDebug("LIBHTP Configuring %p", server_config);
2967  SCReturn;
2968  }
2969  }
2970  SCLogDebug("LIBHTP Configuring %p", server_config);
2971 
2972  ConfNode *si;
2973  /* Server Nodes */
2974  TAILQ_FOREACH(si, &server_config->head, next) {
2975  /* Need the named node, not the index */
2976  ConfNode *s = TAILQ_FIRST(&si->head);
2977  if (NULL == s) {
2978  SCLogDebug("LIBHTP s NULL");
2979  continue;
2980  }
2981 
2982  SCLogDebug("LIBHTP server %s", s->name);
2983 
2984  HTPCfgRec *nextrec = cfglist.next;
2985  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2986  if (NULL == htprec)
2987  exit(EXIT_FAILURE);
2988  memset(htprec, 0x00, sizeof(*htprec));
2989 
2990  cfglist.next = htprec;
2991 
2992  cfglist.next->next = nextrec;
2993  cfglist.next->cfg = htp_config_create();
2994  if (NULL == cfglist.next->cfg) {
2995  FatalError("Failed to create HTP server config");
2996  }
2997 
2998  HTPConfigSetDefaultsPhase1(htprec);
2999  HTPConfigParseParameters(htprec, s, cfgtree);
3000  HTPConfigSetDefaultsPhase2(s->name, htprec);
3001  }
3002 
3003  SCReturn;
3004 }
3005 
3007 {
3008 #ifdef DEBUG
3009  SCMutexLock(&htp_state_mem_lock);
3010  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
3011  SCMutexUnlock(&htp_state_mem_lock);
3012 #endif
3013 }
3014 
3015 /** \internal
3016  * \brief get files callback
3017  * \param state state ptr
3018  * \param direction flow direction
3019  * \retval files files ptr
3020  */
3021 static AppLayerGetFileState HTPGetTxFiles(void *state, void *txv, uint8_t direction)
3022 {
3023  AppLayerGetFileState files = { .fc = NULL, .cfg = &htp_sbcfg };
3024  htp_tx_t *tx = (htp_tx_t *)txv;
3025  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3026  if (tx_ud) {
3027  if (direction & STREAM_TOCLIENT) {
3028  files.fc = &tx_ud->files_tc;
3029  } else {
3030  files.fc = &tx_ud->files_ts;
3031  }
3032  }
3033  return files;
3034 }
3035 
3036 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
3037 {
3038  if (direction & STREAM_TOSERVER)
3039  return ((htp_tx_t *)tx)->request_progress;
3040  else
3041  return ((htp_tx_t *)tx)->response_progress;
3042 }
3043 
3044 static uint64_t HTPStateGetTxCnt(void *alstate)
3045 {
3046  HtpState *http_state = (HtpState *)alstate;
3047 
3048  if (http_state != NULL && http_state->conn != NULL) {
3049  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
3050  if (size < 0)
3051  return 0ULL;
3052  SCLogDebug("size %"PRIu64, size);
3053  return (uint64_t)size;
3054  } else {
3055  return 0ULL;
3056  }
3057 }
3058 
3059 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
3060 {
3061  HtpState *http_state = (HtpState *)alstate;
3062 
3063  if (http_state != NULL && http_state->conn != NULL)
3064  return htp_list_get(http_state->conn->transactions, tx_id);
3065  else
3066  return NULL;
3067 }
3068 
3069 void *HtpGetTxForH2(void *alstate)
3070 {
3071  // gets last transaction
3072  HtpState *http_state = (HtpState *)alstate;
3073  if (http_state != NULL && http_state->conn != NULL) {
3074  size_t txid = htp_list_array_size(http_state->conn->transactions);
3075  if (txid > 0) {
3076  return htp_list_get(http_state->conn->transactions, txid - 1);
3077  }
3078  }
3079  return NULL;
3080 }
3081 
3082 static int HTPStateGetEventInfo(const char *event_name,
3083  int *event_id, AppLayerEventType *event_type)
3084 {
3085  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3086  if (*event_id == -1) {
3087  SCLogError("event \"%s\" not present in "
3088  "http's enum map table.",
3089  event_name);
3090  /* this should be treated as fatal */
3091  return -1;
3092  }
3093 
3094  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3095 
3096  return 0;
3097 }
3098 
3099 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3100  AppLayerEventType *event_type)
3101 {
3102  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3103  if (*event_name == NULL) {
3104  SCLogError("event \"%d\" not present in "
3105  "http's enum map table.",
3106  event_id);
3107  /* this should be treated as fatal */
3108  return -1;
3109  }
3110 
3111  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3112 
3113  return 0;
3114 }
3115 
3116 static AppLayerTxData *HTPGetTxData(void *vtx)
3117 {
3118  htp_tx_t *tx = (htp_tx_t *)vtx;
3119  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3120  if (tx_ud) {
3121  return &tx_ud->tx_data;
3122  }
3123  return NULL;
3124 }
3125 
3126 static AppLayerStateData *HTPGetStateData(void *vstate)
3127 {
3128  HtpState *s = vstate;
3129  return &s->state_data;
3130 }
3131 
3132 static int HTPRegisterPatternsForProtocolDetection(void)
3133 {
3134  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3135  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3136  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3137  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3138  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3139  const char *spacings[] = { "|20|", "|09|", NULL };
3140  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3141 
3142  int methods_pos;
3143  int spacings_pos;
3144  int versions_pos;
3145  int register_result;
3146  char method_buffer[32] = "";
3147 
3148  /* Loop through all the methods ands spacings and register the patterns */
3149  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3150  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3151 
3152  /* Combine the method name and the spacing */
3153  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3154 
3155  /* Register the new method+spacing pattern
3156  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3157  * but the pattern matching should only be one char
3158  */
3159  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3160  method_buffer, (uint16_t)strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3161  if (register_result < 0) {
3162  return -1;
3163  }
3164  }
3165  }
3166 
3167  /* Loop through all the http verions patterns that are TO_CLIENT */
3168  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3169  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3170  versions[versions_pos], (uint16_t)strlen(versions[versions_pos]), 0,
3171  STREAM_TOCLIENT);
3172  if (register_result < 0) {
3173  return -1;
3174  }
3175  }
3176 
3177  return 0;
3178 }
3179 
3180 /**
3181  * \brief Register the HTTP protocol and state handling functions to APP layer
3182  * of the engine.
3183  */
3185 {
3186  SCEnter();
3187 
3188  const char *proto_name = "http";
3189 
3190  /** HTTP */
3191  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3193  if (HTPRegisterPatternsForProtocolDetection() < 0)
3194  return;
3195  } else {
3196  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3197  proto_name);
3198  return;
3199  }
3200 
3201  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3202  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3203  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3204  AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles);
3206  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3207  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3208  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3209 
3211  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3212  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3214  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3215 
3216  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3217  AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData);
3218 
3220  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3221 
3223  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3225  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3226  SC_ATOMIC_INIT(htp_config_flags);
3227  /* This parser accepts gaps. */
3231  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_TOCLIENT);
3232  /* app-layer-frame-documentation tag start: registering relevant callbacks */
3234  IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById);
3235  /* app-layer-frame-documentation tag end: registering relevant callbacks */
3236  HTPConfigure();
3237  } else {
3238  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3239  "still on.", proto_name);
3240  }
3241 #ifdef UNITTESTS
3242  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3243 #endif
3244 
3245  SCReturn;
3246 }
3247 
3248 #ifdef UNITTESTS
3249 #include "detect-engine-alert.h"
3250 
3251 static HTPCfgRec cfglist_backup;
3252 
3254 {
3255  cfglist_backup = cfglist;
3256 
3257  return;
3258 }
3259 
3261 {
3262  cfglist = cfglist_backup;
3263 
3264  return;
3265 }
3266 
3267 /** \test Test case where chunks are sent in smaller chunks and check the
3268  * response of the parser from HTP library. */
3269 static int HTPParserTest01(void)
3270 {
3271  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3272  " Data is c0oL!";
3273  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3274 
3275  TcpSession ssn;
3276  memset(&ssn, 0, sizeof(ssn));
3277 
3280 
3281  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3282  FAIL_IF_NULL(f);
3283  f->protoctx = &ssn;
3284  f->proto = IPPROTO_TCP;
3285  f->alproto = ALPROTO_HTTP1;
3286 
3287  StreamTcpInitConfig(true);
3288 
3289  uint32_t u;
3290  for (u = 0; u < httplen1; u++) {
3291  uint8_t flags = 0;
3292 
3293  if (u == 0)
3294  flags = STREAM_TOSERVER|STREAM_START;
3295  else if (u == (httplen1 - 1))
3296  flags = STREAM_TOSERVER|STREAM_EOF;
3297  else
3298  flags = STREAM_TOSERVER;
3299 
3300  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3301  FAIL_IF(r != 0);
3302  }
3303 
3304  HtpState *htp_state = f->alstate;
3305  FAIL_IF_NULL(htp_state);
3306 
3307  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3308  FAIL_IF_NULL(tx);
3309 
3310  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3311  FAIL_IF_NULL(h);
3312 
3313  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3314  FAIL_IF(tx->request_method_number != HTP_M_POST);
3315  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3316 
3318  StreamTcpFreeConfig(true);
3319  UTHFreeFlow(f);
3320  PASS;
3321 }
3322 
3323 /** \test Test folding in 1 read case */
3324 static int HTPParserTest01b(void)
3325 {
3326  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3327  " Data is c0oL!";
3328  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3329 
3330  TcpSession ssn;
3331  memset(&ssn, 0, sizeof(ssn));
3332 
3335 
3336  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3337  FAIL_IF_NULL(f);
3338  f->protoctx = &ssn;
3339  f->proto = IPPROTO_TCP;
3340  f->alproto = ALPROTO_HTTP1;
3341 
3342  StreamTcpInitConfig(true);
3343 
3344  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3345  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3346  FAIL_IF(r != 0);
3347 
3348  HtpState *htp_state = f->alstate;
3349  FAIL_IF_NULL(htp_state);
3350 
3351  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3352  FAIL_IF_NULL(tx);
3353 
3354  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3355  FAIL_IF_NULL(h);
3356 
3357  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3358  FAIL_IF(tx->request_method_number != HTP_M_POST);
3359  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3360 
3362  StreamTcpFreeConfig(true);
3363  UTHFreeFlow(f);
3364  PASS;
3365 }
3366 
3367 /** \test Test folding in 1byte per read case */
3368 static int HTPParserTest01c(void)
3369 {
3370  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3371  " Data is c0oL!";
3372  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3373 
3374  TcpSession ssn;
3375  memset(&ssn, 0, sizeof(ssn));
3376 
3379 
3380  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3381  FAIL_IF_NULL(f);
3382  f->protoctx = &ssn;
3383  f->proto = IPPROTO_TCP;
3384  f->alproto = ALPROTO_HTTP1;
3385 
3386  StreamTcpInitConfig(true);
3387 
3388  uint32_t u;
3389  for (u = 0; u < httplen1; u++) {
3390  uint8_t flags = 0;
3391 
3392  if (u == 0)
3393  flags = STREAM_TOSERVER|STREAM_START;
3394  else if (u == (httplen1 - 1))
3395  flags = STREAM_TOSERVER|STREAM_EOF;
3396  else
3397  flags = STREAM_TOSERVER;
3398 
3399  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3400  FAIL_IF(r != 0);
3401  }
3402 
3403  HtpState *htp_state = f->alstate;
3404  FAIL_IF_NULL(htp_state);
3405 
3406  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3407  FAIL_IF_NULL(tx);
3408 
3409  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3410  FAIL_IF_NULL(h);
3411 
3412  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3413  FAIL_IF(tx->request_method_number != HTP_M_POST);
3414  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3415 
3417  StreamTcpFreeConfig(true);
3418  UTHFreeFlow(f);
3419  PASS;
3420 }
3421 
3422 /** \test Test case where chunks are sent in smaller chunks and check the
3423  * response of the parser from HTP library. */
3424 static int HTPParserTest01a(void)
3425 {
3426  int result = 0;
3427  Flow *f = NULL;
3428  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3429  " Data is c0oL!";
3430  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3431  TcpSession ssn;
3432  HtpState *htp_state = NULL;
3433  int r = 0;
3435 
3436  memset(&ssn, 0, sizeof(ssn));
3437 
3438  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3439  if (f == NULL)
3440  goto end;
3441  f->protoctx = &ssn;
3442  f->proto = IPPROTO_TCP;
3443  f->alproto = ALPROTO_HTTP1;
3444 
3445  StreamTcpInitConfig(true);
3446 
3447  uint32_t u;
3448  for (u = 0; u < httplen1; u++) {
3449  uint8_t flags = 0;
3450 
3451  if (u == 0)
3452  flags = STREAM_TOSERVER|STREAM_START;
3453  else if (u == (httplen1 - 1))
3454  flags = STREAM_TOSERVER|STREAM_EOF;
3455  else
3456  flags = STREAM_TOSERVER;
3457 
3458  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3459  if (r != 0) {
3460  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3461  " 0: ", u, r);
3462  goto end;
3463  }
3464  }
3465 
3466  htp_state = f->alstate;
3467  if (htp_state == NULL) {
3468  printf("no http state: ");
3469  goto end;
3470  }
3471 
3472  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3473  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3474  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3475  || tx->request_method_number != HTP_M_POST ||
3476  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3477  {
3478  printf("expected header value: Victor/1.0 and got %s: and expected"
3479  " method: POST and got %s, expected protocol number HTTP/1.0"
3480  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3481  bstr_util_strdup_to_c(tx->request_method),
3482  bstr_util_strdup_to_c(tx->request_protocol));
3483  goto end;
3484  }
3485  result = 1;
3486 end:
3487  if (alp_tctx != NULL)
3489  StreamTcpFreeConfig(true);
3490  UTHFreeFlow(f);
3491  return result;
3492 }
3493 
3494 /** \test See how it deals with an incomplete request. */
3495 static int HTPParserTest02(void)
3496 {
3497  int result = 0;
3498  Flow *f = NULL;
3499  uint8_t httpbuf1[] = "POST";
3500  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3501  TcpSession ssn;
3502  HtpState *http_state = NULL;
3504 
3505  memset(&ssn, 0, sizeof(ssn));
3506 
3507  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3508  if (f == NULL)
3509  goto end;
3510  f->protoctx = &ssn;
3511  f->proto = IPPROTO_TCP;
3512  f->alproto = ALPROTO_HTTP1;
3513 
3514  StreamTcpInitConfig(true);
3515 
3516  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3517  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3518  if (r != 0) {
3519  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3520  goto end;
3521  }
3522 
3523  http_state = f->alstate;
3524  if (http_state == NULL) {
3525  printf("no http state: ");
3526  goto end;
3527  }
3528 
3529  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3530  FAIL_IF_NULL(tx);
3531  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3532  FAIL_IF_NOT_NULL(h);
3533 
3534  FAIL_IF_NULL(tx->request_method);
3535  char *method = bstr_util_strdup_to_c(tx->request_method);
3536  FAIL_IF_NULL(method);
3537 
3538  FAIL_IF(strcmp(method, "POST") != 0);
3539  SCFree(method);
3540 
3541  result = 1;
3542 end:
3543  if (alp_tctx != NULL)
3545  StreamTcpFreeConfig(true);
3546  UTHFreeFlow(f);
3547  return result;
3548 }
3549 
3550 /** \test Test case where method is invalid and data is sent in smaller chunks
3551  * and check the response of the parser from HTP library. */
3552 static int HTPParserTest03(void)
3553 {
3554  int result = 0;
3555  Flow *f = NULL;
3556  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3557  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3558  TcpSession ssn;
3559  HtpState *htp_state = NULL;
3560  int r = 0;
3562 
3563  memset(&ssn, 0, sizeof(ssn));
3564 
3565  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3566  if (f == NULL)
3567  goto end;
3568  f->protoctx = &ssn;
3569  f->proto = IPPROTO_TCP;
3570  f->alproto = ALPROTO_HTTP1;
3571 
3572  StreamTcpInitConfig(true);
3573 
3574  uint32_t u;
3575  for (u = 0; u < httplen1; u++) {
3576  uint8_t flags = 0;
3577 
3578  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3579  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3580  else flags = STREAM_TOSERVER;
3581 
3582  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3583  if (r != 0) {
3584  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3585  " 0: ", u, r);
3586  goto end;
3587  }
3588  }
3589  htp_state = f->alstate;
3590  if (htp_state == NULL) {
3591  printf("no http state: ");
3592  goto end;
3593  }
3594 
3595  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3596 
3597  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3598  if (tx->request_method_number != HTP_M_UNKNOWN ||
3599  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3600  {
3601  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3602  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3603  bstr_util_strdup_to_c(tx->request_protocol));
3604  goto end;
3605  }
3606  result = 1;
3607 end:
3608  if (alp_tctx != NULL)
3610  StreamTcpFreeConfig(true);
3611  UTHFreeFlow(f);
3612  return result;
3613 }
3614 
3615 /** \test Test case where invalid data is sent and check the response of the
3616  * parser from HTP library. */
3617 static int HTPParserTest04(void)
3618 {
3619  int result = 0;
3620  Flow *f = NULL;
3621  HtpState *htp_state = NULL;
3622  uint8_t httpbuf1[] = "World!\r\n";
3623  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3624  TcpSession ssn;
3625  int r = 0;
3627 
3628  memset(&ssn, 0, sizeof(ssn));
3629 
3630  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3631  if (f == NULL)
3632  goto end;
3633  f->protoctx = &ssn;
3634  f->proto = IPPROTO_TCP;
3635  f->alproto = ALPROTO_HTTP1;
3636 
3637  StreamTcpInitConfig(true);
3638 
3640  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3641  if (r != 0) {
3642  goto end;
3643  }
3644 
3645  htp_state = f->alstate;
3646  if (htp_state == NULL) {
3647  printf("no http state: ");
3648  goto end;
3649  }
3650 
3651  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3652  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3653  if (tx->request_method_number != HTP_M_UNKNOWN ||
3654  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3655  {
3656  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3657  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3658  bstr_util_strdup_to_c(tx->request_protocol));
3659  goto end;
3660  }
3661  result = 1;
3662 end:
3663  if (alp_tctx != NULL)
3665  StreamTcpFreeConfig(true);
3666  UTHFreeFlow(f);
3667  return result;
3668 }
3669 
3670 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3671  * properly parsed them and also keeps them separated. */
3672 static int HTPParserTest05(void)
3673 {
3674  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3675  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3676  uint8_t httpbuf2[] = "Post D";
3677  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3678  uint8_t httpbuf3[] = "ata is c0oL!";
3679  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3680 
3681  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3682  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3683  uint8_t httpbuf5[] = "post R";
3684  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3685  uint8_t httpbuf6[] = "esults are tha bomb!";
3686  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3687 
3688  TcpSession ssn;
3689  memset(&ssn, 0, sizeof(ssn));
3690 
3693 
3694  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3695  FAIL_IF_NULL(f);
3696  f->protoctx = &ssn;
3697  f->proto = IPPROTO_TCP;
3698  f->alproto = ALPROTO_HTTP1;
3699 
3700  StreamTcpInitConfig(true);
3701 
3702  int r = AppLayerParserParse(
3703  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3704  FAIL_IF(r != 0);
3705 
3706  r = AppLayerParserParse(
3707  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3708  FAIL_IF(r != 0);
3709 
3710  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3711  FAIL_IF(r != 0);
3712 
3713  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3714  FAIL_IF(r != 0);
3715 
3716  r = AppLayerParserParse(
3717  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3718  FAIL_IF(r != 0);
3719 
3720  r = AppLayerParserParse(
3721  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3722  FAIL_IF(r != 0);
3723 
3724  HtpState *http_state = f->alstate;
3725  FAIL_IF_NULL(http_state);
3726 
3727  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3728  FAIL_IF_NULL(tx);
3729  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3730  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3731 
3732  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3733  FAIL_IF_NULL(h);
3734 
3735  FAIL_IF_NOT(tx->response_status_number == 200);
3736 
3738  StreamTcpFreeConfig(true);
3739  UTHFreeFlow(f);
3740  PASS;
3741 }
3742 
3743 /** \test Test proper chunked encoded response body
3744  */
3745 static int HTPParserTest06(void)
3746 {
3747  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3748  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3749  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3750  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3751  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3752  "GMT\r\n"
3753  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3754  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3755  "FrontPage/5.0.2.2510\r\n"
3756  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3757  "chunked\r\n"
3758  "Content-Type: text/html\r\n\r\n"
3759  "580\r\n"
3760  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3761  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3762  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3763  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3764  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3765  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3766  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3767  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3768  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3769  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3770  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3771  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3772  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3773  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3774  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3775  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3776  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3777  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3778  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3779  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3780  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3781  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3782  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3783  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3784  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3785  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3786  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3787  "aHA=\r\n0\r\n\r\n";
3788  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3789  TcpSession ssn;
3790 
3793 
3794  memset(&ssn, 0, sizeof(ssn));
3795 
3796  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3797  FAIL_IF_NULL(f);
3798  f->protoctx = &ssn;
3799  f->proto = IPPROTO_TCP;
3800  f->alproto = ALPROTO_HTTP1;
3801 
3802  StreamTcpInitConfig(true);
3803 
3804  int r = AppLayerParserParse(
3805  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3806  FAIL_IF(r != 0);
3807  r = AppLayerParserParse(
3808  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3809  FAIL_IF(r != 0);
3810 
3811  HtpState *http_state = f->alstate;
3812  FAIL_IF_NULL(http_state);
3813 
3814  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3815  FAIL_IF_NULL(tx);
3816 
3817  FAIL_IF(tx->request_method_number != HTP_M_GET);
3818  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3819 
3820  FAIL_IF(tx->response_status_number != 200);
3821  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3822 
3823  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3824  FAIL_IF_NULL(h);
3825 
3827  StreamTcpFreeConfig(true);
3828  UTHFreeFlow(f);
3829  PASS;
3830 }
3831 
3832 /** \test
3833  */
3834 static int HTPParserTest07(void)
3835 {
3836  int result = 0;
3837  Flow *f = NULL;
3838  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3839  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3840  TcpSession ssn;
3841  HtpState *htp_state = NULL;
3842  int r = 0;
3844 
3845  memset(&ssn, 0, sizeof(ssn));
3846 
3847  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3848  if (f == NULL)
3849  goto end;
3850  f->protoctx = &ssn;
3851  f->proto = IPPROTO_TCP;
3852  f->alproto = ALPROTO_HTTP1;
3853 
3854  StreamTcpInitConfig(true);
3855 
3856  uint32_t u;
3857  for (u = 0; u < httplen1; u++) {
3858  uint8_t flags = 0;
3859 
3860  if (u == 0)
3861  flags = STREAM_TOSERVER|STREAM_START;
3862  else if (u == (httplen1 - 1))
3863  flags = STREAM_TOSERVER|STREAM_EOF;
3864  else
3865  flags = STREAM_TOSERVER;
3866 
3867  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3868  if (r != 0) {
3869  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3870  " 0: ", u, r);
3871  goto end;
3872  }
3873  }
3874 
3875  htp_state = f->alstate;
3876  if (htp_state == NULL) {
3877  printf("no http state: ");
3878  goto end;
3879  }
3880 
3881  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3882  size_t reflen = sizeof(ref) - 1;
3883 
3884  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3885  if (tx == NULL)
3886  goto end;
3887  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3888  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3889  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3890  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3891  (uintmax_t)reflen,
3892  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3893  goto end;
3894  }
3895 
3896  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3897  bstr_len(tx_ud->request_uri_normalized)) != 0)
3898  {
3899  printf("normalized uri \"");
3900  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3901  printf("\" != \"");
3902  PrintRawUriFp(stdout, ref, reflen);
3903  printf("\": ");
3904  goto end;
3905  }
3906  }
3907 
3908  result = 1;
3909 end:
3910  if (alp_tctx != NULL)
3912  StreamTcpFreeConfig(true);
3913  UTHFreeFlow(f);
3914  return result;
3915 }
3916 
3917 #include "conf-yaml-loader.h"
3918 
3919 /** \test Abort
3920  */
3921 static int HTPParserTest08(void)
3922 {
3923  int result = 0;
3924  Flow *f = NULL;
3925  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3926  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3927  TcpSession ssn;
3929 
3930  char input[] = "\
3931 %YAML 1.1\n\
3932 ---\n\
3933 libhtp:\n\
3934 \n\
3935  default-config:\n\
3936  personality: IDS\n\
3937 ";
3938 
3940  ConfInit();
3942 
3943  ConfYamlLoadString(input, strlen(input));
3944  HTPConfigure();
3945 
3946  HtpState *htp_state = NULL;
3947  int r = 0;
3948  memset(&ssn, 0, sizeof(ssn));
3949 
3950  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3951  if (f == NULL)
3952  goto end;
3953  f->protoctx = &ssn;
3954  f->proto = IPPROTO_TCP;
3955  f->alproto = ALPROTO_HTTP1;
3956 
3957  StreamTcpInitConfig(true);
3958 
3959  uint8_t flags = 0;
3960  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3961 
3962  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3963  if (r != 0) {
3964  printf("toserver chunk returned %" PRId32 ", expected"
3965  " 0: ", r);
3966  result = 0;
3967  goto end;
3968  }
3969 
3970  htp_state = f->alstate;
3971  if (htp_state == NULL) {
3972  printf("no http state: ");
3973  result = 0;
3974  goto end;
3975  }
3976 
3977  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3978  if (tx == NULL)
3979  goto end;
3980  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3981  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3982  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3983  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3984  bstr_len(tx_ud->request_uri_normalized));
3985  }
3986 
3987  result = 1;
3988 end:
3989  if (alp_tctx != NULL)
3991  StreamTcpFreeConfig(true);
3992  HTPFreeConfig();
3993  ConfDeInit();
3996  UTHFreeFlow(f);
3997  return result;
3998 }
3999 
4000 /** \test Abort
4001  */
4002 static int HTPParserTest09(void)
4003 {
4004  int result = 0;
4005  Flow *f = NULL;
4006  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
4007  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4008  TcpSession ssn;
4010 
4011  char input[] = "\
4012 %YAML 1.1\n\
4013 ---\n\
4014 libhtp:\n\
4015 \n\
4016  default-config:\n\
4017  personality: Apache_2_2\n\
4018 ";
4019 
4021  ConfInit();
4023 
4024  ConfYamlLoadString(input, strlen(input));
4025  HTPConfigure();
4026 
4027  HtpState *htp_state = NULL;
4028  int r = 0;
4029 
4030  memset(&ssn, 0, sizeof(ssn));
4031 
4032  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4033  if (f == NULL)
4034  goto end;
4035  f->protoctx = &ssn;
4036  f->proto = IPPROTO_TCP;
4037  f->alproto = ALPROTO_HTTP1;
4038 
4039  StreamTcpInitConfig(true);
4040 
4041  uint8_t flags = 0;
4042  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
4043 
4044  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
4045  if (r != 0) {
4046  printf("toserver chunk returned %" PRId32 ", expected"
4047  " 0: ", r);
4048  goto end;
4049  }
4050 
4051  htp_state = f->alstate;
4052  if (htp_state == NULL) {
4053  printf("no http state: ");
4054  goto end;
4055  }
4056 
4057  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4058  if (tx == NULL)
4059  goto end;
4060  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4061  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4062  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4063  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4064  bstr_len(tx_ud->request_uri_normalized));
4065  }
4066 
4067  result = 1;
4068 end:
4069  if (alp_tctx != NULL)
4071  StreamTcpFreeConfig(true);
4072  HTPFreeConfig();
4073  ConfDeInit();
4076  UTHFreeFlow(f);
4077  return result;
4078 }
4079 
4080 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4081  */
4082 static int HTPParserTest10(void)
4083 {
4084  int result = 0;
4085  Flow *f = NULL;
4086  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4087  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4088  TcpSession ssn;
4089  HtpState *htp_state = NULL;
4090  int r = 0;
4092 
4093  memset(&ssn, 0, sizeof(ssn));
4094 
4095  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4096  if (f == NULL)
4097  goto end;
4098  f->protoctx = &ssn;
4099  f->proto = IPPROTO_TCP;
4100  f->alproto = ALPROTO_HTTP1;
4101 
4102  StreamTcpInitConfig(true);
4103 
4104  uint32_t u;
4105  for (u = 0; u < httplen1; u++) {
4106  uint8_t flags = 0;
4107 
4108  if (u == 0)
4109  flags = STREAM_TOSERVER|STREAM_START;
4110  else if (u == (httplen1 - 1))
4111  flags = STREAM_TOSERVER|STREAM_EOF;
4112  else
4113  flags = STREAM_TOSERVER;
4114 
4115  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4116  if (r != 0) {
4117  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4118  " 0: ", u, r);
4119  goto end;
4120  }
4121  }
4122 
4123  htp_state = f->alstate;
4124  if (htp_state == NULL) {
4125  printf("no http state: ");
4126  goto end;
4127  }
4128 
4129  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4130  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4131  if (h == NULL) {
4132  goto end;
4133  }
4134 
4135  char *name = bstr_util_strdup_to_c(h->name);
4136  if (name == NULL) {
4137  goto end;
4138  }
4139 
4140  if (strcmp(name, "Host") != 0) {
4141  printf("header name not \"Host\", instead \"%s\": ", name);
4142  free(name);
4143  goto end;
4144  }
4145  free(name);
4146 
4147  char *value = bstr_util_strdup_to_c(h->value);
4148  if (value == NULL) {
4149  goto end;
4150  }
4151 
4152  if (strcmp(value, "www.google.com") != 0) {
4153  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4154  free(value);
4155  goto end;
4156  }
4157  free(value);
4158 
4159  result = 1;
4160 end:
4161  if (alp_tctx != NULL)
4163  StreamTcpFreeConfig(true);
4164  UTHFreeFlow(f);
4165  return result;
4166 }
4167 
4168 /** \test double encoding in path
4169  */
4170 static int HTPParserTest11(void)
4171 {
4172  int result = 0;
4173  Flow *f = NULL;
4174  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4175  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4176  TcpSession ssn;
4177  HtpState *htp_state = NULL;
4178  int r = 0;
4180 
4181  memset(&ssn, 0, sizeof(ssn));
4182 
4183  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4184  if (f == NULL)
4185  goto end;
4186  f->protoctx = &ssn;
4187  f->proto = IPPROTO_TCP;
4188  f->alproto = ALPROTO_HTTP1;
4189 
4190  StreamTcpInitConfig(true);
4191 
4192  uint32_t u;
4193  for (u = 0; u < httplen1; u++) {
4194  uint8_t flags = 0;
4195 
4196  if (u == 0)
4197  flags = STREAM_TOSERVER|STREAM_START;
4198  else if (u == (httplen1 - 1))
4199  flags = STREAM_TOSERVER|STREAM_EOF;
4200  else
4201  flags = STREAM_TOSERVER;
4202 
4203  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4204  if (r != 0) {
4205  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4206  " 0: ", u, r);
4207  goto end;
4208  }
4209  }
4210 
4211  htp_state = f->alstate;
4212  if (htp_state == NULL) {
4213  printf("no http state: ");
4214  goto end;
4215  }
4216 
4217  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4218  if (tx == NULL)
4219  goto end;
4220  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4221  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4222  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4223  printf("normalized uri len should be 2, is %"PRIuMAX,
4224  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4225  goto end;
4226  }
4227 
4228  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4229  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4230  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4231  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4232  {
4233  printf("normalized uri \"");
4234  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4235  printf("\": ");
4236  goto end;
4237  }
4238  }
4239 
4240  result = 1;
4241 end:
4242  if (alp_tctx != NULL)
4244  StreamTcpFreeConfig(true);
4245  UTHFreeFlow(f);
4246  return result;
4247 }
4248 
4249 /** \test double encoding in query
4250  */
4251 static int HTPParserTest12(void)
4252 {
4253  int result = 0;
4254  Flow *f = NULL;
4255  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4256  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4257  TcpSession ssn;
4258  HtpState *htp_state = NULL;
4259  int r = 0;
4261 
4262  memset(&ssn, 0, sizeof(ssn));
4263 
4264  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4265  if (f == NULL)
4266  goto end;
4267  f->protoctx = &ssn;
4268  f->proto = IPPROTO_TCP;
4269  f->alproto = ALPROTO_HTTP1;
4270 
4271  StreamTcpInitConfig(true);
4272 
4273  uint32_t u;
4274  for (u = 0; u < httplen1; u++) {
4275  uint8_t flags = 0;
4276 
4277  if (u == 0)
4278  flags = STREAM_TOSERVER|STREAM_START;
4279  else if (u == (httplen1 - 1))
4280  flags = STREAM_TOSERVER|STREAM_EOF;
4281  else
4282  flags = STREAM_TOSERVER;
4283 
4284  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4285  if (r != 0) {
4286  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4287  " 0: ", u, r);
4288  goto end;
4289  }
4290  }
4291 
4292  htp_state = f->alstate;
4293  if (htp_state == NULL) {
4294  printf("no http state: ");
4295  goto end;
4296  }
4297 
4298  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4299  if (tx == NULL)
4300  goto end;
4301  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4302  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4303  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4304  printf("normalized uri len should be 5, is %"PRIuMAX,
4305  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4306  goto end;
4307  }
4308 
4309  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4310  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4311  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4312  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4313  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4314  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4315  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4316  {
4317  printf("normalized uri \"");
4318  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4319  printf("\": ");
4320  goto end;
4321  }
4322  }
4323 
4324  result = 1;
4325  end:
4326  if (alp_tctx != NULL)
4328  StreamTcpFreeConfig(true);
4329  UTHFreeFlow(f);
4330  return result;
4331 }
4332 
4333 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4334  */
4335 static int HTPParserTest13(void)
4336 {
4337  int result = 0;
4338  Flow *f = NULL;
4339  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4340  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4341  TcpSession ssn;
4342  HtpState *htp_state = NULL;
4343  int r = 0;
4345 
4346  memset(&ssn, 0, sizeof(ssn));
4347 
4348  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4349  if (f == NULL)
4350  goto end;
4351  f->protoctx = &ssn;
4352  f->proto = IPPROTO_TCP;
4353  f->alproto = ALPROTO_HTTP1;
4354 
4355  StreamTcpInitConfig(true);
4356 
4357  uint32_t u;
4358  for (u = 0; u < httplen1; u++) {
4359  uint8_t flags = 0;
4360 
4361  if (u == 0)
4362  flags = STREAM_TOSERVER|STREAM_START;
4363  else if (u == (httplen1 - 1))
4364  flags = STREAM_TOSERVER|STREAM_EOF;
4365  else
4366  flags = STREAM_TOSERVER;
4367 
4368  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4369  if (r != 0) {
4370  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4371  " 0: ", u, r);
4372  goto end;
4373  }
4374  }
4375 
4376  htp_state = f->alstate;
4377  if (htp_state == NULL) {
4378  printf("no http state: ");
4379  goto end;
4380  }
4381 
4382  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4383  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4384  if (h == NULL) {
4385  goto end;
4386  }
4387 
4388  char *name = bstr_util_strdup_to_c(h->name);
4389  if (name == NULL) {
4390  goto end;
4391  }
4392 
4393  if (strcmp(name, "Host") != 0) {
4394  printf("header name not \"Host\", instead \"%s\": ", name);
4395  free(name);
4396  goto end;
4397  }
4398  free(name);
4399 
4400  char *value = bstr_util_strdup_to_c(h->value);
4401  if (value == NULL) {
4402  goto end;
4403  }
4404 
4405  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4406  printf("header value not \"www.google.com\", instead \"");
4407  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4408  printf("\": ");
4409  free(value);
4410  goto end;
4411  }
4412  free(value);
4413 
4414  result = 1;
4415 end:
4416  if (alp_tctx != NULL)
4418  StreamTcpFreeConfig(true);
4419  UTHFreeFlow(f);
4420  return result;
4421 }
4422 
4423 /** \test Test basic config */
4424 static int HTPParserConfigTest01(void)
4425 {
4426  int ret = 0;
4427  char input[] = "\
4428 %YAML 1.1\n\
4429 ---\n\
4430 libhtp:\n\
4431 \n\
4432  default-config:\n\
4433  personality: IDS\n\
4434 \n\
4435  server-config:\n\
4436 \n\
4437  - apache-tomcat:\n\
4438  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4439  personality: Tomcat_6_0\n\
4440 \n\
4441  - iis7:\n\
4442  address: \n\
4443  - 192.168.0.0/24\n\
4444  - 192.168.10.0/24\n\
4445  personality: IIS_7_0\n\
4446 ";
4447 
4449  ConfInit();
4450 
4451  ConfYamlLoadString(input, strlen(input));
4452 
4453  ConfNode *outputs;
4454  outputs = ConfGetNode("libhtp.default-config.personality");
4455  if (outputs == NULL) {
4456  goto end;
4457  }
4458 
4459  outputs = ConfGetNode("libhtp.server-config");
4460  if (outputs == NULL) {
4461  goto end;
4462  }
4463 
4464  ConfNode *node = TAILQ_FIRST(&outputs->head);
4465  if (node == NULL) {
4466  goto end;
4467  }
4468  if (strcmp(node->name, "0") != 0) {
4469  goto end;
4470  }
4471  node = TAILQ_FIRST(&node->head);
4472  if (node == NULL) {
4473  goto end;
4474  }
4475  if (strcmp(node->name, "apache-tomcat") != 0) {
4476  goto end;
4477  }
4478 
4479  int i = 0;
4480  ConfNode *n;
4481 
4482  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4483  if (node2 == NULL) {
4484  goto end;
4485  }
4486  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4487  goto end;
4488  }
4489 
4490  node = ConfNodeLookupChild(node, "address");
4491  if (node == NULL) {
4492  goto end;
4493  }
4494  TAILQ_FOREACH(n, &node->head, next) {
4495  if (n == NULL) {
4496  goto end;
4497  }
4498 
4499  switch(i) {
4500  case 0:
4501  if (strcmp(n->name, "0") != 0) {
4502  goto end;
4503  }
4504  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4505  goto end;
4506  }
4507  break;
4508  case 1:
4509  if (strcmp(n->name, "1") != 0) {
4510  goto end;
4511  }
4512  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4513  goto end;
4514  }
4515  break;
4516  case 2:
4517  if (strcmp(n->name, "2") != 0) {
4518  goto end;
4519  }
4520  if (strcmp(n->val, "::1") != 0) {
4521  goto end;
4522  }
4523  break;
4524  default:
4525  goto end;
4526  }
4527  i++;
4528  }
4529 
4530  outputs = ConfGetNode("libhtp.server-config");
4531  if (outputs == NULL) {
4532  goto end;
4533  }
4534 
4535  node = TAILQ_FIRST(&outputs->head);
4536  node = TAILQ_NEXT(node, next);
4537  if (node == NULL) {
4538  goto end;
4539  }
4540  if (strcmp(node->name, "1") != 0) {
4541  goto end;
4542  }
4543  node = TAILQ_FIRST(&node->head);
4544  if (node == NULL) {
4545  goto end;
4546  }
4547  if (strcmp(node->name, "iis7") != 0) {
4548  goto end;
4549  }
4550 
4551  node2 = ConfNodeLookupChild(node, "personality");
4552  if (node2 == NULL) {
4553  goto end;
4554  }
4555  if (strcmp(node2->val, "IIS_7_0") != 0) {
4556  goto end;
4557  }
4558 
4559  node = ConfNodeLookupChild(node, "address");
4560  if (node == NULL) {
4561  goto end;
4562  }
4563 
4564  i = 0;
4565  TAILQ_FOREACH(n, &node->head, next) {
4566  if (n == NULL) {
4567  goto end;
4568  }
4569 
4570  switch(i) {
4571  case 0:
4572  if (strcmp(n->name, "0") != 0) {
4573  goto end;
4574  }
4575  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4576  goto end;
4577  }
4578  break;
4579  case 1:
4580  if (strcmp(n->name, "1") != 0) {
4581  goto end;
4582  }
4583  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4584  goto end;
4585  }
4586  break;
4587  default:
4588  goto end;
4589  }
4590  i++;
4591  }
4592 
4593  ret = 1;
4594 
4595 end:
4596  ConfDeInit();
4598 
4599  return ret;
4600 }
4601 
4602 /** \test Test config builds radix correctly */
4603 static int HTPParserConfigTest02(void)
4604 {
4605  int ret = 0;
4606  char input[] = "\
4607 %YAML 1.1\n\
4608 ---\n\
4609 libhtp:\n\
4610 \n\
4611  default-config:\n\
4612  personality: IDS\n\
4613 \n\
4614  server-config:\n\
4615 \n\
4616  - apache-tomcat:\n\
4617  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4618  personality: Tomcat_6_0\n\
4619 \n\
4620  - iis7:\n\
4621  address: \n\
4622  - 192.168.0.0/24\n\
4623  - 192.168.10.0/24\n\
4624  personality: IIS_7_0\n\
4625 ";
4626 
4628  ConfInit();
4630 
4631  ConfYamlLoadString(input, strlen(input));
4632 
4633  HTPConfigure();
4634 
4635  if (cfglist.cfg == NULL) {
4636  printf("No default config created.\n");
4637  goto end;
4638  }
4639 
4640  if (cfgtree == NULL) {
4641  printf("No config tree created.\n");
4642  goto end;
4643  }
4644 
4645  htp_cfg_t *htp = cfglist.cfg;
4646  uint8_t buf[128];
4647  const char *addr;
4648  void *user_data = NULL;
4649 
4650  addr = "192.168.10.42";
4651  if (inet_pton(AF_INET, addr, buf) == 1) {
4652  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4653  if (user_data != NULL) {
4654  HTPCfgRec *htp_cfg_rec = user_data;
4655  htp = htp_cfg_rec->cfg;
4656  SCLogDebug("LIBHTP using config: %p", htp);
4657  }
4658  if (htp == NULL) {
4659  printf("Could not get config for: %s\n", addr);
4660  goto end;
4661  }
4662  }
4663  else {
4664  printf("Failed to parse address: %s\n", addr);
4665  goto end;
4666  }
4667 
4668  user_data = NULL;
4669  addr = "::1";
4670  if (inet_pton(AF_INET6, addr, buf) == 1) {
4671  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4672  if (user_data != NULL) {
4673  HTPCfgRec *htp_cfg_rec = user_data;
4674  htp = htp_cfg_rec->cfg;
4675  SCLogDebug("LIBHTP using config: %p", htp);
4676  }
4677  if (htp == NULL) {
4678  printf("Could not get config for: %s\n", addr);
4679  goto end;
4680  }
4681  }
4682  else {
4683  printf("Failed to parse address: %s\n", addr);
4684  goto end;
4685  }
4686 
4687  ret = 1;
4688 
4689 end:
4690  HTPFreeConfig();
4691  ConfDeInit();
4694 
4695  return ret;
4696 }
4697 
4698 /** \test Test traffic is handled by the correct htp config */
4699 static int HTPParserConfigTest03(void)
4700 {
4701  int result = 1;
4702  Flow *f = NULL;
4703  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4704  " Data is c0oL!";
4705  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4706  TcpSession ssn;
4708 
4709  HtpState *htp_state = NULL;
4710  int r = 0;
4711  char input[] = "\
4712 %YAML 1.1\n\
4713 ---\n\
4714 libhtp:\n\
4715 \n\
4716  default-config:\n\
4717  personality: IDS\n\
4718 \n\
4719  server-config:\n\
4720 \n\
4721  - apache-tomcat:\n\
4722  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4723  personality: Tomcat_6_0\n\
4724 \n\
4725  - iis7:\n\
4726  address: \n\
4727  - 192.168.0.0/24\n\
4728  - 192.168.10.0/24\n\
4729  personality: IIS_7_0\n\
4730 ";
4731 
4733  ConfInit();
4735 
4736  ConfYamlLoadString(input, strlen(input));
4737 
4738  HTPConfigure();
4739 
4740  const char *addr = "192.168.10.42";
4741 
4742  memset(&ssn, 0, sizeof(ssn));
4743 
4744  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4745  if (f == NULL)
4746  goto end;
4747  f->protoctx = &ssn;
4748  f->proto = IPPROTO_TCP;
4749  f->alproto = ALPROTO_HTTP1;
4750 
4751  htp_cfg_t *htp = cfglist.cfg;
4752 
4753  void *user_data = NULL;
4754  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4755  if (user_data != NULL) {
4756  HTPCfgRec *htp_cfg_rec = user_data;
4757  htp = htp_cfg_rec->cfg;
4758  SCLogDebug("LIBHTP using config: %p", htp);
4759  }
4760  if (htp == NULL) {
4761  printf("Could not get config for: %s\n", addr);
4762  goto end;
4763  }
4764 
4765  StreamTcpInitConfig(true);
4766 
4767  uint32_t u;
4768  for (u = 0; u < httplen1; u++) {
4769  uint8_t flags = 0;
4770 
4771  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4772  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4773  else flags = STREAM_TOSERVER;
4774 
4775  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4776  if (r != 0) {
4777  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4778  " 0: ", u, r);
4779  result = 0;
4780  goto end;
4781  }
4782  }
4783 
4784  htp_state = f->alstate;
4785  if (htp_state == NULL) {
4786  printf("no http state: ");
4787  result = 0;
4788  goto end;
4789  }
4790 
4791  if (HTPStateGetTxCnt(htp_state) != 2) {
4792  printf("HTPStateGetTxCnt(htp_state) failure\n");
4793  goto end;
4794  }
4795 
4796  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4797  if (tx == NULL)
4798  goto end;
4799  if (tx->cfg != htp) {
4800  printf("wrong HTP config (%p instead of %p - default=%p): ",
4801  tx->cfg, htp, cfglist.cfg);
4802  goto end;
4803  }
4804  tx = HTPStateGetTx(htp_state, 1);
4805  if (tx == NULL)
4806  goto end;
4807  if (tx->cfg != htp) {
4808  printf("wrong HTP config (%p instead of %p - default=%p): ",
4809  tx->cfg, htp, cfglist.cfg);
4810  goto end;
4811  }
4812 
4813 end:
4814  if (alp_tctx != NULL)
4816  HTPFreeConfig();
4817  ConfDeInit();
4820 
4821  StreamTcpFreeConfig(true);
4822  UTHFreeFlow(f);
4823  return result;
4824 }
4825 
4826 /* disabled when we upgraded to libhtp 0.5.x */
4827 #if 0
4828 static int HTPParserConfigTest04(void)
4829 {
4830  int result = 0;
4831 
4832  char input[] = "\
4833 %YAML 1.1\n\
4834 ---\n\
4835 libhtp:\n\
4836 \n\
4837  default-config:\n\
4838  personality: IDS\n\
4839  path-control-char-handling: status_400\n\
4840  path-convert-utf8: yes\n\
4841  path-invalid-encoding-handling: remove_percent\n\
4842 \n\
4843  server-config:\n\
4844 \n\
4845  - apache-tomcat:\n\
4846  personality: Tomcat_6_0\n\
4847  path-invalid-utf8-handling: none\n\
4848  path-nul-encoded-handling: status_404\n\
4849  path-nul-raw-handling: status_400\n\
4850 \n\
4851  - iis7:\n\
4852  personality: IIS_7_0\n\
4853  path-replacement-char: o\n\
4854  path-unicode-mapping: status_400\n\
4855 ";
4856 
4858  ConfInit();
4860 
4861  ConfYamlLoadString(input, strlen(input));
4862 
4863  HTPConfigure();
4864 
4865  HTPCfgRec *cfg_rec = &cfglist;
4866  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4867  cfg_rec->cfg->path_convert_utf8 != 1 ||
4868  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4869  printf("failed 1\n");
4870  goto end;
4871  }
4872 
4873  cfg_rec = cfg_rec->next;
4874  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4875  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4876  printf("failed 2\n");
4877  goto end;
4878  }
4879 
4880  cfg_rec = cfg_rec->next;
4881  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4882  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4883  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4884  printf("failed 3\n");
4885  goto end;
4886  }
4887 
4888  result = 1;
4889 
4890 end:
4891  HTPFreeConfig();
4892  ConfDeInit();
4895 
4896  return result;
4897 }
4898 #endif
4899 
4900 /** \test Test %2f decoding in profile Apache_2_2
4901  *
4902  * %2f in path is left untouched
4903  * %2f in query string is normalized to %2F
4904  * %252f in query string is decoded/normalized to %2F
4905  */
4906 static int HTPParserDecodingTest01(void)
4907 {
4908  uint8_t httpbuf1[] =
4909  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4910  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4911  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4912  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4913  TcpSession ssn;
4916 
4917  char input[] = "\
4918 %YAML 1.1\n\
4919 ---\n\
4920 libhtp:\n\
4921 \n\
4922  default-config:\n\
4923  personality: Apache_2\n\
4924 ";
4925 
4927  ConfInit();
4929  ConfYamlLoadString(input, strlen(input));
4930  HTPConfigure();
4931  const char *addr = "4.3.2.1";
4932  memset(&ssn, 0, sizeof(ssn));
4933 
4934  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4935  FAIL_IF_NULL(f);
4936  f->protoctx = &ssn;
4937  f->proto = IPPROTO_TCP;
4938  f->alproto = ALPROTO_HTTP1;
4939 
4940  StreamTcpInitConfig(true);
4941 
4942  for (uint32_t u = 0; u < httplen1; u++) {
4943  uint8_t flags = 0;
4944  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4945  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4946  else flags = STREAM_TOSERVER;
4947 
4948  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4949  FAIL_IF(r != 0);
4950  }
4951 
4952  HtpState *htp_state = f->alstate;
4953  FAIL_IF_NULL(htp_state);
4954 
4955  uint8_t ref1[] = "/abc%2fdef";
4956  size_t reflen = sizeof(ref1) - 1;
4957 
4958  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4959  FAIL_IF_NULL(tx);
4960 
4961  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4962  FAIL_IF_NULL(tx_ud);
4964  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4965  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4966  bstr_len(tx_ud->request_uri_normalized)) != 0);
4967 
4968  uint8_t ref2[] = "/abc/def?ghi/jkl";
4969  reflen = sizeof(ref2) - 1;
4970 
4971  tx = HTPStateGetTx(htp_state, 1);
4972  FAIL_IF_NULL(tx);
4973  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4974  FAIL_IF_NULL(tx_ud);
4976  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4977 
4978  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4979  bstr_len(tx_ud->request_uri_normalized)) != 0);
4980 
4981  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4982  reflen = sizeof(ref3) - 1;
4983  tx = HTPStateGetTx(htp_state, 2);
4984  FAIL_IF_NULL(tx);
4985  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4986  FAIL_IF_NULL(tx_ud);
4988  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4989 
4990  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4991  bstr_len(tx_ud->request_uri_normalized)) != 0);
4992 
4994  HTPFreeConfig();
4995  ConfDeInit();
4998 
4999  StreamTcpFreeConfig(true);
5000  UTHFreeFlow(f);
5001  PASS;
5002 }
5003 
5004 static int HTPParserDecodingTest01a(void)
5005 {
5006  uint8_t httpbuf1[] = "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5007  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5008  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5009  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5010  TcpSession ssn;
5013 
5014  char input[] = "\
5015 %YAML 1.1\n\
5016 ---\n\
5017 libhtp:\n\
5018 \n\
5019  default-config:\n\
5020  personality: Apache_2\n\
5021 ";
5022 
5024  ConfInit();
5026  ConfYamlLoadString(input, strlen(input));
5027  HTPConfigure();
5028  const char *addr = "4.3.2.1";
5029  memset(&ssn, 0, sizeof(ssn));
5030 
5031  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5032  FAIL_IF_NULL(f);
5033  f->protoctx = &ssn;
5034  f->proto = IPPROTO_TCP;
5035  f->alproto = ALPROTO_HTTP1;
5036 
5037  StreamTcpInitConfig(true);
5038 
5039  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
5040  (STREAM_TOSERVER | STREAM_START | STREAM_EOF), httpbuf1, httplen1);
5041  FAIL_IF(r != 0);
5042 
5043  HtpState *htp_state = f->alstate;
5044  FAIL_IF_NULL(htp_state);
5045 
5046  uint8_t ref1[] = "/abc%2fdef";
5047  size_t reflen = sizeof(ref1) - 1;
5048 
5049  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5050  FAIL_IF_NULL(tx);
5051 
5052  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5053  FAIL_IF_NULL(tx_ud);
5055  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5056  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5057  bstr_len(tx_ud->request_uri_normalized)) != 0);
5058 
5059  uint8_t ref2[] = "/abc/def?ghi/jkl";
5060  reflen = sizeof(ref2) - 1;
5061 
5062  tx = HTPStateGetTx(htp_state, 1);
5063  FAIL_IF_NULL(tx);
5064  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5065  FAIL_IF_NULL(tx_ud);
5067  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5068 
5069  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5070  bstr_len(tx_ud->request_uri_normalized)) != 0);
5071 
5072  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5073  reflen = sizeof(ref3) - 1;
5074  tx = HTPStateGetTx(htp_state, 2);
5075  FAIL_IF_NULL(tx);
5076  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5077  FAIL_IF_NULL(tx_ud);
5079  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5080 
5081  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5082  bstr_len(tx_ud->request_uri_normalized)) != 0);
5083 
5085  HTPFreeConfig();
5086  ConfDeInit();
5089 
5090  StreamTcpFreeConfig(true);
5091  UTHFreeFlow(f);
5092  PASS;
5093 }
5094 
5095 /** \test Test %2f decoding in profile IDS
5096  *
5097  * %2f in path decoded to /
5098  * %2f in query string is decoded to /
5099  * %252f in query string is decoded to %2F
5100  */
5101 static int HTPParserDecodingTest02(void)
5102 {
5103  int result = 0;
5104  Flow *f = NULL;
5105  uint8_t httpbuf1[] =
5106  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5107  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5108  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5109  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5110  TcpSession ssn;
5112 
5113  HtpState *htp_state = NULL;
5114  int r = 0;
5115  char input[] = "\
5116 %YAML 1.1\n\
5117 ---\n\
5118 libhtp:\n\
5119 \n\
5120  default-config:\n\
5121  personality: IDS\n\
5122  double-decode-path: no\n\
5123  double-decode-query: no\n\
5124 ";
5125 
5127  ConfInit();
5129  ConfYamlLoadString(input, strlen(input));
5130  HTPConfigure();
5131  const char *addr = "4.3.2.1";
5132  memset(&ssn, 0, sizeof(ssn));
5133 
5134  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5135  if (f == NULL)
5136  goto end;
5137  f->protoctx = &ssn;
5138  f->proto = IPPROTO_TCP;
5139  f->alproto = ALPROTO_HTTP1;
5140 
5141  StreamTcpInitConfig(true);
5142 
5143  uint32_t u;
5144  for (u = 0; u < httplen1; u++) {
5145  uint8_t flags = 0;
5146 
5147  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5148  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5149  else flags = STREAM_TOSERVER;
5150 
5151  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5152  if (r != 0) {
5153  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5154  " 0: ", u, r);
5155  goto end;
5156  }
5157  }
5158 
5159  htp_state = f->alstate;
5160  if (htp_state == NULL) {
5161  printf("no http state: ");
5162  goto end;
5163  }
5164 
5165  uint8_t ref1[] = "/abc/def";
5166  size_t reflen = sizeof(ref1) - 1;
5167 
5168  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5169  if (tx == NULL)
5170  goto end;
5171  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5172  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5173  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5174  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5175  (uintmax_t)reflen,
5176  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5177  goto end;
5178  }
5179 
5180  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5181  bstr_len(tx_ud->request_uri_normalized)) != 0)
5182  {
5183  printf("normalized uri \"");
5184  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5185  printf("\" != \"");
5186  PrintRawUriFp(stdout, ref1, reflen);
5187  printf("\": ");
5188  goto end;
5189  }
5190  }
5191 
5192  uint8_t ref2[] = "/abc/def?ghi/jkl";
5193  reflen = sizeof(ref2) - 1;
5194 
5195  tx = HTPStateGetTx(htp_state, 1);
5196  if (tx == NULL)
5197  goto end;
5198  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5199  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5200  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5201  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5202  (uintmax_t)reflen,
5203  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5204  goto end;
5205  }
5206 
5207  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5208  bstr_len(tx_ud->request_uri_normalized)) != 0)
5209  {
5210  printf("normalized uri \"");
5211  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5212  printf("\" != \"");
5213  PrintRawUriFp(stdout, ref2, reflen);
5214  printf("\": ");
5215  goto end;
5216  }
5217  }
5218 
5219  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5220  reflen = sizeof(ref3) - 1;
5221  tx = HTPStateGetTx(htp_state, 2);
5222  if (tx == NULL)
5223  goto end;
5224  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5225  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5226  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5227  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5228  (uintmax_t)reflen,
5229  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5230  goto end;
5231  }
5232 
5233  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5234  bstr_len(tx_ud->request_uri_normalized)) != 0)
5235  {
5236  printf("normalized uri \"");
5237  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5238  printf("\" != \"");
5239  PrintRawUriFp(stdout, ref3, reflen);
5240  printf("\": ");
5241  goto end;
5242  }
5243  }
5244 
5245  result = 1;
5246 
5247 end:
5248  if (alp_tctx != NULL)
5250  HTPFreeConfig();
5251  ConfDeInit();
5254 
5255  StreamTcpFreeConfig(true);
5256  UTHFreeFlow(f);
5257  return result;
5258 }
5259 
5260 /** \test Test %2f decoding in profile IDS with double-decode-* options
5261  *
5262  * %252f in path decoded to /
5263  * %252f in query string is decoded to /
5264  */
5265 static int HTPParserDecodingTest03(void)
5266 {
5267  int result = 0;
5268  Flow *f = NULL;
5269  uint8_t httpbuf1[] =
5270  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5271  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5272  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5273  TcpSession ssn;
5275 
5276  HtpState *htp_state = NULL;
5277  int r = 0;
5278  char input[] = "\
5279 %YAML 1.1\n\
5280 ---\n\
5281 libhtp:\n\
5282 \n\
5283  default-config:\n\
5284  personality: IDS\n\
5285  double-decode-path: yes\n\
5286  double-decode-query: yes\n\
5287 ";
5288 
5290  ConfInit();
5292  ConfYamlLoadString(input, strlen(input));
5293  HTPConfigure();
5294  const char *addr = "4.3.2.1";
5295  memset(&ssn, 0, sizeof(ssn));
5296 
5297  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5298  if (f == NULL)
5299  goto end;
5300  f->protoctx = &ssn;
5301  f->proto = IPPROTO_TCP;
5302  f->alproto = ALPROTO_HTTP1;
5303 
5304  StreamTcpInitConfig(true);
5305 
5306  uint32_t u;
5307  for (u = 0; u < httplen1; u++) {
5308  uint8_t flags = 0;
5309 
5310  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5311  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5312  else flags = STREAM_TOSERVER;
5313 
5314  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5315  if (r != 0) {
5316  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5317  " 0: ", u, r);
5318  goto end;
5319  }
5320  }
5321 
5322  htp_state = f->alstate;
5323  if (htp_state == NULL) {
5324  printf("no http state: ");
5325  goto end;
5326  }
5327 
5328  uint8_t ref1[] = "/abc/def";
5329  size_t reflen = sizeof(ref1) - 1;
5330 
5331  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5332  if (tx == NULL)
5333  goto end;
5334  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5335  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5336  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5337  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5338  (uintmax_t)reflen,
5339  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5340  goto end;
5341  }
5342 
5343  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5344  bstr_len(tx_ud->request_uri_normalized)) != 0)
5345  {
5346  printf("normalized uri \"");
5347  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5348  printf("\" != \"");
5349  PrintRawUriFp(stdout, ref1, reflen);
5350  printf("\": ");
5351  goto end;
5352  }
5353  }
5354 
5355  uint8_t ref2[] = "/abc/def?ghi/jkl";
5356  reflen = sizeof(ref2) - 1;
5357 
5358  tx = HTPStateGetTx(htp_state, 1);
5359  if (tx == NULL)
5360  goto end;
5361  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5362  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5363  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5364  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5365  (uintmax_t)reflen,
5366  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5367  goto end;
5368  }
5369 
5370  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5371  bstr_len(tx_ud->request_uri_normalized)) != 0)
5372  {
5373  printf("normalized uri \"");
5374  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5375  printf("\" != \"");
5376  PrintRawUriFp(stdout, ref2, reflen);
5377  printf("\": ");
5378  goto end;
5379  }
5380  }
5381 
5382  result = 1;
5383 
5384 end:
5385  if (alp_tctx != NULL)
5387  HTPFreeConfig();
5388  ConfDeInit();
5391 
5392  StreamTcpFreeConfig(true);
5393  UTHFreeFlow(f);
5394  return result;
5395 }
5396 
5397 /** \test Test http:// in query profile IDS
5398  */
5399 static int HTPParserDecodingTest04(void)
5400 {
5401  int result = 0;
5402  Flow *f = NULL;
5403  uint8_t httpbuf1[] =
5404  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5405  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5406  TcpSession ssn;
5408 
5409  HtpState *htp_state = NULL;
5410  int r = 0;
5411  char input[] = "\
5412 %YAML 1.1\n\
5413 ---\n\
5414 libhtp:\n\
5415 \n\
5416  default-config:\n\
5417  personality: IDS\n\
5418  double-decode-path: yes\n\
5419  double-decode-query: yes\n\
5420 ";
5421 
5423  ConfInit();
5425  ConfYamlLoadString(input, strlen(input));
5426  HTPConfigure();
5427  const char *addr = "4.3.2.1";
5428  memset(&ssn, 0, sizeof(ssn));
5429 
5430  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5431  if (f == NULL)
5432  goto end;
5433  f->protoctx = &ssn;
5434  f->proto = IPPROTO_TCP;
5435  f->alproto = ALPROTO_HTTP1;
5436 
5437  StreamTcpInitConfig(true);
5438 
5439  uint32_t u;
5440  for (u = 0; u < httplen1; u++) {
5441  uint8_t flags = 0;
5442 
5443  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5444  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5445  else flags = STREAM_TOSERVER;
5446 
5447  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5448  if (r != 0) {
5449  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5450  " 0: ", u, r);
5451  goto end;
5452  }
5453  }
5454 
5455  htp_state = f->alstate;
5456  if (htp_state == NULL) {
5457  printf("no http state: ");
5458  goto end;
5459  }
5460 
5461  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5462  size_t reflen = sizeof(ref1) - 1;
5463 
5464  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5465  if (tx == NULL)
5466  goto end;
5467  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5468  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5469  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5470  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5471  (uintmax_t)reflen,
5472  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5473  goto end;
5474  }
5475 
5476  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5477  bstr_len(tx_ud->request_uri_normalized)) != 0)
5478  {
5479  printf("normalized uri \"");
5480  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5481  printf("\" != \"");
5482  PrintRawUriFp(stdout, ref1, reflen);
5483  printf("\": ");
5484  goto end;
5485  }
5486  }
5487 
5488  result = 1;
5489 
5490 end:
5491  if (alp_tctx != NULL)
5493  HTPFreeConfig();
5494  ConfDeInit();
5497 
5498  StreamTcpFreeConfig(true);
5499  UTHFreeFlow(f);
5500  return result;
5501 }
5502 
5503 /** \test Test \ char in query profile IDS. Bug 739
5504  */
5505 static int HTPParserDecodingTest05(void)
5506 {
5507  int result = 0;
5508  Flow *f = NULL;
5509  uint8_t httpbuf1[] =
5510  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5511  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5512  TcpSession ssn;
5514 
5515  HtpState *htp_state = NULL;
5516  int r = 0;
5517  char input[] = "\
5518 %YAML 1.1\n\
5519 ---\n\
5520 libhtp:\n\
5521 \n\
5522  default-config:\n\
5523  personality: IDS\n\
5524  double-decode-path: yes\n\
5525  double-decode-query: yes\n\
5526 ";
5527 
5529  ConfInit();
5531  ConfYamlLoadString(input, strlen(input));
5532  HTPConfigure();
5533  const char *addr = "4.3.2.1";
5534  memset(&ssn, 0, sizeof(ssn));
5535 
5536  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5537  if (f == NULL)
5538  goto end;
5539  f->protoctx = &ssn;
5540  f->proto = IPPROTO_TCP;
5541  f->alproto = ALPROTO_HTTP1;
5542 
5543  StreamTcpInitConfig(true);
5544 
5545  uint32_t u;
5546  for (u = 0; u < httplen1; u++) {
5547  uint8_t flags = 0;
5548 
5549  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5550  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5551  else flags = STREAM_TOSERVER;
5552 
5553  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5554  if (r != 0) {
5555  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5556  " 0: ", u, r);
5557  goto end;
5558  }
5559  }
5560 
5561  htp_state = f->alstate;
5562  if (htp_state == NULL) {
5563  printf("no http state: ");
5564  goto end;
5565  }
5566 
5567  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5568  size_t reflen = sizeof(ref1) - 1;
5569 
5570  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5571  if (tx == NULL)
5572  goto end;
5573  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5574  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5575  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5576  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5577  (uintmax_t)reflen,
5578  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5579  goto end;
5580  }
5581 
5582  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5583  bstr_len(tx_ud->request_uri_normalized)) != 0)
5584  {
5585  printf("normalized uri \"");
5586  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5587  printf("\" != \"");
5588  PrintRawUriFp(stdout, ref1, reflen);
5589  printf("\": ");
5590  goto end;
5591  }
5592  }
5593 
5594  result = 1;
5595 
5596 end:
5597  if (alp_tctx != NULL)
5599  HTPFreeConfig();
5600  ConfDeInit();
5603 
5604  StreamTcpFreeConfig(true);
5605  UTHFreeFlow(f);
5606  return result;
5607 }
5608 
5609 /** \test Test + char in query. Bug 1035
5610  */
5611 static int HTPParserDecodingTest06(void)
5612 {
5613  int result = 0;
5614  Flow *f = NULL;
5615  uint8_t httpbuf1[] =
5616  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5617  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5618  TcpSession ssn;
5620 
5621  HtpState *htp_state = NULL;
5622  int r = 0;
5623  char input[] = "\
5624 %YAML 1.1\n\
5625 ---\n\
5626 libhtp:\n\
5627 \n\
5628  default-config:\n\
5629  personality: IDS\n\
5630  double-decode-path: yes\n\
5631  double-decode-query: yes\n\
5632 ";
5633 
5635  ConfInit();
5637  ConfYamlLoadString(input, strlen(input));
5638  HTPConfigure();
5639  const char *addr = "4.3.2.1";
5640  memset(&ssn, 0, sizeof(ssn));
5641 
5642  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5643  if (f == NULL)
5644  goto end;
5645  f->protoctx = &ssn;
5646  f->proto = IPPROTO_TCP;
5647  f->alproto = ALPROTO_HTTP1;
5648 
5649  StreamTcpInitConfig(true);
5650 
5651  uint32_t u;
5652  for (u = 0; u < httplen1; u++) {
5653  uint8_t flags = 0;
5654 
5655  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5656  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5657  else flags = STREAM_TOSERVER;
5658 
5659  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5660  if (r != 0) {
5661  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5662  " 0: ", u, r);
5663  goto end;
5664  }
5665  }
5666 
5667  htp_state = f->alstate;
5668  if (htp_state == NULL) {
5669  printf("no http state: ");
5670  goto end;
5671  }
5672 
5673  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5674  size_t reflen = sizeof(ref1) - 1;
5675 
5676  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5677  if (tx == NULL)
5678  goto end;
5679  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5680  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5681  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5682  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5683  (uintmax_t)reflen,
5684  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5685  goto end;
5686  }
5687 
5688  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5689  bstr_len(tx_ud->request_uri_normalized)) != 0)
5690  {
5691  printf("normalized uri \"");
5692  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5693  printf("\" != \"");
5694  PrintRawUriFp(stdout, ref1, reflen);
5695  printf("\": ");
5696  goto end;
5697  }
5698  }
5699 
5700  result = 1;
5701 
5702 end:
5703  if (alp_tctx != NULL)
5705  HTPFreeConfig();
5706  ConfDeInit();
5709 
5710  StreamTcpFreeConfig(true);
5711  UTHFreeFlow(f);
5712  return result;
5713 }
5714 
5715 /** \test Test + char in query. Bug 1035
5716  */
5717 static int HTPParserDecodingTest07(void)
5718 {
5719  int result = 0;
5720  Flow *f = NULL;
5721  uint8_t httpbuf1[] =
5722  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5723  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5724  TcpSession ssn;
5726 
5727  HtpState *htp_state = NULL;
5728  int r = 0;
5729  char input[] = "\
5730 %YAML 1.1\n\
5731 ---\n\
5732 libhtp:\n\
5733 \n\
5734  default-config:\n\
5735  personality: IDS\n\
5736  double-decode-path: yes\n\
5737  double-decode-query: yes\n\
5738  query-plusspace-decode: yes\n\
5739 ";
5740 
5742  ConfInit();
5744  ConfYamlLoadString(input, strlen(input));
5745  HTPConfigure();
5746  const char *addr = "4.3.2.1";
5747  memset(&ssn, 0, sizeof(ssn));
5748 
5749  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5750  if (f == NULL)
5751  goto end;
5752  f->protoctx = &ssn;
5753  f->proto = IPPROTO_TCP;
5754  f->alproto = ALPROTO_HTTP1;
5755 
5756  StreamTcpInitConfig(true);
5757 
5758  uint32_t u;
5759  for (u = 0; u < httplen1; u++) {
5760  uint8_t flags = 0;
5761 
5762  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5763  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5764  else flags = STREAM_TOSERVER;
5765 
5766  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5767  if (r != 0) {
5768  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5769  " 0: ", u, r);
5770  goto end;
5771  }
5772  }
5773 
5774  htp_state = f->alstate;
5775  if (htp_state == NULL) {
5776  printf("no http state: ");
5777  goto end;
5778  }
5779 
5780  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5781  size_t reflen = sizeof(ref1) - 1;
5782 
5783  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5784  if (tx == NULL)
5785  goto end;
5786  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5787  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5788  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5789  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5790  (uintmax_t)reflen,
5791  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5792  goto end;
5793  }
5794 
5795  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5796  bstr_len(tx_ud->request_uri_normalized)) != 0)
5797  {
5798  printf("normalized uri \"");
5799  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5800  printf("\" != \"");
5801  PrintRawUriFp(stdout, ref1, reflen);
5802  printf("\": ");
5803  goto end;
5804  }
5805  }
5806 
5807  result = 1;
5808 
5809 end:
5810  if (alp_tctx != NULL)
5812  HTPFreeConfig();
5813  ConfDeInit();
5816 
5817  StreamTcpFreeConfig(true);
5818  UTHFreeFlow(f);
5819  return result;
5820 }
5821 
5822 /** \test Test 'proxy' URI normalization. Ticket 1008
5823  */
5824 static int HTPParserDecodingTest08(void)
5825 {
5826  int result = 0;
5827  Flow *f = NULL;
5828  uint8_t httpbuf1[] =
5829  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5830  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5831  TcpSession ssn;
5833