suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "decode.h"
40 #include "threads.h"
41 #include "counters.h"
42 
43 #include "util-print.h"
44 #include "util-pool.h"
45 #include "util-radix-tree.h"
46 #include "util-file.h"
47 #include "util-byte.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-detect-proto.h"
59 #include "app-layer-frames.h"
60 #include "app-layer-htp.h"
61 #include "app-layer-htp-body.h"
62 #include "app-layer-htp-file.h"
63 #include "app-layer-htp-libhtp.h"
64 #include "app-layer-htp-xff.h"
65 #include "app-layer-htp-range.h"
66 #include "app-layer-htp-mem.h"
67 
68 #include "util-spm.h"
69 #include "util-debug.h"
70 #include "util-time.h"
71 #include "util-misc.h"
72 #include "util-enum.h"
73 
74 #include "util-unittest.h"
75 #include "util-unittest-helper.h"
76 #include "flow-util.h"
77 
78 #include "detect-engine.h"
79 #include "detect-engine-build.h"
80 #include "detect-engine-state.h"
81 #include "detect-parse.h"
82 
83 #include "decode-events.h"
84 
85 #include "util-memcmp.h"
86 #include "util-random.h"
87 #include "util-validate.h"
88 
89 //#define PRINT
90 
91 /** Fast lookup tree (radix) for the various HTP configurations */
92 static SCRadixTree *cfgtree;
93 /** List of HTP configurations. */
94 static HTPCfgRec cfglist;
95 
97 
98 /** Limit to the number of libhtp messages that can be handled */
99 #define HTP_MAX_MESSAGES 512
100 
101 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
102 
103 #ifdef DEBUG
104 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
105 static uint64_t htp_state_memuse = 0;
106 static uint64_t htp_state_memcnt = 0;
107 #endif
108 
110  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
111  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
112  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
113  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
114  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
115  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
116  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
118  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
120  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
122  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
124  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
126  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
128  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
129  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
131  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
132  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
133  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
134  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
135  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
136  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
137  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
138  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
139  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
140  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
141  { "FILE_NAME_TOO_LONG", HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG },
142  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
143  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
144  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
146  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
147  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
148  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
149  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
150  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
151  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
152  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
153  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
154  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
155  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
156  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
157  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
158  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
160  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
161  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
162  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
163  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
164 
165  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
166  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
167 
168  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
169  { "REQUEST_CHUNK_EXTENSION", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
170 
171  /* suricata warnings/errors */
172  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
173  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
174  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
175 
176  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
177  { "FAILED_PROTOCOL_CHANGE", HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE },
178 
179  { NULL, -1 },
180 };
181 
182 /* app-layer-frame-documentation tag start: HttpFrameTypes */
186 };
187 
189  {
190  "request",
192  },
193  {
194  "response",
196  },
197  { NULL, -1 },
198 };
199 /* app-layer-frame-documentation tag end: HttpFrameTypes */
200 
201 static int HTTPGetFrameIdByName(const char *frame_name)
202 {
203  int id = SCMapEnumNameToValue(frame_name, http_frame_table);
204  if (id < 0) {
205  return -1;
206  }
207  return id;
208 }
209 
210 static const char *HTTPGetFrameNameById(const uint8_t frame_id)
211 {
212  const char *name = SCMapEnumValueToName(frame_id, http_frame_table);
213  return name;
214 }
215 
216 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
217 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
218 static uint64_t HTPStateGetTxCnt(void *alstate);
219 #ifdef UNITTESTS
220 static void HTPParserRegisterTests(void);
221 #endif
222 
223 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
224 {
225  uint64_t id = HTPStateGetTxCnt(s);
226  BUG_ON(id == 0);
227  return id - 1;
228 }
229 
230 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
231 {
232  return s->transaction_cnt;
233 }
234 
235 #ifdef DEBUG
236 /**
237  * \internal
238  *
239  * \brief Lookup the HTP personality string from the numeric personality.
240  *
241  * \todo This needs to be a libhtp function.
242  */
243 static const char *HTPLookupPersonalityString(int p)
244 {
245 #define CASE_HTP_PERSONALITY_STRING(p) \
246  case HTP_SERVER_ ## p: return #p
247 
248  switch (p) {
249  CASE_HTP_PERSONALITY_STRING(MINIMAL);
250  CASE_HTP_PERSONALITY_STRING(GENERIC);
251  CASE_HTP_PERSONALITY_STRING(IDS);
252  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
253  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
254  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
255  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
256  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
257  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
258  CASE_HTP_PERSONALITY_STRING(APACHE_2);
259  }
260 
261  return NULL;
262 }
263 #endif /* DEBUG */
264 
265 /**
266  * \internal
267  *
268  * \brief Lookup the numeric HTP personality from a string.
269  *
270  * \todo This needs to be a libhtp function.
271  */
272 static int HTPLookupPersonality(const char *str)
273 {
274 #define IF_HTP_PERSONALITY_NUM(p) \
275  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
276 
277  IF_HTP_PERSONALITY_NUM(MINIMAL);
278  IF_HTP_PERSONALITY_NUM(GENERIC);
280  IF_HTP_PERSONALITY_NUM(IIS_4_0);
281  IF_HTP_PERSONALITY_NUM(IIS_5_0);
282  IF_HTP_PERSONALITY_NUM(IIS_5_1);
283  IF_HTP_PERSONALITY_NUM(IIS_6_0);
284  IF_HTP_PERSONALITY_NUM(IIS_7_0);
285  IF_HTP_PERSONALITY_NUM(IIS_7_5);
286  IF_HTP_PERSONALITY_NUM(APACHE_2);
287  if (strcasecmp("TOMCAT_6_0", str) == 0) {
288  SCLogError("Personality %s no "
289  "longer supported by libhtp.",
290  str);
291  return -1;
292  } else if ((strcasecmp("APACHE", str) == 0) ||
293  (strcasecmp("APACHE_2_2", str) == 0))
294  {
295  SCLogWarning("Personality %s no "
296  "longer supported by libhtp, failing back to "
297  "Apache2 personality.",
298  str);
299  return HTP_SERVER_APACHE_2;
300  }
301 
302  return -1;
303 }
304 
305 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
306  const uint8_t dir, const uint8_t e)
307 {
308  SCLogDebug("setting event %u", e);
309 
310  if (htud) {
311  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
312  s->events++;
313  return;
314  }
315 
316  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
317  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
318 
319  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
320  if (tx == NULL && tx_id > 0)
321  tx = HTPStateGetTx(s, tx_id - 1);
322  if (tx != NULL) {
323  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
324  if (htud != NULL) {
325  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
326  s->events++;
327  return;
328  }
329  }
330  SCLogDebug("couldn't set event %u", e);
331 }
332 
333 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
334  * connection parser to be used by the HTP library
335  */
336 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
337 {
338  SCEnter();
339 
340  HtpState *s = HTPMalloc(sizeof(HtpState));
341  if (unlikely(s == NULL)) {
342  SCReturnPtr(NULL, "void");
343  }
344 
345  memset(s, 0x00, sizeof(HtpState));
346 
347 #ifdef DEBUG
348  SCMutexLock(&htp_state_mem_lock);
349  htp_state_memcnt++;
350  htp_state_memuse += sizeof(HtpState);
351  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
352  SCMutexUnlock(&htp_state_mem_lock);
353 #endif
354 
355  SCReturnPtr((void *)s, "void");
356 }
357 
358 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
359 {
360  if (likely(htud)) {
361  HtpBodyFree(&state->cfg->request, &htud->request_body);
362  HtpBodyFree(&state->cfg->response, &htud->response_body);
363  bstr_free(htud->request_uri_normalized);
364  if (htud->request_headers_raw)
366  if (htud->response_headers_raw)
369  if (htud->boundary)
370  HTPFree(htud->boundary, htud->boundary_len);
371  if (htud->tx_data.de_state != NULL) {
372  DetectEngineStateFree(htud->tx_data.de_state);
373  }
374  if (htud->file_range) {
375  HTPFileCloseHandleRange(&htp_sbcfg, &htud->files_tc, 0, htud->file_range, NULL, 0);
377  }
380  HTPFree(htud, sizeof(HtpTxUserData));
381  }
382 }
383 
384 /** \brief Function to frees the HTTP state memory and also frees the HTTP
385  * connection parser memory which was used by the HTP library
386  */
387 void HTPStateFree(void *state)
388 {
389  SCEnter();
390 
391  HtpState *s = (HtpState *)state;
392  if (s == NULL) {
393  SCReturn;
394  }
395 
396  /* free the connection parser memory used by HTP library */
397  if (s->connp != NULL) {
398  SCLogDebug("freeing HTP state");
399 
400  uint64_t tx_id;
401  uint64_t total_txs = HTPStateGetTxCnt(state);
402  /* free the list of body chunks */
403  if (s->conn != NULL) {
404  for (tx_id = 0; tx_id < total_txs; tx_id++) {
405  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
406  if (tx != NULL) {
407  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
408  HtpTxUserDataFree(s, htud);
409  htp_tx_set_user_data(tx, NULL);
410  }
411  }
412  }
413  htp_connp_destroy_all(s->connp);
414  }
415 
416  HTPFree(s, sizeof(HtpState));
417 
418 #ifdef DEBUG
419  SCMutexLock(&htp_state_mem_lock);
420  htp_state_memcnt--;
421  htp_state_memuse -= sizeof(HtpState);
422  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
423  SCMutexUnlock(&htp_state_mem_lock);
424 #endif
425 
426  SCReturn;
427 }
428 
429 /**
430  * \brief HTP transaction cleanup callback
431  *
432  * \warning We cannot actually free the transactions here. It seems that
433  * HTP only accepts freeing of transactions in the response callback.
434  */
435 static void HTPStateTransactionFree(void *state, uint64_t id)
436 {
437  SCEnter();
438 
439  HtpState *s = (HtpState *)state;
440 
441  SCLogDebug("state %p, id %"PRIu64, s, id);
442 
443  htp_tx_t *tx = HTPStateGetTx(s, id);
444  if (tx != NULL) {
445  /* This will remove obsolete body chunks */
446  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
447  HtpTxUserDataFree(s, htud);
448  htp_tx_set_user_data(tx, NULL);
449 
450  /* hack: even if libhtp considers the tx incomplete, we want to
451  * free it here. htp_tx_destroy however, will refuse to do this.
452  * As htp_tx_destroy_incomplete isn't available in the public API,
453  * we hack around it here. */
454  if (unlikely(!(
455  tx->request_progress == HTP_REQUEST_COMPLETE &&
456  tx->response_progress == HTP_RESPONSE_COMPLETE)))
457  {
458  tx->request_progress = HTP_REQUEST_COMPLETE;
459  tx->response_progress = HTP_RESPONSE_COMPLETE;
460  }
461  htp_tx_destroy(tx);
462  }
463 }
464 
465 /**
466  * \brief Sets a flag that informs the HTP app layer that some module in the
467  * engine needs the http request body data.
468  * \initonly
469  */
471 {
472  SCEnter();
473 
474  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
475  SCReturn;
476 }
477 
478 /**
479  * \brief Sets a flag that informs the HTP app layer that some module in the
480  * engine needs the http request body data.
481  * \initonly
482  */
484 {
485  SCEnter();
486 
487  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
488  SCReturn;
489 }
490 
491 /**
492  * \brief Sets a flag that informs the HTP app layer that some module in the
493  * engine needs the http request multi part header.
494  *
495  * \initonly
496  */
497 static void AppLayerHtpNeedMultipartHeader(void)
498 {
499  SCEnter();
501 
502  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
503  SCReturn;
504 }
505 
506 /**
507  * \brief Sets a flag that informs the HTP app layer that some module in the
508  * engine needs the http request file.
509  *
510  * \initonly
511  */
513 {
514  SCEnter();
515  AppLayerHtpNeedMultipartHeader();
518 
519  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
520  SCReturn;
521 }
522 
523 static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags)
524 {
525  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
526  if (tx_ud) {
527  SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags);
528  if (flags & STREAM_TOCLIENT) {
529  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
530  } else {
531  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
532  }
533  }
534 }
535 
536 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
537 {
538  SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64,
540  if (flags & HTP_STREAM_DEPTH_SET) {
541  uint32_t stream_depth = FileReassemblyDepth();
542  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
543  SCLogDebug("true");
544  return true;
545  }
546  } else {
547  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
548  return true;
549  }
550  }
551  SCLogDebug("false");
552  return false;
553 }
554 
555 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
556  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
557 {
558  uint32_t chunk_len = 0;
559  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
560  (content_len_so_far < (uint64_t)body_limit) &&
561  (content_len_so_far + (uint64_t)data_len) > body_limit)
562  {
563  chunk_len = body_limit - content_len_so_far;
564  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
565  (content_len_so_far < (uint64_t)stream_depth) &&
566  (content_len_so_far + (uint64_t)data_len) > stream_depth)
567  {
568  chunk_len = stream_depth - content_len_so_far;
569  }
570  SCLogDebug("len %u", chunk_len);
571  return (chunk_len == 0 ? data_len : chunk_len);
572 }
573 
574 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
575 struct {
576  const char *msg;
577  uint8_t de;
578 } htp_errors[] = {
579  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
580  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
581  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
582  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
583  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
584 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
585  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
586 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
587  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
588  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
589  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
590  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
591 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
592  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
593  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
594  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
595  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
596 };
597 
598 struct {
599  const char *msg;
600  uint8_t de;
601 } htp_warnings[] = {
602  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
603  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
604  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
605  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
606  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
607  /* { "Host information in request headers required by HTTP/1.1",
608  HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING { "Host information
609  ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
610  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
611  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
612  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number
613  * differs from the actual TCP port=%d", port, connp->conn->server_port); luckily, "Request
614  * server port=" is unique */
615  /* { "Request server port number differs from the actual TCP port",
616  HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
618  { "Request line: URI contains non-compliant delimiter",
620  { "Request line: non-compliant delimiter between Method and URI",
622  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
623  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
624  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
625  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
626  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
627  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
628  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
629  { "Transfer-encoding has abnormal chunked value",
631  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0",
633  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
634  { "Invalid response line: invalid response status",
636  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
637  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
638  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
639  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST },
640  { "Ambiguous response C-L value",
642  { "Request chunk extension", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
643 };
644 
645 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
646 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
647 
648 /**
649  * \internal
650  *
651  * \brief Get the warning id for the warning msg.
652  *
653  * \param msg warning message
654  *
655  * \retval id the id or 0 in case of not found
656  */
657 static uint8_t HTPHandleWarningGetId(const char *msg)
658 {
659  SCLogDebug("received warning \"%s\"", msg);
660  size_t idx;
661  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
662  if (strncmp(htp_warnings[idx].msg, msg,
663  strlen(htp_warnings[idx].msg)) == 0)
664  {
665  return htp_warnings[idx].de;
666  }
667  }
668 
669  return 0;
670 }
671 
672 /**
673  * \internal
674  *
675  * \brief Get the error id for the error msg.
676  *
677  * \param msg error message
678  *
679  * \retval id the id or 0 in case of not found
680  */
681 static uint8_t HTPHandleErrorGetId(const char *msg)
682 {
683  SCLogDebug("received error \"%s\"", msg);
684 
685  size_t idx;
686  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
687  if (strncmp(htp_errors[idx].msg, msg,
688  strlen(htp_errors[idx].msg)) == 0)
689  {
690  return htp_errors[idx].de;
691  }
692  }
693 
694  return 0;
695 }
696 
697 /**
698  * \internal
699  *
700  * \brief Check state for errors, warnings and add any as events
701  *
702  * \param s state
703  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
704  */
705 static void HTPHandleError(HtpState *s, const uint8_t dir)
706 {
707  if (s == NULL || s->conn == NULL ||
708  s->conn->messages == NULL) {
709  return;
710  }
711 
712  size_t size = htp_list_size(s->conn->messages);
713  size_t msg;
714  if(size >= HTP_MAX_MESSAGES) {
716  //only once per HtpState
717  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
719  //too noisy in fuzzing
720  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
721  }
722  // ignore further messages
723  return;
724  }
725 
726  for (msg = s->htp_messages_offset; msg < size; msg++) {
727  htp_log_t *log = htp_list_get(s->conn->messages, msg);
728  if (log == NULL)
729  continue;
730 
731  HtpTxUserData *htud = NULL;
732  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
733  if (tx != NULL)
734  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
735 
736  SCLogDebug("message %s", log->msg);
737 
738  uint8_t id = HTPHandleErrorGetId(log->msg);
739  if (id == 0) {
740  id = HTPHandleWarningGetId(log->msg);
741  if (id == 0)
743  }
744 
745  if (id > 0) {
746  HTPSetEvent(s, htud, dir, id);
747  }
748  }
749  s->htp_messages_offset = (uint16_t)msg;
750  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
751 }
752 
753 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
754 {
755 #ifdef DEBUG
756  BUG_ON(s == NULL || tx == NULL);
757 #endif
758  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
759  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
760  HTP_HOSTH_INVALID))
761  {
762  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
763  if (htud == NULL)
764  return;
765 
766  if (tx->flags & HTP_REQUEST_INVALID_T_E)
767  HTPSetEvent(s, htud, STREAM_TOSERVER,
769  if (tx->flags & HTP_REQUEST_INVALID_C_L)
770  HTPSetEvent(s, htud, STREAM_TOSERVER,
772  if (tx->flags & HTP_HOST_MISSING)
773  HTPSetEvent(s, htud, STREAM_TOSERVER,
775  if (tx->flags & HTP_HOST_AMBIGUOUS)
776  HTPSetEvent(s, htud, STREAM_TOSERVER,
778  if (tx->flags & HTP_HOSTU_INVALID)
779  HTPSetEvent(s, htud, STREAM_TOSERVER,
781  if (tx->flags & HTP_HOSTH_INVALID)
782  HTPSetEvent(s, htud, STREAM_TOSERVER,
784  }
785  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
786  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
787  if (htud == NULL)
788  return;
789  HTPSetEvent(s, htud, STREAM_TOSERVER,
791  }
792  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
793  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
794  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
795  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
796  if (htud == NULL)
797  return;
798  HTPSetEvent(s, htud, STREAM_TOSERVER,
800  }
801 }
802 
803 static int Setup(Flow *f, HtpState *hstate)
804 {
805  /* store flow ref in state so callbacks can access it */
806  hstate->f = f;
807 
808  HTPCfgRec *htp_cfg_rec = &cfglist;
809  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
810  void *user_data = NULL;
811 
812  if (FLOW_IS_IPV4(f)) {
813  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
814  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
815  }
816  else if (FLOW_IS_IPV6(f)) {
817  SCLogDebug("Looking up HTP config for ipv6");
818  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
819  }
820  else {
821  SCLogError("unknown address family, bug!");
822  goto error;
823  }
824 
825  if (user_data != NULL) {
826  htp_cfg_rec = user_data;
827  htp = htp_cfg_rec->cfg;
828  SCLogDebug("LIBHTP using config: %p", htp);
829  } else {
830  SCLogDebug("Using default HTP config: %p", htp);
831  }
832 
833  if (NULL == htp) {
834 #ifdef DEBUG_VALIDATION
835  BUG_ON(htp == NULL);
836 #endif
837  /* should never happen if HTPConfigure is properly invoked */
838  goto error;
839  }
840 
841  hstate->connp = htp_connp_create(htp);
842  if (hstate->connp == NULL) {
843  goto error;
844  }
845 
846  hstate->conn = htp_connp_get_connection(hstate->connp);
847 
848  htp_connp_set_user_data(hstate->connp, (void *)hstate);
849  hstate->cfg = htp_cfg_rec;
850 
851  SCLogDebug("New hstate->connp %p", hstate->connp);
852 
853  struct timeval tv = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
854  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &tv);
855 
857  htp_cfg_rec->request.inspect_min_size);
859  htp_cfg_rec->response.inspect_min_size);
860  return 0;
861 error:
862  return -1;
863 }
864 
865 /**
866  * \brief Function to handle the reassembled data from client and feed it to
867  * the HTP library to process it.
868  *
869  * \param flow Pointer to the flow the data belong to
870  * \param htp_state Pointer the state in which the parsed value to be stored
871  * \param pstate Application layer parser state for this session
872  *
873  * \retval On success returns 1 or on failure returns -1.
874  */
875 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state, AppLayerParserState *pstate,
876  StreamSlice stream_slice, void *local_data)
877 {
878  SCEnter();
879  int ret = 0;
880  HtpState *hstate = (HtpState *)htp_state;
881 
882  /* On the first invocation, create the connection parser structure to
883  * be used by HTP library. This is looked up via IP in the radix
884  * tree. Failing that, the default HTP config is used.
885  */
886  if (NULL == hstate->conn) {
887  if (Setup(f, hstate) != 0) {
889  }
890  }
891  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
892  hstate->slice = &stream_slice;
893 
894  const uint8_t *input = StreamSliceGetData(&stream_slice);
895  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
896 
897  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
898  /* pass the new data to the htp parser */
899  if (input_len > 0) {
900  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
901  switch (r) {
902  case HTP_STREAM_ERROR:
903  ret = -1;
904  break;
905  default:
906  break;
907  }
908  HTPHandleError(hstate, STREAM_TOSERVER);
909  }
910 
911  /* if the TCP connection is closed, then close the HTTP connection */
913  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
914  {
915  htp_connp_req_close(hstate->connp, &ts);
916  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
917  SCLogDebug("stream eof encountered, closing htp handle for ts");
918  }
919 
920  SCLogDebug("hstate->connp %p", hstate->connp);
921  hstate->slice = NULL;
922 
923  if (ret < 0) {
925  }
927 }
928 
929 /**
930  * \brief Function to handle the reassembled data from server and feed it to
931  * the HTP library to process it.
932  *
933  * \param flow Pointer to the flow the data belong to
934  * \param htp_state Pointer the state in which the parsed value to be stored
935  * \param pstate Application layer parser state for this session
936  * \param input Pointer the received HTTP server data
937  * \param input_len Length in bytes of the received data
938  * \param output Pointer to the output (not used in this function)
939  *
940  * \retval On success returns 1 or on failure returns -1
941  */
942 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state, AppLayerParserState *pstate,
943  StreamSlice stream_slice, void *local_data)
944 {
945  SCEnter();
946  int ret = 0;
947  HtpState *hstate = (HtpState *)htp_state;
948 
949  const uint8_t *input = StreamSliceGetData(&stream_slice);
950  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
951 
952  /* On the first invocation, create the connection parser structure to
953  * be used by HTP library. This is looked up via IP in the radix
954  * tree. Failing that, the default HTP config is used.
955  */
956  if (NULL == hstate->conn) {
957  if (Setup(f, hstate) != 0) {
959  }
960  }
961  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
962  hstate->slice = &stream_slice;
963 
964  htp_time_t ts = { SCTIME_SECS(f->startts), SCTIME_USECS(f->startts) };
965  htp_tx_t *tx = NULL;
966  size_t consumed = 0;
967  if (input_len > 0) {
968  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
969  switch (r) {
970  case HTP_STREAM_ERROR:
971  ret = -1;
972  break;
973  case HTP_STREAM_TUNNEL:
974  tx = htp_connp_get_out_tx(hstate->connp);
975  if (tx != NULL && tx->response_status_number == 101) {
976  htp_header_t *h =
977  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
978  if (h == NULL || bstr_cmp_c(h->value, "h2c") != 0) {
979  break;
980  }
982  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
983  break;
984  }
985  uint16_t dp = 0;
986  if (tx->request_port_number != -1) {
987  dp = (uint16_t)tx->request_port_number;
988  }
989  consumed = htp_connp_res_data_consumed(hstate->connp);
990  hstate->slice = NULL;
991  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_HTTP2)) {
992  HTPSetEvent(hstate, NULL, STREAM_TOCLIENT,
994  }
995  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
996  // and we need to parser the remaining part with HTTP2
997  if (consumed > 0 && consumed < input_len) {
998  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
999  }
1001  }
1002  break;
1003  default:
1004  break;
1005  }
1006  HTPHandleError(hstate, STREAM_TOCLIENT);
1007  }
1008 
1009  /* if we the TCP connection is closed, then close the HTTP connection */
1011  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
1012  {
1013  htp_connp_close(hstate->connp, &ts);
1014  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
1015  }
1016 
1017  SCLogDebug("hstate->connp %p", hstate->connp);
1018  hstate->slice = NULL;
1019 
1020  if (ret < 0) {
1022  }
1024 }
1025 
1026 /**
1027  * \param name /Lowercase/ version of the variable name
1028  */
1029 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1030  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1031 {
1032 #ifdef PRINT
1033  printf("DATA START: \n");
1034  PrintRawDataFp(stdout, data, len);
1035  printf("DATA END: \n");
1036 #endif
1037  size_t x;
1038  int quote = 0;
1039 
1040  for (x = 0; x < len; x++) {
1041  if (!(isspace(data[x])))
1042  break;
1043  }
1044 
1045  if (x >= len)
1046  return 0;
1047 
1048  uint8_t *line = data+x;
1049  size_t line_len = len-x;
1050  size_t offset = 0;
1051 #ifdef PRINT
1052  printf("LINE START: \n");
1053  PrintRawDataFp(stdout, line, line_len);
1054  printf("LINE END: \n");
1055 #endif
1056  for (x = 0 ; x < line_len; x++) {
1057  if (x > 0) {
1058  if (line[x - 1] != '\\' && line[x] == '\"') {
1059  quote++;
1060  }
1061 
1062  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1063  uint8_t *token = line + offset;
1064  size_t token_len = x - offset;
1065 
1066  if ((x + 1) == line_len) {
1067  token_len++;
1068  }
1069 
1070  offset = x + 1;
1071 
1072  while (offset < line_len && isspace(line[offset])) {
1073  x++;
1074  offset++;
1075  }
1076 #ifdef PRINT
1077  printf("TOKEN START: \n");
1078  PrintRawDataFp(stdout, token, token_len);
1079  printf("TOKEN END: \n");
1080 #endif
1081  if (token_len > name_len) {
1082  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1083  uint8_t *value = token + name_len;
1084  size_t value_len = token_len - name_len;
1085 
1086  if (value[0] == '\"') {
1087  value++;
1088  value_len--;
1089  }
1090  if (value[value_len-1] == '\"') {
1091  value_len--;
1092  }
1093 #ifdef PRINT
1094  printf("VALUE START: \n");
1095  PrintRawDataFp(stdout, value, value_len);
1096  printf("VALUE END: \n");
1097 #endif
1098  *retptr = value;
1099  *retlen = value_len;
1100  return 1;
1101  }
1102  }
1103  }
1104  }
1105  }
1106 
1107  return 0;
1108 }
1109 
1110 /**
1111  * \param name /Lowercase/ version of the variable name
1112  */
1113 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1114  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1115 {
1116  SCEnter();
1117 #ifdef PRINT
1118  printf("DATA START: \n");
1119  PrintRawDataFp(stdout, data, len);
1120  printf("DATA END: \n");
1121 #endif
1122  size_t x;
1123  int quote = 0;
1124 
1125  for (x = 0; x < len; x++) {
1126  if (!(isspace(data[x])))
1127  break;
1128  }
1129 
1130  if (x >= len) {
1131  SCReturnInt(0);
1132  }
1133 
1134  uint8_t *line = data+x;
1135  size_t line_len = len-x;
1136  size_t offset = 0;
1137 #ifdef PRINT
1138  printf("LINE START: \n");
1139  PrintRawDataFp(stdout, line, line_len);
1140  printf("LINE END: \n");
1141 #endif
1142  for (x = 0 ; x < line_len; x++) {
1143  if (x > 0) {
1144  if (line[x - 1] != '\\' && line[x] == '\"') {
1145  quote++;
1146  }
1147 
1148  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1149  uint8_t *token = line + offset;
1150  size_t token_len = x - offset;
1151 
1152  if ((x + 1) == line_len) {
1153  token_len++;
1154  }
1155 
1156  offset = x + 1;
1157 
1158  while (offset < line_len && isspace(line[offset])) {
1159  x++;
1160  offset++;
1161  }
1162 #ifdef PRINT
1163  printf("TOKEN START: \n");
1164  PrintRawDataFp(stdout, token, token_len);
1165  printf("TOKEN END: \n");
1166 #endif
1167  if (token_len > name_len) {
1168  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1169  uint8_t *value = token + name_len;
1170  size_t value_len = token_len - name_len;
1171 
1172  if (value[0] == '\"') {
1173  value++;
1174  value_len--;
1175  }
1176  if (value[value_len-1] == '\"') {
1177  value_len--;
1178  }
1179 #ifdef PRINT
1180  printf("VALUE START: \n");
1181  PrintRawDataFp(stdout, value, value_len);
1182  printf("VALUE END: \n");
1183 #endif
1184  *retptr = value;
1185  *retlen = value_len;
1186  SCReturnInt(1);
1187  }
1188  }
1189  }
1190  }
1191  }
1192 
1193  SCReturnInt(0);
1194 }
1195 
1196 /**
1197  * \brief setup multipart parsing: extract boundary and store it
1198  *
1199  * \param d HTTP transaction
1200  * \param htud transaction userdata
1201  *
1202  * \retval 1 ok, multipart set up
1203  * \retval 0 ok, not multipart though
1204  * \retval -1 error: problem with the boundary
1205  *
1206  * If the request contains a multipart message, this function will
1207  * set the HTP_BOUNDARY_SET in the transaction.
1208  */
1209 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1210 {
1211  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1212  "Content-Type");
1213  if (h != NULL && bstr_len(h->value) > 0) {
1214  uint8_t *boundary = NULL;
1215  size_t boundary_len = 0;
1216 
1217  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1218  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1219  &boundary, &boundary_len);
1220  if (r == 1) {
1221 #ifdef PRINT
1222  printf("BOUNDARY START: \n");
1223  PrintRawDataFp(stdout, boundary, boundary_len);
1224  printf("BOUNDARY END: \n");
1225 #endif
1226  if (boundary_len < HTP_BOUNDARY_MAX) {
1227  htud->boundary = HTPMalloc(boundary_len);
1228  if (htud->boundary == NULL) {
1229  return -1;
1230  }
1231  htud->boundary_len = (uint8_t)boundary_len;
1232  memcpy(htud->boundary, boundary, boundary_len);
1233 
1234  htud->tsflags |= HTP_BOUNDARY_SET;
1235  } else {
1236  SCLogDebug("invalid boundary");
1237  return -1;
1238  }
1239  SCReturnInt(1);
1240  }
1241  //SCReturnInt(1);
1242  }
1243  SCReturnInt(0);
1244 }
1245 
1246 #define C_D_HDR "content-disposition:"
1247 #define C_D_HDR_LEN 20
1248 #define C_T_HDR "content-type:"
1249 #define C_T_HDR_LEN 13
1251 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1252  HtpTxUserData *htud,
1253  uint8_t *header, uint32_t header_len,
1254  uint8_t **filename, uint16_t *filename_len,
1255  uint8_t **filetype, uint16_t *filetype_len)
1256 {
1257  uint8_t *fn = NULL;
1258  size_t fn_len = 0;
1259  uint8_t *ft = NULL;
1260  size_t ft_len = 0;
1261 
1262 #ifdef PRINT
1263  printf("HEADER START: \n");
1264  PrintRawDataFp(stdout, header, header_len);
1265  printf("HEADER END: \n");
1266 #endif
1267 
1268  while (header_len > 0) {
1269  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1270  uint8_t *line = header;
1271  uint32_t line_len;
1272 
1273  if (next_line == NULL) {
1274  line_len = header_len;
1275  } else {
1276  line_len = next_line - header;
1277  }
1278  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1279  if (sc == NULL) {
1280  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1282  /* if the : we found is the final char, it means we have
1283  * no value */
1284  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1285  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1287  } else {
1288 #ifdef PRINT
1289  printf("LINE START: \n");
1290  PrintRawDataFp(stdout, line, line_len);
1291  printf("LINE END: \n");
1292 #endif
1293  if (line_len >= C_D_HDR_LEN &&
1294  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1295  uint8_t *value = line + C_D_HDR_LEN;
1296  uint32_t value_len = line_len - C_D_HDR_LEN;
1297 
1298  /* parse content-disposition */
1299  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1300  value, value_len, &fn, &fn_len);
1301  } else if (line_len >= C_T_HDR_LEN &&
1302  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1303  SCLogDebug("content-type line");
1304  uint8_t *value = line + C_T_HDR_LEN;
1305  uint32_t value_len = line_len - C_T_HDR_LEN;
1306 
1307  (void)HTTPParseContentTypeHeader(NULL, 0,
1308  value, value_len, &ft, &ft_len);
1309  }
1310  }
1311 
1312  if (next_line == NULL) {
1313  SCLogDebug("no next_line");
1314  break;
1315  }
1316  header_len -= ((next_line + 2) - header);
1317  header = next_line + 2;
1318  } /* while (header_len > 0) */
1319 
1320  if (fn_len > USHRT_MAX)
1321  fn_len = USHRT_MAX;
1322  if (ft_len > USHRT_MAX)
1323  ft_len = USHRT_MAX;
1324 
1325  *filename = fn;
1326  *filename_len = (uint16_t)fn_len;
1327  *filetype = ft;
1328  *filetype_len = (uint16_t)ft_len;
1329 }
1330 
1331 /**
1332  * \brief Create a single buffer from the HtpBodyChunks in our list
1333  *
1334  * \param htud transaction user data
1335  * \param chunks_buffers pointer to pass back the buffer to the caller
1336  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1337  */
1338 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1339  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1340 {
1342  chunks_buffer, chunks_buffer_len,
1343  htud->request_body.body_parsed);
1344 }
1345 
1346 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1347 {
1348  SCEnter();
1349  if (tx && tx->tx_data.de_state) {
1350  if (dir == STREAM_TOSERVER) {
1351  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1352  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1353  } else if (STREAM_TOCLIENT) {
1354  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1355  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1356  }
1357  }
1358 }
1359 
1360 /**
1361  * \brief Setup boundary buffers
1362  */
1363 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1364  uint8_t *boundary, uint32_t boundary_len)
1365 {
1366  memset(boundary, '-', boundary_len);
1367  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1368 }
1369 
1370 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1371  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1372 {
1373  int result = 0;
1374  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1375  uint16_t expected_boundary_len = htud->boundary_len + 2;
1376  uint16_t expected_boundary_end_len = htud->boundary_len + 4;
1377  int tx_progress = 0;
1378 
1379 #ifdef PRINT
1380  printf("CHUNK START: \n");
1381  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1382  printf("CHUNK END: \n");
1383 #endif
1384 
1385  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1386 
1387  /* search for the header start, header end and form end */
1388  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1389  boundary, expected_boundary_len);
1390  /* end of the multipart form */
1391  const uint8_t *form_end = NULL;
1392  /* end marker belonging to header_start */
1393  const uint8_t *header_end = NULL;
1394  if (header_start != NULL) {
1395  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1396  (uint8_t *)"\r\n\r\n", 4);
1397  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1398  boundary, expected_boundary_end_len);
1399  }
1400 
1401  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1402  header_end, form_end);
1403 
1404  /* we currently only handle multipart for ts. When we support it for tc,
1405  * we will need to supply right direction */
1406  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1407  /* if we're in the file storage process, deal with that now */
1408  if (htud->tsflags & HTP_FILENAME_SET) {
1409  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1410  SCLogDebug("reached the end of the file");
1411 
1412  const uint8_t *filedata = chunks_buffer;
1413  uint32_t filedata_len = 0;
1414  uint8_t flags = 0;
1415 
1416  if (header_start != NULL) {
1417  if (header_start == filedata + 2) {
1418  /* last chunk had all data, but not the boundary */
1419  SCLogDebug("last chunk had all data, but not the boundary");
1420  filedata_len = 0;
1421  } else if (header_start > filedata + 2) {
1422  SCLogDebug("some data from last file before the boundary");
1423  /* some data from last file before the boundary */
1424  filedata_len = header_start - filedata - 2;
1425  }
1426  }
1427  /* body parsing done, we did not get our form end. Use all data
1428  * we still have and signal to files API we have an issue. */
1429  if (tx_progress > HTP_REQUEST_BODY) {
1430  filedata_len = chunks_buffer_len;
1432  }
1433 
1434  if (filedata_len > chunks_buffer_len) {
1435  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1437  goto end;
1438  }
1439 #ifdef PRINT
1440  printf("FILEDATA (final chunk) START: \n");
1441  PrintRawDataFp(stdout, filedata, filedata_len);
1442  printf("FILEDATA (final chunk) END: \n");
1443 #endif
1444  if (!(htud->tsflags & HTP_DONTSTORE)) {
1445  if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
1446  -1) {
1447  goto end;
1448  }
1449  }
1450 
1451  htud->tsflags &=~ HTP_FILENAME_SET;
1452 
1453  /* fall through */
1454  } else {
1455  SCLogDebug("not yet at the end of the file");
1456 
1457  if (chunks_buffer_len > expected_boundary_end_len) {
1458  const uint8_t *filedata = chunks_buffer;
1459  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1460  for (; filedata_len < chunks_buffer_len; filedata_len++) {
1461  // take as much as we can until the beginning of a new line
1462  if (chunks_buffer[filedata_len] == '\r') {
1463  if (filedata_len + 1 == chunks_buffer_len ||
1464  chunks_buffer[filedata_len + 1] == '\n') {
1465  break;
1466  }
1467  }
1468  }
1469 
1470 #ifdef PRINT
1471  printf("FILEDATA (part) START: \n");
1472  PrintRawDataFp(stdout, filedata, filedata_len);
1473  printf("FILEDATA (part) END: \n");
1474 #endif
1475 
1476  if (!(htud->tsflags & HTP_DONTSTORE)) {
1477  result = HTPFileStoreChunk(
1478  hstate, htud, filedata, filedata_len, STREAM_TOSERVER);
1479  if (result == -1) {
1480  goto end;
1481  } else if (result == -2) {
1482  /* we know for sure we're not storing the file */
1483  htud->tsflags |= HTP_DONTSTORE;
1484  }
1485  }
1486 
1487  htud->request_body.body_parsed += filedata_len;
1488  } else {
1489  SCLogDebug("chunk too small to already process in part");
1490  }
1491 
1492  goto end;
1493  }
1494  }
1495 
1496  while (header_start != NULL && header_end != NULL &&
1497  header_end != form_end &&
1498  header_start < (chunks_buffer + chunks_buffer_len) &&
1499  header_end < (chunks_buffer + chunks_buffer_len) &&
1500  header_start < header_end)
1501  {
1502  uint8_t *filename = NULL;
1503  uint16_t filename_len = 0;
1504  uint8_t *filetype = NULL;
1505  uint16_t filetype_len = 0;
1506 
1507  uint32_t header_len = header_end - header_start;
1508  SCLogDebug("header_len %u", header_len);
1509  uint8_t *header = (uint8_t *)header_start;
1510 
1511  /* skip empty records */
1512  if (expected_boundary_len == header_len) {
1513  goto next;
1514  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1515  header_len -= (expected_boundary_len + 2);
1516  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1517  }
1518 
1519  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1520  &filename, &filename_len, &filetype, &filetype_len);
1521 
1522  if (filename != NULL) {
1523  const uint8_t *filedata = NULL;
1524  uint32_t filedata_len = 0;
1525 
1526  SCLogDebug("we have a filename");
1527 
1528  htud->tsflags |= HTP_FILENAME_SET;
1529  htud->tsflags &= ~HTP_DONTSTORE;
1530 
1531  SCLogDebug("header_end %p", header_end);
1532  SCLogDebug("form_end %p", form_end);
1533 
1534  /* everything until the final boundary is the file */
1535  if (form_end != NULL) {
1536  SCLogDebug("have form_end");
1537 
1538  filedata = header_end + 4;
1539  if (form_end == filedata) {
1540  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1542  goto end;
1543  } else if (form_end < filedata) {
1544  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1546  goto end;
1547  }
1548 
1549  filedata_len = form_end - (header_end + 4 + 2);
1550  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1551 
1552  /* or is it? */
1553  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1554  boundary, expected_boundary_len);
1555  if (header_next != NULL) {
1556  filedata_len -= (form_end - header_next);
1557  }
1558 
1559  if (filedata_len > chunks_buffer_len) {
1560  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1562  goto end;
1563  }
1564  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1565 #ifdef PRINT
1566  printf("FILEDATA START: \n");
1567  PrintRawDataFp(stdout, filedata, filedata_len);
1568  printf("FILEDATA END: \n");
1569 #endif
1570 
1571  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata, filedata_len,
1572  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1573  if (result == -1) {
1574  goto end;
1575  } else if (result == -2) {
1576  htud->tsflags |= HTP_DONTSTORE;
1577  } else {
1578  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1579  goto end;
1580  }
1581  }
1582  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1583 
1584  htud->request_body.body_parsed += (header_end - chunks_buffer);
1585  htud->tsflags &= ~HTP_FILENAME_SET;
1586  } else {
1587  SCLogDebug("chunk doesn't contain form end");
1588 
1589  filedata = header_end + 4;
1590  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1591  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1592 
1593  if (filedata_len > chunks_buffer_len) {
1594  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1596  goto end;
1597  }
1598 
1599 #ifdef PRINT
1600  printf("FILEDATA START: \n");
1601  PrintRawDataFp(stdout, filedata, filedata_len);
1602  printf("FILEDATA END: \n");
1603 #endif
1604  /* form doesn't end in this chunk, but the part might. Lets
1605  * see if have another coming up */
1606  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1607  boundary, expected_boundary_len);
1608  SCLogDebug("header_next %p", header_next);
1609  if (header_next == NULL) {
1610  SCLogDebug("more file data to come");
1611 
1612  uint32_t offset = (header_end + 4) - chunks_buffer;
1613  SCLogDebug("offset %u", offset);
1614  htud->request_body.body_parsed += offset;
1615 
1616  if (filedata_len >= (uint32_t)(expected_boundary_len + 2)) {
1617  filedata_len -= (expected_boundary_len + 2 - 1);
1618  // take as much as we can until start of boundary
1619  for (size_t nb = 0; nb < (size_t)expected_boundary_len + 1; nb++) {
1620  if (filedata[filedata_len] == '\r') {
1621  if (nb == expected_boundary_len ||
1622  filedata[filedata_len + 1] == '\n') {
1623  break;
1624  }
1625  }
1626  filedata_len++;
1627  }
1628  SCLogDebug("opening file with partial data");
1629  } else {
1630  filedata = NULL;
1631  filedata_len = 0;
1632  }
1633  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1634  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1635  if (result == -1) {
1636  goto end;
1637  } else if (result == -2) {
1638  htud->tsflags |= HTP_DONTSTORE;
1639  }
1640  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1641  htud->request_body.body_parsed += filedata_len;
1642  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1643 
1644  } else if (header_next - filedata > 2) {
1645  filedata_len = header_next - filedata - 2;
1646  SCLogDebug("filedata_len %u", filedata_len);
1647 
1648  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1649  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1650  if (result == -1) {
1651  goto end;
1652  } else if (result == -2) {
1653  htud->tsflags |= HTP_DONTSTORE;
1654  } else {
1655  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1656  goto end;
1657  }
1658  }
1659  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1660 
1661  htud->tsflags &= ~HTP_FILENAME_SET;
1662  htud->request_body.body_parsed += (header_end - chunks_buffer);
1663  }
1664  }
1665  }
1666 next:
1667  SCLogDebug("header_start %p, header_end %p, form_end %p",
1668  header_start, header_end, form_end);
1669 
1670  /* Search next boundary entry after the start of body */
1671  uint32_t cursizeread = header_end - chunks_buffer;
1672  header_start = Bs2bmSearch(header_end + 4,
1673  chunks_buffer_len - (cursizeread + 4),
1674  boundary, expected_boundary_len);
1675  if (header_start != NULL) {
1676  header_end = Bs2bmSearch(header_end + 4,
1677  chunks_buffer_len - (cursizeread + 4),
1678  (uint8_t *) "\r\n\r\n", 4);
1679  }
1680  }
1681 
1682  /* if we're parsing the multipart and we're not currently processing a
1683  * file, we move the body pointer forward. */
1684  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1685  if (chunks_buffer_len > expected_boundary_end_len) {
1686  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1687 
1688  htud->request_body.body_parsed += move;
1689  SCLogDebug("form not ready, file not set, parsing non-file "
1690  "record: moved %u", move);
1691  }
1692  }
1693 
1694 end:
1695  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1696  return 0;
1697 }
1698 
1699 /** \internal
1700  * \brief Handle POST or PUT, no multipart body data
1701  */
1702 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1703  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1704 {
1705  int result = 0;
1706 
1707  /* see if we need to open the file */
1708  if (!(htud->tsflags & HTP_FILENAME_SET))
1709  {
1710  uint8_t *filename = NULL;
1711  size_t filename_len = 0;
1712 
1713  /* get the name */
1714  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1715  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1716  filename_len = bstr_len(tx->parsed_uri->path);
1717  }
1718 
1719  if (filename != NULL) {
1720  if (filename_len > SC_FILENAME_MAX) {
1721  // explicitly truncate the file name if too long
1722  filename_len = SC_FILENAME_MAX;
1723  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1724  }
1725  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1726  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1727  if (result == -1) {
1728  goto end;
1729  } else if (result == -2) {
1730  htud->tsflags |= HTP_DONTSTORE;
1731  } else {
1732  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1733  htud->tsflags |= HTP_FILENAME_SET;
1734  htud->tsflags &= ~HTP_DONTSTORE;
1735  }
1736  }
1737  }
1738  else
1739  {
1740  /* otherwise, just store the data */
1741 
1742  if (!(htud->tsflags & HTP_DONTSTORE)) {
1743  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOSERVER);
1744  if (result == -1) {
1745  goto end;
1746  } else if (result == -2) {
1747  /* we know for sure we're not storing the file */
1748  htud->tsflags |= HTP_DONTSTORE;
1749  }
1750  }
1751  }
1752 
1753  return 0;
1754 end:
1755  return -1;
1756 }
1757 
1758 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1759  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1760 {
1761  SCEnter();
1762 
1763  int result = 0;
1764 
1765  /* see if we need to open the file
1766  * we check for tx->response_line in case of junk
1767  * interpreted as body before response line
1768  */
1769  if (!(htud->tcflags & HTP_FILENAME_SET)) {
1770  SCLogDebug("setting up file name");
1771 
1772  uint8_t *filename = NULL;
1773  size_t filename_len = 0;
1774 
1775  /* try Content-Disposition header first */
1776  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1777  "Content-Disposition");
1778  if (h != NULL && bstr_len(h->value) > 0) {
1779  /* parse content-disposition */
1780  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1781  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1782  }
1783 
1784  /* fall back to name from the uri */
1785  if (filename == NULL) {
1786  /* get the name */
1787  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1788  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1789  filename_len = bstr_len(tx->parsed_uri->path);
1790  }
1791  }
1792 
1793  if (filename != NULL) {
1794  // set range if present
1795  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1796  if (filename_len > SC_FILENAME_MAX) {
1797  // explicitly truncate the file name if too long
1798  filename_len = SC_FILENAME_MAX;
1799  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1800  }
1801  if (h_content_range != NULL) {
1802  result = HTPFileOpenWithRange(hstate, htud, filename, (uint16_t)filename_len, data,
1803  data_len, HtpGetActiveResponseTxID(hstate), h_content_range->value, htud);
1804  } else {
1805  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1806  HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1807  }
1808  SCLogDebug("result %d", result);
1809  if (result == -1) {
1810  goto end;
1811  } else if (result == -2) {
1812  htud->tcflags |= HTP_DONTSTORE;
1813  } else {
1814  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1815  htud->tcflags |= HTP_FILENAME_SET;
1816  htud->tcflags &= ~HTP_DONTSTORE;
1817  }
1818  }
1819  } else {
1820  /* otherwise, just store the data */
1821 
1822  if (!(htud->tcflags & HTP_DONTSTORE)) {
1823  result = HTPFileStoreChunk(hstate, htud, data, data_len, STREAM_TOCLIENT);
1824  SCLogDebug("result %d", result);
1825  if (result == -1) {
1826  goto end;
1827  } else if (result == -2) {
1828  /* we know for sure we're not storing the file */
1829  htud->tcflags |= HTP_DONTSTORE;
1830  }
1831  }
1832  }
1833 
1834  htud->response_body.body_parsed += data_len;
1835  return 0;
1836 end:
1837  return -1;
1838 }
1839 
1840 /**
1841  * \brief Function callback to append chunks for Requests
1842  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1843  * \retval int HTP_OK if all goes well
1844  */
1845 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1846 {
1847  SCEnter();
1848 
1849  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1850  SCReturnInt(HTP_OK);
1851 
1852  if (d->len == 0)
1853  SCReturnInt(HTP_OK);
1854 
1855 #ifdef PRINT
1856  printf("HTPBODY START: \n");
1857  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1858  printf("HTPBODY END: \n");
1859 #endif
1860 
1861  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1862  if (hstate == NULL) {
1863  SCReturnInt(HTP_ERROR);
1864  }
1865 
1866  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1867  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1868 
1869  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1870  if (tx_ud == NULL) {
1871  SCReturnInt(HTP_OK);
1872  }
1873  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
1874 
1875  if (!tx_ud->response_body_init) {
1876  tx_ud->response_body_init = 1;
1877 
1878  if (d->tx->request_method_number == HTP_M_POST) {
1879  SCLogDebug("POST");
1880  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1881  if (r == 1) {
1883  } else if (r == 0) {
1885  SCLogDebug("not multipart");
1886  }
1887  } else if (d->tx->request_method_number == HTP_M_PUT) {
1889  }
1890  }
1891 
1892  /* see if we can get rid of htp body chunks */
1893  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1894 
1895  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1896  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1897 
1898  /* within limits, add the body chunk to the state. */
1899  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1900  uint32_t stream_depth = FileReassemblyDepth();
1901  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1902  hstate->cfg->request.body_limit,
1903  stream_depth,
1904  tx_ud->tsflags,
1905  (uint32_t)d->len);
1906  BUG_ON(len > (uint32_t)d->len);
1907 
1908  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1909 
1910  const uint8_t *chunks_buffer = NULL;
1911  uint32_t chunks_buffer_len = 0;
1912 
1914  /* multi-part body handling starts here */
1915  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1916  goto end;
1917  }
1918 
1919  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1920  if (chunks_buffer == NULL) {
1921  goto end;
1922  }
1923 #ifdef PRINT
1924  printf("REASSCHUNK START: \n");
1925  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1926  printf("REASSCHUNK END: \n");
1927 #endif
1928 
1929  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1930 
1931  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1933  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1934  }
1935 
1936  } else {
1937  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1938  SCLogDebug("closing file that was being stored");
1939  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1940  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1941  }
1942  }
1943 
1944 end:
1945  if (hstate->conn != NULL) {
1946  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1948  hstate->cfg->request.inspect_min_size,
1949  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1950 
1951  /* if we reach the inspect_min_size we'll trigger inspection,
1952  * so make sure that raw stream is also inspected. Set the
1953  * data to be used to the amount of raw bytes we've seen to
1954  * get here. */
1955  if (tx_ud->request_body.body_inspected == 0 &&
1957  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1958  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1959  {
1960  const uint32_t data_size = (uint32_t)(
1961  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1962  const uint32_t depth = MIN(data_size, hstate->cfg->request.inspect_min_size);
1963 
1964  /* body still in progress, but due to min inspect size we need to inspect now */
1965  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, depth);
1966  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1967  }
1968  /* after the start of the body, disable the depth logic */
1969  } else if (tx_ud->request_body.body_inspected > 0) {
1970  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1971  }
1972  }
1973  SCReturnInt(HTP_OK);
1974 }
1975 
1976 /**
1977  * \brief Function callback to append chunks for Responses
1978  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1979  * \retval int HTP_OK if all goes well
1980  */
1981 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1982 {
1983  SCEnter();
1984 
1985  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1986  SCReturnInt(HTP_OK);
1987 
1988  if (d->len == 0)
1989  SCReturnInt(HTP_OK);
1990 
1991  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1992  if (hstate == NULL) {
1993  SCReturnInt(HTP_ERROR);
1994  }
1995 
1996  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1997  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1998 
1999  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
2000  if (tx_ud == NULL) {
2001  SCReturnInt(HTP_OK);
2002  }
2003  tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
2004  if (!tx_ud->request_body_init) {
2005  tx_ud->request_body_init = 1;
2006  }
2007 
2008  /* see if we can get rid of htp body chunks */
2009  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
2010 
2011  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
2012  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
2013 
2014  /* within limits, add the body chunk to the state. */
2015  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
2016  uint32_t stream_depth = FileReassemblyDepth();
2017  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
2018  hstate->cfg->response.body_limit,
2019  stream_depth,
2020  tx_ud->tcflags,
2021  (uint32_t)d->len);
2022  BUG_ON(len > (uint32_t)d->len);
2023 
2024  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
2025 
2026  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
2027  } else {
2028  if (tx_ud->tcflags & HTP_FILENAME_SET) {
2029  SCLogDebug("closing file that was being stored");
2030  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
2031  tx_ud->tcflags &= ~HTP_FILENAME_SET;
2032  }
2033  }
2034 
2035  if (hstate->conn != NULL) {
2036  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
2038  hstate->cfg->response.inspect_min_size,
2039  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
2040  /* if we reach the inspect_min_size we'll trigger inspection,
2041  * so make sure that raw stream is also inspected. Set the
2042  * data to be used to the amount of raw bytes we've seen to
2043  * get here. */
2044  if (tx_ud->response_body.body_inspected == 0 &&
2046  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2047  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2048  {
2049  const uint32_t data_size = (uint32_t)((uint64_t)hstate->conn->out_data_counter -
2050  hstate->last_response_data_stamp);
2051  const uint32_t depth = MIN(data_size, hstate->cfg->response.inspect_min_size);
2052 
2053  /* body still in progress, but due to min inspect size we need to inspect now */
2054  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, depth);
2055  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2056  }
2057  /* after the start of the body, disable the depth logic */
2058  } else if (tx_ud->response_body.body_inspected > 0) {
2059  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
2060  }
2061  }
2062  SCReturnInt(HTP_OK);
2063 }
2064 
2065 /**
2066  * \brief Print the stats of the HTTP requests
2067  */
2069 {
2070 #ifdef DEBUG
2071  SCEnter();
2072  SCMutexLock(&htp_state_mem_lock);
2073  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2074  htp_state_memcnt, htp_state_memuse);
2075  SCMutexUnlock(&htp_state_mem_lock);
2076  SCReturn;
2077 #endif
2078 }
2079 
2080 /** \brief Clears the HTTP server configuration memory used by HTP library */
2081 void HTPFreeConfig(void)
2082 {
2083  SCEnter();
2084 
2085  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2086  !AppLayerParserConfParserEnabled("tcp", "http"))
2087  {
2088  SCReturn;
2089  }
2090 
2091  HTPCfgRec *nextrec = cfglist.next;
2092  SCRadixReleaseRadixTree(cfgtree);
2093  cfgtree = NULL;
2094  htp_config_destroy(cfglist.cfg);
2095  while (nextrec != NULL) {
2096  HTPCfgRec *htprec = nextrec;
2097  nextrec = nextrec->next;
2098 
2099  htp_config_destroy(htprec->cfg);
2100  SCFree(htprec);
2101  }
2102  SCReturn;
2103 }
2104 
2105 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2106 {
2107  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2108  if (htud != NULL) {
2109  htud->request_has_trailers = 1;
2110  }
2111  return HTP_OK;
2112 }
2113 
2114 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2115 {
2116  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2117  if (htud != NULL) {
2118  htud->response_has_trailers = 1;
2119  }
2120  return HTP_OK;
2121 }
2122 
2123 /**\internal
2124  * \brief called at start of request
2125  * Set min inspect size.
2126  */
2127 static int HTPCallbackRequestStart(htp_tx_t *tx)
2128 {
2129  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2130  if (hstate == NULL) {
2131  SCReturnInt(HTP_ERROR);
2132  }
2133 
2134  uint64_t consumed = hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2135  SCLogDebug("HTTP request start: data offset %" PRIu64 ", in_data_counter %" PRIu64, consumed,
2136  (uint64_t)hstate->conn->in_data_counter);
2137 
2138  /* app-layer-frame-documentation tag start: frame registration http request */
2140  hstate->f, hstate->slice, consumed, -1, 0, HTTP_FRAME_REQUEST);
2141  if (frame) {
2142  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2143  hstate->request_frame_id = frame->id;
2144  AppLayerFrameSetTxId(frame, HtpGetActiveRequestTxID(hstate));
2145  }
2146  /* app-layer-frame-documentation tag end: frame registration http request */
2147 
2148  if (hstate->cfg)
2149  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2150  hstate->cfg->request.inspect_min_size);
2151 
2152  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2153  if (tx_ud == NULL) {
2154  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2155  if (unlikely(tx_ud == NULL)) {
2156  SCReturnInt(HTP_OK);
2157  }
2158  tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files
2159  htp_tx_set_user_data(tx, tx_ud);
2160  }
2161  SCReturnInt(HTP_OK);
2162 }
2163 
2164 /**\internal
2165  * \brief called at start of response
2166  * Set min inspect size.
2167  */
2168 static int HTPCallbackResponseStart(htp_tx_t *tx)
2169 {
2170  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2171  if (hstate == NULL) {
2172  SCReturnInt(HTP_ERROR);
2173  }
2174 
2175  uint64_t consumed = hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2176  SCLogDebug("HTTP response start: data offset %" PRIu64 ", out_data_counter %" PRIu64, consumed,
2177  (uint64_t)hstate->conn->out_data_counter);
2178 
2180  hstate->f, hstate->slice, consumed, -1, 1, HTTP_FRAME_RESPONSE);
2181  if (frame) {
2182  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2183  hstate->response_frame_id = frame->id;
2184  AppLayerFrameSetTxId(frame, HtpGetActiveResponseTxID(hstate));
2185  }
2186 
2187  if (hstate->cfg)
2188  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
2189  hstate->cfg->response.inspect_min_size);
2190 
2191  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2192  if (tx_ud == NULL) {
2193  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2194  if (unlikely(tx_ud == NULL)) {
2195  SCReturnInt(HTP_OK);
2196  }
2197  tx_ud->tx_data.file_tx =
2198  STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed.
2199  htp_tx_set_user_data(tx, tx_ud);
2200  }
2201  SCReturnInt(HTP_OK);
2202 }
2203 
2204 /**
2205  * \brief callback for request to store the recent incoming request
2206  into the recent_in_tx for the given htp state
2207  * \param connp pointer to the current connection parser which has the htp
2208  * state in it as user data
2209  */
2210 static int HTPCallbackRequestComplete(htp_tx_t *tx)
2211 {
2212  SCEnter();
2213 
2214  if (tx == NULL) {
2215  SCReturnInt(HTP_ERROR);
2216  }
2217 
2218  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2219  if (hstate == NULL) {
2220  SCReturnInt(HTP_ERROR);
2221  }
2222 
2223  const uint64_t abs_right_edge =
2224  hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2225 
2226  /* app-layer-frame-documentation tag start: updating frame->len */
2227  if (hstate->request_frame_id > 0) {
2228  Frame *frame = AppLayerFrameGetById(hstate->f, 0, hstate->request_frame_id);
2229  if (frame) {
2230  const uint64_t request_size = abs_right_edge - hstate->last_request_data_stamp;
2231 
2232  SCLogDebug("HTTP request complete: data offset %" PRIu64 ", request_size %" PRIu64,
2233  hstate->last_request_data_stamp, request_size);
2234  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2235  request_size);
2236  frame->len = (int64_t)request_size;
2237  /* app-layer-frame-documentation tag end: updating frame->len */
2238  }
2239  hstate->request_frame_id = 0;
2240  }
2241 
2242  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2243  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2244 
2245  SCLogDebug("HTTP request completed");
2246 
2247  HTPErrorCheckTxRequestFlags(hstate, tx);
2248 
2249  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2250  if (htud != NULL) {
2251  if (htud->tsflags & HTP_FILENAME_SET) {
2252  SCLogDebug("closing file that was being stored");
2253  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
2254  htud->tsflags &= ~HTP_FILENAME_SET;
2255  if (abs_right_edge < (uint64_t)UINT32_MAX) {
2257  hstate->f->protoctx, STREAM_TOSERVER, (uint32_t)abs_right_edge);
2258  }
2259  }
2260  }
2261 
2262  hstate->last_request_data_stamp = abs_right_edge;
2263  /* request done, do raw reassembly now to inspect state and stream
2264  * at the same time. */
2265  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
2266  SCReturnInt(HTP_OK);
2267 }
2268 
2269 /**
2270  * \brief callback for response to remove the recent received requests
2271  from the recent_in_tx for the given htp state
2272  * \param connp pointer to the current connection parser which has the htp
2273  * state in it as user data
2274  */
2275 static int HTPCallbackResponseComplete(htp_tx_t *tx)
2276 {
2277  SCEnter();
2278 
2279  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2280  if (hstate == NULL) {
2281  SCReturnInt(HTP_ERROR);
2282  }
2283 
2284  /* we have one whole transaction now */
2285  hstate->transaction_cnt++;
2286 
2287  const uint64_t abs_right_edge =
2288  hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2289 
2290  if (hstate->response_frame_id > 0) {
2291  Frame *frame = AppLayerFrameGetById(hstate->f, 1, hstate->response_frame_id);
2292  if (frame) {
2293  const uint64_t response_size = abs_right_edge - hstate->last_response_data_stamp;
2294 
2295  SCLogDebug("HTTP response complete: data offset %" PRIu64 ", response_size %" PRIu64,
2296  hstate->last_response_data_stamp, response_size);
2297  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2298  response_size);
2299  frame->len = (int64_t)response_size;
2300  }
2301  hstate->response_frame_id = 0;
2302  }
2303 
2304  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2305  if (htud != NULL) {
2306  if (htud->tcflags & HTP_FILENAME_SET) {
2307  SCLogDebug("closing file that was being stored");
2308  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
2309  htud->tcflags &= ~HTP_FILENAME_SET;
2310  }
2311  }
2312 
2313  /* response done, do raw reassembly now to inspect state and stream
2314  * at the same time. */
2315  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2316 
2317  /* handle HTTP CONNECT */
2318  if (tx->request_method_number == HTP_M_CONNECT) {
2319  /* any 2XX status response implies that the connection will become
2320  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2321  if ((tx->response_status_number >= 200) &&
2322  (tx->response_status_number < 300) &&
2323  (hstate->transaction_cnt == 1)) {
2324  uint16_t dp = 0;
2325  if (tx->request_port_number != -1) {
2326  dp = (uint16_t)tx->request_port_number;
2327  }
2328  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2329  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN)) {
2330  HTPSetEvent(
2331  hstate, htud, STREAM_TOCLIENT, HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE);
2332  }
2333  tx->request_progress = HTP_REQUEST_COMPLETE;
2334  tx->response_progress = HTP_RESPONSE_COMPLETE;
2335  }
2336  }
2337 
2338  hstate->last_response_data_stamp = abs_right_edge;
2339  SCReturnInt(HTP_OK);
2340 }
2341 
2342 static int HTPCallbackRequestLine(htp_tx_t *tx)
2343 {
2344  HtpTxUserData *tx_ud;
2345  bstr *request_uri_normalized;
2346  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2347  const HTPCfgRec *cfg = hstate->cfg;
2348 
2349  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2350  if (request_uri_normalized == NULL)
2351  return HTP_OK;
2352 
2353  tx_ud = htp_tx_get_user_data(tx);
2354  if (unlikely(tx_ud == NULL)) {
2355  bstr_free(request_uri_normalized);
2356  return HTP_OK;
2357  }
2358  if (unlikely(tx_ud->request_uri_normalized != NULL))
2359  bstr_free(tx_ud->request_uri_normalized);
2360  tx_ud->request_uri_normalized = request_uri_normalized;
2361 
2362  if (tx->flags) {
2363  HTPErrorCheckTxRequestFlags(hstate, tx);
2364  }
2365  return HTP_OK;
2366 }
2367 
2368 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2369 {
2370  if (part == NULL)
2371  return HTP_OK;
2372 
2373  uint64_t flags = 0;
2374  size_t prevlen = bstr_len(part);
2375  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2376  // shorter string means that uri was encoded
2377  if (res == HTP_OK && prevlen > bstr_len(part)) {
2378  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2379  if (htud == NULL)
2380  return HTP_OK;
2381  HtpState *s = htp_connp_get_user_data(tx->connp);
2382  if (s == NULL)
2383  return HTP_OK;
2384  HTPSetEvent(s, htud, STREAM_TOSERVER,
2386  }
2387 
2388  return HTP_OK;
2389 }
2390 
2391 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2392 {
2393  if (tx->parsed_uri == NULL)
2394  return HTP_OK;
2395 
2396  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2397 }
2398 
2399 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2400 {
2401  if (tx->parsed_uri == NULL)
2402  return HTP_OK;
2403 
2404  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2405 }
2406 
2407 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2408 {
2409  void *ptmp;
2410  if (tx_data->len == 0 || tx_data->tx == NULL)
2411  return HTP_OK;
2412 
2413  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2414  if (tx_ud == NULL) {
2415  return HTP_OK;
2416  }
2417  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2418  tx_ud->request_headers_raw_len,
2419  tx_ud->request_headers_raw_len + tx_data->len);
2420  if (ptmp == NULL) {
2421  return HTP_OK;
2422  }
2423  tx_ud->request_headers_raw = ptmp;
2424 
2425  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2426  tx_data->data, tx_data->len);
2427  tx_ud->request_headers_raw_len += tx_data->len;
2428 
2429  if (tx_data->tx && tx_data->tx->flags) {
2430  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2431  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2432  }
2433  return HTP_OK;
2434 }
2435 
2436 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2437 {
2438  void *ptmp;
2439  if (tx_data->len == 0 || tx_data->tx == NULL)
2440  return HTP_OK;
2441 
2442  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2443  if (tx_ud == NULL) {
2444  return HTP_OK;
2445  }
2446  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2447  tx_ud->response_headers_raw_len,
2448  tx_ud->response_headers_raw_len + tx_data->len);
2449  if (ptmp == NULL) {
2450  return HTP_OK;
2451  }
2452  tx_ud->response_headers_raw = ptmp;
2453 
2454  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2455  tx_data->data, tx_data->len);
2456  tx_ud->response_headers_raw_len += tx_data->len;
2457 
2458  return HTP_OK;
2459 }
2460 
2461 /*
2462  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2463  */
2464 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2465 {
2466  cfg_prec->uri_include_all = FALSE;
2473 
2474  if (!g_disable_randomness) {
2476  } else {
2477  cfg_prec->randomize = 0;
2478  }
2480 
2481  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2482  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2483  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2484  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2485 
2486  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2487  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2488 
2489  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2490  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2491 
2492  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2493  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequestComplete);
2494 
2495  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2496  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponseComplete);
2497 
2498  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2499 
2500  /* don't convert + to space by default */
2501  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2502  // enables request decompression
2503  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2504 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2505  // disable by default
2506  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2507 #endif
2508 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2509  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2511 #endif
2512 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2513  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2515 #endif
2516 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2517  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2518 #endif
2519  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2520  * only the hard limit. So we set both here to the (current) htp defaults.
2521  * The reason we do this is that if the user sets the hard limit in the
2522  * config, we have to set the soft limit as well. If libhtp starts using
2523  * the soft limit in the future, we at least make sure we control what
2524  * it's value is. */
2525  htp_config_set_field_limits(cfg_prec->cfg,
2528  return;
2529 }
2530 
2531 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2532  * but we can get both <0 and >RAND_MAX values from RandomGet
2533  */
2534 static int RandomGetWrap(void)
2535 {
2536  unsigned long r;
2537 
2538  do {
2539  r = RandomGet();
2540  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2541 
2542  return r % RAND_MAX;
2543 }
2544 
2545 /*
2546  * We have this splitup so that in case double decoding has been enabled
2547  * for query and path, they would be called first on the callback queue,
2548  * before the callback set by Phase2() is called. We need this, since
2549  * the callback in Phase2() generates the normalized uri which utilizes
2550  * the query and path. */
2551 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2552 {
2553  /* randomize inspection size if needed */
2554  if (cfg_prec->randomize) {
2555  int rdrange = cfg_prec->randomize_range;
2556 
2557  long int r = RandomGetWrap();
2558  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2559  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2560 
2561  r = RandomGetWrap();
2562  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2563  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2564  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2565  " %u and 'request-body-inspect-window' set to %u after"
2566  " randomization.",
2567  name, cfg_prec->request.inspect_min_size, cfg_prec->request.inspect_window);
2568 
2569  r = RandomGetWrap();
2570  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2571  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2572 
2573  r = RandomGetWrap();
2574  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2575  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2576 
2577  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2578  " %u and 'response-body-inspect-window' set to %u after"
2579  " randomization.",
2580  name, cfg_prec->response.inspect_min_size, cfg_prec->response.inspect_window);
2581  }
2582 
2583  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2584  return;
2585 }
2586 
2587 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2588  SCRadixTree *tree)
2589 {
2590  if (cfg_prec == NULL || s == NULL || tree == NULL)
2591  return;
2592 
2593  ConfNode *p = NULL;
2594 
2595  /* Default Parameters */
2596  TAILQ_FOREACH(p, &s->head, next) {
2597 
2598  if (strcasecmp("address", p->name) == 0) {
2599  ConfNode *pval;
2600  /* Addresses */
2601  TAILQ_FOREACH(pval, &p->head, next) {
2602  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2603  pval->val);
2604 
2605  /* IPV6 or IPV4? */
2606  if (strchr(pval->val, ':') != NULL) {
2607  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2608  s->name, pval->val, cfg_prec->cfg);
2609  if (!SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec)) {
2610  SCLogWarning("LIBHTP failed to "
2611  "add ipv6 server %s, ignoring",
2612  pval->val);
2613  }
2614  } else {
2615  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2616  s->name, pval->val, cfg_prec->cfg);
2617  if (!SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec)) {
2618  SCLogWarning("LIBHTP failed "
2619  "to add ipv4 server %s, ignoring",
2620  pval->val);
2621  }
2622  } /* else - if (strchr(pval->val, ':') != NULL) */
2623  } /* TAILQ_FOREACH(pval, &p->head, next) */
2624 
2625  } else if (strcasecmp("personality", p->name) == 0) {
2626  /* Personalities */
2627  int personality = HTPLookupPersonality(p->val);
2628  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2629  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2630 
2631  if (personality >= 0) {
2632  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2633  personality);
2634  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2635  SCLogWarning("LIBHTP Failed adding "
2636  "personality \"%s\", ignoring",
2637  p->val);
2638  } else {
2639  SCLogDebug("LIBHTP personality set to %s",
2640  HTPLookupPersonalityString(personality));
2641  }
2642 
2643  /* The IDS personality by default converts the path (and due to
2644  * our query string callback also the query string) to lowercase.
2645  * Signatures do not expect this, so override it. */
2646  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2647  } else {
2648  SCLogWarning("LIBHTP Unknown personality "
2649  "\"%s\", ignoring",
2650  p->val);
2651  continue;
2652  }
2653 
2654  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2655  strcasecmp("request_body_limit", p->name) == 0) {
2656  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2657  SCLogError("Error parsing request-body-limit "
2658  "from conf file - %s. Killing engine",
2659  p->val);
2660  exit(EXIT_FAILURE);
2661  }
2662 
2663  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2664  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2665  SCLogError("Error parsing response-body-limit "
2666  "from conf file - %s. Killing engine",
2667  p->val);
2668  exit(EXIT_FAILURE);
2669  }
2670 
2671  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2672  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2673  SCLogError("Error parsing request-body-minimal-inspect-size "
2674  "from conf file - %s. Killing engine",
2675  p->val);
2676  exit(EXIT_FAILURE);
2677  }
2678 
2679  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2680  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2681  SCLogError("Error parsing request-body-inspect-window "
2682  "from conf file - %s. Killing engine",
2683  p->val);
2684  exit(EXIT_FAILURE);
2685  }
2686 
2687  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2688  if (ConfValIsTrue(p->val)) {
2689  htp_config_register_request_line(cfg_prec->cfg,
2690  HTPCallbackDoubleDecodeQuery);
2691  }
2692 
2693  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2694  if (ConfValIsTrue(p->val)) {
2695  htp_config_register_request_line(cfg_prec->cfg,
2696  HTPCallbackDoubleDecodePath);
2697  }
2698 
2699  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2700  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2701  SCLogError("Error parsing response-body-minimal-inspect-size "
2702  "from conf file - %s. Killing engine",
2703  p->val);
2704  exit(EXIT_FAILURE);
2705  }
2706 
2707  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2708  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2709  SCLogError("Error parsing response-body-inspect-window "
2710  "from conf file - %s. Killing engine",
2711  p->val);
2712  exit(EXIT_FAILURE);
2713  }
2714 
2715  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2716  uint32_t value = 2;
2717  if (ParseSizeStringU32(p->val, &value) < 0) {
2718  SCLogError("Error parsing response-body-inspect-window "
2719  "from conf file - %s. Killing engine",
2720  p->val);
2721  exit(EXIT_FAILURE);
2722  }
2723 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2724  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2725 #else
2726  SCLogWarning("can't set response-body-decompress-layer-limit "
2727  "to %u, libhtp version too old",
2728  value);
2729 #endif
2730  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2731  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2732  HTP_DECODER_URL_PATH,
2733  ConfValIsTrue(p->val));
2734  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2735  if (strlen(p->val) == 1) {
2736  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2737  HTP_DECODER_URL_PATH,
2738  p->val[0]);
2739  } else {
2740  SCLogError("Invalid entry "
2741  "for libhtp param path-bestfit-replacement-char");
2742  }
2743  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2744  htp_config_set_convert_lowercase(cfg_prec->cfg,
2745  HTP_DECODER_URL_PATH,
2746  ConfValIsTrue(p->val));
2747  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2748  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2749  HTP_DECODER_URL_PATH,
2750  ConfValIsTrue(p->val));
2751  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2752  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2753  HTP_DECODER_URL_PATH,
2754  ConfValIsTrue(p->val));
2755  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2756  htp_config_set_path_separators_compress(cfg_prec->cfg,
2757  HTP_DECODER_URL_PATH,
2758  ConfValIsTrue(p->val));
2759  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2760  htp_config_set_path_separators_decode(cfg_prec->cfg,
2761  HTP_DECODER_URL_PATH,
2762  ConfValIsTrue(p->val));
2763  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2764  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2765  HTP_DECODER_URL_PATH,
2766  ConfValIsTrue(p->val));
2767  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2768  enum htp_url_encoding_handling_t handling;
2769  if (strcasecmp(p->val, "preserve_percent") == 0) {
2770  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2771  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2772  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2773  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2774  handling = HTP_URL_DECODE_PROCESS_INVALID;
2775  } else {
2776  SCLogError("Invalid entry "
2777  "for libhtp param path-url-encoding-invalid-handling");
2778  return;
2779  }
2780  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2781  HTP_DECODER_URL_PATH,
2782  handling);
2783  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2784  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2785  HTP_DECODER_URL_PATH,
2786  ConfValIsTrue(p->val));
2787  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2788  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2789  SCLogDebug("uri-include-all %s",
2790  cfg_prec->uri_include_all ? "enabled" : "disabled");
2791  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2792  htp_config_set_plusspace_decode(cfg_prec->cfg,
2793  HTP_DECODER_URLENCODED,
2794  ConfValIsTrue(p->val));
2795  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2796  uint32_t limit = 0;
2797  if (ParseSizeStringU32(p->val, &limit) < 0) {
2798  SCLogError("Error meta-field-limit "
2799  "from conf file - %s. Killing engine",
2800  p->val);
2801  exit(EXIT_FAILURE);
2802  }
2803  if (limit == 0) {
2804  FatalError("Error meta-field-limit "
2805  "from conf file cannot be 0. Killing engine");
2806  }
2807  /* set default soft-limit with our new hard limit */
2808  htp_config_set_field_limits(cfg_prec->cfg,
2810  (size_t)limit);
2811 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2812  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2813  uint32_t limit = 0;
2814  if (ParseSizeStringU32(p->val, &limit) < 0) {
2815  FatalError("failed to parse 'lzma-memlimit' "
2816  "from conf file - %s.",
2817  p->val);
2818  }
2819  if (limit == 0) {
2820  FatalError("'lzma-memlimit' "
2821  "from conf file cannot be 0.");
2822  }
2823  /* set default soft-limit with our new hard limit */
2824  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2825  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2826 #endif
2827 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2828  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2829  if (ConfValIsTrue(p->val)) {
2830  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2831  } else if (!ConfValIsFalse(p->val)) {
2832  int8_t limit;
2833  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2834  FatalError("failed to parse 'lzma-enabled' "
2835  "from conf file - %s.",
2836  p->val);
2837  }
2838  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2839  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2840  }
2841 #endif
2842 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2843  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2844  uint32_t limit = 0;
2845  if (ParseSizeStringU32(p->val, &limit) < 0) {
2846  FatalError("failed to parse 'compression-bomb-limit' "
2847  "from conf file - %s.",
2848  p->val);
2849  }
2850  if (limit == 0) {
2851  FatalError("'compression-bomb-limit' "
2852  "from conf file cannot be 0.");
2853  }
2854  /* set default soft-limit with our new hard limit */
2855  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2856  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2857 #endif
2858 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2859  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2860  uint32_t limit = 0;
2861  // between 1 usec and 1 second
2862  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2863  FatalError("failed to parse 'decompression-time-limit' "
2864  "from conf file - %s.",
2865  p->val);
2866  }
2867  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2868  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2869 #endif
2870  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2871  if (!g_disable_randomness) {
2872  cfg_prec->randomize = ConfValIsTrue(p->val);
2873  }
2874  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2875  uint32_t range;
2876  if (StringParseU32RangeCheck(&range, 10, 0,
2877  (const char *)p->val, 0, 100) < 0) {
2878  SCLogError("Invalid value for randomize"
2879  "-inspection-range setting from conf file - \"%s\"."
2880  " It should be a valid integer less than or equal to 100."
2881  " Killing engine",
2882  p->val);
2883  exit(EXIT_FAILURE);
2884  }
2885  cfg_prec->randomize_range = range;
2886  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2887  if (ConfValIsTrue(p->val)) {
2888  cfg_prec->http_body_inline = 1;
2889  } else if (ConfValIsFalse(p->val)) {
2890  cfg_prec->http_body_inline = 0;
2891  } else {
2892  if (strcmp("auto", p->val) != 0) {
2893  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2894  }
2895  if (EngineModeIsIPS()) {
2896  cfg_prec->http_body_inline = 1;
2897  } else {
2898  cfg_prec->http_body_inline = 0;
2899  }
2900  }
2901  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2902  ConfNode *pval;
2903 
2904  TAILQ_FOREACH(pval, &p->head, next) {
2905  if (strcasecmp("enabled", pval->name) == 0) {
2906  if (ConfValIsTrue(pval->val)) {
2907  cfg_prec->swf_decompression_enabled = 1;
2908  SCLogWarning("Flash decompression is deprecated and will be removed in "
2909  "Suricata 8; see ticket #6179");
2910  } else if (ConfValIsFalse(pval->val)) {
2911  cfg_prec->swf_decompression_enabled = 0;
2912  } else {
2913  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2914  }
2915  } else if (strcasecmp("type", pval->name) == 0) {
2916  if (strcasecmp("no", pval->val) == 0) {
2918  } else if (strcasecmp("deflate", pval->val) == 0) {
2920  } else if (strcasecmp("lzma", pval->val) == 0) {
2922  } else if (strcasecmp("both", pval->val) == 0) {
2924  } else {
2925  SCLogError("Invalid entry for "
2926  "swf-decompression.type: %s - "
2927  "Killing engine",
2928  pval->val);
2929  exit(EXIT_FAILURE);
2930  }
2931  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2932  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2933  SCLogError("Error parsing swf-decompression.compression-depth "
2934  "from conf file - %s. Killing engine",
2935  p->val);
2936  exit(EXIT_FAILURE);
2937  }
2938  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2939  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2940  SCLogError("Error parsing swf-decompression.decompression-depth "
2941  "from conf file - %s. Killing engine",
2942  p->val);
2943  exit(EXIT_FAILURE);
2944  }
2945  } else {
2946  SCLogWarning("Ignoring unknown param %s", pval->name);
2947  }
2948  }
2949  } else {
2950  SCLogWarning("LIBHTP Ignoring unknown "
2951  "default config: %s",
2952  p->name);
2953  }
2954  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2955 
2956  return;
2957 }
2958 
2959 void HTPConfigure(void)
2960 {
2961  SCEnter();
2962 
2963  cfglist.next = NULL;
2964 
2968 
2969  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2970  if (NULL == cfgtree)
2971  exit(EXIT_FAILURE);
2972 
2973  /* Default Config */
2974  cfglist.cfg = htp_config_create();
2975  if (NULL == cfglist.cfg) {
2976  FatalError("Failed to create HTP default config");
2977  }
2978  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2979  HTPConfigSetDefaultsPhase1(&cfglist);
2980  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2981  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2982  cfgtree);
2983  } else {
2984  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2985  }
2986  HTPConfigSetDefaultsPhase2("default", &cfglist);
2987 
2988  HTPParseMemcap();
2989 
2990  /* Read server config and create a parser for each IP in radix tree */
2991  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2992  if (server_config == NULL) {
2993  server_config = ConfGetNode("libhtp.server-config");
2994  if (server_config == NULL) {
2995  SCLogDebug("LIBHTP Configuring %p", server_config);
2996  SCReturn;
2997  }
2998  }
2999  SCLogDebug("LIBHTP Configuring %p", server_config);
3000 
3001  ConfNode *si;
3002  /* Server Nodes */
3003  TAILQ_FOREACH(si, &server_config->head, next) {
3004  /* Need the named node, not the index */
3005  ConfNode *s = TAILQ_FIRST(&si->head);
3006  if (NULL == s) {
3007  SCLogDebug("LIBHTP s NULL");
3008  continue;
3009  }
3010 
3011  SCLogDebug("LIBHTP server %s", s->name);
3012 
3013  HTPCfgRec *nextrec = cfglist.next;
3014  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
3015  if (NULL == htprec)
3016  exit(EXIT_FAILURE);
3017  memset(htprec, 0x00, sizeof(*htprec));
3018 
3019  cfglist.next = htprec;
3020 
3021  cfglist.next->next = nextrec;
3022  cfglist.next->cfg = htp_config_create();
3023  if (NULL == cfglist.next->cfg) {
3024  FatalError("Failed to create HTP server config");
3025  }
3026 
3027  HTPConfigSetDefaultsPhase1(htprec);
3028  HTPConfigParseParameters(htprec, s, cfgtree);
3029  HTPConfigSetDefaultsPhase2(s->name, htprec);
3030  }
3031 
3032  SCReturn;
3033 }
3034 
3036 {
3037 #ifdef DEBUG
3038  SCMutexLock(&htp_state_mem_lock);
3039  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
3040  SCMutexUnlock(&htp_state_mem_lock);
3041 #endif
3042 }
3043 
3044 /** \internal
3045  * \brief get files callback
3046  * \param state state ptr
3047  * \param direction flow direction
3048  * \retval files files ptr
3049  */
3050 static AppLayerGetFileState HTPGetTxFiles(void *state, void *txv, uint8_t direction)
3051 {
3052  AppLayerGetFileState files = { .fc = NULL, .cfg = &htp_sbcfg };
3053  htp_tx_t *tx = (htp_tx_t *)txv;
3054  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3055  if (tx_ud) {
3056  if (direction & STREAM_TOCLIENT) {
3057  files.fc = &tx_ud->files_tc;
3058  } else {
3059  files.fc = &tx_ud->files_ts;
3060  }
3061  }
3062  return files;
3063 }
3064 
3065 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
3066 {
3067  if (direction & STREAM_TOSERVER)
3068  return ((htp_tx_t *)tx)->request_progress;
3069  else
3070  return ((htp_tx_t *)tx)->response_progress;
3071 }
3072 
3073 static uint64_t HTPStateGetTxCnt(void *alstate)
3074 {
3075  HtpState *http_state = (HtpState *)alstate;
3076 
3077  if (http_state != NULL && http_state->conn != NULL) {
3078  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
3079  if (size < 0)
3080  return 0ULL;
3081  SCLogDebug("size %"PRIu64, size);
3082  return (uint64_t)size;
3083  } else {
3084  return 0ULL;
3085  }
3086 }
3087 
3088 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
3089 {
3090  HtpState *http_state = (HtpState *)alstate;
3091 
3092  if (http_state != NULL && http_state->conn != NULL)
3093  return htp_list_get(http_state->conn->transactions, tx_id);
3094  else
3095  return NULL;
3096 }
3097 
3098 void *HtpGetTxForH2(void *alstate)
3099 {
3100  // gets last transaction
3101  HtpState *http_state = (HtpState *)alstate;
3102  if (http_state != NULL && http_state->conn != NULL) {
3103  size_t txid = htp_list_array_size(http_state->conn->transactions);
3104  if (txid > 0) {
3105  return htp_list_get(http_state->conn->transactions, txid - 1);
3106  }
3107  }
3108  return NULL;
3109 }
3110 
3111 static int HTPStateGetEventInfo(const char *event_name,
3112  int *event_id, AppLayerEventType *event_type)
3113 {
3114  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3115  if (*event_id == -1) {
3116  SCLogError("event \"%s\" not present in "
3117  "http's enum map table.",
3118  event_name);
3119  /* this should be treated as fatal */
3120  return -1;
3121  }
3122 
3123  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3124 
3125  return 0;
3126 }
3127 
3128 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3129  AppLayerEventType *event_type)
3130 {
3131  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3132  if (*event_name == NULL) {
3133  SCLogError("event \"%d\" not present in "
3134  "http's enum map table.",
3135  event_id);
3136  /* this should be treated as fatal */
3137  return -1;
3138  }
3139 
3140  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3141 
3142  return 0;
3143 }
3144 
3145 static AppLayerTxData *HTPGetTxData(void *vtx)
3146 {
3147  htp_tx_t *tx = (htp_tx_t *)vtx;
3148  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3149  if (tx_ud) {
3150  return &tx_ud->tx_data;
3151  }
3152  return NULL;
3153 }
3154 
3155 static AppLayerStateData *HTPGetStateData(void *vstate)
3156 {
3157  HtpState *s = vstate;
3158  return &s->state_data;
3159 }
3160 
3161 static int HTPRegisterPatternsForProtocolDetection(void)
3162 {
3163  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3164  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3165  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3166  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3167  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3168  const char *spacings[] = { "|20|", "|09|", NULL };
3169  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3170 
3171  int methods_pos;
3172  int spacings_pos;
3173  int versions_pos;
3174  int register_result;
3175  char method_buffer[32] = "";
3176 
3177  /* Loop through all the methods ands spacings and register the patterns */
3178  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3179  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3180 
3181  /* Combine the method name and the spacing */
3182  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3183 
3184  /* Register the new method+spacing pattern
3185  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3186  * but the pattern matching should only be one char
3187  */
3188  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3189  method_buffer, (uint16_t)strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3190  if (register_result < 0) {
3191  return -1;
3192  }
3193  }
3194  }
3195 
3196  /* Loop through all the http version patterns that are TO_CLIENT */
3197  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3198  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3199  versions[versions_pos], (uint16_t)strlen(versions[versions_pos]), 0,
3200  STREAM_TOCLIENT);
3201  if (register_result < 0) {
3202  return -1;
3203  }
3204  }
3205 
3206  return 0;
3207 }
3208 
3209 /**
3210  * \brief Register the HTTP protocol and state handling functions to APP layer
3211  * of the engine.
3212  */
3214 {
3215  SCEnter();
3216 
3217  const char *proto_name = "http";
3218 
3219  /** HTTP */
3220  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3222  if (HTPRegisterPatternsForProtocolDetection() < 0)
3223  return;
3224  } else {
3225  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3226  proto_name);
3227  return;
3228  }
3229 
3230  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3231  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3232  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3233  AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles);
3235  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3236  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3237  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3238 
3240  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3241  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3243  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3244 
3245  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3246  AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData);
3247 
3249  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3250 
3252  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3254  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3255  SC_ATOMIC_INIT(htp_config_flags);
3256  /* This parser accepts gaps. */
3260  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_TOCLIENT);
3261  /* app-layer-frame-documentation tag start: registering relevant callbacks */
3263  IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById);
3264  /* app-layer-frame-documentation tag end: registering relevant callbacks */
3265  HTPConfigure();
3266  } else {
3267  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3268  "still on.", proto_name);
3269  }
3270 #ifdef UNITTESTS
3271  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3272 #endif
3273 
3274  SCReturn;
3275 }
3276 
3277 #ifdef UNITTESTS
3278 #include "detect-engine-alert.h"
3279 
3280 static HTPCfgRec cfglist_backup;
3281 
3283 {
3284  cfglist_backup = cfglist;
3285 
3286  return;
3287 }
3288 
3290 {
3291  cfglist = cfglist_backup;
3292 
3293  return;
3294 }
3295 
3296 /** \test Test case where chunks are sent in smaller chunks and check the
3297  * response of the parser from HTP library. */
3298 static int HTPParserTest01(void)
3299 {
3300  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3301  " Data is c0oL!";
3302  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3303 
3304  TcpSession ssn;
3305  memset(&ssn, 0, sizeof(ssn));
3306 
3309 
3310  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3311  FAIL_IF_NULL(f);
3312  f->protoctx = &ssn;
3313  f->proto = IPPROTO_TCP;
3314  f->alproto = ALPROTO_HTTP1;
3315 
3316  StreamTcpInitConfig(true);
3317 
3318  uint32_t u;
3319  for (u = 0; u < httplen1; u++) {
3320  uint8_t flags = 0;
3321 
3322  if (u == 0)
3323  flags = STREAM_TOSERVER|STREAM_START;
3324  else if (u == (httplen1 - 1))
3325  flags = STREAM_TOSERVER|STREAM_EOF;
3326  else
3327  flags = STREAM_TOSERVER;
3328 
3329  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3330  FAIL_IF(r != 0);
3331  }
3332 
3333  HtpState *htp_state = f->alstate;
3334  FAIL_IF_NULL(htp_state);
3335 
3336  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3337  FAIL_IF_NULL(tx);
3338 
3339  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3340  FAIL_IF_NULL(h);
3341 
3342  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3343  FAIL_IF(tx->request_method_number != HTP_M_POST);
3344  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3345 
3347  StreamTcpFreeConfig(true);
3348  UTHFreeFlow(f);
3349  PASS;
3350 }
3351 
3352 /** \test Test folding in 1 read case */
3353 static int HTPParserTest01b(void)
3354 {
3355  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3356  " Data is c0oL!";
3357  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3358 
3359  TcpSession ssn;
3360  memset(&ssn, 0, sizeof(ssn));
3361 
3364 
3365  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3366  FAIL_IF_NULL(f);
3367  f->protoctx = &ssn;
3368  f->proto = IPPROTO_TCP;
3369  f->alproto = ALPROTO_HTTP1;
3370 
3371  StreamTcpInitConfig(true);
3372 
3373  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3374  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3375  FAIL_IF(r != 0);
3376 
3377  HtpState *htp_state = f->alstate;
3378  FAIL_IF_NULL(htp_state);
3379 
3380  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3381  FAIL_IF_NULL(tx);
3382 
3383  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3384  FAIL_IF_NULL(h);
3385 
3386  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3387  FAIL_IF(tx->request_method_number != HTP_M_POST);
3388  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3389 
3391  StreamTcpFreeConfig(true);
3392  UTHFreeFlow(f);
3393  PASS;
3394 }
3395 
3396 /** \test Test folding in 1byte per read case */
3397 static int HTPParserTest01c(void)
3398 {
3399  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3400  " Data is c0oL!";
3401  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3402 
3403  TcpSession ssn;
3404  memset(&ssn, 0, sizeof(ssn));
3405 
3408 
3409  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3410  FAIL_IF_NULL(f);
3411  f->protoctx = &ssn;
3412  f->proto = IPPROTO_TCP;
3413  f->alproto = ALPROTO_HTTP1;
3414 
3415  StreamTcpInitConfig(true);
3416 
3417  uint32_t u;
3418  for (u = 0; u < httplen1; u++) {
3419  uint8_t flags = 0;
3420 
3421  if (u == 0)
3422  flags = STREAM_TOSERVER|STREAM_START;
3423  else if (u == (httplen1 - 1))
3424  flags = STREAM_TOSERVER|STREAM_EOF;
3425  else
3426  flags = STREAM_TOSERVER;
3427 
3428  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3429  FAIL_IF(r != 0);
3430  }
3431 
3432  HtpState *htp_state = f->alstate;
3433  FAIL_IF_NULL(htp_state);
3434 
3435  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3436  FAIL_IF_NULL(tx);
3437 
3438  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3439  FAIL_IF_NULL(h);
3440 
3441  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3442  FAIL_IF(tx->request_method_number != HTP_M_POST);
3443  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3444 
3446  StreamTcpFreeConfig(true);
3447  UTHFreeFlow(f);
3448  PASS;
3449 }
3450 
3451 /** \test Test case where chunks are sent in smaller chunks and check the
3452  * response of the parser from HTP library. */
3453 static int HTPParserTest01a(void)
3454 {
3455  int result = 0;
3456  Flow *f = NULL;
3457  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3458  " Data is c0oL!";
3459  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3460  TcpSession ssn;
3461  HtpState *htp_state = NULL;
3462  int r = 0;
3464 
3465  memset(&ssn, 0, sizeof(ssn));
3466 
3467  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3468  if (f == NULL)
3469  goto end;
3470  f->protoctx = &ssn;
3471  f->proto = IPPROTO_TCP;
3472  f->alproto = ALPROTO_HTTP1;
3473 
3474  StreamTcpInitConfig(true);
3475 
3476  uint32_t u;
3477  for (u = 0; u < httplen1; u++) {
3478  uint8_t flags = 0;
3479 
3480  if (u == 0)
3481  flags = STREAM_TOSERVER|STREAM_START;
3482  else if (u == (httplen1 - 1))
3483  flags = STREAM_TOSERVER|STREAM_EOF;
3484  else
3485  flags = STREAM_TOSERVER;
3486 
3487  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3488  if (r != 0) {
3489  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3490  " 0: ", u, r);
3491  goto end;
3492  }
3493  }
3494 
3495  htp_state = f->alstate;
3496  if (htp_state == NULL) {
3497  printf("no http state: ");
3498  goto end;
3499  }
3500 
3501  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3502  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3503  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3504  || tx->request_method_number != HTP_M_POST ||
3505  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3506  {
3507  printf("expected header value: Victor/1.0 and got %s: and expected"
3508  " method: POST and got %s, expected protocol number HTTP/1.0"
3509  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3510  bstr_util_strdup_to_c(tx->request_method),
3511  bstr_util_strdup_to_c(tx->request_protocol));
3512  goto end;
3513  }
3514  result = 1;
3515 end:
3516  if (alp_tctx != NULL)
3518  StreamTcpFreeConfig(true);
3519  UTHFreeFlow(f);
3520  return result;
3521 }
3522 
3523 /** \test See how it deals with an incomplete request. */
3524 static int HTPParserTest02(void)
3525 {
3526  int result = 0;
3527  Flow *f = NULL;
3528  uint8_t httpbuf1[] = "POST";
3529  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3530  TcpSession ssn;
3531  HtpState *http_state = NULL;
3533 
3534  memset(&ssn, 0, sizeof(ssn));
3535 
3536  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3537  if (f == NULL)
3538  goto end;
3539  f->protoctx = &ssn;
3540  f->proto = IPPROTO_TCP;
3541  f->alproto = ALPROTO_HTTP1;
3542 
3543  StreamTcpInitConfig(true);
3544 
3545  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3546  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3547  if (r != 0) {
3548  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3549  goto end;
3550  }
3551 
3552  http_state = f->alstate;
3553  if (http_state == NULL) {
3554  printf("no http state: ");
3555  goto end;
3556  }
3557 
3558  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3559  FAIL_IF_NULL(tx);
3560  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3561  FAIL_IF_NOT_NULL(h);
3562 
3563  FAIL_IF_NULL(tx->request_method);
3564  char *method = bstr_util_strdup_to_c(tx->request_method);
3565  FAIL_IF_NULL(method);
3566 
3567  FAIL_IF(strcmp(method, "POST") != 0);
3568  SCFree(method);
3569 
3570  result = 1;
3571 end:
3572  if (alp_tctx != NULL)
3574  StreamTcpFreeConfig(true);
3575  UTHFreeFlow(f);
3576  return result;
3577 }
3578 
3579 /** \test Test case where method is invalid and data is sent in smaller chunks
3580  * and check the response of the parser from HTP library. */
3581 static int HTPParserTest03(void)
3582 {
3583  int result = 0;
3584  Flow *f = NULL;
3585  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3586  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3587  TcpSession ssn;
3588  HtpState *htp_state = NULL;
3589  int r = 0;
3591 
3592  memset(&ssn, 0, sizeof(ssn));
3593 
3594  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3595  if (f == NULL)
3596  goto end;
3597  f->protoctx = &ssn;
3598  f->proto = IPPROTO_TCP;
3599  f->alproto = ALPROTO_HTTP1;
3600 
3601  StreamTcpInitConfig(true);
3602 
3603  uint32_t u;
3604  for (u = 0; u < httplen1; u++) {
3605  uint8_t flags = 0;
3606 
3607  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3608  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3609  else flags = STREAM_TOSERVER;
3610 
3611  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3612  if (r != 0) {
3613  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3614  " 0: ", u, r);
3615  goto end;
3616  }
3617  }
3618  htp_state = f->alstate;
3619  if (htp_state == NULL) {
3620  printf("no http state: ");
3621  goto end;
3622  }
3623 
3624  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3625 
3626  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3627  if (tx->request_method_number != HTP_M_UNKNOWN ||
3628  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3629  {
3630  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3631  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3632  bstr_util_strdup_to_c(tx->request_protocol));
3633  goto end;
3634  }
3635  result = 1;
3636 end:
3637  if (alp_tctx != NULL)
3639  StreamTcpFreeConfig(true);
3640  UTHFreeFlow(f);
3641  return result;
3642 }
3643 
3644 /** \test Test case where invalid data is sent and check the response of the
3645  * parser from HTP library. */
3646 static int HTPParserTest04(void)
3647 {
3648  int result = 0;
3649  Flow *f = NULL;
3650  HtpState *htp_state = NULL;
3651  uint8_t httpbuf1[] = "World!\r\n";
3652  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3653  TcpSession ssn;
3654  int r = 0;
3656 
3657  memset(&ssn, 0, sizeof(ssn));
3658 
3659  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3660  if (f == NULL)
3661  goto end;
3662  f->protoctx = &ssn;
3663  f->proto = IPPROTO_TCP;
3664  f->alproto = ALPROTO_HTTP1;
3665 
3666  StreamTcpInitConfig(true);
3667 
3669  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3670  if (r != 0) {
3671  goto end;
3672  }
3673 
3674  htp_state = f->alstate;
3675  if (htp_state == NULL) {
3676  printf("no http state: ");
3677  goto end;
3678  }
3679 
3680  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3681  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3682  if (tx->request_method_number != HTP_M_UNKNOWN ||
3683  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3684  {
3685  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3686  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3687  bstr_util_strdup_to_c(tx->request_protocol));
3688  goto end;
3689  }
3690  result = 1;
3691 end:
3692  if (alp_tctx != NULL)
3694  StreamTcpFreeConfig(true);
3695  UTHFreeFlow(f);
3696  return result;
3697 }
3698 
3699 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3700  * properly parsed them and also keeps them separated. */
3701 static int HTPParserTest05(void)
3702 {
3703  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3704  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3705  uint8_t httpbuf2[] = "Post D";
3706  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3707  uint8_t httpbuf3[] = "ata is c0oL!";
3708  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3709 
3710  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3711  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3712  uint8_t httpbuf5[] = "post R";
3713  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3714  uint8_t httpbuf6[] = "esults are tha bomb!";
3715  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3716 
3717  TcpSession ssn;
3718  memset(&ssn, 0, sizeof(ssn));
3719 
3722 
3723  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3724  FAIL_IF_NULL(f);
3725  f->protoctx = &ssn;
3726  f->proto = IPPROTO_TCP;
3727  f->alproto = ALPROTO_HTTP1;
3728 
3729  StreamTcpInitConfig(true);
3730 
3731  int r = AppLayerParserParse(
3732  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3733  FAIL_IF(r != 0);
3734 
3735  r = AppLayerParserParse(
3736  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3737  FAIL_IF(r != 0);
3738 
3739  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3740  FAIL_IF(r != 0);
3741 
3742  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3743  FAIL_IF(r != 0);
3744 
3745  r = AppLayerParserParse(
3746  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3747  FAIL_IF(r != 0);
3748 
3749  r = AppLayerParserParse(
3750  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3751  FAIL_IF(r != 0);
3752 
3753  HtpState *http_state = f->alstate;
3754  FAIL_IF_NULL(http_state);
3755 
3756  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3757  FAIL_IF_NULL(tx);
3758  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3759  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3760 
3761  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3762  FAIL_IF_NULL(h);
3763 
3764  FAIL_IF_NOT(tx->response_status_number == 200);
3765 
3767  StreamTcpFreeConfig(true);
3768  UTHFreeFlow(f);
3769  PASS;
3770 }
3771 
3772 /** \test Test proper chunked encoded response body
3773  */
3774 static int HTPParserTest06(void)
3775 {
3776  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3777  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3778  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3779  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3780  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3781  "GMT\r\n"
3782  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3783  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3784  "FrontPage/5.0.2.2510\r\n"
3785  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3786  "chunked\r\n"
3787  "Content-Type: text/html\r\n\r\n"
3788  "580\r\n"
3789  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3790  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3791  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3792  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3793  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3794  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3795  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3796  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3797  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3798  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3799  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3800  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3801  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3802  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3803  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3804  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3805  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3806  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3807  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3808  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3809  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3810  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3811  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3812  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3813  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3814  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3815  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3816  "aHA=\r\n0\r\n\r\n";
3817  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3818  TcpSession ssn;
3819 
3822 
3823  memset(&ssn, 0, sizeof(ssn));
3824 
3825  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3826  FAIL_IF_NULL(f);
3827  f->protoctx = &ssn;
3828  f->proto = IPPROTO_TCP;
3829  f->alproto = ALPROTO_HTTP1;
3830 
3831  StreamTcpInitConfig(true);
3832 
3833  int r = AppLayerParserParse(
3834  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3835  FAIL_IF(r != 0);
3836  r = AppLayerParserParse(
3837  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3838  FAIL_IF(r != 0);
3839 
3840  HtpState *http_state = f->alstate;
3841  FAIL_IF_NULL(http_state);
3842 
3843  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3844  FAIL_IF_NULL(tx);
3845 
3846  FAIL_IF(tx->request_method_number != HTP_M_GET);
3847  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3848 
3849  FAIL_IF(tx->response_status_number != 200);
3850  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3851 
3852  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3853  FAIL_IF_NULL(h);
3854 
3856  StreamTcpFreeConfig(true);
3857  UTHFreeFlow(f);
3858  PASS;
3859 }
3860 
3861 /** \test
3862  */
3863 static int HTPParserTest07(void)
3864 {
3865  int result = 0;
3866  Flow *f = NULL;
3867  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3868  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3869  TcpSession ssn;
3870  HtpState *htp_state = NULL;
3871  int r = 0;
3873 
3874  memset(&ssn, 0, sizeof(ssn));
3875 
3876  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3877  if (f == NULL)
3878  goto end;
3879  f->protoctx = &ssn;
3880  f->proto = IPPROTO_TCP;
3881  f->alproto = ALPROTO_HTTP1;
3882 
3883  StreamTcpInitConfig(true);
3884 
3885  uint32_t u;
3886  for (u = 0; u < httplen1; u++) {
3887  uint8_t flags = 0;
3888 
3889  if (u == 0)
3890  flags = STREAM_TOSERVER|STREAM_START;
3891  else if (u == (httplen1 - 1))
3892  flags = STREAM_TOSERVER|STREAM_EOF;
3893  else
3894  flags = STREAM_TOSERVER;
3895 
3896  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3897  if (r != 0) {
3898  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3899  " 0: ", u, r);
3900  goto end;
3901  }
3902  }
3903 
3904  htp_state = f->alstate;
3905  if (htp_state == NULL) {
3906  printf("no http state: ");
3907  goto end;
3908  }
3909 
3910  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3911  size_t reflen = sizeof(ref) - 1;
3912 
3913  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3914  if (tx == NULL)
3915  goto end;
3916  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3917  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3918  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3919  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3920  (uintmax_t)reflen,
3921  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3922  goto end;
3923  }
3924 
3925  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3926  bstr_len(tx_ud->request_uri_normalized)) != 0)
3927  {
3928  printf("normalized uri \"");
3929  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3930  printf("\" != \"");
3931  PrintRawUriFp(stdout, ref, reflen);
3932  printf("\": ");
3933  goto end;
3934  }
3935  }
3936 
3937  result = 1;
3938 end:
3939  if (alp_tctx != NULL)
3941  StreamTcpFreeConfig(true);
3942  UTHFreeFlow(f);
3943  return result;
3944 }
3945 
3946 #include "conf-yaml-loader.h"
3947 
3948 /** \test Abort
3949  */
3950 static int HTPParserTest08(void)
3951 {
3952  int result = 0;
3953  Flow *f = NULL;
3954  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3955  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3956  TcpSession ssn;
3958 
3959  char input[] = "\
3960 %YAML 1.1\n\
3961 ---\n\
3962 libhtp:\n\
3963 \n\
3964  default-config:\n\
3965  personality: IDS\n\
3966 ";
3967 
3969  ConfInit();
3971 
3972  ConfYamlLoadString(input, strlen(input));
3973  HTPConfigure();
3974 
3975  HtpState *htp_state = NULL;
3976  int r = 0;
3977  memset(&ssn, 0, sizeof(ssn));
3978 
3979  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3980  if (f == NULL)
3981  goto end;
3982  f->protoctx = &ssn;
3983  f->proto = IPPROTO_TCP;
3984  f->alproto = ALPROTO_HTTP1;
3985 
3986  StreamTcpInitConfig(true);
3987 
3988  uint8_t flags = 0;
3989  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3990 
3991  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3992  if (r != 0) {
3993  printf("toserver chunk returned %" PRId32 ", expected"
3994  " 0: ", r);
3995  result = 0;
3996  goto end;
3997  }
3998 
3999  htp_state = f->alstate;
4000  if (htp_state == NULL) {
4001  printf("no http state: ");
4002  result = 0;
4003  goto end;
4004  }
4005 
4006  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4007  if (tx == NULL)
4008  goto end;
4009  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4010  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4011  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4012  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4013  bstr_len(tx_ud->request_uri_normalized));
4014  }
4015 
4016  result = 1;
4017 end:
4018  if (alp_tctx != NULL)
4020  StreamTcpFreeConfig(true);
4021  HTPFreeConfig();
4022  ConfDeInit();
4025  UTHFreeFlow(f);
4026  return result;
4027 }
4028 
4029 /** \test Abort
4030  */
4031 static int HTPParserTest09(void)
4032 {
4033  int result = 0;
4034  Flow *f = NULL;
4035  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
4036  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4037  TcpSession ssn;
4039 
4040  char input[] = "\
4041 %YAML 1.1\n\
4042 ---\n\
4043 libhtp:\n\
4044 \n\
4045  default-config:\n\
4046  personality: Apache_2_2\n\
4047 ";
4048 
4050  ConfInit();
4052 
4053  ConfYamlLoadString(input, strlen(input));
4054  HTPConfigure();
4055 
4056  HtpState *htp_state = NULL;
4057  int r = 0;
4058 
4059  memset(&ssn, 0, sizeof(ssn));
4060 
4061  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4062  if (f == NULL)
4063  goto end;
4064  f->protoctx = &ssn;
4065  f->proto = IPPROTO_TCP;
4066  f->alproto = ALPROTO_HTTP1;
4067 
4068  StreamTcpInitConfig(true);
4069 
4070  uint8_t flags = 0;
4071  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
4072 
4073  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
4074  if (r != 0) {
4075  printf("toserver chunk returned %" PRId32 ", expected"
4076  " 0: ", r);
4077  goto end;
4078  }
4079 
4080  htp_state = f->alstate;
4081  if (htp_state == NULL) {
4082  printf("no http state: ");
4083  goto end;
4084  }
4085 
4086  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4087  if (tx == NULL)
4088  goto end;
4089  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4090  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4091  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4092  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4093  bstr_len(tx_ud->request_uri_normalized));
4094  }
4095 
4096  result = 1;
4097 end:
4098  if (alp_tctx != NULL)
4100  StreamTcpFreeConfig(true);
4101  HTPFreeConfig();
4102  ConfDeInit();
4105  UTHFreeFlow(f);
4106  return result;
4107 }
4108 
4109 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4110  */
4111 static int HTPParserTest10(void)
4112 {
4113  int result = 0;
4114  Flow *f = NULL;
4115  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4116  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4117  TcpSession ssn;
4118  HtpState *htp_state = NULL;
4119  int r = 0;
4121 
4122  memset(&ssn, 0, sizeof(ssn));
4123 
4124  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4125  if (f == NULL)
4126  goto end;
4127  f->protoctx = &ssn;
4128  f->proto = IPPROTO_TCP;
4129  f->alproto = ALPROTO_HTTP1;
4130 
4131  StreamTcpInitConfig(true);
4132 
4133  uint32_t u;
4134  for (u = 0; u < httplen1; u++) {
4135  uint8_t flags = 0;
4136 
4137  if (u == 0)
4138  flags = STREAM_TOSERVER|STREAM_START;
4139  else if (u == (httplen1 - 1))
4140  flags = STREAM_TOSERVER|STREAM_EOF;
4141  else
4142  flags = STREAM_TOSERVER;
4143 
4144  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4145  if (r != 0) {
4146  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4147  " 0: ", u, r);
4148  goto end;
4149  }
4150  }
4151 
4152  htp_state = f->alstate;
4153  if (htp_state == NULL) {
4154  printf("no http state: ");
4155  goto end;
4156  }
4157 
4158  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4159  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4160  if (h == NULL) {
4161  goto end;
4162  }
4163 
4164  char *name = bstr_util_strdup_to_c(h->name);
4165  if (name == NULL) {
4166  goto end;
4167  }
4168 
4169  if (strcmp(name, "Host") != 0) {
4170  printf("header name not \"Host\", instead \"%s\": ", name);
4171  free(name);
4172  goto end;
4173  }
4174  free(name);
4175 
4176  char *value = bstr_util_strdup_to_c(h->value);
4177  if (value == NULL) {
4178  goto end;
4179  }
4180 
4181  if (strcmp(value, "www.google.com") != 0) {
4182  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4183  free(value);
4184  goto end;
4185  }
4186  free(value);
4187 
4188  result = 1;
4189 end:
4190  if (alp_tctx != NULL)
4192  StreamTcpFreeConfig(true);
4193  UTHFreeFlow(f);
4194  return result;
4195 }
4196 
4197 /** \test double encoding in path
4198  */
4199 static int HTPParserTest11(void)
4200 {
4201  int result = 0;
4202  Flow *f = NULL;
4203  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4204  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4205  TcpSession ssn;
4206  HtpState *htp_state = NULL;
4207  int r = 0;
4209 
4210  memset(&ssn, 0, sizeof(ssn));
4211 
4212  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4213  if (f == NULL)
4214  goto end;
4215  f->protoctx = &ssn;
4216  f->proto = IPPROTO_TCP;
4217  f->alproto = ALPROTO_HTTP1;
4218 
4219  StreamTcpInitConfig(true);
4220 
4221  uint32_t u;
4222  for (u = 0; u < httplen1; u++) {
4223  uint8_t flags = 0;
4224 
4225  if (u == 0)
4226  flags = STREAM_TOSERVER|STREAM_START;
4227  else if (u == (httplen1 - 1))
4228  flags = STREAM_TOSERVER|STREAM_EOF;
4229  else
4230  flags = STREAM_TOSERVER;
4231 
4232  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4233  if (r != 0) {
4234  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4235  " 0: ", u, r);
4236  goto end;
4237  }
4238  }
4239 
4240  htp_state = f->alstate;
4241  if (htp_state == NULL) {
4242  printf("no http state: ");
4243  goto end;
4244  }
4245 
4246  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4247  if (tx == NULL)
4248  goto end;
4249  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4250  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4251  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4252  printf("normalized uri len should be 2, is %"PRIuMAX,
4253  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4254  goto end;
4255  }
4256 
4257  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4258  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4259  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4260  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4261  {
4262  printf("normalized uri \"");
4263  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4264  printf("\": ");
4265  goto end;
4266  }
4267  }
4268 
4269  result = 1;
4270 end:
4271  if (alp_tctx != NULL)
4273  StreamTcpFreeConfig(true);
4274  UTHFreeFlow(f);
4275  return result;
4276 }
4277 
4278 /** \test double encoding in query
4279  */
4280 static int HTPParserTest12(void)
4281 {
4282  int result = 0;
4283  Flow *f = NULL;
4284  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4285  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4286  TcpSession ssn;
4287  HtpState *htp_state = NULL;
4288  int r = 0;
4290 
4291  memset(&ssn, 0, sizeof(ssn));
4292 
4293  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4294  if (f == NULL)
4295  goto end;
4296  f->protoctx = &ssn;
4297  f->proto = IPPROTO_TCP;
4298  f->alproto = ALPROTO_HTTP1;
4299 
4300  StreamTcpInitConfig(true);
4301 
4302  uint32_t u;
4303  for (u = 0; u < httplen1; u++) {
4304  uint8_t flags = 0;
4305 
4306  if (u == 0)
4307  flags = STREAM_TOSERVER|STREAM_START;
4308  else if (u == (httplen1 - 1))
4309  flags = STREAM_TOSERVER|STREAM_EOF;
4310  else
4311  flags = STREAM_TOSERVER;
4312 
4313  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4314  if (r != 0) {
4315  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4316  " 0: ", u, r);
4317  goto end;
4318  }
4319  }
4320 
4321  htp_state = f->alstate;
4322  if (htp_state == NULL) {
4323  printf("no http state: ");
4324  goto end;
4325  }
4326 
4327  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4328  if (tx == NULL)
4329  goto end;
4330  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4331  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4332  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4333  printf("normalized uri len should be 5, is %"PRIuMAX,
4334  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4335  goto end;
4336  }
4337 
4338  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4339  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4340  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4341  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4342  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4343  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4344  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4345  {
4346  printf("normalized uri \"");
4347  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4348  printf("\": ");
4349  goto end;
4350  }
4351  }
4352 
4353  result = 1;
4354  end:
4355  if (alp_tctx != NULL)
4357  StreamTcpFreeConfig(true);
4358  UTHFreeFlow(f);
4359  return result;
4360 }
4361 
4362 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4363  */
4364 static int HTPParserTest13(void)
4365 {
4366  int result = 0;
4367  Flow *f = NULL;
4368  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4369  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4370  TcpSession ssn;
4371  HtpState *htp_state = NULL;
4372  int r = 0;
4374 
4375  memset(&ssn, 0, sizeof(ssn));
4376 
4377  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4378  if (f == NULL)
4379  goto end;
4380  f->protoctx = &ssn;
4381  f->proto = IPPROTO_TCP;
4382  f->alproto = ALPROTO_HTTP1;
4383 
4384  StreamTcpInitConfig(true);
4385 
4386  uint32_t u;
4387  for (u = 0; u < httplen1; u++) {
4388  uint8_t flags = 0;
4389 
4390  if (u == 0)
4391  flags = STREAM_TOSERVER|STREAM_START;
4392  else if (u == (httplen1 - 1))
4393  flags = STREAM_TOSERVER|STREAM_EOF;
4394  else
4395  flags = STREAM_TOSERVER;
4396 
4397  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4398  if (r != 0) {
4399  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4400  " 0: ", u, r);
4401  goto end;
4402  }
4403  }
4404 
4405  htp_state = f->alstate;
4406  if (htp_state == NULL) {
4407  printf("no http state: ");
4408  goto end;
4409  }
4410 
4411  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4412  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4413  if (h == NULL) {
4414  goto end;
4415  }
4416 
4417  char *name = bstr_util_strdup_to_c(h->name);
4418  if (name == NULL) {
4419  goto end;
4420  }
4421 
4422  if (strcmp(name, "Host") != 0) {
4423  printf("header name not \"Host\", instead \"%s\": ", name);
4424  free(name);
4425  goto end;
4426  }
4427  free(name);
4428 
4429  char *value = bstr_util_strdup_to_c(h->value);
4430  if (value == NULL) {
4431  goto end;
4432  }
4433 
4434  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4435  printf("header value not \"www.google.com\", instead \"");
4436  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4437  printf("\": ");
4438  free(value);
4439  goto end;
4440  }
4441  free(value);
4442 
4443  result = 1;
4444 end:
4445  if (alp_tctx != NULL)
4447  StreamTcpFreeConfig(true);
4448  UTHFreeFlow(f);
4449  return result;
4450 }
4451 
4452 /** \test Test basic config */
4453 static int HTPParserConfigTest01(void)
4454 {
4455  int ret = 0;
4456  char input[] = "\
4457 %YAML 1.1\n\
4458 ---\n\
4459 libhtp:\n\
4460 \n\
4461  default-config:\n\
4462  personality: IDS\n\
4463 \n\
4464  server-config:\n\
4465 \n\
4466  - apache-tomcat:\n\
4467  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4468  personality: Tomcat_6_0\n\
4469 \n\
4470  - iis7:\n\
4471  address: \n\
4472  - 192.168.0.0/24\n\
4473  - 192.168.10.0/24\n\
4474  personality: IIS_7_0\n\
4475 ";
4476 
4478  ConfInit();
4479 
4480  ConfYamlLoadString(input, strlen(input));
4481 
4482  ConfNode *outputs;
4483  outputs = ConfGetNode("libhtp.default-config.personality");
4484  if (outputs == NULL) {
4485  goto end;
4486  }
4487 
4488  outputs = ConfGetNode("libhtp.server-config");
4489  if (outputs == NULL) {
4490  goto end;
4491  }
4492 
4493  ConfNode *node = TAILQ_FIRST(&outputs->head);
4494  if (node == NULL) {
4495  goto end;
4496  }
4497  if (strcmp(node->name, "0") != 0) {
4498  goto end;
4499  }
4500  node = TAILQ_FIRST(&node->head);
4501  if (node == NULL) {
4502  goto end;
4503  }
4504  if (strcmp(node->name, "apache-tomcat") != 0) {
4505  goto end;
4506  }
4507 
4508  int i = 0;
4509  ConfNode *n;
4510 
4511  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4512  if (node2 == NULL) {
4513  goto end;
4514  }
4515  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4516  goto end;
4517  }
4518 
4519  node = ConfNodeLookupChild(node, "address");
4520  if (node == NULL) {
4521  goto end;
4522  }
4523  TAILQ_FOREACH(n, &node->head, next) {
4524  if (n == NULL) {
4525  goto end;
4526  }
4527 
4528  switch(i) {
4529  case 0:
4530  if (strcmp(n->name, "0") != 0) {
4531  goto end;
4532  }
4533  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4534  goto end;
4535  }
4536  break;
4537  case 1:
4538  if (strcmp(n->name, "1") != 0) {
4539  goto end;
4540  }
4541  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4542  goto end;
4543  }
4544  break;
4545  case 2:
4546  if (strcmp(n->name, "2") != 0) {
4547  goto end;
4548  }
4549  if (strcmp(n->val, "::1") != 0) {
4550  goto end;
4551  }
4552  break;
4553  default:
4554  goto end;
4555  }
4556  i++;
4557  }
4558 
4559  outputs = ConfGetNode("libhtp.server-config");
4560  if (outputs == NULL) {
4561  goto end;
4562  }
4563 
4564  node = TAILQ_FIRST(&outputs->head);
4565  node = TAILQ_NEXT(node, next);
4566  if (node == NULL) {
4567  goto end;
4568  }
4569  if (strcmp(node->name, "1") != 0) {
4570  goto end;
4571  }
4572  node = TAILQ_FIRST(&node->head);
4573  if (node == NULL) {
4574  goto end;
4575  }
4576  if (strcmp(node->name, "iis7") != 0) {
4577  goto end;
4578  }
4579 
4580  node2 = ConfNodeLookupChild(node, "personality");
4581  if (node2 == NULL) {
4582  goto end;
4583  }
4584  if (strcmp(node2->val, "IIS_7_0") != 0) {
4585  goto end;
4586  }
4587 
4588  node = ConfNodeLookupChild(node, "address");
4589  if (node == NULL) {
4590  goto end;
4591  }
4592 
4593  i = 0;
4594  TAILQ_FOREACH(n, &node->head, next) {
4595  if (n == NULL) {
4596  goto end;
4597  }
4598 
4599  switch(i) {
4600  case 0:
4601  if (strcmp(n->name, "0") != 0) {
4602  goto end;
4603  }
4604  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4605  goto end;
4606  }
4607  break;
4608  case 1:
4609  if (strcmp(n->name, "1") != 0) {
4610  goto end;
4611  }
4612  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4613  goto end;
4614  }
4615  break;
4616  default:
4617  goto end;
4618  }
4619  i++;
4620  }
4621 
4622  ret = 1;
4623 
4624 end:
4625  ConfDeInit();
4627 
4628  return ret;
4629 }
4630 
4631 /** \test Test config builds radix correctly */
4632 static int HTPParserConfigTest02(void)
4633 {
4634  int ret = 0;
4635  char input[] = "\
4636 %YAML 1.1\n\
4637 ---\n\
4638 libhtp:\n\
4639 \n\
4640  default-config:\n\
4641  personality: IDS\n\
4642 \n\
4643  server-config:\n\
4644 \n\
4645  - apache-tomcat:\n\
4646  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4647  personality: Tomcat_6_0\n\
4648 \n\
4649  - iis7:\n\
4650  address: \n\
4651  - 192.168.0.0/24\n\
4652  - 192.168.10.0/24\n\
4653  personality: IIS_7_0\n\
4654 ";
4655 
4657  ConfInit();
4659 
4660  ConfYamlLoadString(input, strlen(input));
4661 
4662  HTPConfigure();
4663 
4664  if (cfglist.cfg == NULL) {
4665  printf("No default config created.\n");
4666  goto end;
4667  }
4668 
4669  if (cfgtree == NULL) {
4670  printf("No config tree created.\n");
4671  goto end;
4672  }
4673 
4674  htp_cfg_t *htp = cfglist.cfg;
4675  uint8_t buf[128];
4676  const char *addr;
4677  void *user_data = NULL;
4678 
4679  addr = "192.168.10.42";
4680  if (inet_pton(AF_INET, addr, buf) == 1) {
4681  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4682  if (user_data != NULL) {
4683  HTPCfgRec *htp_cfg_rec = user_data;
4684  htp = htp_cfg_rec->cfg;
4685  SCLogDebug("LIBHTP using config: %p", htp);
4686  }
4687  if (htp == NULL) {
4688  printf("Could not get config for: %s\n", addr);
4689  goto end;
4690  }
4691  }
4692  else {
4693  printf("Failed to parse address: %s\n", addr);
4694  goto end;
4695  }
4696 
4697  user_data = NULL;
4698  addr = "::1";
4699  if (inet_pton(AF_INET6, addr, buf) == 1) {
4700  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4701  if (user_data != NULL) {
4702  HTPCfgRec *htp_cfg_rec = user_data;
4703  htp = htp_cfg_rec->cfg;
4704  SCLogDebug("LIBHTP using config: %p", htp);
4705  }
4706  if (htp == NULL) {
4707  printf("Could not get config for: %s\n", addr);
4708  goto end;
4709  }
4710  }
4711  else {
4712  printf("Failed to parse address: %s\n", addr);
4713  goto end;
4714  }
4715 
4716  ret = 1;
4717 
4718 end:
4719  HTPFreeConfig();
4720  ConfDeInit();
4723 
4724  return ret;
4725 }
4726 
4727 /** \test Test traffic is handled by the correct htp config */
4728 static int HTPParserConfigTest03(void)
4729 {
4730  int result = 1;
4731  Flow *f = NULL;
4732  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4733  " Data is c0oL!";
4734  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4735  TcpSession ssn;
4737 
4738  HtpState *htp_state = NULL;
4739  int r = 0;
4740  char input[] = "\
4741 %YAML 1.1\n\
4742 ---\n\
4743 libhtp:\n\
4744 \n\
4745  default-config:\n\
4746  personality: IDS\n\
4747 \n\
4748  server-config:\n\
4749 \n\
4750  - apache-tomcat:\n\
4751  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4752  personality: Tomcat_6_0\n\
4753 \n\
4754  - iis7:\n\
4755  address: \n\
4756  - 192.168.0.0/24\n\
4757  - 192.168.10.0/24\n\
4758  personality: IIS_7_0\n\
4759 ";
4760 
4762  ConfInit();
4764 
4765  ConfYamlLoadString(input, strlen(input));
4766 
4767  HTPConfigure();
4768 
4769  const char *addr = "192.168.10.42";
4770 
4771  memset(&ssn, 0, sizeof(ssn));
4772 
4773  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4774  if (f == NULL)
4775  goto end;
4776  f->protoctx = &ssn;
4777  f->proto = IPPROTO_TCP;
4778  f->alproto = ALPROTO_HTTP1;
4779 
4780  htp_cfg_t *htp = cfglist.cfg;
4781 
4782  void *user_data = NULL;
4783  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4784  if (user_data != NULL) {
4785  HTPCfgRec *htp_cfg_rec = user_data;
4786  htp = htp_cfg_rec->cfg;
4787  SCLogDebug("LIBHTP using config: %p", htp);
4788  }
4789  if (htp == NULL) {
4790  printf("Could not get config for: %s\n", addr);
4791  goto end;
4792  }
4793 
4794  StreamTcpInitConfig(true);
4795 
4796  uint32_t u;
4797  for (u = 0; u < httplen1; u++) {
4798  uint8_t flags = 0;
4799 
4800  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4801  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4802  else flags = STREAM_TOSERVER;
4803 
4804  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4805  if (r != 0) {
4806  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4807  " 0: ", u, r);
4808  result = 0;
4809  goto end;
4810  }
4811  }
4812 
4813  htp_state = f->alstate;
4814  if (htp_state == NULL) {
4815  printf("no http state: ");
4816  result = 0;
4817  goto end;
4818  }
4819 
4820  if (HTPStateGetTxCnt(htp_state) != 2) {
4821  printf("HTPStateGetTxCnt(htp_state) failure\n");
4822  goto end;
4823  }
4824 
4825  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4826  if (tx == NULL)
4827  goto end;
4828  if (tx->cfg != htp) {
4829  printf("wrong HTP config (%p instead of %p - default=%p): ",
4830  tx->cfg, htp, cfglist.cfg);
4831  goto end;
4832  }
4833  tx = HTPStateGetTx(htp_state, 1);
4834  if (tx == NULL)
4835  goto end;
4836  if (tx->cfg != htp) {
4837  printf("wrong HTP config (%p instead of %p - default=%p): ",
4838  tx->cfg, htp, cfglist.cfg);
4839  goto end;
4840  }
4841 
4842 end:
4843  if (alp_tctx != NULL)
4845  HTPFreeConfig();
4846  ConfDeInit();
4849 
4850  StreamTcpFreeConfig(true);
4851  UTHFreeFlow(f);
4852  return result;
4853 }
4854 
4855 /* disabled when we upgraded to libhtp 0.5.x */
4856 #if 0
4857 static int HTPParserConfigTest04(void)
4858 {
4859  int result = 0;
4860 
4861  char input[] = "\
4862 %YAML 1.1\n\
4863 ---\n\
4864 libhtp:\n\
4865 \n\
4866  default-config:\n\
4867  personality: IDS\n\
4868  path-control-char-handling: status_400\n\
4869  path-convert-utf8: yes\n\
4870  path-invalid-encoding-handling: remove_percent\n\
4871 \n\
4872  server-config:\n\
4873 \n\
4874  - apache-tomcat:\n\
4875  personality: Tomcat_6_0\n\
4876  path-invalid-utf8-handling: none\n\
4877  path-nul-encoded-handling: status_404\n\
4878  path-nul-raw-handling: status_400\n\
4879 \n\
4880  - iis7:\n\
4881  personality: IIS_7_0\n\
4882  path-replacement-char: o\n\
4883  path-unicode-mapping: status_400\n\
4884 ";
4885 
4887  ConfInit();
4889 
4890  ConfYamlLoadString(input, strlen(input));
4891 
4892  HTPConfigure();
4893 
4894  HTPCfgRec *cfg_rec = &cfglist;
4895  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4896  cfg_rec->cfg->path_convert_utf8 != 1 ||
4897  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4898  printf("failed 1\n");
4899  goto end;
4900  }
4901 
4902  cfg_rec = cfg_rec->next;
4903  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4904  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4905  printf("failed 2\n");
4906  goto end;
4907  }
4908 
4909  cfg_rec = cfg_rec->next;
4910  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4911  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4912  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4913  printf("failed 3\n");
4914  goto end;
4915  }
4916 
4917  result = 1;
4918 
4919 end:
4920  HTPFreeConfig();
4921  ConfDeInit();
4924 
4925  return result;
4926 }
4927 #endif
4928 
4929 /** \test Test %2f decoding in profile Apache_2_2
4930  *
4931  * %2f in path is left untouched
4932  * %2f in query string is normalized to %2F
4933  * %252f in query string is decoded/normalized to %2F
4934  */
4935 static int HTPParserDecodingTest01(void)
4936 {
4937  uint8_t httpbuf1[] =
4938  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4939  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4940  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4941  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4942  TcpSession ssn;
4945 
4946  char input[] = "\
4947 %YAML 1.1\n\
4948 ---\n\
4949 libhtp:\n\
4950 \n\
4951  default-config:\n\
4952  personality: Apache_2\n\
4953 ";
4954 
4956  ConfInit();
4958  ConfYamlLoadString(input, strlen(input));
4959  HTPConfigure();
4960  const char *addr = "4.3.2.1";
4961  memset(&ssn, 0, sizeof(ssn));
4962 
4963  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4964  FAIL_IF_NULL(f);
4965  f->protoctx = &ssn;
4966  f->proto = IPPROTO_TCP;
4967  f->alproto = ALPROTO_HTTP1;
4968 
4969  StreamTcpInitConfig(true);
4970 
4971  for (uint32_t u = 0; u < httplen1; u++) {
4972  uint8_t flags = 0;
4973  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4974  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4975  else flags = STREAM_TOSERVER;
4976 
4977  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4978  FAIL_IF(r != 0);
4979  }
4980 
4981  HtpState *htp_state = f->alstate;
4982  FAIL_IF_NULL(htp_state);
4983 
4984  uint8_t ref1[] = "/abc%2fdef";
4985  size_t reflen = sizeof(ref1) - 1;
4986 
4987  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4988  FAIL_IF_NULL(tx);
4989 
4990  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4991  FAIL_IF_NULL(tx_ud);
4993  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4994  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4995  bstr_len(tx_ud->request_uri_normalized)) != 0);
4996 
4997  uint8_t ref2[] = "/abc/def?ghi/jkl";
4998  reflen = sizeof(ref2) - 1;
4999 
5000  tx = HTPStateGetTx(htp_state, 1);
5001  FAIL_IF_NULL(tx);
5002  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5003  FAIL_IF_NULL(tx_ud);
5005  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5006 
5007  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5008  bstr_len(tx_ud->request_uri_normalized)) != 0);
5009 
5010  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5011  reflen = sizeof(ref3) - 1;
5012  tx = HTPStateGetTx(htp_state, 2);
5013  FAIL_IF_NULL(tx);
5014  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5015  FAIL_IF_NULL(tx_ud);
5017  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5018 
5019  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5020  bstr_len(tx_ud->request_uri_normalized)) != 0);
5021 
5023  HTPFreeConfig();
5024  ConfDeInit();
5027 
5028  StreamTcpFreeConfig(true);
5029  UTHFreeFlow(f);
5030  PASS;
5031 }
5032 
5033 static int HTPParserDecodingTest01a(void)
5034 {
5035  uint8_t httpbuf1[] = "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5036  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5037  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5038  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5039  TcpSession ssn;
5042 
5043  char input[] = "\
5044 %YAML 1.1\n\
5045 ---\n\
5046 libhtp:\n\
5047 \n\
5048  default-config:\n\
5049  personality: Apache_2\n\
5050 ";
5051 
5053  ConfInit();
5055  ConfYamlLoadString(input, strlen(input));
5056  HTPConfigure();
5057  const char *addr = "4.3.2.1";
5058  memset(&ssn, 0, sizeof(ssn));
5059 
5060  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5061  FAIL_IF_NULL(f);
5062  f->protoctx = &ssn;
5063  f->proto = IPPROTO_TCP;
5064  f->alproto = ALPROTO_HTTP1;
5065 
5066  StreamTcpInitConfig(true);
5067 
5068  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
5069  (STREAM_TOSERVER | STREAM_START | STREAM_EOF), httpbuf1, httplen1);
5070  FAIL_IF(r != 0);
5071 
5072  HtpState *htp_state = f->alstate;
5073  FAIL_IF_NULL(htp_state);
5074 
5075  uint8_t ref1[] = "/abc%2fdef";
5076  size_t reflen = sizeof(ref1) - 1;
5077 
5078  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5079  FAIL_IF_NULL(tx);
5080 
5081  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5082  FAIL_IF_NULL(tx_ud);
5084  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5085  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5086  bstr_len(tx_ud->request_uri_normalized)) != 0);
5087 
5088  uint8_t ref2[] = "/abc/def?ghi/jkl";
5089  reflen = sizeof(ref2) - 1;
5090 
5091  tx = HTPStateGetTx(htp_state, 1);
5092  FAIL_IF_NULL(tx);
5093  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5094  FAIL_IF_NULL(tx_ud);
5096  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5097 
5098  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5099  bstr_len(tx_ud->request_uri_normalized)) != 0);
5100 
5101  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5102  reflen = sizeof(ref3) - 1;
5103  tx = HTPStateGetTx(htp_state, 2);
5104  FAIL_IF_NULL(tx);
5105  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5106  FAIL_IF_NULL(tx_ud);
5108  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5109 
5110  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5111  bstr_len(tx_ud->request_uri_normalized)) != 0);
5112 
5114  HTPFreeConfig();
5115  ConfDeInit();
5118 
5119  StreamTcpFreeConfig(true);
5120  UTHFreeFlow(f);
5121  PASS;
5122 }
5123 
5124 /** \test Test %2f decoding in profile IDS
5125  *
5126  * %2f in path decoded to /
5127  * %2f in query string is decoded to /
5128  * %252f in query string is decoded to %2F
5129  */
5130 static int HTPParserDecodingTest02(void)
5131 {
5132  int result = 0;
5133  Flow *f = NULL;
5134  uint8_t httpbuf1[] =
5135  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5136  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5137  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5138  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5139  TcpSession ssn;
5141 
5142  HtpState *htp_state = NULL;
5143  int r = 0;
5144  char input[] = "\
5145 %YAML 1.1\n\
5146 ---\n\
5147 libhtp:\n\
5148 \n\
5149  default-config:\n\
5150  personality: IDS\n\
5151  double-decode-path: no\n\
5152  double-decode-query: no\n\
5153 ";
5154 
5156  ConfInit();
5158  ConfYamlLoadString(input, strlen(input));
5159  HTPConfigure();
5160  const char *addr = "4.3.2.1";
5161  memset(&ssn, 0, sizeof(ssn));
5162 
5163  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5164  if (f == NULL)
5165  goto end;
5166  f->protoctx = &ssn;
5167  f->proto = IPPROTO_TCP;
5168  f->alproto = ALPROTO_HTTP1;
5169 
5170  StreamTcpInitConfig(true);
5171 
5172  uint32_t u;
5173  for (u = 0; u < httplen1; u++) {
5174  uint8_t flags = 0;
5175 
5176  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5177  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5178  else flags = STREAM_TOSERVER;
5179 
5180  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5181  if (r != 0) {
5182  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5183  " 0: ", u, r);
5184  goto end;
5185  }
5186  }
5187 
5188  htp_state = f->alstate;
5189  if (htp_state == NULL) {
5190  printf("no http state: ");
5191  goto end;
5192  }
5193 
5194  uint8_t ref1[] = "/abc/def";
5195  size_t reflen = sizeof(ref1) - 1;
5196 
5197  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5198  if (tx == NULL)
5199  goto end;
5200  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5201  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5202  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5203  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5204  (uintmax_t)reflen,
5205  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5206  goto end;
5207  }
5208 
5209  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5210  bstr_len(tx_ud->request_uri_normalized)) != 0)
5211  {
5212  printf("normalized uri \"");
5213  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5214  printf("\" != \"");
5215  PrintRawUriFp(stdout, ref1, reflen);
5216  printf("\": ");
5217  goto end;
5218  }
5219  }
5220 
5221  uint8_t ref2[] = "/abc/def?ghi/jkl";
5222  reflen = sizeof(ref2) - 1;
5223 
5224  tx = HTPStateGetTx(htp_state, 1);
5225  if (tx == NULL)
5226  goto end;
5227  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5228  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5229  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5230  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5231  (uintmax_t)reflen,
5232  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5233  goto end;
5234  }
5235 
5236  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5237  bstr_len(tx_ud->request_uri_normalized)) != 0)
5238  {
5239  printf("normalized uri \"");
5240  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5241  printf("\" != \"");
5242  PrintRawUriFp(stdout, ref2, reflen);
5243  printf("\": ");
5244  goto end;
5245  }
5246  }
5247 
5248  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5249  reflen = sizeof(ref3) - 1;
5250  tx = HTPStateGetTx(htp_state, 2);
5251  if (tx == NULL)
5252  goto end;
5253  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5254  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5255  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5256  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5257  (uintmax_t)reflen,
5258  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5259  goto end;
5260  }
5261 
5262  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5263  bstr_len(tx_ud->request_uri_normalized)) != 0)
5264  {
5265  printf("normalized uri \"");
5266  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5267  printf("\" != \"");
5268  PrintRawUriFp(stdout, ref3, reflen);
5269  printf("\": ");
5270  goto end;
5271  }
5272  }
5273 
5274  result = 1;
5275 
5276 end:
5277  if (alp_tctx != NULL)
5279  HTPFreeConfig();
5280  ConfDeInit();
5283 
5284  StreamTcpFreeConfig(true);
5285  UTHFreeFlow(f);
5286  return result;
5287 }
5288 
5289 /** \test Test %2f decoding in profile IDS with double-decode-* options
5290  *
5291  * %252f in path decoded to /
5292  * %252f in query string is decoded to /
5293  */
5294 static int HTPParserDecodingTest03(void)
5295 {
5296  int result = 0;
5297  Flow *f = NULL;
5298  uint8_t httpbuf1[] =
5299  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5300  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5301  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5302  TcpSession ssn;
5304 
5305  HtpState *htp_state = NULL;
5306  int r = 0;
5307  char input[] = "\
5308 %YAML 1.1\n\
5309 ---\n\
5310 libhtp:\n\
5311 \n\
5312  default-config:\n\
5313  personality: IDS\n\
5314  double-decode-path: yes\n\
5315  double-decode-query: yes\n\
5316 ";
5317 
5319  ConfInit();
5321  ConfYamlLoadString(input, strlen(input));
5322  HTPConfigure();
5323  const char *addr = "4.3.2.1";
5324  memset(&ssn, 0, sizeof(ssn));
5325 
5326  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5327  if (f == NULL)
5328  goto end;
5329  f->protoctx = &ssn;
5330  f->proto = IPPROTO_TCP;
5331  f->alproto = ALPROTO_HTTP1;
5332 
5333  StreamTcpInitConfig(true);
5334 
5335  uint32_t u;
5336  for (u = 0; u < httplen1; u++) {
5337  uint8_t flags = 0;
5338 
5339  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5340  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5341  else flags = STREAM_TOSERVER;
5342 
5343  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5344  if (r != 0) {
5345  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5346  " 0: ", u, r);
5347  goto end;
5348  }
5349  }
5350 
5351  htp_state = f->alstate;
5352  if (htp_state == NULL) {
5353  printf("no http state: ");
5354  goto end;
5355  }
5356 
5357  uint8_t ref1[] = "/abc/def";
5358  size_t reflen = sizeof(ref1) - 1;
5359 
5360  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5361  if (tx == NULL)
5362  goto end;
5363  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5364  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5365  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5366  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5367  (uintmax_t)reflen,
5368  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5369  goto end;
5370  }
5371 
5372  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5373  bstr_len(tx_ud->request_uri_normalized)) != 0)
5374  {
5375  printf("normalized uri \"");
5376  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5377  printf("\" != \"");
5378  PrintRawUriFp(stdout, ref1, reflen);
5379  printf("\": ");
5380  goto end;
5381  }
5382  }
5383 
5384  uint8_t ref2[] = "/abc/def?ghi/jkl";
5385  reflen = sizeof(ref2) - 1;
5386 
5387  tx = HTPStateGetTx(htp_state, 1);
5388  if (tx == NULL)
5389  goto end;
5390  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5391  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5392  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5393  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5394  (uintmax_t)reflen,
5395  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5396  goto end;
5397  }
5398 
5399  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5400  bstr_len(tx_ud->request_uri_normalized)) != 0)
5401  {
5402  printf("normalized uri \"");
5403  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5404  printf("\" != \"");
5405  PrintRawUriFp(stdout, ref2, reflen);
5406  printf("\": ");
5407  goto end;
5408  }
5409  }
5410 
5411  result = 1;
5412 
5413 end:
5414  if (alp_tctx != NULL)
5416  HTPFreeConfig();
5417  ConfDeInit();
5420 
5421  StreamTcpFreeConfig(true);
5422  UTHFreeFlow(f);
5423  return result;
5424 }
5425 
5426 /** \test Test http:// in query profile IDS
5427  */
5428 static int HTPParserDecodingTest04(void)
5429 {
5430  int result = 0;
5431  Flow *f = NULL;
5432  uint8_t httpbuf1[] =
5433  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5434  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5435  TcpSession ssn;
5437 
5438  HtpState *htp_state = NULL;
5439  int r = 0;
5440  char input[] = "\
5441 %YAML 1.1\n\
5442 ---\n\
5443 libhtp:\n\
5444 \n\
5445  default-config:\n\
5446  personality: IDS\n\
5447  double-decode-path: yes\n\
5448  double-decode-query: yes\n\
5449 ";
5450 
5452  ConfInit();
5454  ConfYamlLoadString(input, strlen(input));
5455  HTPConfigure();
5456  const char *addr = "4.3.2.1";
5457  memset(&ssn, 0, sizeof(ssn));
5458 
5459  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5460  if (f == NULL)
5461  goto end;
5462  f->protoctx = &ssn;
5463  f->proto = IPPROTO_TCP;
5464  f->alproto = ALPROTO_HTTP1;
5465 
5466  StreamTcpInitConfig(true);
5467 
5468  uint32_t u;
5469  for (u = 0; u < httplen1; u++) {
5470  uint8_t flags = 0;
5471 
5472  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5473  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5474  else flags = STREAM_TOSERVER;
5475 
5476  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5477  if (r != 0) {
5478  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5479  " 0: ", u, r);
5480  goto end;
5481  }
5482  }
5483 
5484  htp_state = f->alstate;
5485  if (htp_state == NULL) {
5486  printf("no http state: ");
5487  goto end;
5488  }
5489 
5490  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5491  size_t reflen = sizeof(ref1) - 1;
5492 
5493  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5494  if (tx == NULL)
5495  goto end;
5496  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5497  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5498  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5499  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5500  (uintmax_t)reflen,
5501  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5502  goto end;
5503  }
5504 
5505  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5506  bstr_len(tx_ud->request_uri_normalized)) != 0)
5507  {
5508  printf("normalized uri \"");
5509  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5510  printf("\" != \"");
5511  PrintRawUriFp(stdout, ref1, reflen);
5512  printf("\": ");
5513  goto end;
5514  }
5515  }
5516 
5517  result = 1;
5518 
5519 end:
5520  if (alp_tctx != NULL)
5522  HTPFreeConfig();
5523  ConfDeInit();
5526 
5527  StreamTcpFreeConfig(true);
5528  UTHFreeFlow(f);
5529  return result;
5530 }
5531 
5532 /** \test Test \ char in query profile IDS. Bug 739
5533  */
5534 static int HTPParserDecodingTest05(void)
5535 {
5536  int result = 0;
5537  Flow *f = NULL;
5538  uint8_t httpbuf1[] =
5539  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5540  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5541  TcpSession ssn;
5543 
5544  HtpState *htp_state = NULL;
5545  int r = 0;
5546  char input[] = "\
5547 %YAML 1.1\n\
5548 ---\n\
5549 libhtp:\n\
5550 \n\
5551  default-config:\n\
5552  personality: IDS\n\
5553  double-decode-path: yes\n\
5554  double-decode-query: yes\n\
5555 ";
5556 
5558  ConfInit();
5560  ConfYamlLoadString(input, strlen(input));
5561  HTPConfigure();
5562  const char *addr = "4.3.2.1";
5563  memset(&ssn, 0, sizeof(ssn));
5564 
5565  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5566  if (f == NULL)
5567  goto end;
5568  f->protoctx = &ssn;
5569  f->proto = IPPROTO_TCP;
5570  f->alproto = ALPROTO_HTTP1;
5571 
5572  StreamTcpInitConfig(true);
5573 
5574  uint32_t u;
5575  for (u = 0; u < httplen1; u++) {
5576  uint8_t flags = 0;
5577 
5578  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5579  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5580  else flags = STREAM_TOSERVER;
5581 
5582  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5583  if (r != 0) {
5584  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5585  " 0: ", u, r);
5586  goto end;
5587  }
5588  }
5589 
5590  htp_state = f->alstate;
5591  if (htp_state == NULL) {
5592  printf("no http state: ");
5593  goto end;
5594  }
5595 
5596  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5597  size_t reflen = sizeof(ref1) - 1;
5598 
5599  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5600  if (tx == NULL)
5601  goto end;
5602  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5603  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5604  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5605  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5606  (uintmax_t)reflen,
5607  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5608  goto end;
5609  }
5610 
5611  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5612  bstr_len(tx_ud->request_uri_normalized)) != 0)
5613  {
5614  printf("normalized uri \"");
5615  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5616  printf("\" != \"");
5617  PrintRawUriFp(stdout, ref1, reflen);
5618  printf("\": ");
5619  goto end;
5620  }
5621  }
5622 
5623  result = 1;
5624 
5625 end:
5626  if (alp_tctx != NULL)
5628  HTPFreeConfig();
5629  ConfDeInit();
5632 
5633  StreamTcpFreeConfig(true);
5634  UTHFreeFlow(f);
5635  return result;
5636 }
5637 
5638 /** \test Test + char in query. Bug 1035
5639  */
5640 static int HTPParserDecodingTest06(void)
5641 {
5642  int result = 0;
5643  Flow *f = NULL;
5644  uint8_t httpbuf1[] =
5645  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5646  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5647  TcpSession ssn;
5649 
5650  HtpState *htp_state = NULL;
5651  int r = 0;
5652  char input[] = "\
5653 %YAML 1.1\n\
5654 ---\n\
5655 libhtp:\n\
5656 \n\
5657  default-config:\n\
5658  personality: IDS\n\
5659  double-decode-path: yes\n\
5660  double-decode-query: yes\n\
5661 ";
5662 
5664  ConfInit();
5666  ConfYamlLoadString(input, strlen(input));
5667  HTPConfigure();
5668  const char *addr = "4.3.2.1";
5669  memset(&ssn, 0, sizeof(ssn));
5670 
5671  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5672  if (f == NULL)
5673  goto end;
5674  f->protoctx = &ssn;
5675  f->proto = IPPROTO_TCP;
5676  f->alproto = ALPROTO_HTTP1;
5677 
5678  StreamTcpInitConfig(true);
5679 
5680  uint32_t u;
5681  for (u = 0; u < httplen1; u++) {
5682  uint8_t flags = 0;
5683 
5684  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5685  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5686  else flags = STREAM_TOSERVER;
5687 
5688  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5689  if (r != 0) {
5690  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5691  " 0: ", u, r);
5692  goto end;
5693  }
5694  }
5695 
5696  htp_state = f->alstate;
5697  if (htp_state == NULL) {
5698  printf("no http state: ");
5699  goto end;
5700  }
5701 
5702  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5703  size_t reflen = sizeof(ref1) - 1;
5704 
5705  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5706  if (tx == NULL)
5707  goto end;
5708  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5709  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5710  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5711  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5712  (uintmax_t)reflen,
5713  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5714  goto end;
5715  }
5716 
5717  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5718  bstr_len(tx_ud->request_uri_normalized)) != 0)
5719  {
5720  printf("normalized uri \"");
5721  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5722  printf("\" != \"");
5723  PrintRawUriFp(stdout, ref1, reflen);
5724  printf("\": ");
5725  goto end;
5726  }
5727  }
5728 
5729  result = 1;
5730 
5731 end:
5732  if (alp_tctx != NULL)
5734  HTPFreeConfig();
5735  ConfDeInit();
5738 
5739  StreamTcpFreeConfig(true);
5740  UTHFreeFlow(f);
5741  return result;
5742 }
5743 
5744 /** \test Test + char in query. Bug 1035
5745  */
5746 static int HTPParserDecodingTest07(void)
5747 {
5748  int result = 0;
5749  Flow *f = NULL;
5750  uint8_t httpbuf1[] =
5751  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5752  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5753  TcpSession ssn;
5755 
5756  HtpState *htp_state = NULL;
5757  int r = 0;
5758  char input[] = "\
5759 %YAML 1.1\n\
5760 ---\n\
5761 libhtp:\n\
5762 \n\
5763  default-config:\n\
5764  personality: IDS\n\
5765  double-decode-path: yes\n\
5766  double-decode-query: yes\n\
5767  query-plusspace-decode: yes\n\
5768 ";
5769 
5771  ConfInit();
5773  ConfYamlLoadString(input, strlen(input));
5774  HTPConfigure();
5775  const char *addr = "4.3.2.1";
5776  memset(&ssn, 0, sizeof(ssn));
5777 
5778  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5779  if (f == NULL)
5780  goto end;
5781  f->protoctx = &ssn;
5782  f->proto = IPPROTO_TCP;
5783  f->alproto = ALPROTO_HTTP1;
5784 
5785  StreamTcpInitConfig(true);
5786 
5787  uint32_t u;
5788  for (u = 0; u < httplen1; u++) {
5789  uint8_t flags = 0;
5790 
5791  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5792  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5793  else flags = STREAM_TOSERVER;
5794 
5795  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5796  if (r != 0) {
5797  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5798  " 0: ", u, r);
5799  goto end;
5800  }
5801  }
5802 
5803  htp_state = f->alstate;
5804  if (htp_state == NULL) {
5805  printf("no http state: ");
5806  goto end;
5807  }
5808 
5809  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5810  size_t reflen = sizeof(ref1) - 1;
5811 
5812  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5813  if (tx == NULL)
5814  goto end;
5815  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5816  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5817  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5818  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5819  (uintmax_t)reflen,
5820  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5821  goto end;
5822  }
5823 
5824  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5825  bstr_len(tx_ud->request_uri_normalized)) != 0)
5826  {
5827  printf("normalized uri \"");
5828  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5829  printf("\" != \"");
5830  PrintRawUriFp(stdout, ref1, reflen);
5831  printf("\": ");
5832  goto end;
5833  }
5834  }
5835 
5836&