suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "decode.h"
40 #include "threads.h"
41 #include "counters.h"
42 
43 #include "util-print.h"
44 #include "util-pool.h"
45 #include "util-radix-tree.h"
46 #include "util-file.h"
47 #include "util-byte.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-detect-proto.h"
59 #include "app-layer-frames.h"
60 #include "app-layer-htp.h"
61 #include "app-layer-htp-body.h"
62 #include "app-layer-htp-file.h"
63 #include "app-layer-htp-libhtp.h"
64 #include "app-layer-htp-xff.h"
65 #include "app-layer-htp-range.h"
66 #include "app-layer-htp-mem.h"
67 
68 #include "util-spm.h"
69 #include "util-debug.h"
70 #include "util-time.h"
71 #include "util-misc.h"
72 #include "util-enum.h"
73 
74 #include "util-unittest.h"
75 #include "util-unittest-helper.h"
76 #include "flow-util.h"
77 
78 #include "detect-engine.h"
79 #include "detect-engine-build.h"
80 #include "detect-engine-state.h"
81 #include "detect-parse.h"
82 
83 #include "decode-events.h"
84 
85 #include "util-memcmp.h"
86 #include "util-random.h"
87 #include "util-validate.h"
88 
89 //#define PRINT
90 
91 /** Fast lookup tree (radix) for the various HTP configurations */
92 static SCRadixTree *cfgtree;
93 /** List of HTP configurations. */
94 static HTPCfgRec cfglist;
95 
96 /** Limit to the number of libhtp messages that can be handled */
97 #define HTP_MAX_MESSAGES 512
98 
99 SC_ATOMIC_DECLARE(uint32_t, htp_config_flags);
100 
101 #ifdef DEBUG
102 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
103 static uint64_t htp_state_memuse = 0;
104 static uint64_t htp_state_memcnt = 0;
105 #endif
106 
108  { "UNKNOWN_ERROR", HTTP_DECODER_EVENT_UNKNOWN_ERROR },
109  { "GZIP_DECOMPRESSION_FAILED", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED },
110  { "REQUEST_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON },
111  { "RESPONSE_FIELD_MISSING_COLON", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON },
112  { "INVALID_REQUEST_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN },
113  { "INVALID_RESPONSE_CHUNK_LEN", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN },
114  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
116  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
118  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
120  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
122  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST",
124  { "DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE",
126  { "100_CONTINUE_ALREADY_SEEN", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN },
127  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
129  { "INVALID_SERVER_PORT_IN_REQUEST", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST },
130  { "INVALID_AUTHORITY_PORT", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT },
131  { "REQUEST_HEADER_INVALID", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID },
132  { "RESPONSE_HEADER_INVALID", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID },
133  { "MISSING_HOST_HEADER", HTTP_DECODER_EVENT_MISSING_HOST_HEADER },
134  { "HOST_HEADER_AMBIGUOUS", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS },
135  { "INVALID_REQUEST_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING },
136  { "INVALID_RESPONSE_FIELD_FOLDING", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING },
137  { "REQUEST_FIELD_TOO_LONG", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG },
138  { "RESPONSE_FIELD_TOO_LONG", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG },
139  { "FILE_NAME_TOO_LONG", HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG },
140  { "REQUEST_LINE_INVALID", HTTP_DECODER_EVENT_REQUEST_LINE_INVALID },
141  { "REQUEST_BODY_UNEXPECTED", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED },
142  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
144  { "REQUEST_URI_HOST_INVALID", HTTP_DECODER_EVENT_URI_HOST_INVALID },
145  { "REQUEST_HEADER_HOST_INVALID", HTTP_DECODER_EVENT_HEADER_HOST_INVALID },
146  { "REQUEST_AUTH_UNRECOGNIZED", HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED },
147  { "REQUEST_HEADER_REPETITION", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION },
148  { "RESPONSE_HEADER_REPETITION", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION },
149  { "DOUBLE_ENCODED_URI", HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI },
150  { "URI_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT },
151  { "METHOD_DELIM_NON_COMPLIANT", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT },
152  { "REQUEST_LINE_LEADING_WHITESPACE", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE },
153  { "TOO_MANY_ENCODING_LAYERS", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS },
154  { "ABNORMAL_CE_HEADER", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER },
155  { "RESPONSE_MULTIPART_BYTERANGES", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES },
156  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
158  { "RESPONSE_CHUNKED_OLD_PROTO", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO },
159  { "RESPONSE_INVALID_PROTOCOL", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL },
160  { "RESPONSE_INVALID_STATUS", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS },
161  { "REQUEST_LINE_INCOMPLETE", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE },
162 
163  { "LZMA_MEMLIMIT_REACHED", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED },
164  { "COMPRESSION_BOMB", HTTP_DECODER_EVENT_COMPRESSION_BOMB },
165 
166  { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
167 
168  /* suricata warnings/errors */
169  { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
170  { "MULTIPART_NO_FILEDATA", HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA },
171  { "MULTIPART_INVALID_HEADER", HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER },
172 
173  { "TOO_MANY_WARNINGS", HTTP_DECODER_EVENT_TOO_MANY_WARNINGS },
174  { "FAILED_PROTOCOL_CHANGE", HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE },
175 
176  { NULL, -1 },
177 };
178 
179 /* app-layer-frame-documentation tag start: HttpFrameTypes */
183 };
184 
186  {
187  "request",
189  },
190  {
191  "response",
193  },
194  { NULL, -1 },
195 };
196 /* app-layer-frame-documentation tag end: HttpFrameTypes */
197 
198 static int HTTPGetFrameIdByName(const char *frame_name)
199 {
200  int id = SCMapEnumNameToValue(frame_name, http_frame_table);
201  if (id < 0) {
202  return -1;
203  }
204  return id;
205 }
206 
207 static const char *HTTPGetFrameNameById(const uint8_t frame_id)
208 {
209  const char *name = SCMapEnumValueToName(frame_id, http_frame_table);
210  return name;
211 }
212 
213 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
214 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
215 static uint64_t HTPStateGetTxCnt(void *alstate);
216 #ifdef UNITTESTS
217 static void HTPParserRegisterTests(void);
218 #endif
219 
220 static inline uint64_t HtpGetActiveRequestTxID(HtpState *s)
221 {
222  uint64_t id = HTPStateGetTxCnt(s);
223  BUG_ON(id == 0);
224  return id - 1;
225 }
226 
227 static inline uint64_t HtpGetActiveResponseTxID(HtpState *s)
228 {
229  return s->transaction_cnt;
230 }
231 
232 #ifdef DEBUG
233 /**
234  * \internal
235  *
236  * \brief Lookup the HTP personality string from the numeric personality.
237  *
238  * \todo This needs to be a libhtp function.
239  */
240 static const char *HTPLookupPersonalityString(int p)
241 {
242 #define CASE_HTP_PERSONALITY_STRING(p) \
243  case HTP_SERVER_ ## p: return #p
244 
245  switch (p) {
246  CASE_HTP_PERSONALITY_STRING(MINIMAL);
247  CASE_HTP_PERSONALITY_STRING(GENERIC);
248  CASE_HTP_PERSONALITY_STRING(IDS);
249  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
250  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
251  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
252  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
253  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
254  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
255  CASE_HTP_PERSONALITY_STRING(APACHE_2);
256  }
257 
258  return NULL;
259 }
260 #endif /* DEBUG */
261 
262 /**
263  * \internal
264  *
265  * \brief Lookup the numeric HTP personality from a string.
266  *
267  * \todo This needs to be a libhtp function.
268  */
269 static int HTPLookupPersonality(const char *str)
270 {
271 #define IF_HTP_PERSONALITY_NUM(p) \
272  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
273 
274  IF_HTP_PERSONALITY_NUM(MINIMAL);
275  IF_HTP_PERSONALITY_NUM(GENERIC);
277  IF_HTP_PERSONALITY_NUM(IIS_4_0);
278  IF_HTP_PERSONALITY_NUM(IIS_5_0);
279  IF_HTP_PERSONALITY_NUM(IIS_5_1);
280  IF_HTP_PERSONALITY_NUM(IIS_6_0);
281  IF_HTP_PERSONALITY_NUM(IIS_7_0);
282  IF_HTP_PERSONALITY_NUM(IIS_7_5);
283  IF_HTP_PERSONALITY_NUM(APACHE_2);
284  if (strcasecmp("TOMCAT_6_0", str) == 0) {
285  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
286  "longer supported by libhtp.", str);
287  return -1;
288  } else if ((strcasecmp("APACHE", str) == 0) ||
289  (strcasecmp("APACHE_2_2", str) == 0))
290  {
291  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
292  "longer supported by libhtp, failing back to "
293  "Apache2 personality.", str);
294  return HTP_SERVER_APACHE_2;
295  }
296 
297  return -1;
298 }
299 
300 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud,
301  const uint8_t dir, const uint8_t e)
302 {
303  SCLogDebug("setting event %u", e);
304 
305  if (htud) {
306  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
307  s->events++;
308  return;
309  }
310 
311  const uint64_t tx_id = (dir == STREAM_TOSERVER) ?
312  HtpGetActiveRequestTxID(s) : HtpGetActiveResponseTxID(s);
313 
314  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
315  if (tx == NULL && tx_id > 0)
316  tx = HTPStateGetTx(s, tx_id - 1);
317  if (tx != NULL) {
318  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
319  if (htud != NULL) {
320  AppLayerDecoderEventsSetEventRaw(&htud->tx_data.events, e);
321  s->events++;
322  return;
323  }
324  }
325  SCLogDebug("couldn't set event %u", e);
326 }
327 
328 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
329  * connection parser to be used by the HTP library
330  */
331 static void *HTPStateAlloc(void *orig_state, AppProto proto_orig)
332 {
333  SCEnter();
334 
335  HtpState *s = HTPMalloc(sizeof(HtpState));
336  if (unlikely(s == NULL)) {
337  SCReturnPtr(NULL, "void");
338  }
339 
340  memset(s, 0x00, sizeof(HtpState));
341 
342 #ifdef DEBUG
343  SCMutexLock(&htp_state_mem_lock);
344  htp_state_memcnt++;
345  htp_state_memuse += sizeof(HtpState);
346  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
347  SCMutexUnlock(&htp_state_mem_lock);
348 #endif
349 
350  SCReturnPtr((void *)s, "void");
351 }
352 
353 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
354 {
355  if (likely(htud)) {
356  HtpBodyFree(&htud->request_body);
357  HtpBodyFree(&htud->response_body);
358  bstr_free(htud->request_uri_normalized);
359  if (htud->request_headers_raw)
361  if (htud->response_headers_raw)
364  if (htud->boundary)
365  HTPFree(htud->boundary, htud->boundary_len);
366  if (htud->tx_data.de_state != NULL) {
367  DetectEngineStateFree(htud->tx_data.de_state);
368  }
369  HTPFree(htud, sizeof(HtpTxUserData));
370  }
371 }
372 
373 /** \brief Function to frees the HTTP state memory and also frees the HTTP
374  * connection parser memory which was used by the HTP library
375  */
376 void HTPStateFree(void *state)
377 {
378  SCEnter();
379 
380  HtpState *s = (HtpState *)state;
381  if (s == NULL) {
382  SCReturn;
383  }
384 
385  /* free the connection parser memory used by HTP library */
386  if (s->connp != NULL) {
387  SCLogDebug("freeing HTP state");
388 
389  uint64_t tx_id;
390  uint64_t total_txs = HTPStateGetTxCnt(state);
391  /* free the list of body chunks */
392  if (s->conn != NULL) {
393  for (tx_id = 0; tx_id < total_txs; tx_id++) {
394  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
395  if (tx != NULL) {
396  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
397  HtpTxUserDataFree(s, htud);
398  htp_tx_set_user_data(tx, NULL);
399  }
400  }
401  }
402  htp_connp_destroy_all(s->connp);
403  }
404 
405  if (s->file_range) {
406  HTPFileCloseHandleRange(s->files_tc, 0, s->file_range, NULL, 0);
408  }
409 
412  HTPFree(s, sizeof(HtpState));
413 
414 #ifdef DEBUG
415  SCMutexLock(&htp_state_mem_lock);
416  htp_state_memcnt--;
417  htp_state_memuse -= sizeof(HtpState);
418  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
419  SCMutexUnlock(&htp_state_mem_lock);
420 #endif
421 
422  SCReturn;
423 }
424 
425 /**
426  * \brief HTP transaction cleanup callback
427  *
428  * \warning We cannot actually free the transactions here. It seems that
429  * HTP only accepts freeing of transactions in the response callback.
430  */
431 static void HTPStateTransactionFree(void *state, uint64_t id)
432 {
433  SCEnter();
434 
435  HtpState *s = (HtpState *)state;
436 
437  SCLogDebug("state %p, id %"PRIu64, s, id);
438 
439  htp_tx_t *tx = HTPStateGetTx(s, id);
440  if (tx != NULL) {
441  /* This will remove obsolete body chunks */
442  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
443  HtpTxUserDataFree(s, htud);
444  htp_tx_set_user_data(tx, NULL);
445 
446  /* hack: even if libhtp considers the tx incomplete, we want to
447  * free it here. htp_tx_destroy however, will refuse to do this.
448  * As htp_tx_destroy_incomplete isn't available in the public API,
449  * we hack around it here. */
450  if (unlikely(!(
451  tx->request_progress == HTP_REQUEST_COMPLETE &&
452  tx->response_progress == HTP_RESPONSE_COMPLETE)))
453  {
454  tx->request_progress = HTP_REQUEST_COMPLETE;
455  tx->response_progress = HTP_RESPONSE_COMPLETE;
456  }
457  htp_tx_destroy(tx);
458  }
459 }
460 
461 /**
462  * \brief Sets a flag that informs the HTP app layer that some module in the
463  * engine needs the http request body data.
464  * \initonly
465  */
467 {
468  SCEnter();
469 
470  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
471  SCReturn;
472 }
473 
474 /**
475  * \brief Sets a flag that informs the HTP app layer that some module in the
476  * engine needs the http request body data.
477  * \initonly
478  */
480 {
481  SCEnter();
482 
483  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
484  SCReturn;
485 }
486 
487 /**
488  * \brief Sets a flag that informs the HTP app layer that some module in the
489  * engine needs the http request multi part header.
490  *
491  * \initonly
492  */
493 static void AppLayerHtpNeedMultipartHeader(void)
494 {
495  SCEnter();
497 
498  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
499  SCReturn;
500 }
501 
502 /**
503  * \brief Sets a flag that informs the HTP app layer that some module in the
504  * engine needs the http request file.
505  *
506  * \initonly
507  */
509 {
510  SCEnter();
511  AppLayerHtpNeedMultipartHeader();
514 
515  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
516  SCReturn;
517 }
518 
519 static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
520 {
521  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
522  if (tx_ud) {
523  if (flags & STREAM_TOCLIENT) {
524  tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
525  } else {
526  tx_ud->tsflags |= HTP_STREAM_DEPTH_SET;
527  }
528  }
529 }
530 
531 static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
532 {
533  if (flags & HTP_STREAM_DEPTH_SET) {
534  uint32_t stream_depth = FileReassemblyDepth();
535  if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
536  return true;
537  }
538  } else {
539  if (cfg->body_limit == 0 || body->content_len_so_far < cfg->body_limit) {
540  return true;
541  }
542  }
543  return false;
544 }
545 
546 static uint32_t AppLayerHtpComputeChunkLength(uint64_t content_len_so_far, uint32_t body_limit,
547  uint32_t stream_depth, uint8_t flags, uint32_t data_len)
548 {
549  uint32_t chunk_len = 0;
550  if (!(flags & HTP_STREAM_DEPTH_SET) && body_limit > 0 &&
551  (content_len_so_far < (uint64_t)body_limit) &&
552  (content_len_so_far + (uint64_t)data_len) > body_limit)
553  {
554  chunk_len = body_limit - content_len_so_far;
555  } else if ((flags & HTP_STREAM_DEPTH_SET) && stream_depth > 0 &&
556  (content_len_so_far < (uint64_t)stream_depth) &&
557  (content_len_so_far + (uint64_t)data_len) > stream_depth)
558  {
559  chunk_len = stream_depth - content_len_so_far;
560  }
561  SCLogDebug("len %u", chunk_len);
562  return (chunk_len == 0 ? data_len : chunk_len);
563 }
564 
565 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
566 struct {
567  const char *msg;
568  uint8_t de;
569 } htp_errors[] = {
570  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
571  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
572  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
573  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
574  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
575 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
576  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
577 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
578  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
579  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
580  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
581  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
582 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
583  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
584  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
585  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
586  { "Compression bomb:", HTTP_DECODER_EVENT_COMPRESSION_BOMB},
587 };
588 
589 struct {
590  const char *msg;
591  uint8_t de;
592 } htp_warnings[] = {
593  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
594  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
595  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
596  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
597  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
598 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
599  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
600  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
601  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
602  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
603  * luckily, "Request server port=" is unique */
604 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
606  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
607  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
608  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
609  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
610  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
611  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
612  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
613  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
614  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
615  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
616  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
617  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
618  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
619  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
620  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
621  { "LZMA decompressor: memory limit reached", HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED},
622  { "Ambiguous request C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST},
623  { "Ambiguous response C-L value", HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE},
624 };
625 
626 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
627 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
628 
629 /**
630  * \internal
631  *
632  * \brief Get the warning id for the warning msg.
633  *
634  * \param msg warning message
635  *
636  * \retval id the id or 0 in case of not found
637  */
638 static uint8_t HTPHandleWarningGetId(const char *msg)
639 {
640  SCLogDebug("received warning \"%s\"", msg);
641  size_t idx;
642  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
643  if (strncmp(htp_warnings[idx].msg, msg,
644  strlen(htp_warnings[idx].msg)) == 0)
645  {
646  return htp_warnings[idx].de;
647  }
648  }
649 
650  return 0;
651 }
652 
653 /**
654  * \internal
655  *
656  * \brief Get the error id for the error msg.
657  *
658  * \param msg error message
659  *
660  * \retval id the id or 0 in case of not found
661  */
662 static uint8_t HTPHandleErrorGetId(const char *msg)
663 {
664  SCLogDebug("received error \"%s\"", msg);
665 
666  size_t idx;
667  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
668  if (strncmp(htp_errors[idx].msg, msg,
669  strlen(htp_errors[idx].msg)) == 0)
670  {
671  return htp_errors[idx].de;
672  }
673  }
674 
675  return 0;
676 }
677 
678 /**
679  * \internal
680  *
681  * \brief Check state for errors, warnings and add any as events
682  *
683  * \param s state
684  * \param dir direction: STREAM_TOSERVER or STREAM_TOCLIENT
685  */
686 static void HTPHandleError(HtpState *s, const uint8_t dir)
687 {
688  if (s == NULL || s->conn == NULL ||
689  s->conn->messages == NULL) {
690  return;
691  }
692 
693  size_t size = htp_list_size(s->conn->messages);
694  size_t msg;
695  if(size >= HTP_MAX_MESSAGES) {
697  //only once per HtpState
698  HTPSetEvent(s, NULL, dir, HTTP_DECODER_EVENT_TOO_MANY_WARNINGS);
700  //too noisy in fuzzing
701  //DEBUG_VALIDATE_BUG_ON("Too many libhtp messages");
702  }
703  // ignore further messages
704  return;
705  }
706 
707  for (msg = s->htp_messages_offset; msg < size; msg++) {
708  htp_log_t *log = htp_list_get(s->conn->messages, msg);
709  if (log == NULL)
710  continue;
711 
712  HtpTxUserData *htud = NULL;
713  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
714  if (tx != NULL)
715  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
716 
717  SCLogDebug("message %s", log->msg);
718 
719  uint8_t id = HTPHandleErrorGetId(log->msg);
720  if (id == 0) {
721  id = HTPHandleWarningGetId(log->msg);
722  if (id == 0)
724  }
725 
726  if (id > 0) {
727  HTPSetEvent(s, htud, dir, id);
728  }
729  }
730  s->htp_messages_offset = (uint16_t)msg;
731  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
732 }
733 
734 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
735 {
736 #ifdef DEBUG
737  BUG_ON(s == NULL || tx == NULL);
738 #endif
739  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
740  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
741  HTP_HOSTH_INVALID))
742  {
743  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
744  if (htud == NULL)
745  return;
746 
747  if (tx->flags & HTP_REQUEST_INVALID_T_E)
748  HTPSetEvent(s, htud, STREAM_TOSERVER,
750  if (tx->flags & HTP_REQUEST_INVALID_C_L)
751  HTPSetEvent(s, htud, STREAM_TOSERVER,
753  if (tx->flags & HTP_HOST_MISSING)
754  HTPSetEvent(s, htud, STREAM_TOSERVER,
756  if (tx->flags & HTP_HOST_AMBIGUOUS)
757  HTPSetEvent(s, htud, STREAM_TOSERVER,
759  if (tx->flags & HTP_HOSTU_INVALID)
760  HTPSetEvent(s, htud, STREAM_TOSERVER,
762  if (tx->flags & HTP_HOSTH_INVALID)
763  HTPSetEvent(s, htud, STREAM_TOSERVER,
765  }
766  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
767  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
768  if (htud == NULL)
769  return;
770  HTPSetEvent(s, htud, STREAM_TOSERVER,
772  }
773  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
774  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
775  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
776  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
777  if (htud == NULL)
778  return;
779  HTPSetEvent(s, htud, STREAM_TOSERVER,
781  }
782 }
783 
784 static int Setup(Flow *f, HtpState *hstate)
785 {
786  /* store flow ref in state so callbacks can access it */
787  hstate->f = f;
788 
789  HTPCfgRec *htp_cfg_rec = &cfglist;
790  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
791  void *user_data = NULL;
792 
793  if (FLOW_IS_IPV4(f)) {
794  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
795  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
796  }
797  else if (FLOW_IS_IPV6(f)) {
798  SCLogDebug("Looking up HTP config for ipv6");
799  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
800  }
801  else {
802  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
803  goto error;
804  }
805 
806  if (user_data != NULL) {
807  htp_cfg_rec = user_data;
808  htp = htp_cfg_rec->cfg;
809  SCLogDebug("LIBHTP using config: %p", htp);
810  } else {
811  SCLogDebug("Using default HTP config: %p", htp);
812  }
813 
814  if (NULL == htp) {
815 #ifdef DEBUG_VALIDATION
816  BUG_ON(htp == NULL);
817 #endif
818  /* should never happen if HTPConfigure is properly invoked */
819  goto error;
820  }
821 
822  hstate->connp = htp_connp_create(htp);
823  if (hstate->connp == NULL) {
824  goto error;
825  }
826 
827  hstate->conn = htp_connp_get_connection(hstate->connp);
828 
829  htp_connp_set_user_data(hstate->connp, (void *)hstate);
830  hstate->cfg = htp_cfg_rec;
831 
832  SCLogDebug("New hstate->connp %p", hstate->connp);
833 
834  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
835 
837  htp_cfg_rec->request.inspect_min_size);
839  htp_cfg_rec->response.inspect_min_size);
840  return 0;
841 error:
842  return -1;
843 }
844 
845 /**
846  * \brief Function to handle the reassembled data from client and feed it to
847  * the HTP library to process it.
848  *
849  * \param flow Pointer to the flow the data belong to
850  * \param htp_state Pointer the state in which the parsed value to be stored
851  * \param pstate Application layer parser state for this session
852  *
853  * \retval On success returns 1 or on failure returns -1.
854  */
855 static AppLayerResult HTPHandleRequestData(Flow *f, void *htp_state, AppLayerParserState *pstate,
856  StreamSlice stream_slice, void *local_data)
857 {
858  SCEnter();
859  int ret = 0;
860  HtpState *hstate = (HtpState *)htp_state;
861 
862  /* On the first invocation, create the connection parser structure to
863  * be used by HTP library. This is looked up via IP in the radix
864  * tree. Failing that, the default HTP config is used.
865  */
866  if (NULL == hstate->conn) {
867  if (Setup(f, hstate) != 0) {
869  }
870  }
871  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
872  hstate->slice = &stream_slice;
873 
874  const uint8_t *input = StreamSliceGetData(&stream_slice);
875  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
876 
877  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
878  /* pass the new data to the htp parser */
879  if (input_len > 0) {
880  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
881  switch (r) {
882  case HTP_STREAM_ERROR:
883  ret = -1;
884  break;
885  default:
886  break;
887  }
888  HTPHandleError(hstate, STREAM_TOSERVER);
889  }
890 
891  /* if the TCP connection is closed, then close the HTTP connection */
893  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
894  {
895  htp_connp_req_close(hstate->connp, &ts);
896  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
897  SCLogDebug("stream eof encountered, closing htp handle for ts");
898  }
899 
900  SCLogDebug("hstate->connp %p", hstate->connp);
901  hstate->slice = NULL;
902 
903  if (ret < 0) {
905  }
907 }
908 
909 /**
910  * \brief Function to handle the reassembled data from server and feed it to
911  * the HTP library to process it.
912  *
913  * \param flow Pointer to the flow the data belong to
914  * \param htp_state Pointer the state in which the parsed value to be stored
915  * \param pstate Application layer parser state for this session
916  * \param input Pointer the received HTTP server data
917  * \param input_len Length in bytes of the received data
918  * \param output Pointer to the output (not used in this function)
919  *
920  * \retval On success returns 1 or on failure returns -1
921  */
922 static AppLayerResult HTPHandleResponseData(Flow *f, void *htp_state, AppLayerParserState *pstate,
923  StreamSlice stream_slice, void *local_data)
924 {
925  SCEnter();
926  int ret = 0;
927  HtpState *hstate = (HtpState *)htp_state;
928 
929  const uint8_t *input = StreamSliceGetData(&stream_slice);
930  uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
931 
932  /* On the first invocation, create the connection parser structure to
933  * be used by HTP library. This is looked up via IP in the radix
934  * tree. Failing that, the default HTP config is used.
935  */
936  if (NULL == hstate->conn) {
937  if (Setup(f, hstate) != 0) {
939  }
940  }
941  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
942  hstate->slice = &stream_slice;
943 
944  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
945  htp_tx_t *tx = NULL;
946  size_t consumed = 0;
947  if (input_len > 0) {
948  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
949  switch (r) {
950  case HTP_STREAM_ERROR:
951  ret = -1;
952  break;
953  case HTP_STREAM_TUNNEL:
954  tx = htp_connp_get_out_tx(hstate->connp);
955  if (tx != NULL && tx->response_status_number == 101) {
956  htp_header_t *h =
957  (htp_header_t *)htp_table_get_c(tx->response_headers, "Upgrade");
958  if (h == NULL || bstr_cmp_c(h->value, "h2c") != 0) {
959  break;
960  }
962  // if HTTP2 is disabled, keep the HTP_STREAM_TUNNEL mode
963  break;
964  }
965  uint16_t dp = 0;
966  if (tx->request_port_number != -1) {
967  dp = (uint16_t)tx->request_port_number;
968  }
969  consumed = htp_connp_res_data_consumed(hstate->connp);
970  hstate->slice = NULL;
971  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_HTTP2)) {
972  HTPSetEvent(hstate, NULL, STREAM_TOCLIENT,
974  }
975  // During HTTP2 upgrade, we may consume the HTTP1 part of the data
976  // and we need to parser the remaining part with HTTP2
977  if (consumed > 0 && consumed < input_len) {
978  SCReturnStruct(APP_LAYER_INCOMPLETE(consumed, input_len - consumed));
979  }
981  }
982  break;
983  default:
984  break;
985  }
986  HTPHandleError(hstate, STREAM_TOCLIENT);
987  }
988 
989  /* if we the TCP connection is closed, then close the HTTP connection */
991  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
992  {
993  htp_connp_close(hstate->connp, &ts);
994  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
995  }
996 
997  SCLogDebug("hstate->connp %p", hstate->connp);
998  hstate->slice = NULL;
999 
1000  if (ret < 0) {
1002  }
1004 }
1005 
1006 /**
1007  * \param name /Lowercase/ version of the variable name
1008  */
1009 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
1010  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1011 {
1012 #ifdef PRINT
1013  printf("DATA START: \n");
1014  PrintRawDataFp(stdout, data, len);
1015  printf("DATA END: \n");
1016 #endif
1017  size_t x;
1018  int quote = 0;
1019 
1020  for (x = 0; x < len; x++) {
1021  if (!(isspace(data[x])))
1022  break;
1023  }
1024 
1025  if (x >= len)
1026  return 0;
1027 
1028  uint8_t *line = data+x;
1029  size_t line_len = len-x;
1030  size_t offset = 0;
1031 #ifdef PRINT
1032  printf("LINE START: \n");
1033  PrintRawDataFp(stdout, line, line_len);
1034  printf("LINE END: \n");
1035 #endif
1036  for (x = 0 ; x < line_len; x++) {
1037  if (x > 0) {
1038  if (line[x - 1] != '\\' && line[x] == '\"') {
1039  quote++;
1040  }
1041 
1042  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1043  uint8_t *token = line + offset;
1044  size_t token_len = x - offset;
1045 
1046  if ((x + 1) == line_len) {
1047  token_len++;
1048  }
1049 
1050  offset = x + 1;
1051 
1052  while (offset < line_len && isspace(line[offset])) {
1053  x++;
1054  offset++;
1055  }
1056 #ifdef PRINT
1057  printf("TOKEN START: \n");
1058  PrintRawDataFp(stdout, token, token_len);
1059  printf("TOKEN END: \n");
1060 #endif
1061  if (token_len > name_len) {
1062  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1063  uint8_t *value = token + name_len;
1064  size_t value_len = token_len - name_len;
1065 
1066  if (value[0] == '\"') {
1067  value++;
1068  value_len--;
1069  }
1070  if (value[value_len-1] == '\"') {
1071  value_len--;
1072  }
1073 #ifdef PRINT
1074  printf("VALUE START: \n");
1075  PrintRawDataFp(stdout, value, value_len);
1076  printf("VALUE END: \n");
1077 #endif
1078  *retptr = value;
1079  *retlen = value_len;
1080  return 1;
1081  }
1082  }
1083  }
1084  }
1085  }
1086 
1087  return 0;
1088 }
1089 
1090 /**
1091  * \param name /Lowercase/ version of the variable name
1092  */
1093 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
1094  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
1095 {
1096  SCEnter();
1097 #ifdef PRINT
1098  printf("DATA START: \n");
1099  PrintRawDataFp(stdout, data, len);
1100  printf("DATA END: \n");
1101 #endif
1102  size_t x;
1103  int quote = 0;
1104 
1105  for (x = 0; x < len; x++) {
1106  if (!(isspace(data[x])))
1107  break;
1108  }
1109 
1110  if (x >= len) {
1111  SCReturnInt(0);
1112  }
1113 
1114  uint8_t *line = data+x;
1115  size_t line_len = len-x;
1116  size_t offset = 0;
1117 #ifdef PRINT
1118  printf("LINE START: \n");
1119  PrintRawDataFp(stdout, line, line_len);
1120  printf("LINE END: \n");
1121 #endif
1122  for (x = 0 ; x < line_len; x++) {
1123  if (x > 0) {
1124  if (line[x - 1] != '\\' && line[x] == '\"') {
1125  quote++;
1126  }
1127 
1128  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1129  uint8_t *token = line + offset;
1130  size_t token_len = x - offset;
1131 
1132  if ((x + 1) == line_len) {
1133  token_len++;
1134  }
1135 
1136  offset = x + 1;
1137 
1138  while (offset < line_len && isspace(line[offset])) {
1139  x++;
1140  offset++;
1141  }
1142 #ifdef PRINT
1143  printf("TOKEN START: \n");
1144  PrintRawDataFp(stdout, token, token_len);
1145  printf("TOKEN END: \n");
1146 #endif
1147  if (token_len > name_len) {
1148  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1149  uint8_t *value = token + name_len;
1150  size_t value_len = token_len - name_len;
1151 
1152  if (value[0] == '\"') {
1153  value++;
1154  value_len--;
1155  }
1156  if (value[value_len-1] == '\"') {
1157  value_len--;
1158  }
1159 #ifdef PRINT
1160  printf("VALUE START: \n");
1161  PrintRawDataFp(stdout, value, value_len);
1162  printf("VALUE END: \n");
1163 #endif
1164  *retptr = value;
1165  *retlen = value_len;
1166  SCReturnInt(1);
1167  }
1168  }
1169  }
1170  }
1171  }
1172 
1173  SCReturnInt(0);
1174 }
1175 
1176 /**
1177  * \brief setup multipart parsing: extract boundary and store it
1178  *
1179  * \param d HTTP transaction
1180  * \param htud transaction userdata
1181  *
1182  * \retval 1 ok, multipart set up
1183  * \retval 0 ok, not multipart though
1184  * \retval -1 error: problem with the boundary
1185  *
1186  * If the request contains a multipart message, this function will
1187  * set the HTP_BOUNDARY_SET in the transaction.
1188  */
1189 static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
1190 {
1191  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
1192  "Content-Type");
1193  if (h != NULL && bstr_len(h->value) > 0) {
1194  uint8_t *boundary = NULL;
1195  size_t boundary_len = 0;
1196 
1197  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1198  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1199  &boundary, &boundary_len);
1200  if (r == 1) {
1201 #ifdef PRINT
1202  printf("BOUNDARY START: \n");
1203  PrintRawDataFp(stdout, boundary, boundary_len);
1204  printf("BOUNDARY END: \n");
1205 #endif
1206  if (boundary_len < HTP_BOUNDARY_MAX) {
1207  htud->boundary = HTPMalloc(boundary_len);
1208  if (htud->boundary == NULL) {
1209  return -1;
1210  }
1211  htud->boundary_len = (uint8_t)boundary_len;
1212  memcpy(htud->boundary, boundary, boundary_len);
1213 
1214  htud->tsflags |= HTP_BOUNDARY_SET;
1215  } else {
1216  SCLogDebug("invalid boundary");
1217  return -1;
1218  }
1219  SCReturnInt(1);
1220  }
1221  //SCReturnInt(1);
1222  }
1223  SCReturnInt(0);
1224 }
1225 
1226 #define C_D_HDR "content-disposition:"
1227 #define C_D_HDR_LEN 20
1228 #define C_T_HDR "content-type:"
1229 #define C_T_HDR_LEN 13
1231 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1232  HtpTxUserData *htud,
1233  uint8_t *header, uint32_t header_len,
1234  uint8_t **filename, uint16_t *filename_len,
1235  uint8_t **filetype, uint16_t *filetype_len)
1236 {
1237  uint8_t *fn = NULL;
1238  size_t fn_len = 0;
1239  uint8_t *ft = NULL;
1240  size_t ft_len = 0;
1241 
1242 #ifdef PRINT
1243  printf("HEADER START: \n");
1244  PrintRawDataFp(stdout, header, header_len);
1245  printf("HEADER END: \n");
1246 #endif
1247 
1248  while (header_len > 0) {
1249  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1250  uint8_t *line = header;
1251  uint32_t line_len;
1252 
1253  if (next_line == NULL) {
1254  line_len = header_len;
1255  } else {
1256  line_len = next_line - header;
1257  }
1258  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1259  if (sc == NULL) {
1260  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1262  /* if the : we found is the final char, it means we have
1263  * no value */
1264  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1265  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1267  } else {
1268 #ifdef PRINT
1269  printf("LINE START: \n");
1270  PrintRawDataFp(stdout, line, line_len);
1271  printf("LINE END: \n");
1272 #endif
1273  if (line_len >= C_D_HDR_LEN &&
1274  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1275  uint8_t *value = line + C_D_HDR_LEN;
1276  uint32_t value_len = line_len - C_D_HDR_LEN;
1277 
1278  /* parse content-disposition */
1279  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1280  value, value_len, &fn, &fn_len);
1281  } else if (line_len >= C_T_HDR_LEN &&
1282  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1283  SCLogDebug("content-type line");
1284  uint8_t *value = line + C_T_HDR_LEN;
1285  uint32_t value_len = line_len - C_T_HDR_LEN;
1286 
1287  (void)HTTPParseContentTypeHeader(NULL, 0,
1288  value, value_len, &ft, &ft_len);
1289  }
1290  }
1291 
1292  if (next_line == NULL) {
1293  SCLogDebug("no next_line");
1294  break;
1295  }
1296  header_len -= ((next_line + 2) - header);
1297  header = next_line + 2;
1298  } /* while (header_len > 0) */
1299 
1300  if (fn_len > USHRT_MAX)
1301  fn_len = USHRT_MAX;
1302  if (ft_len > USHRT_MAX)
1303  ft_len = USHRT_MAX;
1304 
1305  *filename = fn;
1306  *filename_len = (uint16_t)fn_len;
1307  *filetype = ft;
1308  *filetype_len = (uint16_t)ft_len;
1309 }
1310 
1311 /**
1312  * \brief Create a single buffer from the HtpBodyChunks in our list
1313  *
1314  * \param htud transaction user data
1315  * \param chunks_buffers pointer to pass back the buffer to the caller
1316  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1317  */
1318 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1319  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1320 {
1322  chunks_buffer, chunks_buffer_len,
1323  htud->request_body.body_parsed);
1324 }
1325 
1326 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1327 {
1328  SCEnter();
1329  if (tx && tx->tx_data.de_state) {
1330  if (dir == STREAM_TOSERVER) {
1331  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1332  tx->tx_data.de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1333  } else if (STREAM_TOCLIENT) {
1334  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1335  tx->tx_data.de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1336  }
1337  }
1338 }
1339 
1340 /**
1341  * \brief Setup boundary buffers
1342  */
1343 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1344  uint8_t *boundary, uint32_t boundary_len)
1345 {
1346  memset(boundary, '-', boundary_len);
1347  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1348 }
1349 
1350 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1351  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1352 {
1353  int result = 0;
1354  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1355  uint16_t expected_boundary_len = htud->boundary_len + 2;
1356  uint16_t expected_boundary_end_len = htud->boundary_len + 4;
1357  int tx_progress = 0;
1358 
1359 #ifdef PRINT
1360  printf("CHUNK START: \n");
1361  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1362  printf("CHUNK END: \n");
1363 #endif
1364 
1365  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1366 
1367  /* search for the header start, header end and form end */
1368  const uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1369  boundary, expected_boundary_len);
1370  /* end of the multipart form */
1371  const uint8_t *form_end = NULL;
1372  /* end marker belonging to header_start */
1373  const uint8_t *header_end = NULL;
1374  if (header_start != NULL) {
1375  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1376  (uint8_t *)"\r\n\r\n", 4);
1377  form_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1378  boundary, expected_boundary_end_len);
1379  }
1380 
1381  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1382  header_end, form_end);
1383 
1384  /* we currently only handle multipart for ts. When we support it for tc,
1385  * we will need to supply right direction */
1386  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, STREAM_TOSERVER);
1387  /* if we're in the file storage process, deal with that now */
1388  if (htud->tsflags & HTP_FILENAME_SET) {
1389  if (header_start != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1390  SCLogDebug("reached the end of the file");
1391 
1392  const uint8_t *filedata = chunks_buffer;
1393  uint32_t filedata_len = 0;
1394  uint8_t flags = 0;
1395 
1396  if (header_start != NULL) {
1397  if (header_start == filedata + 2) {
1398  /* last chunk had all data, but not the boundary */
1399  SCLogDebug("last chunk had all data, but not the boundary");
1400  filedata_len = 0;
1401  } else if (header_start > filedata + 2) {
1402  SCLogDebug("some data from last file before the boundary");
1403  /* some data from last file before the boundary */
1404  filedata_len = header_start - filedata - 2;
1405  }
1406  }
1407  /* body parsing done, we did not get our form end. Use all data
1408  * we still have and signal to files API we have an issue. */
1409  if (tx_progress > HTP_REQUEST_BODY) {
1410  filedata_len = chunks_buffer_len;
1412  }
1413 
1414  if (filedata_len > chunks_buffer_len) {
1415  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1417  goto end;
1418  }
1419 #ifdef PRINT
1420  printf("FILEDATA (final chunk) START: \n");
1421  PrintRawDataFp(stdout, filedata, filedata_len);
1422  printf("FILEDATA (final chunk) END: \n");
1423 #endif
1424  if (!(htud->tsflags & HTP_DONTSTORE)) {
1425  if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
1426  -1) {
1427  goto end;
1428  }
1429  }
1430 
1431  htud->tsflags &=~ HTP_FILENAME_SET;
1432 
1433  /* fall through */
1434  } else {
1435  SCLogDebug("not yet at the end of the file");
1436 
1437  if (chunks_buffer_len > expected_boundary_end_len) {
1438  const uint8_t *filedata = chunks_buffer;
1439  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1440 #ifdef PRINT
1441  printf("FILEDATA (part) START: \n");
1442  PrintRawDataFp(stdout, filedata, filedata_len);
1443  printf("FILEDATA (part) END: \n");
1444 #endif
1445 
1446  if (!(htud->tsflags & HTP_DONTSTORE)) {
1447  result = HTPFileStoreChunk(hstate, filedata,
1448  filedata_len, STREAM_TOSERVER);
1449  if (result == -1) {
1450  goto end;
1451  } else if (result == -2) {
1452  /* we know for sure we're not storing the file */
1453  htud->tsflags |= HTP_DONTSTORE;
1454  }
1455  }
1456 
1457  htud->request_body.body_parsed += filedata_len;
1458  } else {
1459  SCLogDebug("chunk too small to already process in part");
1460  }
1461 
1462  goto end;
1463  }
1464  }
1465 
1466  while (header_start != NULL && header_end != NULL &&
1467  header_end != form_end &&
1468  header_start < (chunks_buffer + chunks_buffer_len) &&
1469  header_end < (chunks_buffer + chunks_buffer_len) &&
1470  header_start < header_end)
1471  {
1472  uint8_t *filename = NULL;
1473  uint16_t filename_len = 0;
1474  uint8_t *filetype = NULL;
1475  uint16_t filetype_len = 0;
1476 
1477  uint32_t header_len = header_end - header_start;
1478  SCLogDebug("header_len %u", header_len);
1479  uint8_t *header = (uint8_t *)header_start;
1480 
1481  /* skip empty records */
1482  if (expected_boundary_len == header_len) {
1483  goto next;
1484  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1485  header_len -= (expected_boundary_len + 2);
1486  header = (uint8_t *)header_start + (expected_boundary_len + 2); // + for 0d 0a
1487  }
1488 
1489  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1490  &filename, &filename_len, &filetype, &filetype_len);
1491 
1492  if (filename != NULL) {
1493  const uint8_t *filedata = NULL;
1494  uint32_t filedata_len = 0;
1495 
1496  SCLogDebug("we have a filename");
1497 
1498  htud->tsflags |= HTP_FILENAME_SET;
1499  htud->tsflags &= ~HTP_DONTSTORE;
1500 
1501  SCLogDebug("header_end %p", header_end);
1502  SCLogDebug("form_end %p", form_end);
1503 
1504  /* everything until the final boundary is the file */
1505  if (form_end != NULL) {
1506  SCLogDebug("have form_end");
1507 
1508  filedata = header_end + 4;
1509  if (form_end == filedata) {
1510  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1512  goto end;
1513  } else if (form_end < filedata) {
1514  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1516  goto end;
1517  }
1518 
1519  filedata_len = form_end - (header_end + 4 + 2);
1520  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1521 
1522  /* or is it? */
1523  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1524  boundary, expected_boundary_len);
1525  if (header_next != NULL) {
1526  filedata_len -= (form_end - header_next);
1527  }
1528 
1529  if (filedata_len > chunks_buffer_len) {
1530  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1532  goto end;
1533  }
1534  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1535 #ifdef PRINT
1536  printf("FILEDATA START: \n");
1537  PrintRawDataFp(stdout, filedata, filedata_len);
1538  printf("FILEDATA END: \n");
1539 #endif
1540 
1541  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata, filedata_len,
1542  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1543  if (result == -1) {
1544  goto end;
1545  } else if (result == -2) {
1546  htud->tsflags |= HTP_DONTSTORE;
1547  } else {
1548  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1549  goto end;
1550  }
1551  }
1552  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1553 
1554  htud->request_body.body_parsed += (header_end - chunks_buffer);
1555  htud->tsflags &= ~HTP_FILENAME_SET;
1556  } else {
1557  SCLogDebug("chunk doesn't contain form end");
1558 
1559  filedata = header_end + 4;
1560  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1561  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1562 
1563  if (filedata_len > chunks_buffer_len) {
1564  HTPSetEvent(hstate, htud, STREAM_TOSERVER,
1566  goto end;
1567  }
1568 
1569 #ifdef PRINT
1570  printf("FILEDATA START: \n");
1571  PrintRawDataFp(stdout, filedata, filedata_len);
1572  printf("FILEDATA END: \n");
1573 #endif
1574  /* form doesn't end in this chunk, but the part might. Lets
1575  * see if have another coming up */
1576  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1577  boundary, expected_boundary_len);
1578  SCLogDebug("header_next %p", header_next);
1579  if (header_next == NULL) {
1580  SCLogDebug("more file data to come");
1581 
1582  uint32_t offset = (header_end + 4) - chunks_buffer;
1583  SCLogDebug("offset %u", offset);
1584  htud->request_body.body_parsed += offset;
1585 
1586  if (filedata_len >= (uint32_t)(expected_boundary_len + 2)) {
1587  filedata_len -= (expected_boundary_len + 2 - 1);
1588  SCLogDebug("opening file with partial data");
1589  } else {
1590  filedata = NULL;
1591  filedata_len = 0;
1592  }
1593  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1594  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1595  if (result == -1) {
1596  goto end;
1597  } else if (result == -2) {
1598  htud->tsflags |= HTP_DONTSTORE;
1599  }
1600  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1601  htud->request_body.body_parsed += filedata_len;
1602  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1603 
1604  } else if (header_next - filedata > 2) {
1605  filedata_len = header_next - filedata - 2;
1606  SCLogDebug("filedata_len %u", filedata_len);
1607 
1608  result = HTPFileOpen(hstate, htud, filename, filename_len, filedata,
1609  filedata_len, HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1610  if (result == -1) {
1611  goto end;
1612  } else if (result == -2) {
1613  htud->tsflags |= HTP_DONTSTORE;
1614  } else {
1615  if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1616  goto end;
1617  }
1618  }
1619  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1620 
1621  htud->tsflags &= ~HTP_FILENAME_SET;
1622  htud->request_body.body_parsed += (header_end - chunks_buffer);
1623  }
1624  }
1625  }
1626 next:
1627  SCLogDebug("header_start %p, header_end %p, form_end %p",
1628  header_start, header_end, form_end);
1629 
1630  /* Search next boundary entry after the start of body */
1631  uint32_t cursizeread = header_end - chunks_buffer;
1632  header_start = Bs2bmSearch(header_end + 4,
1633  chunks_buffer_len - (cursizeread + 4),
1634  boundary, expected_boundary_len);
1635  if (header_start != NULL) {
1636  header_end = Bs2bmSearch(header_end + 4,
1637  chunks_buffer_len - (cursizeread + 4),
1638  (uint8_t *) "\r\n\r\n", 4);
1639  }
1640  }
1641 
1642  /* if we're parsing the multipart and we're not currently processing a
1643  * file, we move the body pointer forward. */
1644  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1645  if (chunks_buffer_len > expected_boundary_end_len) {
1646  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1647 
1648  htud->request_body.body_parsed += move;
1649  SCLogDebug("form not ready, file not set, parsing non-file "
1650  "record: moved %u", move);
1651  }
1652  }
1653 
1654 end:
1655  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1656  return 0;
1657 }
1658 
1659 /** \internal
1660  * \brief Handle POST or PUT, no multipart body data
1661  */
1662 static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
1663  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1664 {
1665  int result = 0;
1666 
1667  /* see if we need to open the file */
1668  if (!(htud->tsflags & HTP_FILENAME_SET))
1669  {
1670  uint8_t *filename = NULL;
1671  size_t filename_len = 0;
1672 
1673  /* get the name */
1674  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1675  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1676  filename_len = bstr_len(tx->parsed_uri->path);
1677  }
1678 
1679  if (filename != NULL) {
1680  if (filename_len > SC_FILENAME_MAX) {
1681  // explicitly truncate the file name if too long
1682  filename_len = SC_FILENAME_MAX;
1683  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1684  }
1685  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1686  HtpGetActiveRequestTxID(hstate), STREAM_TOSERVER);
1687  if (result == -1) {
1688  goto end;
1689  } else if (result == -2) {
1690  htud->tsflags |= HTP_DONTSTORE;
1691  } else {
1692  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1693  htud->tsflags |= HTP_FILENAME_SET;
1694  htud->tsflags &= ~HTP_DONTSTORE;
1695  }
1696  }
1697  }
1698  else
1699  {
1700  /* otherwise, just store the data */
1701 
1702  if (!(htud->tsflags & HTP_DONTSTORE)) {
1703  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1704  if (result == -1) {
1705  goto end;
1706  } else if (result == -2) {
1707  /* we know for sure we're not storing the file */
1708  htud->tsflags |= HTP_DONTSTORE;
1709  }
1710  }
1711  }
1712 
1713  return 0;
1714 end:
1715  return -1;
1716 }
1717 
1718 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1719  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1720 {
1721  SCEnter();
1722 
1723  int result = 0;
1724 
1725  /* see if we need to open the file
1726  * we check for tx->response_line in case of junk
1727  * interpreted as body before response line
1728  */
1729  if (!(htud->tcflags & HTP_FILENAME_SET) &&
1730  (tx->response_line != NULL || tx->is_protocol_0_9))
1731  {
1732  SCLogDebug("setting up file name");
1733 
1734  uint8_t *filename = NULL;
1735  size_t filename_len = 0;
1736 
1737  /* try Content-Disposition header first */
1738  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1739  "Content-Disposition");
1740  if (h != NULL && bstr_len(h->value) > 0) {
1741  /* parse content-disposition */
1742  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1743  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1744  }
1745 
1746  /* fall back to name from the uri */
1747  if (filename == NULL) {
1748  /* get the name */
1749  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1750  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1751  filename_len = bstr_len(tx->parsed_uri->path);
1752  }
1753  }
1754 
1755  if (filename != NULL) {
1756  // set range if present
1757  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1758  if (filename_len > SC_FILENAME_MAX) {
1759  // explicitly truncate the file name if too long
1760  filename_len = SC_FILENAME_MAX;
1761  HTPSetEvent(hstate, htud, STREAM_TOSERVER, HTTP_DECODER_EVENT_FILE_NAME_TOO_LONG);
1762  }
1763  if (h_content_range != NULL) {
1764  result = HTPFileOpenWithRange(hstate, htud, filename, (uint16_t)filename_len, data,
1765  data_len, HtpGetActiveResponseTxID(hstate), h_content_range->value, htud);
1766  } else {
1767  result = HTPFileOpen(hstate, htud, filename, (uint16_t)filename_len, data, data_len,
1768  HtpGetActiveResponseTxID(hstate), STREAM_TOCLIENT);
1769  }
1770  SCLogDebug("result %d", result);
1771  if (result == -1) {
1772  goto end;
1773  } else if (result == -2) {
1774  htud->tcflags |= HTP_DONTSTORE;
1775  } else {
1776  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1777  htud->tcflags |= HTP_FILENAME_SET;
1778  htud->tcflags &= ~HTP_DONTSTORE;
1779  }
1780  }
1781  }
1782  else if (tx->response_line != NULL || tx->is_protocol_0_9)
1783  {
1784  /* otherwise, just store the data */
1785 
1786  if (!(htud->tcflags & HTP_DONTSTORE)) {
1787  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1788  SCLogDebug("result %d", result);
1789  if (result == -1) {
1790  goto end;
1791  } else if (result == -2) {
1792  /* we know for sure we're not storing the file */
1793  htud->tcflags |= HTP_DONTSTORE;
1794  }
1795  }
1796  }
1797 
1798  htud->response_body.body_parsed += data_len;
1799  return 0;
1800 end:
1801  return -1;
1802 }
1803 
1804 /**
1805  * \brief Function callback to append chunks for Requests
1806  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1807  * \retval int HTP_OK if all goes well
1808  */
1809 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1810 {
1811  SCEnter();
1812 
1813  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1814  SCReturnInt(HTP_OK);
1815 
1816  if (d->len == 0)
1817  SCReturnInt(HTP_OK);
1818 
1819 #ifdef PRINT
1820  printf("HTPBODY START: \n");
1821  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1822  printf("HTPBODY END: \n");
1823 #endif
1824 
1825  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1826  if (hstate == NULL) {
1827  SCReturnInt(HTP_ERROR);
1828  }
1829 
1830  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1831  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1832 
1833  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1834  if (tx_ud == NULL) {
1835  SCReturnInt(HTP_OK);
1836  }
1837  if (!tx_ud->response_body_init) {
1838  tx_ud->response_body_init = 1;
1839 
1840  if (d->tx->request_method_number == HTP_M_POST) {
1841  SCLogDebug("POST");
1842  int r = HtpRequestBodySetupMultipart(d->tx, tx_ud);
1843  if (r == 1) {
1845  } else if (r == 0) {
1847  SCLogDebug("not multipart");
1848  }
1849  } else if (d->tx->request_method_number == HTP_M_PUT) {
1851  }
1852  }
1853 
1854  /* see if we can get rid of htp body chunks */
1855  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1856 
1857  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1858  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1859 
1860  /* within limits, add the body chunk to the state. */
1861  if (AppLayerHtpCheckDepth(&hstate->cfg->request, &tx_ud->request_body, tx_ud->tsflags)) {
1862  uint32_t stream_depth = FileReassemblyDepth();
1863  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->request_body.content_len_so_far,
1864  hstate->cfg->request.body_limit,
1865  stream_depth,
1866  tx_ud->tsflags,
1867  (uint32_t)d->len);
1868  BUG_ON(len > (uint32_t)d->len);
1869 
1870  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1871 
1872  const uint8_t *chunks_buffer = NULL;
1873  uint32_t chunks_buffer_len = 0;
1874 
1876  /* multi-part body handling starts here */
1877  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1878  goto end;
1879  }
1880 
1881  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1882  if (chunks_buffer == NULL) {
1883  goto end;
1884  }
1885 #ifdef PRINT
1886  printf("REASSCHUNK START: \n");
1887  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1888  printf("REASSCHUNK END: \n");
1889 #endif
1890 
1891  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1892 
1893  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST ||
1895  HtpRequestBodyHandlePOSTorPUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1896  }
1897 
1898  } else {
1899  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1900  SCLogDebug("closing file that was being stored");
1901  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1902  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1903  }
1904  }
1905 
1906 end:
1907  if (hstate->conn != NULL) {
1908  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1910  hstate->cfg->request.inspect_min_size,
1911  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1912 
1913  /* if we reach the inspect_min_size we'll trigger inspection,
1914  * so make sure that raw stream is also inspected. Set the
1915  * data to be used to the amount of raw bytes we've seen to
1916  * get here. */
1917  if (tx_ud->request_body.body_inspected == 0 &&
1919  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1920  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1921  {
1922  const uint32_t data_size = (uint32_t)(
1923  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1924  const uint32_t depth = MIN(data_size, hstate->cfg->request.inspect_min_size);
1925 
1926  /* body still in progress, but due to min inspect size we need to inspect now */
1927  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, depth);
1928  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1929  }
1930  /* after the start of the body, disable the depth logic */
1931  } else if (tx_ud->request_body.body_inspected > 0) {
1932  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1933  }
1934  }
1935  SCReturnInt(HTP_OK);
1936 }
1937 
1938 /**
1939  * \brief Function callback to append chunks for Responses
1940  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1941  * \retval int HTP_OK if all goes well
1942  */
1943 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1944 {
1945  SCEnter();
1946 
1947  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1948  SCReturnInt(HTP_OK);
1949 
1950  if (d->len == 0)
1951  SCReturnInt(HTP_OK);
1952 
1953  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1954  if (hstate == NULL) {
1955  SCReturnInt(HTP_ERROR);
1956  }
1957 
1958  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1959  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1960 
1961  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1962  if (tx_ud == NULL) {
1963  SCReturnInt(HTP_OK);
1964  }
1965  if (!tx_ud->request_body_init) {
1966  tx_ud->request_body_init = 1;
1967  }
1968 
1969  /* see if we can get rid of htp body chunks */
1970  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1971 
1972  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1973  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1974 
1975  /* within limits, add the body chunk to the state. */
1976  if (AppLayerHtpCheckDepth(&hstate->cfg->response, &tx_ud->response_body, tx_ud->tcflags)) {
1977  uint32_t stream_depth = FileReassemblyDepth();
1978  uint32_t len = AppLayerHtpComputeChunkLength(tx_ud->response_body.content_len_so_far,
1979  hstate->cfg->response.body_limit,
1980  stream_depth,
1981  tx_ud->tcflags,
1982  (uint32_t)d->len);
1983  BUG_ON(len > (uint32_t)d->len);
1984 
1985  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1986 
1987  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, len);
1988  } else {
1989  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1990  SCLogDebug("closing file that was being stored");
1991  (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1992  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1993  }
1994  }
1995 
1996  if (hstate->conn != NULL) {
1997  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1999  hstate->cfg->response.inspect_min_size,
2000  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
2001  /* if we reach the inspect_min_size we'll trigger inspection,
2002  * so make sure that raw stream is also inspected. Set the
2003  * data to be used to the amount of raw bytes we've seen to
2004  * get here. */
2005  if (tx_ud->response_body.body_inspected == 0 &&
2007  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
2008  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
2009  {
2010  const uint32_t data_size = (uint32_t)((uint64_t)hstate->conn->out_data_counter -
2011  hstate->last_response_data_stamp);
2012  const uint32_t depth = MIN(data_size, hstate->cfg->response.inspect_min_size);
2013 
2014  /* body still in progress, but due to min inspect size we need to inspect now */
2015  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, depth);
2016  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2017  }
2018  /* after the start of the body, disable the depth logic */
2019  } else if (tx_ud->response_body.body_inspected > 0) {
2020  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
2021  }
2022  }
2023  SCReturnInt(HTP_OK);
2024 }
2025 
2026 /**
2027  * \brief Print the stats of the HTTP requests
2028  */
2030 {
2031 #ifdef DEBUG
2032  SCEnter();
2033  SCMutexLock(&htp_state_mem_lock);
2034  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
2035  htp_state_memcnt, htp_state_memuse);
2036  SCMutexUnlock(&htp_state_mem_lock);
2037  SCReturn;
2038 #endif
2039 }
2040 
2041 /** \brief Clears the HTTP server configuration memory used by HTP library */
2042 void HTPFreeConfig(void)
2043 {
2044  SCEnter();
2045 
2046  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
2047  !AppLayerParserConfParserEnabled("tcp", "http"))
2048  {
2049  SCReturn;
2050  }
2051 
2052  HTPCfgRec *nextrec = cfglist.next;
2053  SCRadixReleaseRadixTree(cfgtree);
2054  cfgtree = NULL;
2055  htp_config_destroy(cfglist.cfg);
2056  while (nextrec != NULL) {
2057  HTPCfgRec *htprec = nextrec;
2058  nextrec = nextrec->next;
2059 
2060  htp_config_destroy(htprec->cfg);
2061  SCFree(htprec);
2062  }
2063  SCReturn;
2064 }
2065 
2066 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
2067 {
2068  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2069  if (htud != NULL) {
2070  htud->request_has_trailers = 1;
2071  }
2072  return HTP_OK;
2073 }
2074 
2075 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2076 {
2077  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2078  if (htud != NULL) {
2079  htud->response_has_trailers = 1;
2080  }
2081  return HTP_OK;
2082 }
2083 
2084 /**\internal
2085  * \brief called at start of request
2086  * Set min inspect size.
2087  */
2088 static int HTPCallbackRequestStart(htp_tx_t *tx)
2089 {
2090  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2091  if (hstate == NULL) {
2092  SCReturnInt(HTP_ERROR);
2093  }
2094 
2095  uint64_t consumed = hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2096  SCLogDebug("HTTP request start: data offset %" PRIu64 ", in_data_counter %" PRIu64, consumed,
2097  (uint64_t)hstate->conn->in_data_counter);
2098 
2099  /* app-layer-frame-documentation tag start: frame registration http request */
2101  hstate->f, hstate->slice, consumed, -1, 0, HTTP_FRAME_REQUEST);
2102  if (frame) {
2103  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2104  hstate->request_frame_id = frame->id;
2105  AppLayerFrameSetTxId(frame, HtpGetActiveRequestTxID(hstate));
2106  }
2107  /* app-layer-frame-documentation tag end: frame registration http request */
2108 
2109  if (hstate->cfg)
2110  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2111  hstate->cfg->request.inspect_min_size);
2112 
2113  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2114  if (tx_ud == NULL) {
2115  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2116  if (unlikely(tx_ud == NULL)) {
2117  SCReturnInt(HTP_OK);
2118  }
2119  htp_tx_set_user_data(tx, tx_ud);
2120  }
2121  SCReturnInt(HTP_OK);
2122 }
2123 
2124 /**\internal
2125  * \brief called at start of response
2126  * Set min inspect size.
2127  */
2128 static int HTPCallbackResponseStart(htp_tx_t *tx)
2129 {
2130  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2131  if (hstate == NULL) {
2132  SCReturnInt(HTP_ERROR);
2133  }
2134 
2135  uint64_t consumed = hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2136  SCLogDebug("HTTP response start: data offset %" PRIu64 ", out_data_counter %" PRIu64, consumed,
2137  (uint64_t)hstate->conn->out_data_counter);
2138 
2140  hstate->f, hstate->slice, consumed, -1, 1, HTTP_FRAME_RESPONSE);
2141  if (frame) {
2142  SCLogDebug("frame %p/%" PRIi64, frame, frame->id);
2143  hstate->response_frame_id = frame->id;
2144  AppLayerFrameSetTxId(frame, HtpGetActiveResponseTxID(hstate));
2145  }
2146 
2147  if (hstate->cfg)
2148  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
2149  hstate->cfg->response.inspect_min_size);
2150 
2151  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2152  if (tx_ud == NULL) {
2153  tx_ud = HTPCalloc(1, sizeof(HtpTxUserData));
2154  if (unlikely(tx_ud == NULL)) {
2155  SCReturnInt(HTP_OK);
2156  }
2157  htp_tx_set_user_data(tx, tx_ud);
2158  }
2159  SCReturnInt(HTP_OK);
2160 }
2161 
2162 
2163 /**
2164  * \brief callback for request to store the recent incoming request
2165  in to the recent_in_tx for the given htp state
2166  * \param connp pointer to the current connection parser which has the htp
2167  * state in it as user data
2168  */
2169 static int HTPCallbackRequestComplete(htp_tx_t *tx)
2170 {
2171  SCEnter();
2172 
2173  if (tx == NULL) {
2174  SCReturnInt(HTP_ERROR);
2175  }
2176 
2177  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2178  if (hstate == NULL) {
2179  SCReturnInt(HTP_ERROR);
2180  }
2181 
2182  const uint64_t abs_right_edge =
2183  hstate->slice->offset + htp_connp_req_data_consumed(hstate->connp);
2184 
2185  /* app-layer-frame-documentation tag start: updating frame->len */
2186  if (hstate->request_frame_id > 0) {
2187  Frame *frame = AppLayerFrameGetById(hstate->f, 0, hstate->request_frame_id);
2188  if (frame) {
2189  const uint64_t request_size = abs_right_edge - hstate->last_request_data_stamp;
2190 
2191  SCLogDebug("HTTP request complete: data offset %" PRIu64 ", request_size %" PRIu64,
2192  hstate->last_request_data_stamp, request_size);
2193  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2194  request_size);
2195  frame->len = (int64_t)request_size;
2196  /* app-layer-frame-documentation tag end: updating frame->len */
2197  }
2198  hstate->request_frame_id = 0;
2199  }
2200 
2201  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2202  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2203 
2204  SCLogDebug("HTTP request completed");
2205 
2206  HTPErrorCheckTxRequestFlags(hstate, tx);
2207 
2208  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2209  if (htud != NULL) {
2210  if (htud->tsflags & HTP_FILENAME_SET) {
2211  SCLogDebug("closing file that was being stored");
2212  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
2213  htud->tsflags &= ~HTP_FILENAME_SET;
2214  if (abs_right_edge < (uint64_t)UINT32_MAX) {
2216  hstate->f->protoctx, STREAM_TOSERVER, (uint32_t)abs_right_edge);
2217  }
2218  }
2219  }
2220 
2221  hstate->last_request_data_stamp = abs_right_edge;
2222  /* request done, do raw reassembly now to inspect state and stream
2223  * at the same time. */
2224  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
2225  SCReturnInt(HTP_OK);
2226 }
2227 
2228 /**
2229  * \brief callback for response to remove the recent received requests
2230  from the recent_in_tx for the given htp state
2231  * \param connp pointer to the current connection parser which has the htp
2232  * state in it as user data
2233  */
2234 static int HTPCallbackResponseComplete(htp_tx_t *tx)
2235 {
2236  SCEnter();
2237 
2238  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2239  if (hstate == NULL) {
2240  SCReturnInt(HTP_ERROR);
2241  }
2242 
2243  /* we have one whole transaction now */
2244  hstate->transaction_cnt++;
2245 
2246  const uint64_t abs_right_edge =
2247  hstate->slice->offset + htp_connp_res_data_consumed(hstate->connp);
2248 
2249  if (hstate->response_frame_id > 0) {
2250  Frame *frame = AppLayerFrameGetById(hstate->f, 1, hstate->response_frame_id);
2251  if (frame) {
2252  const uint64_t response_size = abs_right_edge - hstate->last_response_data_stamp;
2253 
2254  SCLogDebug("HTTP response complete: data offset %" PRIu64 ", response_size %" PRIu64,
2255  hstate->last_response_data_stamp, response_size);
2256  SCLogDebug("frame %p/%" PRIi64 " setting len to %" PRIu64, frame, frame->id,
2257  response_size);
2258  frame->len = (int64_t)response_size;
2259  }
2260  hstate->response_frame_id = 0;
2261  }
2262 
2263  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2264  if (htud != NULL) {
2265  if (htud->tcflags & HTP_FILENAME_SET) {
2266  SCLogDebug("closing file that was being stored");
2267  (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
2268  htud->tcflags &= ~HTP_FILENAME_SET;
2269  }
2270  }
2271 
2272  /* response done, do raw reassembly now to inspect state and stream
2273  * at the same time. */
2274  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2275 
2276  /* handle HTTP CONNECT */
2277  if (tx->request_method_number == HTP_M_CONNECT) {
2278  /* any 2XX status response implies that the connection will become
2279  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2280  if ((tx->response_status_number >= 200) &&
2281  (tx->response_status_number < 300) &&
2282  (hstate->transaction_cnt == 1)) {
2283  uint16_t dp = 0;
2284  if (tx->request_port_number != -1) {
2285  dp = (uint16_t)tx->request_port_number;
2286  }
2287  // both ALPROTO_HTTP1 and ALPROTO_TLS are normal options
2288  if (!AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN)) {
2289  HTPSetEvent(
2290  hstate, htud, STREAM_TOCLIENT, HTTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE);
2291  }
2292  tx->request_progress = HTP_REQUEST_COMPLETE;
2293  tx->response_progress = HTP_RESPONSE_COMPLETE;
2294  }
2295  }
2296 
2297  hstate->last_response_data_stamp = abs_right_edge;
2298  SCReturnInt(HTP_OK);
2299 }
2300 
2301 static int HTPCallbackRequestLine(htp_tx_t *tx)
2302 {
2303  HtpTxUserData *tx_ud;
2304  bstr *request_uri_normalized;
2305  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2306  const HTPCfgRec *cfg = hstate->cfg;
2307 
2308  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2309  if (request_uri_normalized == NULL)
2310  return HTP_OK;
2311 
2312  tx_ud = htp_tx_get_user_data(tx);
2313  if (unlikely(tx_ud == NULL)) {
2314  return HTP_OK;
2315  }
2316  if (unlikely(tx_ud->request_uri_normalized != NULL))
2317  bstr_free(tx_ud->request_uri_normalized);
2318  tx_ud->request_uri_normalized = request_uri_normalized;
2319 
2320  if (tx->flags) {
2321  HTPErrorCheckTxRequestFlags(hstate, tx);
2322  }
2323  return HTP_OK;
2324 }
2325 
2326 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2327 {
2328  if (part == NULL)
2329  return HTP_OK;
2330 
2331  uint64_t flags = 0;
2332  size_t prevlen = bstr_len(part);
2333  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2334  // shorter string means that uri was encoded
2335  if (res == HTP_OK && prevlen > bstr_len(part)) {
2336  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2337  if (htud == NULL)
2338  return HTP_OK;
2339  HtpState *s = htp_connp_get_user_data(tx->connp);
2340  if (s == NULL)
2341  return HTP_OK;
2342  HTPSetEvent(s, htud, STREAM_TOSERVER,
2344  }
2345 
2346  return HTP_OK;
2347 }
2348 
2349 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2350 {
2351  if (tx->parsed_uri == NULL)
2352  return HTP_OK;
2353 
2354  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2355 }
2356 
2357 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2358 {
2359  if (tx->parsed_uri == NULL)
2360  return HTP_OK;
2361 
2362  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2363 }
2364 
2365 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2366 {
2367  void *ptmp;
2368  if (tx_data->len == 0 || tx_data->tx == NULL)
2369  return HTP_OK;
2370 
2371  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2372  if (tx_ud == NULL) {
2373  return HTP_OK;
2374  }
2375  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2376  tx_ud->request_headers_raw_len,
2377  tx_ud->request_headers_raw_len + tx_data->len);
2378  if (ptmp == NULL) {
2379  return HTP_OK;
2380  }
2381  tx_ud->request_headers_raw = ptmp;
2382 
2383  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2384  tx_data->data, tx_data->len);
2385  tx_ud->request_headers_raw_len += tx_data->len;
2386 
2387  if (tx_data->tx && tx_data->tx->flags) {
2388  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2389  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2390  }
2391  return HTP_OK;
2392 }
2393 
2394 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2395 {
2396  void *ptmp;
2397  if (tx_data->len == 0 || tx_data->tx == NULL)
2398  return HTP_OK;
2399 
2400  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2401  if (tx_ud == NULL) {
2402  return HTP_OK;
2403  }
2404  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2405  tx_ud->response_headers_raw_len,
2406  tx_ud->response_headers_raw_len + tx_data->len);
2407  if (ptmp == NULL) {
2408  return HTP_OK;
2409  }
2410  tx_ud->response_headers_raw = ptmp;
2411 
2412  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2413  tx_data->data, tx_data->len);
2414  tx_ud->response_headers_raw_len += tx_data->len;
2415 
2416  return HTP_OK;
2417 }
2418 
2419 /*
2420  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2421  */
2422 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2423 {
2424  cfg_prec->uri_include_all = FALSE;
2431 
2432  if (!g_disable_randomness) {
2434  } else {
2435  cfg_prec->randomize = 0;
2436  }
2438 
2439  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2440  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2441  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2442  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2443 
2444  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2445  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2446 
2447  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2448  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2449 
2450  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2451  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequestComplete);
2452 
2453  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2454  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponseComplete);
2455 
2456  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2457 
2458  /* don't convert + to space by default */
2459  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2460  // enables request decompression
2461  htp_config_set_request_decompression(cfg_prec->cfg, 1);
2462 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2463  // disable by default
2464  htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
2465 #endif
2466 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2467  htp_config_set_lzma_memlimit(cfg_prec->cfg,
2469 #endif
2470 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2471  htp_config_set_compression_bomb_limit(cfg_prec->cfg,
2473 #endif
2474 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2475  htp_config_set_compression_time_limit(cfg_prec->cfg, HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT);
2476 #endif
2477  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2478  * only the hard limit. So we set both here to the (current) htp defaults.
2479  * The reason we do this is that if the user sets the hard limit in the
2480  * config, we have to set the soft limit as well. If libhtp starts using
2481  * the soft limit in the future, we at least make sure we control what
2482  * it's value is. */
2483  htp_config_set_field_limits(cfg_prec->cfg,
2486  return;
2487 }
2488 
2489 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2490  * but we can get both <0 and >RAND_MAX values from RandomGet
2491  */
2492 static int RandomGetWrap(void)
2493 {
2494  unsigned long r;
2495 
2496  do {
2497  r = RandomGet();
2498  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2499 
2500  return r % RAND_MAX;
2501 }
2502 
2503 /*
2504  * We have this splitup so that in case double decoding has been enabled
2505  * for query and path, they would be called first on the callback queue,
2506  * before the callback set by Phase2() is called. We need this, since
2507  * the callback in Phase2() generates the normalized uri which utilizes
2508  * the query and path. */
2509 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2510 {
2511  /* randomize inspection size if needed */
2512  if (cfg_prec->randomize) {
2513  int rdrange = cfg_prec->randomize_range;
2514 
2515  long int r = RandomGetWrap();
2516  cfg_prec->request.inspect_min_size += (int)(cfg_prec->request.inspect_min_size *
2517  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2518 
2519  r = RandomGetWrap();
2520  cfg_prec->request.inspect_window += (int)(cfg_prec->request.inspect_window *
2521  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2522  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2523  " %u and 'request-body-inspect-window' set to %u after"
2524  " randomization.",
2525  name, cfg_prec->request.inspect_min_size, cfg_prec->request.inspect_window);
2526 
2527  r = RandomGetWrap();
2528  cfg_prec->response.inspect_min_size += (int)(cfg_prec->response.inspect_min_size *
2529  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2530 
2531  r = RandomGetWrap();
2532  cfg_prec->response.inspect_window += (int)(cfg_prec->response.inspect_window *
2533  ((double)r / RAND_MAX - 0.5) * rdrange / 100);
2534 
2535  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2536  " %u and 'response-body-inspect-window' set to %u after"
2537  " randomization.",
2538  name, cfg_prec->response.inspect_min_size, cfg_prec->response.inspect_window);
2539  }
2540 
2541  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2542 
2543  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2544  cfg_prec->request.inspect_window : 256;
2545  cfg_prec->request.sbcfg.buf_slide = 0;
2546  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2547  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2548  cfg_prec->request.sbcfg.Free = HTPFree;
2549 
2550  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2551  cfg_prec->response.inspect_window : 256;
2552  cfg_prec->response.sbcfg.buf_slide = 0;
2553  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2554  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2555  cfg_prec->response.sbcfg.Free = HTPFree;
2556  return;
2557 }
2558 
2559 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2560  SCRadixTree *tree)
2561 {
2562  if (cfg_prec == NULL || s == NULL || tree == NULL)
2563  return;
2564 
2565  ConfNode *p = NULL;
2566 
2567  /* Default Parameters */
2568  TAILQ_FOREACH(p, &s->head, next) {
2569 
2570  if (strcasecmp("address", p->name) == 0) {
2571  ConfNode *pval;
2572  /* Addresses */
2573  TAILQ_FOREACH(pval, &p->head, next) {
2574  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2575  pval->val);
2576 
2577  /* IPV6 or IPV4? */
2578  if (strchr(pval->val, ':') != NULL) {
2579  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2580  s->name, pval->val, cfg_prec->cfg);
2581  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2582  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2583  "add ipv6 server %s, ignoring", pval->val);
2584  }
2585  } else {
2586  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2587  s->name, pval->val, cfg_prec->cfg);
2588  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2589  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2590  "to add ipv4 server %s, ignoring",
2591  pval->val);
2592  }
2593  } /* else - if (strchr(pval->val, ':') != NULL) */
2594  } /* TAILQ_FOREACH(pval, &p->head, next) */
2595 
2596  } else if (strcasecmp("personality", p->name) == 0) {
2597  /* Personalities */
2598  int personality = HTPLookupPersonality(p->val);
2599  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2600  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2601 
2602  if (personality >= 0) {
2603  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2604  personality);
2605  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2606  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2607  "personality \"%s\", ignoring", p->val);
2608  } else {
2609  SCLogDebug("LIBHTP personality set to %s",
2610  HTPLookupPersonalityString(personality));
2611  }
2612 
2613  /* The IDS personality by default converts the path (and due to
2614  * our query string callback also the query string) to lowercase.
2615  * Signatures do not expect this, so override it. */
2616  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2617  } else {
2618  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2619  "\"%s\", ignoring", p->val);
2620  continue;
2621  }
2622 
2623  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2624  strcasecmp("request_body_limit", p->name) == 0) {
2625  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2626  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2627  "from conf file - %s. Killing engine", p->val);
2628  exit(EXIT_FAILURE);
2629  }
2630 
2631  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2632  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2633  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2634  "from conf file - %s. Killing engine", p->val);
2635  exit(EXIT_FAILURE);
2636  }
2637 
2638  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2639  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2640  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2641  "from conf file - %s. Killing engine", p->val);
2642  exit(EXIT_FAILURE);
2643  }
2644 
2645  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2646  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2647  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2648  "from conf file - %s. Killing engine", p->val);
2649  exit(EXIT_FAILURE);
2650  }
2651 
2652  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2653  if (ConfValIsTrue(p->val)) {
2654  htp_config_register_request_line(cfg_prec->cfg,
2655  HTPCallbackDoubleDecodeQuery);
2656  }
2657 
2658  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2659  if (ConfValIsTrue(p->val)) {
2660  htp_config_register_request_line(cfg_prec->cfg,
2661  HTPCallbackDoubleDecodePath);
2662  }
2663 
2664  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2665  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2666  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2667  "from conf file - %s. Killing engine", p->val);
2668  exit(EXIT_FAILURE);
2669  }
2670 
2671  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2672  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2673  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2674  "from conf file - %s. Killing engine", p->val);
2675  exit(EXIT_FAILURE);
2676  }
2677 
2678  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2679  uint32_t value = 2;
2680  if (ParseSizeStringU32(p->val, &value) < 0) {
2681  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2682  "from conf file - %s. Killing engine", p->val);
2683  exit(EXIT_FAILURE);
2684  }
2685 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2686  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2687 #else
2688  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2689  "to %u, libhtp version too old", value);
2690 #endif
2691  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2692  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2693  HTP_DECODER_URL_PATH,
2694  ConfValIsTrue(p->val));
2695  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2696  if (strlen(p->val) == 1) {
2697  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2698  HTP_DECODER_URL_PATH,
2699  p->val[0]);
2700  } else {
2701  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2702  "for libhtp param path-bestfit-replacement-char");
2703  }
2704  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2705  htp_config_set_convert_lowercase(cfg_prec->cfg,
2706  HTP_DECODER_URL_PATH,
2707  ConfValIsTrue(p->val));
2708  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2709  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2710  HTP_DECODER_URL_PATH,
2711  ConfValIsTrue(p->val));
2712  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2713  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2714  HTP_DECODER_URL_PATH,
2715  ConfValIsTrue(p->val));
2716  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2717  htp_config_set_path_separators_compress(cfg_prec->cfg,
2718  HTP_DECODER_URL_PATH,
2719  ConfValIsTrue(p->val));
2720  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2721  htp_config_set_path_separators_decode(cfg_prec->cfg,
2722  HTP_DECODER_URL_PATH,
2723  ConfValIsTrue(p->val));
2724  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2725  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2726  HTP_DECODER_URL_PATH,
2727  ConfValIsTrue(p->val));
2728  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2729  enum htp_url_encoding_handling_t handling;
2730  if (strcasecmp(p->val, "preserve_percent") == 0) {
2731  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2732  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2733  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2734  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2735  handling = HTP_URL_DECODE_PROCESS_INVALID;
2736  } else {
2737  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2738  "for libhtp param path-url-encoding-invalid-handling");
2739  return;
2740  }
2741  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2742  HTP_DECODER_URL_PATH,
2743  handling);
2744  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2745  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2746  HTP_DECODER_URL_PATH,
2747  ConfValIsTrue(p->val));
2748  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2749  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2750  SCLogDebug("uri-include-all %s",
2751  cfg_prec->uri_include_all ? "enabled" : "disabled");
2752  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2753  htp_config_set_plusspace_decode(cfg_prec->cfg,
2754  HTP_DECODER_URLENCODED,
2755  ConfValIsTrue(p->val));
2756  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2757  uint32_t limit = 0;
2758  if (ParseSizeStringU32(p->val, &limit) < 0) {
2759  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2760  "from conf file - %s. Killing engine", p->val);
2761  exit(EXIT_FAILURE);
2762  }
2763  if (limit == 0) {
2764  FatalError(SC_ERR_FATAL, "Error meta-field-limit "
2765  "from conf file cannot be 0. Killing engine");
2766  }
2767  /* set default soft-limit with our new hard limit */
2768  htp_config_set_field_limits(cfg_prec->cfg,
2770  (size_t)limit);
2771 #ifdef HAVE_HTP_CONFIG_SET_LZMA_MEMLIMIT
2772  } else if (strcasecmp("lzma-memlimit", p->name) == 0) {
2773  uint32_t limit = 0;
2774  if (ParseSizeStringU32(p->val, &limit) < 0) {
2775  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'lzma-memlimit' "
2776  "from conf file - %s.", p->val);
2777  }
2778  if (limit == 0) {
2779  FatalError(SC_ERR_SIZE_PARSE, "'lzma-memlimit' "
2780  "from conf file cannot be 0.");
2781  }
2782  /* set default soft-limit with our new hard limit */
2783  SCLogConfig("Setting HTTP LZMA memory limit to %"PRIu32" bytes", limit);
2784  htp_config_set_lzma_memlimit(cfg_prec->cfg, (size_t)limit);
2785 #endif
2786 #ifdef HAVE_HTP_CONFIG_SET_LZMA_LAYERS
2787  } else if (strcasecmp("lzma-enabled", p->name) == 0) {
2788  if (ConfValIsTrue(p->val)) {
2789  htp_config_set_lzma_layers(cfg_prec->cfg, 1);
2790  } else if (!ConfValIsFalse(p->val)) {
2791  int8_t limit;
2792  if (StringParseInt8(&limit, 10, 0, (const char *)p->val) < 0) {
2794  "failed to parse 'lzma-enabled' "
2795  "from conf file - %s.",
2796  p->val);
2797  }
2798  SCLogConfig("Setting HTTP LZMA decompression layers to %" PRIu32 "", (int)limit);
2799  htp_config_set_lzma_layers(cfg_prec->cfg, limit);
2800  }
2801 #endif
2802 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_BOMB_LIMIT
2803  } else if (strcasecmp("compression-bomb-limit", p->name) == 0) {
2804  uint32_t limit = 0;
2805  if (ParseSizeStringU32(p->val, &limit) < 0) {
2806  FatalError(SC_ERR_SIZE_PARSE, "failed to parse 'compression-bomb-limit' "
2807  "from conf file - %s.", p->val);
2808  }
2809  if (limit == 0) {
2810  FatalError(SC_ERR_SIZE_PARSE, "'compression-bomb-limit' "
2811  "from conf file cannot be 0.");
2812  }
2813  /* set default soft-limit with our new hard limit */
2814  SCLogConfig("Setting HTTP compression bomb limit to %"PRIu32" bytes", limit);
2815  htp_config_set_compression_bomb_limit(cfg_prec->cfg, (size_t)limit);
2816 #endif
2817 #ifdef HAVE_HTP_CONFIG_SET_COMPRESSION_TIME_LIMIT
2818  } else if (strcasecmp("decompression-time-limit", p->name) == 0) {
2819  uint32_t limit = 0;
2820  // between 1 usec and 1 second
2821  if (StringParseU32RangeCheck(&limit, 10, 0, p->val, 1, 1000000) < 0) {
2823  "failed to parse 'decompression-time-limit' "
2824  "from conf file - %s.",
2825  p->val);
2826  }
2827  SCLogConfig("Setting HTTP decompression time limit to %" PRIu32 " usec", limit);
2828  htp_config_set_compression_time_limit(cfg_prec->cfg, (size_t)limit);
2829 #endif
2830  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2831  if (!g_disable_randomness) {
2832  cfg_prec->randomize = ConfValIsTrue(p->val);
2833  }
2834  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2835  uint32_t range;
2836  if (StringParseU32RangeCheck(&range, 10, 0,
2837  (const char *)p->val, 0, 100) < 0) {
2838  SCLogError(SC_ERR_INVALID_VALUE, "Invalid value for randomize"
2839  "-inspection-range setting from conf file - \"%s\"."
2840  " It should be a valid integer less than or equal to 100."
2841  " Killing engine",
2842  p->val);
2843  exit(EXIT_FAILURE);
2844  }
2845  cfg_prec->randomize_range = range;
2846  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2847  if (ConfValIsTrue(p->val)) {
2848  cfg_prec->http_body_inline = 1;
2849  } else if (ConfValIsFalse(p->val)) {
2850  cfg_prec->http_body_inline = 0;
2851  } else {
2852  if (strcmp("auto", p->val) != 0) {
2853  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2854  }
2855  if (EngineModeIsIPS()) {
2856  cfg_prec->http_body_inline = 1;
2857  } else {
2858  cfg_prec->http_body_inline = 0;
2859  }
2860  }
2861  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2862  ConfNode *pval;
2863 
2864  TAILQ_FOREACH(pval, &p->head, next) {
2865  if (strcasecmp("enabled", pval->name) == 0) {
2866  if (ConfValIsTrue(pval->val)) {
2867  cfg_prec->swf_decompression_enabled = 1;
2868  } else if (ConfValIsFalse(pval->val)) {
2869  cfg_prec->swf_decompression_enabled = 0;
2870  } else {
2871  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2872  }
2873  } else if (strcasecmp("type", pval->name) == 0) {
2874  if (strcasecmp("no", pval->val) == 0) {
2876  } else if (strcasecmp("deflate", pval->val) == 0) {
2878  } else if (strcasecmp("lzma", pval->val) == 0) {
2880  } else if (strcasecmp("both", pval->val) == 0) {
2882  } else {
2884  "Invalid entry for "
2885  "swf-decompression.type: %s - "
2886  "Killing engine", pval->val);
2887  exit(EXIT_FAILURE);
2888  }
2889  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2890  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2892  "Error parsing swf-decompression.compression-depth "
2893  "from conf file - %s. Killing engine", p->val);
2894  exit(EXIT_FAILURE);
2895  }
2896  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2897  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2899  "Error parsing swf-decompression.decompression-depth "
2900  "from conf file - %s. Killing engine", p->val);
2901  exit(EXIT_FAILURE);
2902  }
2903  } else {
2904  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2905  }
2906  }
2907  } else {
2908  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2909  "default config: %s", p->name);
2910  }
2911  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2912 
2913  return;
2914 }
2915 
2916 void HTPConfigure(void)
2917 {
2918  SCEnter();
2919 
2920  cfglist.next = NULL;
2921 
2922  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2923  if (NULL == cfgtree)
2924  exit(EXIT_FAILURE);
2925 
2926  /* Default Config */
2927  cfglist.cfg = htp_config_create();
2928  if (NULL == cfglist.cfg) {
2929  FatalError(SC_ERR_FATAL, "Failed to create HTP default config");
2930  }
2931  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2932  HTPConfigSetDefaultsPhase1(&cfglist);
2933  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2934  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2935  cfgtree);
2936  } else {
2937  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2938  }
2939  HTPConfigSetDefaultsPhase2("default", &cfglist);
2940 
2941  HTPParseMemcap();
2942 
2943  /* Read server config and create a parser for each IP in radix tree */
2944  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2945  if (server_config == NULL) {
2946  server_config = ConfGetNode("libhtp.server-config");
2947  if (server_config == NULL) {
2948  SCLogDebug("LIBHTP Configuring %p", server_config);
2949  SCReturn;
2950  }
2951  }
2952  SCLogDebug("LIBHTP Configuring %p", server_config);
2953 
2954  ConfNode *si;
2955  /* Server Nodes */
2956  TAILQ_FOREACH(si, &server_config->head, next) {
2957  /* Need the named node, not the index */
2958  ConfNode *s = TAILQ_FIRST(&si->head);
2959  if (NULL == s) {
2960  SCLogDebug("LIBHTP s NULL");
2961  continue;
2962  }
2963 
2964  SCLogDebug("LIBHTP server %s", s->name);
2965 
2966  HTPCfgRec *nextrec = cfglist.next;
2967  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2968  if (NULL == htprec)
2969  exit(EXIT_FAILURE);
2970  memset(htprec, 0x00, sizeof(*htprec));
2971 
2972  cfglist.next = htprec;
2973 
2974  cfglist.next->next = nextrec;
2975  cfglist.next->cfg = htp_config_create();
2976  if (NULL == cfglist.next->cfg) {
2977  FatalError(SC_ERR_FATAL, "Failed to create HTP server config");
2978  }
2979 
2980  HTPConfigSetDefaultsPhase1(htprec);
2981  HTPConfigParseParameters(htprec, s, cfgtree);
2982  HTPConfigSetDefaultsPhase2(s->name, htprec);
2983  }
2984 
2985  SCReturn;
2986 }
2987 
2989 {
2990 #ifdef DEBUG
2991  SCMutexLock(&htp_state_mem_lock);
2992  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2993  SCMutexUnlock(&htp_state_mem_lock);
2994 #endif
2995 }
2996 
2997 /** \internal
2998  * \brief get files callback
2999  * \param state state ptr
3000  * \param direction flow direction
3001  * \retval files files ptr
3002  */
3003 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
3004 {
3005  if (state == NULL)
3006  return NULL;
3007 
3008  HtpState *http_state = (HtpState *)state;
3009 
3010  if (direction & STREAM_TOCLIENT) {
3011  SCReturnPtr(http_state->files_tc, "FileContainer");
3012  } else {
3013  SCReturnPtr(http_state->files_ts, "FileContainer");
3014  }
3015 }
3016 
3017 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
3018 {
3019  if (direction & STREAM_TOSERVER)
3020  return ((htp_tx_t *)tx)->request_progress;
3021  else
3022  return ((htp_tx_t *)tx)->response_progress;
3023 }
3024 
3025 static uint64_t HTPStateGetTxCnt(void *alstate)
3026 {
3027  HtpState *http_state = (HtpState *)alstate;
3028 
3029  if (http_state != NULL && http_state->conn != NULL) {
3030  const int64_t size = (int64_t)htp_list_size(http_state->conn->transactions);
3031  if (size < 0)
3032  return 0ULL;
3033  SCLogDebug("size %"PRIu64, size);
3034  return (uint64_t)size;
3035  } else {
3036  return 0ULL;
3037  }
3038 }
3039 
3040 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
3041 {
3042  HtpState *http_state = (HtpState *)alstate;
3043 
3044  if (http_state != NULL && http_state->conn != NULL)
3045  return htp_list_get(http_state->conn->transactions, tx_id);
3046  else
3047  return NULL;
3048 }
3049 
3050 void *HtpGetTxForH2(void *alstate)
3051 {
3052  // gets last transaction
3053  HtpState *http_state = (HtpState *)alstate;
3054  if (http_state != NULL && http_state->conn != NULL) {
3055  size_t txid = htp_list_array_size(http_state->conn->transactions);
3056  if (txid > 0) {
3057  return htp_list_get(http_state->conn->transactions, txid - 1);
3058  }
3059  }
3060  return NULL;
3061 }
3062 
3063 static int HTPStateGetEventInfo(const char *event_name,
3064  int *event_id, AppLayerEventType *event_type)
3065 {
3066  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
3067  if (*event_id == -1) {
3068  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
3069  "http's enum map table.", event_name);
3070  /* this should be treated as fatal */
3071  return -1;
3072  }
3073 
3074  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3075 
3076  return 0;
3077 }
3078 
3079 static int HTPStateGetEventInfoById(int event_id, const char **event_name,
3080  AppLayerEventType *event_type)
3081 {
3082  *event_name = SCMapEnumValueToName(event_id, http_decoder_event_table);
3083  if (*event_name == NULL) {
3084  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%d\" not present in "
3085  "http's enum map table.", event_id);
3086  /* this should be treated as fatal */
3087  return -1;
3088  }
3089 
3090  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
3091 
3092  return 0;
3093 }
3094 
3095 static void HTPStateTruncate(void *state, uint8_t direction)
3096 {
3097  FileContainer *fc = HTPStateGetFiles(state, direction);
3098  if (fc != NULL) {
3100  }
3101 }
3102 
3103 static AppLayerTxData *HTPGetTxData(void *vtx)
3104 {
3105  htp_tx_t *tx = (htp_tx_t *)vtx;
3106  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
3107  if (tx_ud) {
3108  return &tx_ud->tx_data;
3109  }
3110  return NULL;
3111 }
3112 
3113 static int HTPRegisterPatternsForProtocolDetection(void)
3114 {
3115  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
3116  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
3117  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
3118  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
3119  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
3120  const char *spacings[] = { "|20|", "|09|", NULL };
3121  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
3122 
3123  int methods_pos;
3124  int spacings_pos;
3125  int versions_pos;
3126  int register_result;
3127  char method_buffer[32] = "";
3128 
3129  /* Loop through all the methods ands spacings and register the patterns */
3130  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
3131  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
3132 
3133  /* Combine the method name and the spacing */
3134  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
3135 
3136  /* Register the new method+spacing pattern
3137  * 3 is subtracted from the length since the spacing is hex typed as |xx|
3138  * but the pattern matching should only be one char
3139  */
3140  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3141  method_buffer, (uint16_t)strlen(method_buffer) - 3, 0, STREAM_TOSERVER);
3142  if (register_result < 0) {
3143  return -1;
3144  }
3145  }
3146  }
3147 
3148  /* Loop through all the http verions patterns that are TO_CLIENT */
3149  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
3150  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP, ALPROTO_HTTP1,
3151  versions[versions_pos], (uint16_t)strlen(versions[versions_pos]), 0,
3152  STREAM_TOCLIENT);
3153  if (register_result < 0) {
3154  return -1;
3155  }
3156  }
3157 
3158  return 0;
3159 }
3160 
3161 /**
3162  * \brief Register the HTTP protocol and state handling functions to APP layer
3163  * of the engine.
3164  */
3166 {
3167  SCEnter();
3168 
3169  const char *proto_name = "http";
3170 
3171  /** HTTP */
3172  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3174  if (HTPRegisterPatternsForProtocolDetection() < 0)
3175  return;
3176  } else {
3177  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3178  proto_name);
3179  return;
3180  }
3181 
3182  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3183  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
3184  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
3185  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetFiles);
3187  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
3188  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
3189  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTx);
3190 
3192  ALPROTO_HTTP1, HTP_REQUEST_COMPLETE, HTP_RESPONSE_COMPLETE);
3193  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfo);
3195  IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
3196 
3197  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTruncate);
3198  AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
3199 
3201  IPPROTO_TCP, ALPROTO_HTTP1, AppLayerHtpSetStreamDepthFlag);
3202 
3204  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER, HTPHandleRequestData);
3206  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOCLIENT, HTPHandleResponseData);
3207  SC_ATOMIC_INIT(htp_config_flags);
3208  /* This parser accepts gaps. */
3212  IPPROTO_TCP, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_TOCLIENT);
3213  /* app-layer-frame-documentation tag start: registering relevant callbacks */
3215  IPPROTO_TCP, ALPROTO_HTTP1, HTTPGetFrameIdByName, HTTPGetFrameNameById);
3216  /* app-layer-frame-documentation tag end: registering relevant callbacks */
3217  HTPConfigure();
3218  } else {
3219  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3220  "still on.", proto_name);
3221  }
3222 #ifdef UNITTESTS
3223  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP1, HTPParserRegisterTests);
3224 #endif
3225 
3226  SCReturn;
3227 }
3228 
3229 #ifdef UNITTESTS
3230 static HTPCfgRec cfglist_backup;
3231 
3233 {
3234  cfglist_backup = cfglist;
3235 
3236  return;
3237 }
3238 
3240 {
3241  cfglist = cfglist_backup;
3242 
3243  return;
3244 }
3245 
3246 /** \test Test case where chunks are sent in smaller chunks and check the
3247  * response of the parser from HTP library. */
3248 static int HTPParserTest01(void)
3249 {
3250  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3251  " Data is c0oL!";
3252  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3253 
3254  TcpSession ssn;
3255  memset(&ssn, 0, sizeof(ssn));
3256 
3259 
3260  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3261  FAIL_IF_NULL(f);
3262  f->protoctx = &ssn;
3263  f->proto = IPPROTO_TCP;
3264  f->alproto = ALPROTO_HTTP1;
3265 
3266  StreamTcpInitConfig(true);
3267 
3268  uint32_t u;
3269  for (u = 0; u < httplen1; u++) {
3270  uint8_t flags = 0;
3271 
3272  if (u == 0)
3273  flags = STREAM_TOSERVER|STREAM_START;
3274  else if (u == (httplen1 - 1))
3275  flags = STREAM_TOSERVER|STREAM_EOF;
3276  else
3277  flags = STREAM_TOSERVER;
3278 
3279  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3280  FAIL_IF(r != 0);
3281  }
3282 
3283  HtpState *htp_state = f->alstate;
3284  FAIL_IF_NULL(htp_state);
3285 
3286  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3287  FAIL_IF_NULL(tx);
3288 
3289  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3290  FAIL_IF_NULL(h);
3291 
3292  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3293  FAIL_IF(tx->request_method_number != HTP_M_POST);
3294  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3295 
3297  StreamTcpFreeConfig(true);
3298  UTHFreeFlow(f);
3299  PASS;
3300 }
3301 
3302 /** \test Test folding in 1 read case */
3303 static int HTPParserTest01b(void)
3304 {
3305  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3306  " Data is c0oL!";
3307  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3308 
3309  TcpSession ssn;
3310  memset(&ssn, 0, sizeof(ssn));
3311 
3314 
3315  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3316  FAIL_IF_NULL(f);
3317  f->protoctx = &ssn;
3318  f->proto = IPPROTO_TCP;
3319  f->alproto = ALPROTO_HTTP1;
3320 
3321  StreamTcpInitConfig(true);
3322 
3323  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3324  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3325  FAIL_IF(r != 0);
3326 
3327  HtpState *htp_state = f->alstate;
3328  FAIL_IF_NULL(htp_state);
3329 
3330  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3331  FAIL_IF_NULL(tx);
3332 
3333  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3334  FAIL_IF_NULL(h);
3335 
3336  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3337  FAIL_IF(tx->request_method_number != HTP_M_POST);
3338  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3339 
3341  StreamTcpFreeConfig(true);
3342  UTHFreeFlow(f);
3343  PASS;
3344 }
3345 
3346 /** \test Test folding in 1byte per read case */
3347 static int HTPParserTest01c(void)
3348 {
3349  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3350  " Data is c0oL!";
3351  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3352 
3353  TcpSession ssn;
3354  memset(&ssn, 0, sizeof(ssn));
3355 
3358 
3359  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3360  FAIL_IF_NULL(f);
3361  f->protoctx = &ssn;
3362  f->proto = IPPROTO_TCP;
3363  f->alproto = ALPROTO_HTTP1;
3364 
3365  StreamTcpInitConfig(true);
3366 
3367  uint32_t u;
3368  for (u = 0; u < httplen1; u++) {
3369  uint8_t flags = 0;
3370 
3371  if (u == 0)
3372  flags = STREAM_TOSERVER|STREAM_START;
3373  else if (u == (httplen1 - 1))
3374  flags = STREAM_TOSERVER|STREAM_EOF;
3375  else
3376  flags = STREAM_TOSERVER;
3377 
3378  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3379  FAIL_IF(r != 0);
3380  }
3381 
3382  HtpState *htp_state = f->alstate;
3383  FAIL_IF_NULL(htp_state);
3384 
3385  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3386  FAIL_IF_NULL(tx);
3387 
3388  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3389  FAIL_IF_NULL(h);
3390 
3391  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3392  FAIL_IF(tx->request_method_number != HTP_M_POST);
3393  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3394 
3396  StreamTcpFreeConfig(true);
3397  UTHFreeFlow(f);
3398  PASS;
3399 }
3400 
3401 /** \test Test case where chunks are sent in smaller chunks and check the
3402  * response of the parser from HTP library. */
3403 static int HTPParserTest01a(void)
3404 {
3405  int result = 0;
3406  Flow *f = NULL;
3407  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3408  " Data is c0oL!";
3409  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3410  TcpSession ssn;
3411  HtpState *htp_state = NULL;
3412  int r = 0;
3414 
3415  memset(&ssn, 0, sizeof(ssn));
3416 
3417  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3418  if (f == NULL)
3419  goto end;
3420  f->protoctx = &ssn;
3421  f->proto = IPPROTO_TCP;
3422  f->alproto = ALPROTO_HTTP1;
3423 
3424  StreamTcpInitConfig(true);
3425 
3426  uint32_t u;
3427  for (u = 0; u < httplen1; u++) {
3428  uint8_t flags = 0;
3429 
3430  if (u == 0)
3431  flags = STREAM_TOSERVER|STREAM_START;
3432  else if (u == (httplen1 - 1))
3433  flags = STREAM_TOSERVER|STREAM_EOF;
3434  else
3435  flags = STREAM_TOSERVER;
3436 
3437  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3438  if (r != 0) {
3439  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3440  " 0: ", u, r);
3441  goto end;
3442  }
3443  }
3444 
3445  htp_state = f->alstate;
3446  if (htp_state == NULL) {
3447  printf("no http state: ");
3448  goto end;
3449  }
3450 
3451  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3452  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3453  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3454  || tx->request_method_number != HTP_M_POST ||
3455  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3456  {
3457  printf("expected header value: Victor/1.0 and got %s: and expected"
3458  " method: POST and got %s, expected protocol number HTTP/1.0"
3459  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3460  bstr_util_strdup_to_c(tx->request_method),
3461  bstr_util_strdup_to_c(tx->request_protocol));
3462  goto end;
3463  }
3464  result = 1;
3465 end:
3466  if (alp_tctx != NULL)
3468  StreamTcpFreeConfig(true);
3469  UTHFreeFlow(f);
3470  return result;
3471 }
3472 
3473 /** \test See how it deals with an incomplete request. */
3474 static int HTPParserTest02(void)
3475 {
3476  int result = 0;
3477  Flow *f = NULL;
3478  uint8_t httpbuf1[] = "POST";
3479  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3480  TcpSession ssn;
3481  HtpState *http_state = NULL;
3483 
3484  memset(&ssn, 0, sizeof(ssn));
3485 
3486  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3487  if (f == NULL)
3488  goto end;
3489  f->protoctx = &ssn;
3490  f->proto = IPPROTO_TCP;
3491  f->alproto = ALPROTO_HTTP1;
3492 
3493  StreamTcpInitConfig(true);
3494 
3495  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
3496  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3497  if (r != 0) {
3498  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3499  goto end;
3500  }
3501 
3502  http_state = f->alstate;
3503  if (http_state == NULL) {
3504  printf("no http state: ");
3505  goto end;
3506  }
3507 
3508  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3509  FAIL_IF_NULL(tx);
3510  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3511  FAIL_IF_NOT_NULL(h);
3512 
3513  FAIL_IF_NULL(tx->request_method);
3514  char *method = bstr_util_strdup_to_c(tx->request_method);
3515  FAIL_IF_NULL(method);
3516 
3517  FAIL_IF(strcmp(method, "POST") != 0);
3518  SCFree(method);
3519 
3520  result = 1;
3521 end:
3522  if (alp_tctx != NULL)
3524  StreamTcpFreeConfig(true);
3525  UTHFreeFlow(f);
3526  return result;
3527 }
3528 
3529 /** \test Test case where method is invalid and data is sent in smaller chunks
3530  * and check the response of the parser from HTP library. */
3531 static int HTPParserTest03(void)
3532 {
3533  int result = 0;
3534  Flow *f = NULL;
3535  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3536  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3537  TcpSession ssn;
3538  HtpState *htp_state = NULL;
3539  int r = 0;
3541 
3542  memset(&ssn, 0, sizeof(ssn));
3543 
3544  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3545  if (f == NULL)
3546  goto end;
3547  f->protoctx = &ssn;
3548  f->proto = IPPROTO_TCP;
3549  f->alproto = ALPROTO_HTTP1;
3550 
3551  StreamTcpInitConfig(true);
3552 
3553  uint32_t u;
3554  for (u = 0; u < httplen1; u++) {
3555  uint8_t flags = 0;
3556 
3557  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3558  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3559  else flags = STREAM_TOSERVER;
3560 
3561  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3562  if (r != 0) {
3563  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3564  " 0: ", u, r);
3565  goto end;
3566  }
3567  }
3568  htp_state = f->alstate;
3569  if (htp_state == NULL) {
3570  printf("no http state: ");
3571  goto end;
3572  }
3573 
3574  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3575 
3576  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3577  if (tx->request_method_number != HTP_M_UNKNOWN ||
3578  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3579  {
3580  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3581  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3582  bstr_util_strdup_to_c(tx->request_protocol));
3583  goto end;
3584  }
3585  result = 1;
3586 end:
3587  if (alp_tctx != NULL)
3589  StreamTcpFreeConfig(true);
3590  UTHFreeFlow(f);
3591  return result;
3592 }
3593 
3594 /** \test Test case where invalid data is sent and check the response of the
3595  * parser from HTP library. */
3596 static int HTPParserTest04(void)
3597 {
3598  int result = 0;
3599  Flow *f = NULL;
3600  HtpState *htp_state = NULL;
3601  uint8_t httpbuf1[] = "World!\r\n";
3602  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3603  TcpSession ssn;
3604  int r = 0;
3606 
3607  memset(&ssn, 0, sizeof(ssn));
3608 
3609  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3610  if (f == NULL)
3611  goto end;
3612  f->protoctx = &ssn;
3613  f->proto = IPPROTO_TCP;
3614  f->alproto = ALPROTO_HTTP1;
3615 
3616  StreamTcpInitConfig(true);
3617 
3619  STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
3620  if (r != 0) {
3621  goto end;
3622  }
3623 
3624  htp_state = f->alstate;
3625  if (htp_state == NULL) {
3626  printf("no http state: ");
3627  goto end;
3628  }
3629 
3630  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3631  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3632  if (tx->request_method_number != HTP_M_UNKNOWN ||
3633  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3634  {
3635  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3636  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3637  bstr_util_strdup_to_c(tx->request_protocol));
3638  goto end;
3639  }
3640  result = 1;
3641 end:
3642  if (alp_tctx != NULL)
3644  StreamTcpFreeConfig(true);
3645  UTHFreeFlow(f);
3646  return result;
3647 }
3648 
3649 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3650  * properly parsed them and also keeps them separated. */
3651 static int HTPParserTest05(void)
3652 {
3653  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3654  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3655  uint8_t httpbuf2[] = "Post D";
3656  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3657  uint8_t httpbuf3[] = "ata is c0oL!";
3658  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3659 
3660  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3661  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3662  uint8_t httpbuf5[] = "post R";
3663  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3664  uint8_t httpbuf6[] = "esults are tha bomb!";
3665  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3666 
3667  TcpSession ssn;
3668  memset(&ssn, 0, sizeof(ssn));
3669 
3672 
3673  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3674  FAIL_IF_NULL(f);
3675  f->protoctx = &ssn;
3676  f->proto = IPPROTO_TCP;
3677  f->alproto = ALPROTO_HTTP1;
3678 
3679  StreamTcpInitConfig(true);
3680 
3681  int r = AppLayerParserParse(
3682  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3683  FAIL_IF(r != 0);
3684 
3685  r = AppLayerParserParse(
3686  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf4, httplen4);
3687  FAIL_IF(r != 0);
3688 
3689  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT, httpbuf5, httplen5);
3690  FAIL_IF(r != 0);
3691 
3692  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER, httpbuf2, httplen2);
3693  FAIL_IF(r != 0);
3694 
3695  r = AppLayerParserParse(
3696  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3697  FAIL_IF(r != 0);
3698 
3699  r = AppLayerParserParse(
3700  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3701  FAIL_IF(r != 0);
3702 
3703  HtpState *http_state = f->alstate;
3704  FAIL_IF_NULL(http_state);
3705 
3706  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3707  FAIL_IF_NULL(tx);
3708  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3709  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3710 
3711  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3712  FAIL_IF_NULL(h);
3713 
3714  FAIL_IF_NOT(tx->response_status_number == 200);
3715 
3717  StreamTcpFreeConfig(true);
3718  UTHFreeFlow(f);
3719  PASS;
3720 }
3721 
3722 /** \test Test proper chunked encoded response body
3723  */
3724 static int HTPParserTest06(void)
3725 {
3726  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3727  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3728  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3729  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3730  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3731  "GMT\r\n"
3732  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3733  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3734  "FrontPage/5.0.2.2510\r\n"
3735  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3736  "chunked\r\n"
3737  "Content-Type: text/html\r\n\r\n"
3738  "580\r\n"
3739  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3740  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3741  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3742  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3743  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3744  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3745  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3746  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3747  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3748  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3749  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3750  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3751  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3752  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3753  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3754  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3755  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3756  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3757  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3758  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3759  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3760  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3761  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3762  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3763  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3764  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3765  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3766  "aHA=\r\n0\r\n\r\n";
3767  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3768  TcpSession ssn;
3769 
3772 
3773  memset(&ssn, 0, sizeof(ssn));
3774 
3775  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3776  FAIL_IF_NULL(f);
3777  f->protoctx = &ssn;
3778  f->proto = IPPROTO_TCP;
3779  f->alproto = ALPROTO_HTTP1;
3780 
3781  StreamTcpInitConfig(true);
3782 
3783  int r = AppLayerParserParse(
3784  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START, httpbuf1, httplen1);
3785  FAIL_IF(r != 0);
3786  r = AppLayerParserParse(
3787  NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOCLIENT | STREAM_START, httpbuf2, httplen2);
3788  FAIL_IF(r != 0);
3789 
3790  HtpState *http_state = f->alstate;
3791  FAIL_IF_NULL(http_state);
3792 
3793  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3794  FAIL_IF_NULL(tx);
3795 
3796  FAIL_IF(tx->request_method_number != HTP_M_GET);
3797  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3798 
3799  FAIL_IF(tx->response_status_number != 200);
3800  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3801 
3802  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3803  FAIL_IF_NULL(h);
3804 
3806  StreamTcpFreeConfig(true);
3807  UTHFreeFlow(f);
3808  PASS;
3809 }
3810 
3811 /** \test
3812  */
3813 static int HTPParserTest07(void)
3814 {
3815  int result = 0;
3816  Flow *f = NULL;
3817  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3818  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3819  TcpSession ssn;
3820  HtpState *htp_state = NULL;
3821  int r = 0;
3823 
3824  memset(&ssn, 0, sizeof(ssn));
3825 
3826  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3827  if (f == NULL)
3828  goto end;
3829  f->protoctx = &ssn;
3830  f->proto = IPPROTO_TCP;
3831  f->alproto = ALPROTO_HTTP1;
3832 
3833  StreamTcpInitConfig(true);
3834 
3835  uint32_t u;
3836  for (u = 0; u < httplen1; u++) {
3837  uint8_t flags = 0;
3838 
3839  if (u == 0)
3840  flags = STREAM_TOSERVER|STREAM_START;
3841  else if (u == (httplen1 - 1))
3842  flags = STREAM_TOSERVER|STREAM_EOF;
3843  else
3844  flags = STREAM_TOSERVER;
3845 
3846  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
3847  if (r != 0) {
3848  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3849  " 0: ", u, r);
3850  goto end;
3851  }
3852  }
3853 
3854  htp_state = f->alstate;
3855  if (htp_state == NULL) {
3856  printf("no http state: ");
3857  goto end;
3858  }
3859 
3860  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3861  size_t reflen = sizeof(ref) - 1;
3862 
3863  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3864  if (tx == NULL)
3865  goto end;
3866  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3867  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3868  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3869  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3870  (uintmax_t)reflen,
3871  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3872  goto end;
3873  }
3874 
3875  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3876  bstr_len(tx_ud->request_uri_normalized)) != 0)
3877  {
3878  printf("normalized uri \"");
3879  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3880  printf("\" != \"");
3881  PrintRawUriFp(stdout, ref, reflen);
3882  printf("\": ");
3883  goto end;
3884  }
3885  }
3886 
3887  result = 1;
3888 end:
3889  if (alp_tctx != NULL)
3891  StreamTcpFreeConfig(true);
3892  UTHFreeFlow(f);
3893  return result;
3894 }
3895 
3896 #include "conf-yaml-loader.h"
3897 
3898 /** \test Abort
3899  */
3900 static int HTPParserTest08(void)
3901 {
3902  int result = 0;
3903  Flow *f = NULL;
3904  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3905  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3906  TcpSession ssn;
3908 
3909  char input[] = "\
3910 %YAML 1.1\n\
3911 ---\n\
3912 libhtp:\n\
3913 \n\
3914  default-config:\n\
3915  personality: IDS\n\
3916 ";
3917 
3919  ConfInit();
3921 
3922  ConfYamlLoadString(input, strlen(input));
3923  HTPConfigure();
3924 
3925  HtpState *htp_state = NULL;
3926  int r = 0;
3927  memset(&ssn, 0, sizeof(ssn));
3928 
3929  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3930  if (f == NULL)
3931  goto end;
3932  f->protoctx = &ssn;
3933  f->proto = IPPROTO_TCP;
3934  f->alproto = ALPROTO_HTTP1;
3935 
3936  StreamTcpInitConfig(true);
3937 
3938  uint8_t flags = 0;
3939  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3940 
3941  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
3942  if (r != 0) {
3943  printf("toserver chunk returned %" PRId32 ", expected"
3944  " 0: ", r);
3945  result = 0;
3946  goto end;
3947  }
3948 
3949  htp_state = f->alstate;
3950  if (htp_state == NULL) {
3951  printf("no http state: ");
3952  result = 0;
3953  goto end;
3954  }
3955 
3956  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3957  if (tx == NULL)
3958  goto end;
3959  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3960  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3961  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3962  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3963  bstr_len(tx_ud->request_uri_normalized));
3964  }
3965 
3966  result = 1;
3967 end:
3968  if (alp_tctx != NULL)
3970  StreamTcpFreeConfig(true);
3971  HTPFreeConfig();
3972  ConfDeInit();
3975  UTHFreeFlow(f);
3976  return result;
3977 }
3978 
3979 /** \test Abort
3980  */
3981 static int HTPParserTest09(void)
3982 {
3983  int result = 0;
3984  Flow *f = NULL;
3985  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3986  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3987  TcpSession ssn;
3989 
3990  char input[] = "\
3991 %YAML 1.1\n\
3992 ---\n\
3993 libhtp:\n\
3994 \n\
3995  default-config:\n\
3996  personality: Apache_2_2\n\
3997 ";
3998 
4000  ConfInit();
4002 
4003  ConfYamlLoadString(input, strlen(input));
4004  HTPConfigure();
4005 
4006  HtpState *htp_state = NULL;
4007  int r = 0;
4008 
4009  memset(&ssn, 0, sizeof(ssn));
4010 
4011  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4012  if (f == NULL)
4013  goto end;
4014  f->protoctx = &ssn;
4015  f->proto = IPPROTO_TCP;
4016  f->alproto = ALPROTO_HTTP1;
4017 
4018  StreamTcpInitConfig(true);
4019 
4020  uint8_t flags = 0;
4021  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
4022 
4023  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, httpbuf1, httplen1);
4024  if (r != 0) {
4025  printf("toserver chunk returned %" PRId32 ", expected"
4026  " 0: ", r);
4027  goto end;
4028  }
4029 
4030  htp_state = f->alstate;
4031  if (htp_state == NULL) {
4032  printf("no http state: ");
4033  goto end;
4034  }
4035 
4036  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4037  if (tx == NULL)
4038  goto end;
4039  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4040  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4041  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
4042  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
4043  bstr_len(tx_ud->request_uri_normalized));
4044  }
4045 
4046  result = 1;
4047 end:
4048  if (alp_tctx != NULL)
4050  StreamTcpFreeConfig(true);
4051  HTPFreeConfig();
4052  ConfDeInit();
4055  UTHFreeFlow(f);
4056  return result;
4057 }
4058 
4059 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
4060  */
4061 static int HTPParserTest10(void)
4062 {
4063  int result = 0;
4064  Flow *f = NULL;
4065  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
4066  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4067  TcpSession ssn;
4068  HtpState *htp_state = NULL;
4069  int r = 0;
4071 
4072  memset(&ssn, 0, sizeof(ssn));
4073 
4074  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4075  if (f == NULL)
4076  goto end;
4077  f->protoctx = &ssn;
4078  f->proto = IPPROTO_TCP;
4079  f->alproto = ALPROTO_HTTP1;
4080 
4081  StreamTcpInitConfig(true);
4082 
4083  uint32_t u;
4084  for (u = 0; u < httplen1; u++) {
4085  uint8_t flags = 0;
4086 
4087  if (u == 0)
4088  flags = STREAM_TOSERVER|STREAM_START;
4089  else if (u == (httplen1 - 1))
4090  flags = STREAM_TOSERVER|STREAM_EOF;
4091  else
4092  flags = STREAM_TOSERVER;
4093 
4094  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4095  if (r != 0) {
4096  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4097  " 0: ", u, r);
4098  goto end;
4099  }
4100  }
4101 
4102  htp_state = f->alstate;
4103  if (htp_state == NULL) {
4104  printf("no http state: ");
4105  goto end;
4106  }
4107 
4108  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4109  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4110  if (h == NULL) {
4111  goto end;
4112  }
4113 
4114  char *name = bstr_util_strdup_to_c(h->name);
4115  if (name == NULL) {
4116  goto end;
4117  }
4118 
4119  if (strcmp(name, "Host") != 0) {
4120  printf("header name not \"Host\", instead \"%s\": ", name);
4121  free(name);
4122  goto end;
4123  }
4124  free(name);
4125 
4126  char *value = bstr_util_strdup_to_c(h->value);
4127  if (value == NULL) {
4128  goto end;
4129  }
4130 
4131  if (strcmp(value, "www.google.com") != 0) {
4132  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4133  free(value);
4134  goto end;
4135  }
4136  free(value);
4137 
4138  result = 1;
4139 end:
4140  if (alp_tctx != NULL)
4142  StreamTcpFreeConfig(true);
4143  UTHFreeFlow(f);
4144  return result;
4145 }
4146 
4147 /** \test double encoding in path
4148  */
4149 static int HTPParserTest11(void)
4150 {
4151  int result = 0;
4152  Flow *f = NULL;
4153  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4154  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4155  TcpSession ssn;
4156  HtpState *htp_state = NULL;
4157  int r = 0;
4159 
4160  memset(&ssn, 0, sizeof(ssn));
4161 
4162  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4163  if (f == NULL)
4164  goto end;
4165  f->protoctx = &ssn;
4166  f->proto = IPPROTO_TCP;
4167  f->alproto = ALPROTO_HTTP1;
4168 
4169  StreamTcpInitConfig(true);
4170 
4171  uint32_t u;
4172  for (u = 0; u < httplen1; u++) {
4173  uint8_t flags = 0;
4174 
4175  if (u == 0)
4176  flags = STREAM_TOSERVER|STREAM_START;
4177  else if (u == (httplen1 - 1))
4178  flags = STREAM_TOSERVER|STREAM_EOF;
4179  else
4180  flags = STREAM_TOSERVER;
4181 
4182  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4183  if (r != 0) {
4184  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4185  " 0: ", u, r);
4186  goto end;
4187  }
4188  }
4189 
4190  htp_state = f->alstate;
4191  if (htp_state == NULL) {
4192  printf("no http state: ");
4193  goto end;
4194  }
4195 
4196  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4197  if (tx == NULL)
4198  goto end;
4199  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4200  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4201  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4202  printf("normalized uri len should be 2, is %"PRIuMAX,
4203  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4204  goto end;
4205  }
4206 
4207  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4208  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4209  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4210  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4211  {
4212  printf("normalized uri \"");
4213  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4214  printf("\": ");
4215  goto end;
4216  }
4217  }
4218 
4219  result = 1;
4220 end:
4221  if (alp_tctx != NULL)
4223  StreamTcpFreeConfig(true);
4224  UTHFreeFlow(f);
4225  return result;
4226 }
4227 
4228 /** \test double encoding in query
4229  */
4230 static int HTPParserTest12(void)
4231 {
4232  int result = 0;
4233  Flow *f = NULL;
4234  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4235  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4236  TcpSession ssn;
4237  HtpState *htp_state = NULL;
4238  int r = 0;
4240 
4241  memset(&ssn, 0, sizeof(ssn));
4242 
4243  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4244  if (f == NULL)
4245  goto end;
4246  f->protoctx = &ssn;
4247  f->proto = IPPROTO_TCP;
4248  f->alproto = ALPROTO_HTTP1;
4249 
4250  StreamTcpInitConfig(true);
4251 
4252  uint32_t u;
4253  for (u = 0; u < httplen1; u++) {
4254  uint8_t flags = 0;
4255 
4256  if (u == 0)
4257  flags = STREAM_TOSERVER|STREAM_START;
4258  else if (u == (httplen1 - 1))
4259  flags = STREAM_TOSERVER|STREAM_EOF;
4260  else
4261  flags = STREAM_TOSERVER;
4262 
4263  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4264  if (r != 0) {
4265  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4266  " 0: ", u, r);
4267  goto end;
4268  }
4269  }
4270 
4271  htp_state = f->alstate;
4272  if (htp_state == NULL) {
4273  printf("no http state: ");
4274  goto end;
4275  }
4276 
4277  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4278  if (tx == NULL)
4279  goto end;
4280  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4281  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4282  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4283  printf("normalized uri len should be 5, is %"PRIuMAX,
4284  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4285  goto end;
4286  }
4287 
4288  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4289  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4290  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4291  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4292  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4293  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4294  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4295  {
4296  printf("normalized uri \"");
4297  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4298  printf("\": ");
4299  goto end;
4300  }
4301  }
4302 
4303  result = 1;
4304  end:
4305  if (alp_tctx != NULL)
4307  StreamTcpFreeConfig(true);
4308  UTHFreeFlow(f);
4309  return result;
4310 }
4311 
4312 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4313  */
4314 static int HTPParserTest13(void)
4315 {
4316  int result = 0;
4317  Flow *f = NULL;
4318  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4319  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4320  TcpSession ssn;
4321  HtpState *htp_state = NULL;
4322  int r = 0;
4324 
4325  memset(&ssn, 0, sizeof(ssn));
4326 
4327  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4328  if (f == NULL)
4329  goto end;
4330  f->protoctx = &ssn;
4331  f->proto = IPPROTO_TCP;
4332  f->alproto = ALPROTO_HTTP1;
4333 
4334  StreamTcpInitConfig(true);
4335 
4336  uint32_t u;
4337  for (u = 0; u < httplen1; u++) {
4338  uint8_t flags = 0;
4339 
4340  if (u == 0)
4341  flags = STREAM_TOSERVER|STREAM_START;
4342  else if (u == (httplen1 - 1))
4343  flags = STREAM_TOSERVER|STREAM_EOF;
4344  else
4345  flags = STREAM_TOSERVER;
4346 
4347  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4348  if (r != 0) {
4349  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4350  " 0: ", u, r);
4351  goto end;
4352  }
4353  }
4354 
4355  htp_state = f->alstate;
4356  if (htp_state == NULL) {
4357  printf("no http state: ");
4358  goto end;
4359  }
4360 
4361  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4362  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4363  if (h == NULL) {
4364  goto end;
4365  }
4366 
4367  char *name = bstr_util_strdup_to_c(h->name);
4368  if (name == NULL) {
4369  goto end;
4370  }
4371 
4372  if (strcmp(name, "Host") != 0) {
4373  printf("header name not \"Host\", instead \"%s\": ", name);
4374  free(name);
4375  goto end;
4376  }
4377  free(name);
4378 
4379  char *value = bstr_util_strdup_to_c(h->value);
4380  if (value == NULL) {
4381  goto end;
4382  }
4383 
4384  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4385  printf("header value not \"www.google.com\", instead \"");
4386  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4387  printf("\": ");
4388  free(value);
4389  goto end;
4390  }
4391  free(value);
4392 
4393  result = 1;
4394 end:
4395  if (alp_tctx != NULL)
4397  StreamTcpFreeConfig(true);
4398  UTHFreeFlow(f);
4399  return result;
4400 }
4401 
4402 /** \test Test basic config */
4403 static int HTPParserConfigTest01(void)
4404 {
4405  int ret = 0;
4406  char input[] = "\
4407 %YAML 1.1\n\
4408 ---\n\
4409 libhtp:\n\
4410 \n\
4411  default-config:\n\
4412  personality: IDS\n\
4413 \n\
4414  server-config:\n\
4415 \n\
4416  - apache-tomcat:\n\
4417  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4418  personality: Tomcat_6_0\n\
4419 \n\
4420  - iis7:\n\
4421  address: \n\
4422  - 192.168.0.0/24\n\
4423  - 192.168.10.0/24\n\
4424  personality: IIS_7_0\n\
4425 ";
4426 
4428  ConfInit();
4429 
4430  ConfYamlLoadString(input, strlen(input));
4431 
4432  ConfNode *outputs;
4433  outputs = ConfGetNode("libhtp.default-config.personality");
4434  if (outputs == NULL) {
4435  goto end;
4436  }
4437 
4438  outputs = ConfGetNode("libhtp.server-config");
4439  if (outputs == NULL) {
4440  goto end;
4441  }
4442 
4443  ConfNode *node = TAILQ_FIRST(&outputs->head);
4444  if (node == NULL) {
4445  goto end;
4446  }
4447  if (strcmp(node->name, "0") != 0) {
4448  goto end;
4449  }
4450  node = TAILQ_FIRST(&node->head);
4451  if (node == NULL) {
4452  goto end;
4453  }
4454  if (strcmp(node->name, "apache-tomcat") != 0) {
4455  goto end;
4456  }
4457 
4458  int i = 0;
4459  ConfNode *n;
4460 
4461  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4462  if (node2 == NULL) {
4463  goto end;
4464  }
4465  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4466  goto end;
4467  }
4468 
4469  node = ConfNodeLookupChild(node, "address");
4470  if (node == NULL) {
4471  goto end;
4472  }
4473  TAILQ_FOREACH(n, &node->head, next) {
4474  if (n == NULL) {
4475  goto end;
4476  }
4477 
4478  switch(i) {
4479  case 0:
4480  if (strcmp(n->name, "0") != 0) {
4481  goto end;
4482  }
4483  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4484  goto end;
4485  }
4486  break;
4487  case 1:
4488  if (strcmp(n->name, "1") != 0) {
4489  goto end;
4490  }
4491  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4492  goto end;
4493  }
4494  break;
4495  case 2:
4496  if (strcmp(n->name, "2") != 0) {
4497  goto end;
4498  }
4499  if (strcmp(n->val, "::1") != 0) {
4500  goto end;
4501  }
4502  break;
4503  default:
4504  goto end;
4505  }
4506  i++;
4507  }
4508 
4509  outputs = ConfGetNode("libhtp.server-config");
4510  if (outputs == NULL) {
4511  goto end;
4512  }
4513 
4514  node = TAILQ_FIRST(&outputs->head);
4515  node = TAILQ_NEXT(node, next);
4516  if (node == NULL) {
4517  goto end;
4518  }
4519  if (strcmp(node->name, "1") != 0) {
4520  goto end;
4521  }
4522  node = TAILQ_FIRST(&node->head);
4523  if (node == NULL) {
4524  goto end;
4525  }
4526  if (strcmp(node->name, "iis7") != 0) {
4527  goto end;
4528  }
4529 
4530  node2 = ConfNodeLookupChild(node, "personality");
4531  if (node2 == NULL) {
4532  goto end;
4533  }
4534  if (strcmp(node2->val, "IIS_7_0") != 0) {
4535  goto end;
4536  }
4537 
4538  node = ConfNodeLookupChild(node, "address");
4539  if (node == NULL) {
4540  goto end;
4541  }
4542 
4543  i = 0;
4544  TAILQ_FOREACH(n, &node->head, next) {
4545  if (n == NULL) {
4546  goto end;
4547  }
4548 
4549  switch(i) {
4550  case 0:
4551  if (strcmp(n->name, "0") != 0) {
4552  goto end;
4553  }
4554  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4555  goto end;
4556  }
4557  break;
4558  case 1:
4559  if (strcmp(n->name, "1") != 0) {
4560  goto end;
4561  }
4562  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4563  goto end;
4564  }
4565  break;
4566  default:
4567  goto end;
4568  }
4569  i++;
4570  }
4571 
4572  ret = 1;
4573 
4574 end:
4575  ConfDeInit();
4577 
4578  return ret;
4579 }
4580 
4581 /** \test Test config builds radix correctly */
4582 static int HTPParserConfigTest02(void)
4583 {
4584  int ret = 0;
4585  char input[] = "\
4586 %YAML 1.1\n\
4587 ---\n\
4588 libhtp:\n\
4589 \n\
4590  default-config:\n\
4591  personality: IDS\n\
4592 \n\
4593  server-config:\n\
4594 \n\
4595  - apache-tomcat:\n\
4596  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4597  personality: Tomcat_6_0\n\
4598 \n\
4599  - iis7:\n\
4600  address: \n\
4601  - 192.168.0.0/24\n\
4602  - 192.168.10.0/24\n\
4603  personality: IIS_7_0\n\
4604 ";
4605 
4607  ConfInit();
4609 
4610  ConfYamlLoadString(input, strlen(input));
4611 
4612  HTPConfigure();
4613 
4614  if (cfglist.cfg == NULL) {
4615  printf("No default config created.\n");
4616  goto end;
4617  }
4618 
4619  if (cfgtree == NULL) {
4620  printf("No config tree created.\n");
4621  goto end;
4622  }
4623 
4624  htp_cfg_t *htp = cfglist.cfg;
4625  uint8_t buf[128];
4626  const char *addr;
4627  void *user_data = NULL;
4628 
4629  addr = "192.168.10.42";
4630  if (inet_pton(AF_INET, addr, buf) == 1) {
4631  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4632  if (user_data != NULL) {
4633  HTPCfgRec *htp_cfg_rec = user_data;
4634  htp = htp_cfg_rec->cfg;
4635  SCLogDebug("LIBHTP using config: %p", htp);
4636  }
4637  if (htp == NULL) {
4638  printf("Could not get config for: %s\n", addr);
4639  goto end;
4640  }
4641  }
4642  else {
4643  printf("Failed to parse address: %s\n", addr);
4644  goto end;
4645  }
4646 
4647  user_data = NULL;
4648  addr = "::1";
4649  if (inet_pton(AF_INET6, addr, buf) == 1) {
4650  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4651  if (user_data != NULL) {
4652  HTPCfgRec *htp_cfg_rec = user_data;
4653  htp = htp_cfg_rec->cfg;
4654  SCLogDebug("LIBHTP using config: %p", htp);
4655  }
4656  if (htp == NULL) {
4657  printf("Could not get config for: %s\n", addr);
4658  goto end;
4659  }
4660  }
4661  else {
4662  printf("Failed to parse address: %s\n", addr);
4663  goto end;
4664  }
4665 
4666  ret = 1;
4667 
4668 end:
4669  HTPFreeConfig();
4670  ConfDeInit();
4673 
4674  return ret;
4675 }
4676 
4677 /** \test Test traffic is handled by the correct htp config */
4678 static int HTPParserConfigTest03(void)
4679 {
4680  int result = 1;
4681  Flow *f = NULL;
4682  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4683  " Data is c0oL!";
4684  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4685  TcpSession ssn;
4687 
4688  HtpState *htp_state = NULL;
4689  int r = 0;
4690  char input[] = "\
4691 %YAML 1.1\n\
4692 ---\n\
4693 libhtp:\n\
4694 \n\
4695  default-config:\n\
4696  personality: IDS\n\
4697 \n\
4698  server-config:\n\
4699 \n\
4700  - apache-tomcat:\n\
4701  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4702  personality: Tomcat_6_0\n\
4703 \n\
4704  - iis7:\n\
4705  address: \n\
4706  - 192.168.0.0/24\n\
4707  - 192.168.10.0/24\n\
4708  personality: IIS_7_0\n\
4709 ";
4710 
4712  ConfInit();
4714 
4715  ConfYamlLoadString(input, strlen(input));
4716 
4717  HTPConfigure();
4718 
4719  const char *addr = "192.168.10.42";
4720 
4721  memset(&ssn, 0, sizeof(ssn));
4722 
4723  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4724  if (f == NULL)
4725  goto end;
4726  f->protoctx = &ssn;
4727  f->proto = IPPROTO_TCP;
4728  f->alproto = ALPROTO_HTTP1;
4729 
4730  htp_cfg_t *htp = cfglist.cfg;
4731 
4732  void *user_data = NULL;
4733  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4734  if (user_data != NULL) {
4735  HTPCfgRec *htp_cfg_rec = user_data;
4736  htp = htp_cfg_rec->cfg;
4737  SCLogDebug("LIBHTP using config: %p", htp);
4738  }
4739  if (htp == NULL) {
4740  printf("Could not get config for: %s\n", addr);
4741  goto end;
4742  }
4743 
4744  StreamTcpInitConfig(true);
4745 
4746  uint32_t u;
4747  for (u = 0; u < httplen1; u++) {
4748  uint8_t flags = 0;
4749 
4750  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4751  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4752  else flags = STREAM_TOSERVER;
4753 
4754  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4755  if (r != 0) {
4756  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4757  " 0: ", u, r);
4758  result = 0;
4759  goto end;
4760  }
4761  }
4762 
4763  htp_state = f->alstate;
4764  if (htp_state == NULL) {
4765  printf("no http state: ");
4766  result = 0;
4767  goto end;
4768  }
4769 
4770  if (HTPStateGetTxCnt(htp_state) != 2) {
4771  printf("HTPStateGetTxCnt(htp_state) failure\n");
4772  goto end;
4773  }
4774 
4775  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4776  if (tx == NULL)
4777  goto end;
4778  if (tx->cfg != htp) {
4779  printf("wrong HTP config (%p instead of %p - default=%p): ",
4780  tx->cfg, htp, cfglist.cfg);
4781  goto end;
4782  }
4783  tx = HTPStateGetTx(htp_state, 1);
4784  if (tx == NULL)
4785  goto end;
4786  if (tx->cfg != htp) {
4787  printf("wrong HTP config (%p instead of %p - default=%p): ",
4788  tx->cfg, htp, cfglist.cfg);
4789  goto end;
4790  }
4791 
4792 end:
4793  if (alp_tctx != NULL)
4795  HTPFreeConfig();
4796  ConfDeInit();
4799 
4800  StreamTcpFreeConfig(true);
4801  UTHFreeFlow(f);
4802  return result;
4803 }
4804 
4805 /* disabled when we upgraded to libhtp 0.5.x */
4806 #if 0
4807 static int HTPParserConfigTest04(void)
4808 {
4809  int result = 0;
4810 
4811  char input[] = "\
4812 %YAML 1.1\n\
4813 ---\n\
4814 libhtp:\n\
4815 \n\
4816  default-config:\n\
4817  personality: IDS\n\
4818  path-control-char-handling: status_400\n\
4819  path-convert-utf8: yes\n\
4820  path-invalid-encoding-handling: remove_percent\n\
4821 \n\
4822  server-config:\n\
4823 \n\
4824  - apache-tomcat:\n\
4825  personality: Tomcat_6_0\n\
4826  path-invalid-utf8-handling: none\n\
4827  path-nul-encoded-handling: status_404\n\
4828  path-nul-raw-handling: status_400\n\
4829 \n\
4830  - iis7:\n\
4831  personality: IIS_7_0\n\
4832  path-replacement-char: o\n\
4833  path-unicode-mapping: status_400\n\
4834 ";
4835 
4837  ConfInit();
4839 
4840  ConfYamlLoadString(input, strlen(input));
4841 
4842  HTPConfigure();
4843 
4844  HTPCfgRec *cfg_rec = &cfglist;
4845  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4846  cfg_rec->cfg->path_convert_utf8 != 1 ||
4847  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4848  printf("failed 1\n");
4849  goto end;
4850  }
4851 
4852  cfg_rec = cfg_rec->next;
4853  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4854  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4855  printf("failed 2\n");
4856  goto end;
4857  }
4858 
4859  cfg_rec = cfg_rec->next;
4860  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4861  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4862  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4863  printf("failed 3\n");
4864  goto end;
4865  }
4866 
4867  result = 1;
4868 
4869 end:
4870  HTPFreeConfig();
4871  ConfDeInit();
4874 
4875  return result;
4876 }
4877 #endif
4878 
4879 /** \test Test %2f decoding in profile Apache_2_2
4880  *
4881  * %2f in path is left untouched
4882  * %2f in query string is normalized to %2F
4883  * %252f in query string is decoded/normalized to %2F
4884  */
4885 static int HTPParserDecodingTest01(void)
4886 {
4887  uint8_t httpbuf1[] =
4888  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4889  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4890  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4891  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4892  TcpSession ssn;
4895 
4896  char input[] = "\
4897 %YAML 1.1\n\
4898 ---\n\
4899 libhtp:\n\
4900 \n\
4901  default-config:\n\
4902  personality: Apache_2\n\
4903 ";
4904 
4906  ConfInit();
4908  ConfYamlLoadString(input, strlen(input));
4909  HTPConfigure();
4910  const char *addr = "4.3.2.1";
4911  memset(&ssn, 0, sizeof(ssn));
4912 
4913  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4914  FAIL_IF_NULL(f);
4915  f->protoctx = &ssn;
4916  f->proto = IPPROTO_TCP;
4917  f->alproto = ALPROTO_HTTP1;
4918 
4919  StreamTcpInitConfig(true);
4920 
4921  for (uint32_t u = 0; u < httplen1; u++) {
4922  uint8_t flags = 0;
4923  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4924  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4925  else flags = STREAM_TOSERVER;
4926 
4927  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
4928  FAIL_IF(r != 0);
4929  }
4930 
4931  HtpState *htp_state = f->alstate;
4932  FAIL_IF_NULL(htp_state);
4933 
4934  uint8_t ref1[] = "/abc%2fdef";
4935  size_t reflen = sizeof(ref1) - 1;
4936 
4937  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4938  FAIL_IF_NULL(tx);
4939 
4940  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4941  FAIL_IF_NULL(tx_ud);
4943  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4944  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4945  bstr_len(tx_ud->request_uri_normalized)) != 0);
4946 
4947  uint8_t ref2[] = "/abc/def?ghi/jkl";
4948  reflen = sizeof(ref2) - 1;
4949 
4950  tx = HTPStateGetTx(htp_state, 1);
4951  FAIL_IF_NULL(tx);
4952  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4953  FAIL_IF_NULL(tx_ud);
4955  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4956 
4957  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4958  bstr_len(tx_ud->request_uri_normalized)) != 0);
4959 
4960  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4961  reflen = sizeof(ref3) - 1;
4962  tx = HTPStateGetTx(htp_state, 2);
4963  FAIL_IF_NULL(tx);
4964  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4965  FAIL_IF_NULL(tx_ud);
4967  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
4968 
4969  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4970  bstr_len(tx_ud->request_uri_normalized)) != 0);
4971 
4973  HTPFreeConfig();
4974  ConfDeInit();
4977 
4978  StreamTcpFreeConfig(true);
4979  UTHFreeFlow(f);
4980  PASS;
4981 }
4982 
4983 static int HTPParserDecodingTest01a(void)
4984 {
4985  uint8_t httpbuf1[] = "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4986  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4987  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4988  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4989  TcpSession ssn;
4992 
4993  char input[] = "\
4994 %YAML 1.1\n\
4995 ---\n\
4996 libhtp:\n\
4997 \n\
4998  default-config:\n\
4999  personality: Apache_2\n\
5000 ";
5001 
5003  ConfInit();
5005  ConfYamlLoadString(input, strlen(input));
5006  HTPConfigure();
5007  const char *addr = "4.3.2.1";
5008  memset(&ssn, 0, sizeof(ssn));
5009 
5010  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5011  FAIL_IF_NULL(f);
5012  f->protoctx = &ssn;
5013  f->proto = IPPROTO_TCP;
5014  f->alproto = ALPROTO_HTTP1;
5015 
5016  StreamTcpInitConfig(true);
5017 
5018  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
5019  (STREAM_TOSERVER | STREAM_START | STREAM_EOF), httpbuf1, httplen1);
5020  FAIL_IF(r != 0);
5021 
5022  HtpState *htp_state = f->alstate;
5023  FAIL_IF_NULL(htp_state);
5024 
5025  uint8_t ref1[] = "/abc%2fdef";
5026  size_t reflen = sizeof(ref1) - 1;
5027 
5028  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5029  FAIL_IF_NULL(tx);
5030 
5031  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5032  FAIL_IF_NULL(tx_ud);
5034  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5035  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5036  bstr_len(tx_ud->request_uri_normalized)) != 0);
5037 
5038  uint8_t ref2[] = "/abc/def?ghi/jkl";
5039  reflen = sizeof(ref2) - 1;
5040 
5041  tx = HTPStateGetTx(htp_state, 1);
5042  FAIL_IF_NULL(tx);
5043  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5044  FAIL_IF_NULL(tx_ud);
5046  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5047 
5048  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5049  bstr_len(tx_ud->request_uri_normalized)) != 0);
5050 
5051  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5052  reflen = sizeof(ref3) - 1;
5053  tx = HTPStateGetTx(htp_state, 2);
5054  FAIL_IF_NULL(tx);
5055  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5056  FAIL_IF_NULL(tx_ud);
5058  FAIL_IF(reflen != bstr_len(tx_ud->request_uri_normalized));
5059 
5060  FAIL_IF(memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5061  bstr_len(tx_ud->request_uri_normalized)) != 0);
5062 
5064  HTPFreeConfig();
5065  ConfDeInit();
5068 
5069  StreamTcpFreeConfig(true);
5070  UTHFreeFlow(f);
5071  PASS;
5072 }
5073 
5074 /** \test Test %2f decoding in profile IDS
5075  *
5076  * %2f in path decoded to /
5077  * %2f in query string is decoded to /
5078  * %252f in query string is decoded to %2F
5079  */
5080 static int HTPParserDecodingTest02(void)
5081 {
5082  int result = 0;
5083  Flow *f = NULL;
5084  uint8_t httpbuf1[] =
5085  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5086  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5087  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5088  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5089  TcpSession ssn;
5091 
5092  HtpState *htp_state = NULL;
5093  int r = 0;
5094  char input[] = "\
5095 %YAML 1.1\n\
5096 ---\n\
5097 libhtp:\n\
5098 \n\
5099  default-config:\n\
5100  personality: IDS\n\
5101  double-decode-path: no\n\
5102  double-decode-query: no\n\
5103 ";
5104 
5106  ConfInit();
5108  ConfYamlLoadString(input, strlen(input));
5109  HTPConfigure();
5110  const char *addr = "4.3.2.1";
5111  memset(&ssn, 0, sizeof(ssn));
5112 
5113  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5114  if (f == NULL)
5115  goto end;
5116  f->protoctx = &ssn;
5117  f->proto = IPPROTO_TCP;
5118  f->alproto = ALPROTO_HTTP1;
5119 
5120  StreamTcpInitConfig(true);
5121 
5122  uint32_t u;
5123  for (u = 0; u < httplen1; u++) {
5124  uint8_t flags = 0;
5125 
5126  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5127  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5128  else flags = STREAM_TOSERVER;
5129 
5130  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5131  if (r != 0) {
5132  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5133  " 0: ", u, r);
5134  goto end;
5135  }
5136  }
5137 
5138  htp_state = f->alstate;
5139  if (htp_state == NULL) {
5140  printf("no http state: ");
5141  goto end;
5142  }
5143 
5144  uint8_t ref1[] = "/abc/def";
5145  size_t reflen = sizeof(ref1) - 1;
5146 
5147  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5148  if (tx == NULL)
5149  goto end;
5150  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5151  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5152  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5153  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5154  (uintmax_t)reflen,
5155  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5156  goto end;
5157  }
5158 
5159  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5160  bstr_len(tx_ud->request_uri_normalized)) != 0)
5161  {
5162  printf("normalized uri \"");
5163  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5164  printf("\" != \"");
5165  PrintRawUriFp(stdout, ref1, reflen);
5166  printf("\": ");
5167  goto end;
5168  }
5169  }
5170 
5171  uint8_t ref2[] = "/abc/def?ghi/jkl";
5172  reflen = sizeof(ref2) - 1;
5173 
5174  tx = HTPStateGetTx(htp_state, 1);
5175  if (tx == NULL)
5176  goto end;
5177  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5178  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5179  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5180  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5181  (uintmax_t)reflen,
5182  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5183  goto end;
5184  }
5185 
5186  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5187  bstr_len(tx_ud->request_uri_normalized)) != 0)
5188  {
5189  printf("normalized uri \"");
5190  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5191  printf("\" != \"");
5192  PrintRawUriFp(stdout, ref2, reflen);
5193  printf("\": ");
5194  goto end;
5195  }
5196  }
5197 
5198  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5199  reflen = sizeof(ref3) - 1;
5200  tx = HTPStateGetTx(htp_state, 2);
5201  if (tx == NULL)
5202  goto end;
5203  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5204  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5205  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5206  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5207  (uintmax_t)reflen,
5208  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5209  goto end;
5210  }
5211 
5212  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5213  bstr_len(tx_ud->request_uri_normalized)) != 0)
5214  {
5215  printf("normalized uri \"");
5216  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5217  printf("\" != \"");
5218  PrintRawUriFp(stdout, ref3, reflen);
5219  printf("\": ");
5220  goto end;
5221  }
5222  }
5223 
5224  result = 1;
5225 
5226 end:
5227  if (alp_tctx != NULL)
5229  HTPFreeConfig();
5230  ConfDeInit();
5233 
5234  StreamTcpFreeConfig(true);
5235  UTHFreeFlow(f);
5236  return result;
5237 }
5238 
5239 /** \test Test %2f decoding in profile IDS with double-decode-* options
5240  *
5241  * %252f in path decoded to /
5242  * %252f in query string is decoded to /
5243  */
5244 static int HTPParserDecodingTest03(void)
5245 {
5246  int result = 0;
5247  Flow *f = NULL;
5248  uint8_t httpbuf1[] =
5249  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5250  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5251  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5252  TcpSession ssn;
5254 
5255  HtpState *htp_state = NULL;
5256  int r = 0;
5257  char input[] = "\
5258 %YAML 1.1\n\
5259 ---\n\
5260 libhtp:\n\
5261 \n\
5262  default-config:\n\
5263  personality: IDS\n\
5264  double-decode-path: yes\n\
5265  double-decode-query: yes\n\
5266 ";
5267 
5269  ConfInit();
5271  ConfYamlLoadString(input, strlen(input));
5272  HTPConfigure();
5273  const char *addr = "4.3.2.1";
5274  memset(&ssn, 0, sizeof(ssn));
5275 
5276  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5277  if (f == NULL)
5278  goto end;
5279  f->protoctx = &ssn;
5280  f->proto = IPPROTO_TCP;
5281  f->alproto = ALPROTO_HTTP1;
5282 
5283  StreamTcpInitConfig(true);
5284 
5285  uint32_t u;
5286  for (u = 0; u < httplen1; u++) {
5287  uint8_t flags = 0;
5288 
5289  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5290  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5291  else flags = STREAM_TOSERVER;
5292 
5293  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5294  if (r != 0) {
5295  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5296  " 0: ", u, r);
5297  goto end;
5298  }
5299  }
5300 
5301  htp_state = f->alstate;
5302  if (htp_state == NULL) {
5303  printf("no http state: ");
5304  goto end;
5305  }
5306 
5307  uint8_t ref1[] = "/abc/def";
5308  size_t reflen = sizeof(ref1) - 1;
5309 
5310  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5311  if (tx == NULL)
5312  goto end;
5313  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5314  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5315  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5316  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5317  (uintmax_t)reflen,
5318  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5319  goto end;
5320  }
5321 
5322  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5323  bstr_len(tx_ud->request_uri_normalized)) != 0)
5324  {
5325  printf("normalized uri \"");
5326  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5327  printf("\" != \"");
5328  PrintRawUriFp(stdout, ref1, reflen);
5329  printf("\": ");
5330  goto end;
5331  }
5332  }
5333 
5334  uint8_t ref2[] = "/abc/def?ghi/jkl";
5335  reflen = sizeof(ref2) - 1;
5336 
5337  tx = HTPStateGetTx(htp_state, 1);
5338  if (tx == NULL)
5339  goto end;
5340  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5341  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5342  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5343  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5344  (uintmax_t)reflen,
5345  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5346  goto end;
5347  }
5348 
5349  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5350  bstr_len(tx_ud->request_uri_normalized)) != 0)
5351  {
5352  printf("normalized uri \"");
5353  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5354  printf("\" != \"");
5355  PrintRawUriFp(stdout, ref2, reflen);
5356  printf("\": ");
5357  goto end;
5358  }
5359  }
5360 
5361  result = 1;
5362 
5363 end:
5364  if (alp_tctx != NULL)
5366  HTPFreeConfig();
5367  ConfDeInit();
5370 
5371  StreamTcpFreeConfig(true);
5372  UTHFreeFlow(f);
5373  return result;
5374 }
5375 
5376 /** \test Test http:// in query profile IDS
5377  */
5378 static int HTPParserDecodingTest04(void)
5379 {
5380  int result = 0;
5381  Flow *f = NULL;
5382  uint8_t httpbuf1[] =
5383  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5384  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5385  TcpSession ssn;
5387 
5388  HtpState *htp_state = NULL;
5389  int r = 0;
5390  char input[] = "\
5391 %YAML 1.1\n\
5392 ---\n\
5393 libhtp:\n\
5394 \n\
5395  default-config:\n\
5396  personality: IDS\n\
5397  double-decode-path: yes\n\
5398  double-decode-query: yes\n\
5399 ";
5400 
5402  ConfInit();
5404  ConfYamlLoadString(input, strlen(input));
5405  HTPConfigure();
5406  const char *addr = "4.3.2.1";
5407  memset(&ssn, 0, sizeof(ssn));
5408 
5409  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5410  if (f == NULL)
5411  goto end;
5412  f->protoctx = &ssn;
5413  f->proto = IPPROTO_TCP;
5414  f->alproto = ALPROTO_HTTP1;
5415 
5416  StreamTcpInitConfig(true);
5417 
5418  uint32_t u;
5419  for (u = 0; u < httplen1; u++) {
5420  uint8_t flags = 0;
5421 
5422  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5423  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5424  else flags = STREAM_TOSERVER;
5425 
5426  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5427  if (r != 0) {
5428  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5429  " 0: ", u, r);
5430  goto end;
5431  }
5432  }
5433 
5434  htp_state = f->alstate;
5435  if (htp_state == NULL) {
5436  printf("no http state: ");
5437  goto end;
5438  }
5439 
5440  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5441  size_t reflen = sizeof(ref1) - 1;
5442 
5443  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5444  if (tx == NULL)
5445  goto end;
5446  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5447  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5448  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5449  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5450  (uintmax_t)reflen,
5451  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5452  goto end;
5453  }
5454 
5455  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5456  bstr_len(tx_ud->request_uri_normalized)) != 0)
5457  {
5458  printf("normalized uri \"");
5459  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5460  printf("\" != \"");
5461  PrintRawUriFp(stdout, ref1, reflen);
5462  printf("\": ");
5463  goto end;
5464  }
5465  }
5466 
5467  result = 1;
5468 
5469 end:
5470  if (alp_tctx != NULL)
5472  HTPFreeConfig();
5473  ConfDeInit();
5476 
5477  StreamTcpFreeConfig(true);
5478  UTHFreeFlow(f);
5479  return result;
5480 }
5481 
5482 /** \test Test \ char in query profile IDS. Bug 739
5483  */
5484 static int HTPParserDecodingTest05(void)
5485 {
5486  int result = 0;
5487  Flow *f = NULL;
5488  uint8_t httpbuf1[] =
5489  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5490  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5491  TcpSession ssn;
5493 
5494  HtpState *htp_state = NULL;
5495  int r = 0;
5496  char input[] = "\
5497 %YAML 1.1\n\
5498 ---\n\
5499 libhtp:\n\
5500 \n\
5501  default-config:\n\
5502  personality: IDS\n\
5503  double-decode-path: yes\n\
5504  double-decode-query: yes\n\
5505 ";
5506 
5508  ConfInit();
5510  ConfYamlLoadString(input, strlen(input));
5511  HTPConfigure();
5512  const char *addr = "4.3.2.1";
5513  memset(&ssn, 0, sizeof(ssn));
5514 
5515  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5516  if (f == NULL)
5517  goto end;
5518  f->protoctx = &ssn;
5519  f->proto = IPPROTO_TCP;
5520  f->alproto = ALPROTO_HTTP1;
5521 
5522  StreamTcpInitConfig(true);
5523 
5524  uint32_t u;
5525  for (u = 0; u < httplen1; u++) {
5526  uint8_t flags = 0;
5527 
5528  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5529  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5530  else flags = STREAM_TOSERVER;
5531 
5532  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5533  if (r != 0) {
5534  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5535  " 0: ", u, r);
5536  goto end;
5537  }
5538  }
5539 
5540  htp_state = f->alstate;
5541  if (htp_state == NULL) {
5542  printf("no http state: ");
5543  goto end;
5544  }
5545 
5546  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5547  size_t reflen = sizeof(ref1) - 1;
5548 
5549  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5550  if (tx == NULL)
5551  goto end;
5552  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5553  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5554  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5555  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5556  (uintmax_t)reflen,
5557  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5558  goto end;
5559  }
5560 
5561  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5562  bstr_len(tx_ud->request_uri_normalized)) != 0)
5563  {
5564  printf("normalized uri \"");
5565  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5566  printf("\" != \"");
5567  PrintRawUriFp(stdout, ref1, reflen);
5568  printf("\": ");
5569  goto end;
5570  }
5571  }
5572 
5573  result = 1;
5574 
5575 end:
5576  if (alp_tctx != NULL)
5578  HTPFreeConfig();
5579  ConfDeInit();
5582 
5583  StreamTcpFreeConfig(true);
5584  UTHFreeFlow(f);
5585  return result;
5586 }
5587 
5588 /** \test Test + char in query. Bug 1035
5589  */
5590 static int HTPParserDecodingTest06(void)
5591 {
5592  int result = 0;
5593  Flow *f = NULL;
5594  uint8_t httpbuf1[] =
5595  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5596  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5597  TcpSession ssn;
5599 
5600  HtpState *htp_state = NULL;
5601  int r = 0;
5602  char input[] = "\
5603 %YAML 1.1\n\
5604 ---\n\
5605 libhtp:\n\
5606 \n\
5607  default-config:\n\
5608  personality: IDS\n\
5609  double-decode-path: yes\n\
5610  double-decode-query: yes\n\
5611 ";
5612 
5614  ConfInit();
5616  ConfYamlLoadString(input, strlen(input));
5617  HTPConfigure();
5618  const char *addr = "4.3.2.1";
5619  memset(&ssn, 0, sizeof(ssn));
5620 
5621  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5622  if (f == NULL)
5623  goto end;
5624  f->protoctx = &ssn;
5625  f->proto = IPPROTO_TCP;
5626  f->alproto = ALPROTO_HTTP1;
5627 
5628  StreamTcpInitConfig(true);
5629 
5630  uint32_t u;
5631  for (u = 0; u < httplen1; u++) {
5632  uint8_t flags = 0;
5633 
5634  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5635  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5636  else flags = STREAM_TOSERVER;
5637 
5638  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5639  if (r != 0) {
5640  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5641  " 0: ", u, r);
5642  goto end;
5643  }
5644  }
5645 
5646  htp_state = f->alstate;
5647  if (htp_state == NULL) {
5648  printf("no http state: ");
5649  goto end;
5650  }
5651 
5652  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5653  size_t reflen = sizeof(ref1) - 1;
5654 
5655  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5656  if (tx == NULL)
5657  goto end;
5658  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5659  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5660  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5661  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5662  (uintmax_t)reflen,
5663  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5664  goto end;
5665  }
5666 
5667  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5668  bstr_len(tx_ud->request_uri_normalized)) != 0)
5669  {
5670  printf("normalized uri \"");
5671  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5672  printf("\" != \"");
5673  PrintRawUriFp(stdout, ref1, reflen);
5674  printf("\": ");
5675  goto end;
5676  }
5677  }
5678 
5679  result = 1;
5680 
5681 end:
5682  if (alp_tctx != NULL)
5684  HTPFreeConfig();
5685  ConfDeInit();
5688 
5689  StreamTcpFreeConfig(true);
5690  UTHFreeFlow(f);
5691  return result;
5692 }
5693 
5694 /** \test Test + char in query. Bug 1035
5695  */
5696 static int HTPParserDecodingTest07(void)
5697 {
5698  int result = 0;
5699  Flow *f = NULL;
5700  uint8_t httpbuf1[] =
5701  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5702  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5703  TcpSession ssn;
5705 
5706  HtpState *htp_state = NULL;
5707  int r = 0;
5708  char input[] = "\
5709 %YAML 1.1\n\
5710 ---\n\
5711 libhtp:\n\
5712 \n\
5713  default-config:\n\
5714  personality: IDS\n\
5715  double-decode-path: yes\n\
5716  double-decode-query: yes\n\
5717  query-plusspace-decode: yes\n\
5718 ";
5719 
5721  ConfInit();
5723  ConfYamlLoadString(input, strlen(input));
5724  HTPConfigure();
5725  const char *addr = "4.3.2.1";
5726  memset(&ssn, 0, sizeof(ssn));
5727 
5728  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5729  if (f == NULL)
5730  goto end;
5731  f->protoctx = &ssn;
5732  f->proto = IPPROTO_TCP;
5733  f->alproto = ALPROTO_HTTP1;
5734 
5735  StreamTcpInitConfig(true);
5736 
5737  uint32_t u;
5738  for (u = 0; u < httplen1; u++) {
5739  uint8_t flags = 0;
5740 
5741  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5742  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5743  else flags = STREAM_TOSERVER;
5744 
5745  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, flags, &httpbuf1[u], 1);
5746  if (r != 0) {
5747  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5748  " 0: ", u, r);
5749  goto end;
5750  }
5751  }
5752 
5753  htp_state = f->alstate;
5754  if (htp_state == NULL) {
5755  printf("no http state: ");
5756  goto end;
5757  }
5758 
5759  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5760  size_t reflen = sizeof(ref1) - 1;
5761 
5762  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5763  if (tx == NULL)
5764  goto end;
5765  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5766  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5767  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5768  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5769  (uintmax_t)reflen,
5770  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5771  goto end;
5772  }
5773 
5774  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5775  bstr_len(tx_ud->request_uri_normalized)) != 0)
5776  {
5777  printf("normalized uri \"");
5778  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5779  printf("\" != \"");
5780  PrintRawUriFp(stdout, ref1, reflen);
5781  printf("\": ");
5782  goto end;
5783  }
5784  }
5785 
5786  result = 1;
5787 
5788 end:
5789  if (alp_tctx != NULL)
5791  HTPFreeConfig();
5792  ConfDeInit();
5795 
5796  StreamTcpFreeConfig(true);
5797  UTHFreeFlow(f);
5798  return result;