suricata
app-layer-htp.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \ingroup httplayer
20  *
21  * @{
22  */
23 
24 /**
25  * \file
26  *
27  * \author Victor Julien <victor@inliniac.net>
28  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
29  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
30  * \author Brian Rectanus <brectanu@gmail.com>
31  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
32  *
33  * This file provides a HTTP protocol support for the engine using HTP library.
34  */
35 
36 #include "suricata.h"
37 #include "suricata-common.h"
38 #include "conf.h"
39 #include "debug.h"
40 #include "decode.h"
41 #include "threads.h"
42 #include "counters.h"
43 
44 #include "util-print.h"
45 #include "util-pool.h"
46 #include "util-radix-tree.h"
47 #include "util-file.h"
48 
49 #include "stream-tcp-private.h"
50 #include "stream-tcp-reassemble.h"
51 #include "stream-tcp.h"
52 #include "stream.h"
53 
54 #include "app-layer-protos.h"
55 #include "app-layer-parser.h"
56 
57 #include "app-layer.h"
58 #include "app-layer-htp.h"
59 #include "app-layer-htp-body.h"
60 #include "app-layer-htp-file.h"
61 #include "app-layer-htp-libhtp.h"
62 #include "app-layer-htp-xff.h"
63 
64 #include "util-spm.h"
65 #include "util-debug.h"
66 #include "util-time.h"
67 #include "util-misc.h"
68 
69 #include "util-unittest.h"
70 #include "util-unittest-helper.h"
71 #include "flow-util.h"
72 
73 #include "detect-engine.h"
74 #include "detect-engine-state.h"
75 #include "detect-parse.h"
76 
77 #include "decode-events.h"
78 
79 #include "util-memcmp.h"
80 #include "util-random.h"
81 #include "util-validate.h"
82 
83 //#define PRINT
84 
85 /** Fast lookup tree (radix) for the various HTP configurations */
86 static SCRadixTree *cfgtree;
87 /** List of HTP configurations. */
88 static HTPCfgRec cfglist;
89 
90 #ifdef DEBUG
91 static SCMutex htp_state_mem_lock = SCMUTEX_INITIALIZER;
92 static uint64_t htp_state_memuse = 0;
93 static uint64_t htp_state_memcnt = 0;
94 #endif
95 
96 SCEnumCharMap http_decoder_event_table[ ] = {
97  { "UNKNOWN_ERROR",
98  HTTP_DECODER_EVENT_UNKNOWN_ERROR},
99  { "GZIP_DECOMPRESSION_FAILED",
100  HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
101  { "REQUEST_FIELD_MISSING_COLON",
102  HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
103  { "RESPONSE_FIELD_MISSING_COLON",
104  HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
105  { "INVALID_REQUEST_CHUNK_LEN",
106  HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
107  { "INVALID_RESPONSE_CHUNK_LEN",
108  HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
109  { "INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST",
110  HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST},
111  { "INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE",
112  HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE},
113  { "INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST",
114  HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST},
115  { "INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE",
116  HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
117  { "100_CONTINUE_ALREADY_SEEN",
118  HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
119  { "UNABLE_TO_MATCH_RESPONSE_TO_REQUEST",
120  HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
121  { "INVALID_SERVER_PORT_IN_REQUEST",
122  HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
123  { "INVALID_AUTHORITY_PORT",
124  HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT},
125  { "REQUEST_HEADER_INVALID",
126  HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
127  { "RESPONSE_HEADER_INVALID",
128  HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
129  { "MISSING_HOST_HEADER",
130  HTTP_DECODER_EVENT_MISSING_HOST_HEADER},
131  { "HOST_HEADER_AMBIGUOUS",
132  HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS},
133  { "INVALID_REQUEST_FIELD_FOLDING",
134  HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
135  { "INVALID_RESPONSE_FIELD_FOLDING",
136  HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
137  { "REQUEST_FIELD_TOO_LONG",
138  HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
139  { "RESPONSE_FIELD_TOO_LONG",
140  HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
141  { "REQUEST_LINE_INVALID",
142  HTTP_DECODER_EVENT_REQUEST_LINE_INVALID},
143  { "REQUEST_BODY_UNEXPECTED",
144  HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
145  { "REQUEST_SERVER_PORT_TCP_PORT_MISMATCH",
146  HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH},
147  { "REQUEST_URI_HOST_INVALID",
148  HTTP_DECODER_EVENT_URI_HOST_INVALID},
149  { "REQUEST_HEADER_HOST_INVALID",
150  HTTP_DECODER_EVENT_HEADER_HOST_INVALID},
151  { "REQUEST_AUTH_UNRECOGNIZED",
152  HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED},
153  { "REQUEST_HEADER_REPETITION",
154  HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
155  { "RESPONSE_HEADER_REPETITION",
156  HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
157  { "DOUBLE_ENCODED_URI",
158  HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI},
159  { "URI_DELIM_NON_COMPLIANT",
160  HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
161  { "METHOD_DELIM_NON_COMPLIANT",
162  HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
163  { "REQUEST_LINE_LEADING_WHITESPACE",
164  HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
165  { "TOO_MANY_ENCODING_LAYERS",
166  HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
167  { "ABNORMAL_CE_HEADER",
168  HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
169  { "RESPONSE_MULTIPART_BYTERANGES",
170  HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
171  { "RESPONSE_ABNORMAL_TRANSFER_ENCODING",
172  HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
173  { "RESPONSE_CHUNKED_OLD_PROTO",
174  HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
175  { "RESPONSE_INVALID_PROTOCOL",
176  HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
177  { "RESPONSE_INVALID_STATUS",
178  HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
179  { "REQUEST_LINE_INCOMPLETE",
180  HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
181 
182  /* suricata warnings/errors */
183  { "MULTIPART_GENERIC_ERROR",
184  HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR},
185  { "MULTIPART_NO_FILEDATA",
186  HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA},
187  { "MULTIPART_INVALID_HEADER",
188  HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER},
189 
190  { NULL, -1 },
191 };
192 
193 static void *HTPStateGetTx(void *alstate, uint64_t tx_id);
194 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction);
195 static uint64_t HTPStateGetTxCnt(void *alstate);
196 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction);
197 
198 #ifdef DEBUG
199 /**
200  * \internal
201  *
202  * \brief Lookup the HTP personality string from the numeric personality.
203  *
204  * \todo This needs to be a libhtp function.
205  */
206 static const char *HTPLookupPersonalityString(int p)
207 {
208 #define CASE_HTP_PERSONALITY_STRING(p) \
209  case HTP_SERVER_ ## p: return #p
210 
211  switch (p) {
212  CASE_HTP_PERSONALITY_STRING(MINIMAL);
213  CASE_HTP_PERSONALITY_STRING(GENERIC);
214  CASE_HTP_PERSONALITY_STRING(IDS);
215  CASE_HTP_PERSONALITY_STRING(IIS_4_0);
216  CASE_HTP_PERSONALITY_STRING(IIS_5_0);
217  CASE_HTP_PERSONALITY_STRING(IIS_5_1);
218  CASE_HTP_PERSONALITY_STRING(IIS_6_0);
219  CASE_HTP_PERSONALITY_STRING(IIS_7_0);
220  CASE_HTP_PERSONALITY_STRING(IIS_7_5);
221  CASE_HTP_PERSONALITY_STRING(APACHE_2);
222  }
223 
224  return NULL;
225 }
226 #endif /* DEBUG */
227 
228 /**
229  * \internal
230  *
231  * \brief Lookup the numeric HTP personality from a string.
232  *
233  * \todo This needs to be a libhtp function.
234  */
235 static int HTPLookupPersonality(const char *str)
236 {
237 #define IF_HTP_PERSONALITY_NUM(p) \
238  if (strcasecmp(#p, str) == 0) return HTP_SERVER_ ## p
239 
240  IF_HTP_PERSONALITY_NUM(MINIMAL);
241  IF_HTP_PERSONALITY_NUM(GENERIC);
242  IF_HTP_PERSONALITY_NUM(IDS);
243  IF_HTP_PERSONALITY_NUM(IIS_4_0);
244  IF_HTP_PERSONALITY_NUM(IIS_5_0);
245  IF_HTP_PERSONALITY_NUM(IIS_5_1);
246  IF_HTP_PERSONALITY_NUM(IIS_6_0);
247  IF_HTP_PERSONALITY_NUM(IIS_7_0);
248  IF_HTP_PERSONALITY_NUM(IIS_7_5);
249  IF_HTP_PERSONALITY_NUM(APACHE_2);
250  if (strcasecmp("TOMCAT_6_0", str) == 0) {
251  SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
252  "longer supported by libhtp.", str);
253  return -1;
254  } else if ((strcasecmp("APACHE", str) == 0) ||
255  (strcasecmp("APACHE_2_2", str) == 0))
256  {
257  SCLogWarning(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
258  "longer supported by libhtp, failing back to "
259  "Apache2 personality.", str);
260  return HTP_SERVER_APACHE_2;
261  }
262 
263  return -1;
264 }
265 
266 static void HTPSetEvent(HtpState *s, HtpTxUserData *htud, uint8_t e)
267 {
268  SCLogDebug("setting event %u", e);
269 
270  if (htud) {
271  AppLayerDecoderEventsSetEventRaw(&htud->decoder_events, e);
272  s->events++;
273  return;
274  }
275 
276  htp_tx_t *tx = HTPStateGetTx(s, s->transaction_cnt);
277  if (tx == NULL && s->transaction_cnt > 0)
278  tx = HTPStateGetTx(s, s->transaction_cnt - 1);
279  if (tx != NULL) {
280  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
281  if (htud != NULL) {
282  AppLayerDecoderEventsSetEventRaw(&htud->decoder_events, e);
283  s->events++;
284  return;
285  }
286  }
287  SCLogDebug("couldn't set event %u", e);
288 }
289 
290 static AppLayerDecoderEvents *HTPGetEvents(void *state, uint64_t tx_id)
291 {
292  SCLogDebug("get HTTP events for TX %"PRIu64, tx_id);
293 
294  HtpState *s = (HtpState *)state;
295  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
296  if (tx != NULL) {
297  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
298  if (htud != NULL) {
299  SCLogDebug("has htud, htud->decoder_events %p", htud->decoder_events);
300  return htud->decoder_events;
301  }
302  }
303  return NULL;
304 }
305 
306 /** \brief Function to allocates the HTTP state memory and also creates the HTTP
307  * connection parser to be used by the HTP library
308  */
309 static void *HTPStateAlloc(void)
310 {
311  SCEnter();
312 
313  HtpState *s = HTPMalloc(sizeof(HtpState));
314  if (unlikely(s == NULL)) {
315  SCReturnPtr(NULL, "void");
316  }
317 
318  memset(s, 0x00, sizeof(HtpState));
319 
320 #ifdef DEBUG
321  SCMutexLock(&htp_state_mem_lock);
322  htp_state_memcnt++;
323  htp_state_memuse += sizeof(HtpState);
324  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
325  SCMutexUnlock(&htp_state_mem_lock);
326 #endif
327 
328  SCReturnPtr((void *)s, "void");
329 }
330 
331 static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
332 {
333  if (likely(htud)) {
334  HtpBodyFree(&htud->request_body);
335  HtpBodyFree(&htud->response_body);
336  bstr_free(htud->request_uri_normalized);
337  if (htud->request_headers_raw)
338  HTPFree(htud->request_headers_raw, htud->request_headers_raw_len);
339  if (htud->response_headers_raw)
340  HTPFree(htud->response_headers_raw, htud->response_headers_raw_len);
341  AppLayerDecoderEventsFreeEvents(&htud->decoder_events);
342  if (htud->boundary)
343  HTPFree(htud->boundary, htud->boundary_len);
344  if (htud->de_state != NULL) {
345  DetectEngineStateFree(htud->de_state);
346  }
347  HTPFree(htud, sizeof(HtpTxUserData));
348  }
349 }
350 
351 /** \brief Function to frees the HTTP state memory and also frees the HTTP
352  * connection parser memory which was used by the HTP library
353  */
354 void HTPStateFree(void *state)
355 {
356  SCEnter();
357 
358  HtpState *s = (HtpState *)state;
359  if (s == NULL) {
360  SCReturn;
361  }
362 
363  /* free the connection parser memory used by HTP library */
364  if (s->connp != NULL) {
365  SCLogDebug("freeing HTP state");
366 
367  uint64_t tx_id;
368  uint64_t total_txs = HTPStateGetTxCnt(state);
369  /* free the list of body chunks */
370  if (s->conn != NULL) {
371  for (tx_id = 0; tx_id < total_txs; tx_id++) {
372  htp_tx_t *tx = HTPStateGetTx(s, tx_id);
373  if (tx != NULL) {
374  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
375  HtpTxUserDataFree(s, htud);
376  htp_tx_set_user_data(tx, NULL);
377  }
378  }
379  }
380  htp_connp_destroy_all(s->connp);
381  }
382 
383  FileContainerFree(s->files_ts);
384  FileContainerFree(s->files_tc);
385  HTPFree(s, sizeof(HtpState));
386 
387 #ifdef DEBUG
388  SCMutexLock(&htp_state_mem_lock);
389  htp_state_memcnt--;
390  htp_state_memuse -= sizeof(HtpState);
391  SCLogDebug("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
392  SCMutexUnlock(&htp_state_mem_lock);
393 #endif
394 
395  SCReturn;
396 }
397 
398 /**
399  * \brief HTP transaction cleanup callback
400  *
401  * \warning We cannot actually free the transactions here. It seems that
402  * HTP only accepts freeing of transactions in the response callback.
403  */
404 static void HTPStateTransactionFree(void *state, uint64_t id)
405 {
406  SCEnter();
407 
408  HtpState *s = (HtpState *)state;
409 
410  SCLogDebug("state %p, id %"PRIu64, s, id);
411 
412  htp_tx_t *tx = HTPStateGetTx(s, id);
413  if (tx != NULL) {
414  /* This will remove obsolete body chunks */
415  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
416  HtpTxUserDataFree(s, htud);
417  htp_tx_set_user_data(tx, NULL);
418 
419  /* hack: even if libhtp considers the tx incomplete, we want to
420  * free it here. htp_tx_destroy however, will refuse to do this.
421  * As htp_tx_destroy_incomplete isn't available in the public API,
422  * we hack around it here. */
423  if (unlikely(!(
424  tx->request_progress == HTP_REQUEST_COMPLETE &&
425  tx->response_progress == HTP_RESPONSE_COMPLETE)))
426  {
427  tx->request_progress = HTP_REQUEST_COMPLETE;
428  tx->response_progress = HTP_RESPONSE_COMPLETE;
429  }
430  htp_tx_destroy(tx);
431  }
432 }
433 
434 /**
435  * \brief Sets a flag that informs the HTP app layer that some module in the
436  * engine needs the http request body data.
437  * \initonly
438  */
439 void AppLayerHtpEnableRequestBodyCallback(void)
440 {
441  SCEnter();
442 
443  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_BODY);
444  SCReturn;
445 }
446 
447 /**
448  * \brief Sets a flag that informs the HTP app layer that some module in the
449  * engine needs the http request body data.
450  * \initonly
451  */
452 void AppLayerHtpEnableResponseBodyCallback(void)
453 {
454  SCEnter();
455 
456  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_RESPONSE_BODY);
457  SCReturn;
458 }
459 
460 /**
461  * \brief Sets a flag that informs the HTP app layer that some module in the
462  * engine needs the http request multi part header.
463  *
464  * \initonly
465  */
466 static void AppLayerHtpNeedMultipartHeader(void)
467 {
468  SCEnter();
469  AppLayerHtpEnableRequestBodyCallback();
470 
471  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_MULTIPART);
472  SCReturn;
473 }
474 
475 /**
476  * \brief Sets a flag that informs the HTP app layer that some module in the
477  * engine needs the http request file.
478  *
479  * \initonly
480  */
481 void AppLayerHtpNeedFileInspection(void)
482 {
483  SCEnter();
484  AppLayerHtpNeedMultipartHeader();
485  AppLayerHtpEnableRequestBodyCallback();
486  AppLayerHtpEnableResponseBodyCallback();
487 
488  SC_ATOMIC_OR(htp_config_flags, HTP_REQUIRE_REQUEST_FILE);
489  SCReturn;
490 }
491 
492 /* below error messages updated up to libhtp 0.5.7 (git 379632278b38b9a792183694a4febb9e0dbd1e7a) */
493 struct {
494  const char *msg;
495  int de;
496 } htp_errors[] = {
497  { "GZip decompressor: inflateInit2 failed", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
498  { "Request field invalid: colon missing", HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON},
499  { "Response field invalid: missing colon", HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON},
500  { "Request chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN},
501  { "Response chunk encoding: Invalid chunk length", HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN},
502 /* { "Invalid T-E value in request", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_T_E
503  { "Invalid T-E value in response", HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE}, <- nothing to replace it */
504 /* { "Invalid C-L field in request", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST}, <- tx flag HTP_REQUEST_INVALID_C_L */
505  { "Invalid C-L field in response", HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE},
506  { "Already seen 100-Continue", HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN},
507  { "Unable to match response to request", HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST},
508  { "Invalid server port information in request", HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST},
509 /* { "Invalid authority port", HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT}, htp no longer returns this error */
510  { "Request buffer over", HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG},
511  { "Response buffer over", HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG},
512  { "C-T multipart/byteranges in responses not supported", HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES},
513 };
514 
515 struct {
516  const char *msg;
517  int de;
518 } htp_warnings[] = {
519  { "GZip decompressor:", HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED},
520  { "Request field invalid", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
521  { "Response field invalid", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
522  { "Request header name is not a token", HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID},
523  { "Response header name is not a token", HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID},
524 /* { "Host information in request headers required by HTTP/1.1", HTTP_DECODER_EVENT_MISSING_HOST_HEADER}, <- tx flag HTP_HOST_MISSING
525  { "Host information ambiguous", HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS}, <- tx flag HTP_HOST_AMBIGUOUS */
526  { "Invalid request field folding", HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING},
527  { "Invalid response field folding", HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING},
528  /* line is now: htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Request server port=%d number differs from the actual TCP port=%d", port, connp->conn->server_port);
529  * luckily, "Request server port=" is unique */
530 /* { "Request server port number differs from the actual TCP port", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH}, */
531  { "Request server port=", HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH},
532  { "Request line: URI contains non-compliant delimiter", HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT},
533  { "Request line: non-compliant delimiter between Method and URI", HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT},
534  { "Request line: leading whitespace", HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE},
535  { "Too many response content encoding layers", HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS},
536  { "C-E gzip has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
537  { "C-E deflate has abnormal value", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
538  { "C-E unknown setting", HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER},
539  { "Excessive request header repetitions", HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION},
540  { "Excessive response header repetitions", HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION},
541  { "Transfer-encoding has abnormal chunked value", HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING},
542  { "Chunked transfer-encoding on HTTP/0.9 or HTTP/1.0", HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO},
543  { "Invalid response line: invalid protocol", HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL},
544  { "Invalid response line: invalid response status", HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS},
545  { "Request line incomplete", HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE},
546  { "Unexpected request body", HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED},
547 };
548 
549 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))
550 #define HTP_WARNING_MAX (sizeof(htp_warnings) / sizeof(htp_warnings[0]))
551 
552 /**
553  * \internal
554  *
555  * \brief Get the warning id for the warning msg.
556  *
557  * \param msg warning message
558  *
559  * \retval id the id or 0 in case of not found
560  */
561 static int HTPHandleWarningGetId(const char *msg)
562 {
563  SCLogDebug("received warning \"%s\"", msg);
564  size_t idx;
565  for (idx = 0; idx < HTP_WARNING_MAX; idx++) {
566  if (strncmp(htp_warnings[idx].msg, msg,
567  strlen(htp_warnings[idx].msg)) == 0)
568  {
569  return htp_warnings[idx].de;
570  }
571  }
572 
573  return 0;
574 }
575 
576 /**
577  * \internal
578  *
579  * \brief Get the error id for the error msg.
580  *
581  * \param msg error message
582  *
583  * \retval id the id or 0 in case of not found
584  */
585 static int HTPHandleErrorGetId(const char *msg)
586 {
587  SCLogDebug("received error \"%s\"", msg);
588 
589  size_t idx;
590  for (idx = 0; idx < HTP_ERROR_MAX; idx++) {
591  if (strncmp(htp_errors[idx].msg, msg,
592  strlen(htp_errors[idx].msg)) == 0)
593  {
594  return htp_errors[idx].de;
595  }
596  }
597 
598  return 0;
599 }
600 
601 /**
602  * \internal
603  *
604  * \brief Check state for errors, warnings and add any as events
605  *
606  * \param s state
607  */
608 static void HTPHandleError(HtpState *s)
609 {
610  if (s == NULL || s->conn == NULL ||
611  s->conn->messages == NULL) {
612  return;
613  }
614 
615  size_t size = htp_list_size(s->conn->messages);
616  size_t msg;
617 
618  for (msg = s->htp_messages_offset; msg < size; msg++) {
619  htp_log_t *log = htp_list_get(s->conn->messages, msg);
620  if (log == NULL)
621  continue;
622 
623  HtpTxUserData *htud = NULL;
624  htp_tx_t *tx = log->tx; // will be NULL in <=0.5.9
625  if (tx != NULL)
626  htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
627 
628  SCLogDebug("message %s", log->msg);
629 
630  int id = HTPHandleErrorGetId(log->msg);
631  if (id == 0) {
632  id = HTPHandleWarningGetId(log->msg);
633  if (id == 0)
634  id = HTTP_DECODER_EVENT_UNKNOWN_ERROR;
635  }
636 
637  if (id > 0) {
638  HTPSetEvent(s, htud, id);
639  }
640  }
641  s->htp_messages_offset = (uint16_t)msg;
642  SCLogDebug("s->htp_messages_offset %u", s->htp_messages_offset);
643 }
644 
645 static inline void HTPErrorCheckTxRequestFlags(HtpState *s, htp_tx_t *tx)
646 {
647 #ifdef DEBUG
648  BUG_ON(s == NULL || tx == NULL);
649 #endif
650  if (tx->flags & ( HTP_REQUEST_INVALID_T_E|HTP_REQUEST_INVALID_C_L|
651  HTP_HOST_MISSING|HTP_HOST_AMBIGUOUS|HTP_HOSTU_INVALID|
652  HTP_HOSTH_INVALID))
653  {
654  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
655  if (htud == NULL)
656  return;
657 
658  if (tx->flags & HTP_REQUEST_INVALID_T_E)
659  HTPSetEvent(s, htud,
660  HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST);
661  if (tx->flags & HTP_REQUEST_INVALID_C_L)
662  HTPSetEvent(s, htud,
663  HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST);
664  if (tx->flags & HTP_HOST_MISSING)
665  HTPSetEvent(s, htud,
666  HTTP_DECODER_EVENT_MISSING_HOST_HEADER);
667  if (tx->flags & HTP_HOST_AMBIGUOUS)
668  HTPSetEvent(s, htud,
669  HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS);
670  if (tx->flags & HTP_HOSTU_INVALID)
671  HTPSetEvent(s, htud,
672  HTTP_DECODER_EVENT_URI_HOST_INVALID);
673  if (tx->flags & HTP_HOSTH_INVALID)
674  HTPSetEvent(s, htud,
675  HTTP_DECODER_EVENT_HEADER_HOST_INVALID);
676  }
677  if (tx->request_auth_type == HTP_AUTH_UNRECOGNIZED) {
678  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
679  if (htud == NULL)
680  return;
681  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED);
682  }
683  if (tx->is_protocol_0_9 && tx->request_method_number == HTP_M_UNKNOWN &&
684  (tx->request_protocol_number == HTP_PROTOCOL_INVALID ||
685  tx->request_protocol_number == HTP_PROTOCOL_UNKNOWN)) {
686  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
687  if (htud == NULL)
688  return;
689  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_REQUEST_LINE_INVALID);
690  }
691 }
692 
693 static int Setup(Flow *f, HtpState *hstate)
694 {
695  /* store flow ref in state so callbacks can access it */
696  hstate->f = f;
697 
698  HTPCfgRec *htp_cfg_rec = &cfglist;
699  htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
700  void *user_data = NULL;
701 
702  if (FLOW_IS_IPV4(f)) {
703  SCLogDebug("Looking up HTP config for ipv4 %08x", *GET_IPV4_DST_ADDR_PTR(f));
704  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)GET_IPV4_DST_ADDR_PTR(f), cfgtree, &user_data);
705  }
706  else if (FLOW_IS_IPV6(f)) {
707  SCLogDebug("Looking up HTP config for ipv6");
708  (void)SCRadixFindKeyIPV6BestMatch((uint8_t *)GET_IPV6_DST_ADDR(f), cfgtree, &user_data);
709  }
710  else {
711  SCLogError(SC_ERR_INVALID_ARGUMENT, "unknown address family, bug!");
712  goto error;
713  }
714 
715  if (user_data != NULL) {
716  htp_cfg_rec = user_data;
717  htp = htp_cfg_rec->cfg;
718  SCLogDebug("LIBHTP using config: %p", htp);
719  } else {
720  SCLogDebug("Using default HTP config: %p", htp);
721  }
722 
723  if (NULL == htp) {
724 #ifdef DEBUG_VALIDATION
725  BUG_ON(htp == NULL);
726 #endif
727  /* should never happen if HTPConfigure is properly invoked */
728  goto error;
729  }
730 
731  hstate->connp = htp_connp_create(htp);
732  if (hstate->connp == NULL) {
733  goto error;
734  }
735 
736  hstate->conn = htp_connp_get_connection(hstate->connp);
737 
738  htp_connp_set_user_data(hstate->connp, (void *)hstate);
739  hstate->cfg = htp_cfg_rec;
740 
741  SCLogDebug("New hstate->connp %p", hstate->connp);
742 
743  htp_connp_open(hstate->connp, NULL, f->sp, NULL, f->dp, &f->startts);
744 
745  StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER,
746  htp_cfg_rec->request.inspect_min_size);
747  StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOCLIENT,
748  htp_cfg_rec->response.inspect_min_size);
749  return 0;
750 error:
751  return -1;
752 }
753 
754 /**
755  * \brief Function to handle the reassembled data from client and feed it to
756  * the HTP library to process it.
757  *
758  * \param flow Pointer to the flow the data belong to
759  * \param htp_state Pointer the state in which the parsed value to be stored
760  * \param pstate Application layer parser state for this session
761  * \param input Pointer the received HTTP client data
762  * \param input_len Length in bytes of the received data
763  * \param output Pointer to the output (not used in this function)
764  *
765  * \retval On success returns 1 or on failure returns -1.
766  */
767 static int HTPHandleRequestData(Flow *f, void *htp_state,
768  AppLayerParserState *pstate,
769  uint8_t *input, uint32_t input_len,
770  void *local_data, const uint8_t flags)
771 {
772  SCEnter();
773  int ret = 1;
774  HtpState *hstate = (HtpState *)htp_state;
775 
776  /* On the first invocation, create the connection parser structure to
777  * be used by HTP library. This is looked up via IP in the radix
778  * tree. Failing that, the default HTP config is used.
779  */
780  if (NULL == hstate->conn) {
781  if (Setup(f, hstate) != 0) {
782  goto error;
783  }
784  }
785  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
786 
787  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
788  /* pass the new data to the htp parser */
789  if (input_len > 0) {
790  const int r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
791  switch (r) {
792  case HTP_STREAM_ERROR:
793  ret = -1;
794  break;
795  default:
796  break;
797  }
798  HTPHandleError(hstate);
799  }
800 
801  /* if the TCP connection is closed, then close the HTTP connection */
802  if (AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF) &&
803  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS))
804  {
805  htp_connp_close(hstate->connp, &ts);
806  hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
807  SCLogDebug("stream eof encountered, closing htp handle for ts");
808  }
809 
810  SCLogDebug("hstate->connp %p", hstate->connp);
811  SCReturnInt(ret);
812 
813 error:
814  SCReturnInt(-1);
815 }
816 
817 /**
818  * \brief Function to handle the reassembled data from server and feed it to
819  * the HTP library to process it.
820  *
821  * \param flow Pointer to the flow the data belong to
822  * \param htp_state Pointer the state in which the parsed value to be stored
823  * \param pstate Application layer parser state for this session
824  * \param input Pointer the received HTTP server data
825  * \param input_len Length in bytes of the received data
826  * \param output Pointer to the output (not used in this function)
827  *
828  * \retval On success returns 1 or on failure returns -1
829  */
830 static int HTPHandleResponseData(Flow *f, void *htp_state,
831  AppLayerParserState *pstate,
832  uint8_t *input, uint32_t input_len,
833  void *local_data, const uint8_t flags)
834 {
835  SCEnter();
836  int ret = 1;
837  HtpState *hstate = (HtpState *)htp_state;
838 
839  /* On the first invocation, create the connection parser structure to
840  * be used by HTP library. This is looked up via IP in the radix
841  * tree. Failing that, the default HTP config is used.
842  */
843  if (NULL == hstate->conn) {
844  if (Setup(f, hstate) != 0) {
845  goto error;
846  }
847  }
848  DEBUG_VALIDATE_BUG_ON(hstate->connp == NULL);
849 
850  htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
851  if (input_len > 0) {
852  const int r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
853  switch (r) {
854  case HTP_STREAM_ERROR:
855  ret = -1;
856  break;
857  default:
858  break;
859  }
860  HTPHandleError(hstate);
861  }
862 
863  /* if we the TCP connection is closed, then close the HTTP connection */
864  if (AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF) &&
865  !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC))
866  {
867  htp_connp_close(hstate->connp, &ts);
868  hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
869  }
870 
871  SCLogDebug("hstate->connp %p", hstate->connp);
872  SCReturnInt(ret);
873 error:
874  SCReturnInt(-1);
875 }
876 
877 /**
878  * \param name /Lowercase/ version of the variable name
879  */
880 static int HTTPParseContentDispositionHeader(uint8_t *name, size_t name_len,
881  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
882 {
883 #ifdef PRINT
884  printf("DATA START: \n");
885  PrintRawDataFp(stdout, data, len);
886  printf("DATA END: \n");
887 #endif
888  size_t x;
889  int quote = 0;
890 
891  for (x = 0; x < len; x++) {
892  if (!(isspace(data[x])))
893  break;
894  }
895 
896  if (x >= len)
897  return 0;
898 
899  uint8_t *line = data+x;
900  size_t line_len = len-x;
901  size_t offset = 0;
902 #ifdef PRINT
903  printf("LINE START: \n");
904  PrintRawDataFp(stdout, line, line_len);
905  printf("LINE END: \n");
906 #endif
907  for (x = 0 ; x < line_len; x++) {
908  if (x > 0) {
909  if (line[x - 1] != '\\' && line[x] == '\"') {
910  quote++;
911  }
912 
913  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
914  uint8_t *token = line + offset;
915  size_t token_len = x - offset;
916 
917  if ((x + 1) == line_len) {
918  token_len++;
919  }
920 
921  offset = x + 1;
922 
923  while (offset < line_len && isspace(line[offset])) {
924  x++;
925  offset++;
926  }
927 #ifdef PRINT
928  printf("TOKEN START: \n");
929  PrintRawDataFp(stdout, token, token_len);
930  printf("TOKEN END: \n");
931 #endif
932  if (token_len > name_len) {
933  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
934  uint8_t *value = token + name_len;
935  size_t value_len = token_len - name_len;
936 
937  if (value[0] == '\"') {
938  value++;
939  value_len--;
940  }
941  if (value[value_len-1] == '\"') {
942  value_len--;
943  }
944 #ifdef PRINT
945  printf("VALUE START: \n");
946  PrintRawDataFp(stdout, value, value_len);
947  printf("VALUE END: \n");
948 #endif
949  *retptr = value;
950  *retlen = value_len;
951  return 1;
952  }
953  }
954  }
955  }
956  }
957 
958  return 0;
959 }
960 
961 /**
962  * \param name /Lowercase/ version of the variable name
963  */
964 static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
965  uint8_t *data, size_t len, uint8_t **retptr, size_t *retlen)
966 {
967  SCEnter();
968 #ifdef PRINT
969  printf("DATA START: \n");
970  PrintRawDataFp(stdout, data, len);
971  printf("DATA END: \n");
972 #endif
973  size_t x;
974  int quote = 0;
975 
976  for (x = 0; x < len; x++) {
977  if (!(isspace(data[x])))
978  break;
979  }
980 
981  if (x >= len) {
982  SCReturnInt(0);
983  }
984 
985  uint8_t *line = data+x;
986  size_t line_len = len-x;
987  size_t offset = 0;
988 #ifdef PRINT
989  printf("LINE START: \n");
990  PrintRawDataFp(stdout, line, line_len);
991  printf("LINE END: \n");
992 #endif
993  for (x = 0 ; x < line_len; x++) {
994  if (x > 0) {
995  if (line[x - 1] != '\\' && line[x] == '\"') {
996  quote++;
997  }
998 
999  if (((line[x - 1] != '\\' && line[x] == ';') || ((x + 1) == line_len)) && (quote == 0 || quote % 2 == 0)) {
1000  uint8_t *token = line + offset;
1001  size_t token_len = x - offset;
1002 
1003  if ((x + 1) == line_len) {
1004  token_len++;
1005  }
1006 
1007  offset = x + 1;
1008 
1009  while (offset < line_len && isspace(line[offset])) {
1010  x++;
1011  offset++;
1012  }
1013 #ifdef PRINT
1014  printf("TOKEN START: \n");
1015  PrintRawDataFp(stdout, token, token_len);
1016  printf("TOKEN END: \n");
1017 #endif
1018  if (token_len > name_len) {
1019  if (name == NULL || SCMemcmpLowercase(name, token, name_len) == 0) {
1020  uint8_t *value = token + name_len;
1021  size_t value_len = token_len - name_len;
1022 
1023  if (value[0] == '\"') {
1024  value++;
1025  value_len--;
1026  }
1027  if (value[value_len-1] == '\"') {
1028  value_len--;
1029  }
1030 #ifdef PRINT
1031  printf("VALUE START: \n");
1032  PrintRawDataFp(stdout, value, value_len);
1033  printf("VALUE END: \n");
1034 #endif
1035  *retptr = value;
1036  *retlen = value_len;
1037  SCReturnInt(1);
1038  }
1039  }
1040  }
1041  }
1042  }
1043 
1044  SCReturnInt(0);
1045 }
1046 
1047 /**
1048  * \brief setup multipart parsing: extract boundary and store it
1049  *
1050  * \param d HTTP transaction
1051  * \param htud transaction userdata
1052  *
1053  * \retval 1 ok, multipart set up
1054  * \retval 0 ok, not multipart though
1055  * \retval -1 error: problem with the boundary
1056  *
1057  * If the request contains a multipart message, this function will
1058  * set the HTP_BOUNDARY_SET in the transaction.
1059  */
1060 static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud)
1061 {
1062  htp_header_t *h = (htp_header_t *)htp_table_get_c(d->tx->request_headers,
1063  "Content-Type");
1064  if (h != NULL && bstr_len(h->value) > 0) {
1065  uint8_t *boundary = NULL;
1066  size_t boundary_len = 0;
1067 
1068  int r = HTTPParseContentTypeHeader((uint8_t *)"boundary=", 9,
1069  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value),
1070  &boundary, &boundary_len);
1071  if (r == 1) {
1072 #ifdef PRINT
1073  printf("BOUNDARY START: \n");
1074  PrintRawDataFp(stdout, boundary, boundary_len);
1075  printf("BOUNDARY END: \n");
1076 #endif
1077  if (boundary_len < HTP_BOUNDARY_MAX) {
1078  htud->boundary = HTPMalloc(boundary_len);
1079  if (htud->boundary == NULL) {
1080  return -1;
1081  }
1082  htud->boundary_len = (uint8_t)boundary_len;
1083  memcpy(htud->boundary, boundary, boundary_len);
1084 
1085  htud->tsflags |= HTP_BOUNDARY_SET;
1086  } else {
1087  SCLogDebug("invalid boundary");
1088  return -1;
1089  }
1090  SCReturnInt(1);
1091  }
1092  //SCReturnInt(1);
1093  }
1094  SCReturnInt(0);
1095 }
1096 
1097 #define C_D_HDR "content-disposition:"
1098 #define C_D_HDR_LEN 20
1099 #define C_T_HDR "content-type:"
1100 #define C_T_HDR_LEN 13
1101 
1102 static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
1103  HtpTxUserData *htud,
1104  uint8_t *header, uint32_t header_len,
1105  uint8_t **filename, uint16_t *filename_len,
1106  uint8_t **filetype, uint16_t *filetype_len)
1107 {
1108  uint8_t *fn = NULL;
1109  size_t fn_len = 0;
1110  uint8_t *ft = NULL;
1111  size_t ft_len = 0;
1112 
1113 #ifdef PRINT
1114  printf("HEADER START: \n");
1115  PrintRawDataFp(stdout, header, header_len);
1116  printf("HEADER END: \n");
1117 #endif
1118 
1119  while (header_len > 0) {
1120  uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
1121  uint8_t *line = header;
1122  uint32_t line_len;
1123 
1124  if (next_line == NULL) {
1125  line_len = header_len;
1126  } else {
1127  line_len = next_line - header;
1128  }
1129  uint8_t *sc = (uint8_t *)memchr(line, ':', line_len);
1130  if (sc == NULL) {
1131  HTPSetEvent(hstate, htud,
1132  HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER);
1133  /* if the : we found is the final char, it means we have
1134  * no value */
1135  } else if (line_len > 0 && sc == &line[line_len - 1]) {
1136  HTPSetEvent(hstate, htud,
1137  HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER);
1138  } else {
1139 #ifdef PRINT
1140  printf("LINE START: \n");
1141  PrintRawDataFp(stdout, line, line_len);
1142  printf("LINE END: \n");
1143 #endif
1144  if (line_len >= C_D_HDR_LEN &&
1145  SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) {
1146  uint8_t *value = line + C_D_HDR_LEN;
1147  uint32_t value_len = line_len - C_D_HDR_LEN;
1148 
1149  /* parse content-disposition */
1150  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1151  value, value_len, &fn, &fn_len);
1152  } else if (line_len >= C_T_HDR_LEN &&
1153  SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) {
1154  SCLogDebug("content-type line");
1155  uint8_t *value = line + C_T_HDR_LEN;
1156  uint32_t value_len = line_len - C_T_HDR_LEN;
1157 
1158  (void)HTTPParseContentTypeHeader(NULL, 0,
1159  value, value_len, &ft, &ft_len);
1160  }
1161  }
1162 
1163  if (next_line == NULL) {
1164  SCLogDebug("no next_line");
1165  break;
1166  }
1167  header_len -= ((next_line + 2) - header);
1168  header = next_line + 2;
1169  } /* while (header_len > 0) */
1170 
1171  if (fn_len > USHRT_MAX)
1172  fn_len = USHRT_MAX;
1173  if (ft_len > USHRT_MAX)
1174  ft_len = USHRT_MAX;
1175 
1176  *filename = fn;
1177  *filename_len = fn_len;
1178  *filetype = ft;
1179  *filetype_len = ft_len;
1180 }
1181 
1182 /**
1183  * \brief Create a single buffer from the HtpBodyChunks in our list
1184  *
1185  * \param htud transaction user data
1186  * \param chunks_buffers pointer to pass back the buffer to the caller
1187  * \param chunks_buffer_len pointer to pass back the buffer length to the caller
1188  */
1189 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
1190  const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
1191 {
1192  StreamingBufferGetDataAtOffset(htud->request_body.sb,
1193  chunks_buffer, chunks_buffer_len,
1194  htud->request_body.body_parsed);
1195 }
1196 
1197 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
1198 {
1199  SCEnter();
1200  if (tx && tx->de_state) {
1201  if (dir == STREAM_TOSERVER) {
1202  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1203  tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1204  } else if (STREAM_TOCLIENT) {
1205  SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
1206  tx->de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
1207  }
1208  }
1209 }
1210 
1211 /**
1212  * \brief Setup boundary buffers
1213  */
1214 static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
1215  uint8_t *boundary, uint32_t boundary_len)
1216 {
1217  memset(boundary, '-', boundary_len);
1218  memcpy(boundary + 2, htud->boundary, htud->boundary_len);
1219 }
1220 
1221 static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
1222  const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
1223 {
1224  int result = 0;
1225  uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
1226  uint8_t expected_boundary_len = htud->boundary_len + 2;
1227  uint8_t expected_boundary_end_len = htud->boundary_len + 4;
1228  int tx_progress = 0;
1229 
1230 #ifdef PRINT
1231  printf("CHUNK START: \n");
1232  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1233  printf("CHUNK END: \n");
1234 #endif
1235 
1236  HtpRequestBodySetupBoundary(htud, boundary, htud->boundary_len + 4);
1237 
1238  /* search for the header start, header end and form end */
1239  uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1240  boundary, expected_boundary_len);
1241  uint8_t *header_end = NULL;
1242  if (header_start != NULL) {
1243  header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer),
1244  (uint8_t *)"\r\n\r\n", 4);
1245  }
1246  uint8_t *form_end = Bs2bmSearch(chunks_buffer, chunks_buffer_len,
1247  boundary, expected_boundary_end_len);
1248 
1249  SCLogDebug("header_start %p, header_end %p, form_end %p", header_start,
1250  header_end, form_end);
1251 
1252  /* we currently only handle multipart for ts. When we support it for tc,
1253  * we will need to supply right direction */
1254  tx_progress = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, STREAM_TOSERVER);
1255  /* if we're in the file storage process, deal with that now */
1256  if (htud->tsflags & HTP_FILENAME_SET) {
1257  if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
1258  SCLogDebug("reached the end of the file");
1259 
1260  const uint8_t *filedata = chunks_buffer;
1261  uint32_t filedata_len = 0;
1262  uint8_t flags = 0;
1263 
1264  if (header_start < form_end || (header_start != NULL && form_end == NULL)) {
1265  filedata_len = header_start - filedata - 2; /* 0d 0a */
1266  } else if (form_end != NULL && form_end < header_start) {
1267  filedata_len = form_end - filedata;
1268  } else if (form_end != NULL && form_end == header_start) {
1269  filedata_len = form_end - filedata - 2; /* 0d 0a */
1270  } else if (tx_progress > HTP_RESPONSE_BODY) {
1271  filedata_len = chunks_buffer_len;
1272  flags = FILE_TRUNCATED;
1273  }
1274 
1275  if (filedata_len > chunks_buffer_len) {
1276  HTPSetEvent(hstate, htud,
1277  HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR);
1278  goto end;
1279  }
1280 #ifdef PRINT
1281  printf("FILEDATA (final chunk) START: \n");
1282  PrintRawDataFp(stdout, filedata, filedata_len);
1283  printf("FILEDATA (final chunk) END: \n");
1284 #endif
1285  if (!(htud->tsflags & HTP_DONTSTORE)) {
1286  if (HTPFileClose(hstate, filedata, filedata_len, flags,
1287  STREAM_TOSERVER) == -1)
1288  {
1289  goto end;
1290  }
1291  }
1292 
1293  htud->tsflags &=~ HTP_FILENAME_SET;
1294 
1295  /* fall through */
1296  } else {
1297  SCLogDebug("not yet at the end of the file");
1298 
1299  if (chunks_buffer_len > expected_boundary_end_len) {
1300  const uint8_t *filedata = chunks_buffer;
1301  uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
1302 #ifdef PRINT
1303  printf("FILEDATA (part) START: \n");
1304  PrintRawDataFp(stdout, filedata, filedata_len);
1305  printf("FILEDATA (part) END: \n");
1306 #endif
1307 
1308  if (!(htud->tsflags & HTP_DONTSTORE)) {
1309  result = HTPFileStoreChunk(hstate, filedata,
1310  filedata_len, STREAM_TOSERVER);
1311  if (result == -1) {
1312  goto end;
1313  } else if (result == -2) {
1314  /* we know for sure we're not storing the file */
1315  htud->tsflags |= HTP_DONTSTORE;
1316  }
1317  }
1318 
1319  htud->request_body.body_parsed += filedata_len;
1320  } else {
1321  SCLogDebug("chunk too small to already process in part");
1322  }
1323 
1324  goto end;
1325  }
1326  }
1327 
1328  while (header_start != NULL && header_end != NULL &&
1329  header_end != form_end &&
1330  header_start < (chunks_buffer + chunks_buffer_len) &&
1331  header_end < (chunks_buffer + chunks_buffer_len) &&
1332  header_start < header_end)
1333  {
1334  uint8_t *filename = NULL;
1335  uint16_t filename_len = 0;
1336  uint8_t *filetype = NULL;
1337  uint16_t filetype_len = 0;
1338 
1339  uint32_t header_len = header_end - header_start;
1340  SCLogDebug("header_len %u", header_len);
1341  uint8_t *header = header_start;
1342 
1343  /* skip empty records */
1344  if (expected_boundary_len == header_len) {
1345  goto next;
1346  } else if ((uint32_t)(expected_boundary_len + 2) <= header_len) {
1347  header_len -= (expected_boundary_len + 2);
1348  header = header_start + (expected_boundary_len + 2); // + for 0d 0a
1349  }
1350 
1351  HtpRequestBodyMultipartParseHeader(hstate, htud, header, header_len,
1352  &filename, &filename_len, &filetype, &filetype_len);
1353 
1354  if (filename != NULL) {
1355  uint8_t *filedata = NULL;
1356  uint32_t filedata_len = 0;
1357 
1358  SCLogDebug("we have a filename");
1359 
1360  htud->tsflags |= HTP_FILENAME_SET;
1361  htud->tsflags &= ~HTP_DONTSTORE;
1362 
1363  SCLogDebug("header_end %p", header_end);
1364  SCLogDebug("form_end %p", form_end);
1365 
1366  /* everything until the final boundary is the file */
1367  if (form_end != NULL) {
1368  filedata = header_end + 4;
1369  if (form_end == filedata) {
1370  HTPSetEvent(hstate, htud,
1371  HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA);
1372  goto end;
1373  } else if (form_end < filedata) {
1374  HTPSetEvent(hstate, htud,
1375  HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR);
1376  goto end;
1377  }
1378 
1379  filedata_len = form_end - (header_end + 4 + 2);
1380  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1381 
1382  /* or is it? */
1383  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1384  boundary, expected_boundary_len);
1385  if (header_next != NULL) {
1386  filedata_len -= (form_end - header_next);
1387  }
1388 
1389  if (filedata_len > chunks_buffer_len) {
1390  HTPSetEvent(hstate, htud,
1391  HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR);
1392  goto end;
1393  }
1394  SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
1395 #ifdef PRINT
1396  printf("FILEDATA START: \n");
1397  PrintRawDataFp(stdout, filedata, filedata_len);
1398  printf("FILEDATA END: \n");
1399 #endif
1400 
1401  result = HTPFileOpen(hstate, filename, filename_len,
1402  filedata, filedata_len, hstate->transaction_cnt,
1403  STREAM_TOSERVER);
1404  if (result == -1) {
1405  goto end;
1406  } else if (result == -2) {
1407  htud->tsflags |= HTP_DONTSTORE;
1408  } else {
1409  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1410  goto end;
1411  }
1412  }
1413  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1414 
1415  htud->request_body.body_parsed += (header_end - chunks_buffer);
1416  htud->tsflags &= ~HTP_FILENAME_SET;
1417  } else {
1418  SCLogDebug("chunk doesn't contain form end");
1419 
1420  filedata = header_end + 4;
1421  filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
1422  SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
1423 
1424  if (filedata_len > chunks_buffer_len) {
1425  HTPSetEvent(hstate, htud,
1426  HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR);
1427  goto end;
1428  }
1429 
1430 #ifdef PRINT
1431  printf("FILEDATA START: \n");
1432  PrintRawDataFp(stdout, filedata, filedata_len);
1433  printf("FILEDATA END: \n");
1434 #endif
1435  /* form doesn't end in this chunk, but part might. Lets
1436  * see if have another coming up */
1437  uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
1438  boundary, expected_boundary_len);
1439  SCLogDebug("header_next %p", header_next);
1440  if (header_next == NULL) {
1441  /* no, but we'll handle the file data when we see the
1442  * form_end */
1443 
1444  SCLogDebug("more file data to come");
1445 
1446  uint32_t offset = (header_end + 4) - chunks_buffer;
1447  SCLogDebug("offset %u", offset);
1448  htud->request_body.body_parsed += offset;
1449 
1450  result = HTPFileOpen(hstate, filename, filename_len,
1451  NULL, 0, hstate->transaction_cnt,
1452  STREAM_TOSERVER);
1453  if (result == -1) {
1454  goto end;
1455  } else if (result == -2) {
1456  htud->tsflags |= HTP_DONTSTORE;
1457  }
1458  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1459 
1460  } else if (header_next - filedata > 2) {
1461  filedata_len = header_next - filedata - 2;
1462  SCLogDebug("filedata_len %u", filedata_len);
1463 
1464  result = HTPFileOpen(hstate, filename, filename_len,
1465  filedata, filedata_len, hstate->transaction_cnt,
1466  STREAM_TOSERVER);
1467  if (result == -1) {
1468  goto end;
1469  } else if (result == -2) {
1470  htud->tsflags |= HTP_DONTSTORE;
1471  } else {
1472  if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
1473  goto end;
1474  }
1475  }
1476  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1477 
1478  htud->tsflags &= ~HTP_FILENAME_SET;
1479  htud->request_body.body_parsed += (header_end - chunks_buffer);
1480  }
1481  }
1482  }
1483 next:
1484  SCLogDebug("header_start %p, header_end %p, form_end %p",
1485  header_start, header_end, form_end);
1486 
1487  /* Search next boundary entry after the start of body */
1488  uint32_t cursizeread = header_end - chunks_buffer;
1489  header_start = Bs2bmSearch(header_end + 4,
1490  chunks_buffer_len - (cursizeread + 4),
1491  boundary, expected_boundary_len);
1492  if (header_start != NULL) {
1493  header_end = Bs2bmSearch(header_end + 4,
1494  chunks_buffer_len - (cursizeread + 4),
1495  (uint8_t *) "\r\n\r\n", 4);
1496  }
1497  }
1498 
1499  /* if we're parsing the multipart and we're not currently processing a
1500  * file, we move the body pointer forward. */
1501  if (form_end == NULL && !(htud->tsflags & HTP_FILENAME_SET) && header_start == NULL) {
1502  if (chunks_buffer_len > expected_boundary_end_len) {
1503  uint32_t move = chunks_buffer_len - expected_boundary_end_len + 1;
1504 
1505  htud->request_body.body_parsed += move;
1506  SCLogDebug("form not ready, file not set, parsing non-file "
1507  "record: moved %u", move);
1508  }
1509  }
1510 
1511 end:
1512  SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
1513  return 0;
1514 }
1515 
1516 /** \brief setup things for put request
1517  * \todo really needed? */
1518 static int HtpRequestBodySetupPUT(htp_tx_data_t *d, HtpTxUserData *htud)
1519 {
1520 // if (d->tx->parsed_uri == NULL || d->tx->parsed_uri->path == NULL) {
1521 // return -1;
1522 // }
1523 
1524  /* filename is d->tx->parsed_uri->path */
1525 
1526  return 0;
1527 }
1528 
1529 /** \internal
1530  * \brief Handle POST, no multipart body data
1531  */
1532 static int HtpRequestBodyHandlePOST(HtpState *hstate, HtpTxUserData *htud,
1533  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1534 {
1535  int result = 0;
1536 
1537  /* see if we need to open the file */
1538  if (!(htud->tsflags & HTP_FILENAME_SET))
1539  {
1540  uint8_t *filename = NULL;
1541  size_t filename_len = 0;
1542 
1543  /* get the name */
1544  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1545  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1546  filename_len = bstr_len(tx->parsed_uri->path);
1547  }
1548 
1549  if (filename != NULL) {
1550  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1551  hstate->transaction_cnt, STREAM_TOSERVER);
1552  if (result == -1) {
1553  goto end;
1554  } else if (result == -2) {
1555  htud->tsflags |= HTP_DONTSTORE;
1556  } else {
1557  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1558  htud->tsflags |= HTP_FILENAME_SET;
1559  htud->tsflags &= ~HTP_DONTSTORE;
1560  }
1561  }
1562  }
1563  else
1564  {
1565  /* otherwise, just store the data */
1566 
1567  if (!(htud->tsflags & HTP_DONTSTORE)) {
1568  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1569  if (result == -1) {
1570  goto end;
1571  } else if (result == -2) {
1572  /* we know for sure we're not storing the file */
1573  htud->tsflags |= HTP_DONTSTORE;
1574  }
1575  }
1576  }
1577 
1578  return 0;
1579 end:
1580  return -1;
1581 }
1582 
1583 /** \internal
1584  * \brief Handle PUT body data
1585  */
1586 static int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
1587  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1588 {
1589  int result = 0;
1590 
1591  /* see if we need to open the file */
1592  if (!(htud->tsflags & HTP_FILENAME_SET))
1593  {
1594  uint8_t *filename = NULL;
1595  size_t filename_len = 0;
1596 
1597  /* get the name */
1598  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1599  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1600  filename_len = bstr_len(tx->parsed_uri->path);
1601  }
1602 
1603  if (filename != NULL) {
1604  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len, data, data_len,
1605  hstate->transaction_cnt, STREAM_TOSERVER);
1606  if (result == -1) {
1607  goto end;
1608  } else if (result == -2) {
1609  htud->tsflags |= HTP_DONTSTORE;
1610  } else {
1611  FlagDetectStateNewFile(htud, STREAM_TOSERVER);
1612  htud->tsflags |= HTP_FILENAME_SET;
1613  htud->tsflags &= ~HTP_DONTSTORE;
1614  }
1615  }
1616  }
1617  else
1618  {
1619  /* otherwise, just store the data */
1620 
1621  if (!(htud->tsflags & HTP_DONTSTORE)) {
1622  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
1623  if (result == -1) {
1624  goto end;
1625  } else if (result == -2) {
1626  /* we know for sure we're not storing the file */
1627  htud->tsflags |= HTP_DONTSTORE;
1628  }
1629  }
1630  }
1631 
1632  return 0;
1633 end:
1634  return -1;
1635 }
1636 
1637 static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
1638  htp_tx_t *tx, uint8_t *data, uint32_t data_len)
1639 {
1640  SCEnter();
1641 
1642  int result = 0;
1643 
1644  /* see if we need to open the file */
1645  if (!(htud->tcflags & HTP_FILENAME_SET))
1646  {
1647  SCLogDebug("setting up file name");
1648 
1649  uint8_t *filename = NULL;
1650  size_t filename_len = 0;
1651 
1652  /* try Content-Disposition header first */
1653  htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
1654  "Content-Disposition");
1655  if (h != NULL && bstr_len(h->value) > 0) {
1656  /* parse content-disposition */
1657  (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9,
1658  (uint8_t *) bstr_ptr(h->value), bstr_len(h->value), &filename, &filename_len);
1659  }
1660 
1661  /* fall back to name from the uri */
1662  if (filename == NULL) {
1663  /* get the name */
1664  if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
1665  filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
1666  filename_len = bstr_len(tx->parsed_uri->path);
1667  }
1668  }
1669 
1670  if (filename != NULL) {
1671  result = HTPFileOpen(hstate, filename, (uint32_t)filename_len,
1672  data, data_len, hstate->transaction_cnt, STREAM_TOCLIENT);
1673  SCLogDebug("result %d", result);
1674  if (result == -1) {
1675  goto end;
1676  } else if (result == -2) {
1677  htud->tcflags |= HTP_DONTSTORE;
1678  } else {
1679  FlagDetectStateNewFile(htud, STREAM_TOCLIENT);
1680  htud->tcflags |= HTP_FILENAME_SET;
1681  htud->tcflags &= ~HTP_DONTSTORE;
1682  }
1683  //set range if present
1684  htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range");
1685  if (h_content_range != NULL) {
1686  HTPFileSetRange(hstate, h_content_range->value);
1687  }
1688  }
1689  }
1690  else
1691  {
1692  /* otherwise, just store the data */
1693 
1694  if (!(htud->tcflags & HTP_DONTSTORE)) {
1695  result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
1696  SCLogDebug("result %d", result);
1697  if (result == -1) {
1698  goto end;
1699  } else if (result == -2) {
1700  /* we know for sure we're not storing the file */
1701  htud->tcflags |= HTP_DONTSTORE;
1702  }
1703  }
1704  }
1705 
1706  htud->response_body.body_parsed += data_len;
1707  return 0;
1708 end:
1709  return -1;
1710 }
1711 
1712 /**
1713  * \brief Function callback to append chunks for Requests
1714  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1715  * \retval int HTP_OK if all goes well
1716  */
1717 static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
1718 {
1719  SCEnter();
1720 
1721  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
1722  SCReturnInt(HTP_OK);
1723 
1724  if (d->data == NULL || d->len == 0)
1725  SCReturnInt(HTP_OK);
1726 
1727 #ifdef PRINT
1728  printf("HTPBODY START: \n");
1729  PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
1730  printf("HTPBODY END: \n");
1731 #endif
1732 
1733  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1734  if (hstate == NULL) {
1735  SCReturnInt(HTP_ERROR);
1736  }
1737 
1738  SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
1739  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1740 
1741  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1742  if (tx_ud == NULL) {
1743  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1744  if (unlikely(tx_ud == NULL)) {
1745  SCReturnInt(HTP_OK);
1746  }
1747  memset(tx_ud, 0, sizeof(HtpTxUserData));
1748 
1749  /* Set the user data for handling body chunks on this transaction */
1750  htp_tx_set_user_data(d->tx, tx_ud);
1751  }
1752  if (!tx_ud->response_body_init) {
1753  tx_ud->response_body_init = 1;
1754 
1755  if (d->tx->request_method_number == HTP_M_POST) {
1756  SCLogDebug("POST");
1757  int r = HtpRequestBodySetupMultipart(d, tx_ud);
1758  if (r == 1) {
1759  tx_ud->request_body_type = HTP_BODY_REQUEST_MULTIPART;
1760  } else if (r == 0) {
1761  tx_ud->request_body_type = HTP_BODY_REQUEST_POST;
1762  SCLogDebug("not multipart");
1763  }
1764  } else if (d->tx->request_method_number == HTP_M_PUT) {
1765  if (HtpRequestBodySetupPUT(d, tx_ud) == 0) {
1766  tx_ud->request_body_type = HTP_BODY_REQUEST_PUT;
1767  }
1768  }
1769  }
1770 
1771  /* see if we can get rid of htp body chunks */
1772  HtpBodyPrune(hstate, &tx_ud->request_body, STREAM_TOSERVER);
1773 
1774  SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
1775  SCLogDebug("hstate->cfg->request.body_limit %u", hstate->cfg->request.body_limit);
1776 
1777  /* within limits, add the body chunk to the state. */
1778  if (hstate->cfg->request.body_limit == 0 || tx_ud->request_body.content_len_so_far < hstate->cfg->request.body_limit)
1779  {
1780  uint32_t len = (uint32_t)d->len;
1781 
1782  if (hstate->cfg->request.body_limit > 0 &&
1783  (tx_ud->request_body.content_len_so_far + len) > hstate->cfg->request.body_limit)
1784  {
1785  len = hstate->cfg->request.body_limit - tx_ud->request_body.content_len_so_far;
1786  BUG_ON(len > (uint32_t)d->len);
1787  }
1788  SCLogDebug("len %u", len);
1789 
1790  HtpBodyAppendChunk(&hstate->cfg->request, &tx_ud->request_body, d->data, len);
1791 
1792  const uint8_t *chunks_buffer = NULL;
1793  uint32_t chunks_buffer_len = 0;
1794 
1795  if (tx_ud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
1796  /* multi-part body handling starts here */
1797  if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
1798  goto end;
1799  }
1800 
1801  HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
1802  if (chunks_buffer == NULL) {
1803  goto end;
1804  }
1805 #ifdef PRINT
1806  printf("REASSCHUNK START: \n");
1807  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
1808  printf("REASSCHUNK END: \n");
1809 #endif
1810 
1811  HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
1812 
1813  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
1814  HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1815  } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
1816  HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1817  }
1818 
1819  } else {
1820  if (tx_ud->tsflags & HTP_FILENAME_SET) {
1821  SCLogDebug("closing file that was being stored");
1822  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
1823  tx_ud->tsflags &= ~HTP_FILENAME_SET;
1824  }
1825  }
1826 
1827 end:
1828  if (hstate->conn != NULL) {
1829  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1830  tx_ud->request_body.content_len_so_far,
1831  hstate->cfg->request.inspect_min_size,
1832  (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
1833 
1834  /* if we reach the inspect_min_size we'll trigger inspection,
1835  * so make sure that raw stream is also inspected. Set the
1836  * data to be used to the amount of raw bytes we've seen to
1837  * get here. */
1838  if (tx_ud->request_body.body_inspected == 0 &&
1839  tx_ud->request_body.content_len_so_far >= hstate->cfg->request.inspect_min_size) {
1840  if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
1841  (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
1842  {
1843  uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
1844 
1845  /* body still in progress, but due to min inspect size we need to inspect now */
1846  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, x);
1847  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
1848  }
1849  /* after the start of the body, disable the depth logic */
1850  } else if (tx_ud->request_body.body_inspected > 0) {
1851  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
1852  }
1853  }
1854  SCReturnInt(HTP_OK);
1855 }
1856 
1857 /**
1858  * \brief Function callback to append chunks for Responses
1859  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
1860  * \retval int HTP_OK if all goes well
1861  */
1862 static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
1863 {
1864  SCEnter();
1865 
1866  if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
1867  SCReturnInt(HTP_OK);
1868 
1869  if (d->data == NULL || d->len == 0)
1870  SCReturnInt(HTP_OK);
1871 
1872  HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
1873  if (hstate == NULL) {
1874  SCReturnInt(HTP_ERROR);
1875  }
1876 
1877  SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
1878  "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
1879 
1880  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
1881  if (tx_ud == NULL) {
1882  tx_ud = HTPMalloc(sizeof(HtpTxUserData));
1883  if (unlikely(tx_ud == NULL)) {
1884  SCReturnInt(HTP_OK);
1885  }
1886  memset(tx_ud, 0, sizeof(HtpTxUserData));
1887 
1888  /* Set the user data for handling body chunks on this transaction */
1889  htp_tx_set_user_data(d->tx, tx_ud);
1890  }
1891  if (!tx_ud->request_body_init) {
1892  tx_ud->request_body_init = 1;
1893  }
1894 
1895  /* see if we can get rid of htp body chunks */
1896  HtpBodyPrune(hstate, &tx_ud->response_body, STREAM_TOCLIENT);
1897 
1898  SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
1899  SCLogDebug("hstate->cfg->response.body_limit %u", hstate->cfg->response.body_limit);
1900 
1901  /* within limits, add the body chunk to the state. */
1902  if (hstate->cfg->response.body_limit == 0 || tx_ud->response_body.content_len_so_far < hstate->cfg->response.body_limit)
1903  {
1904  uint32_t len = (uint32_t)d->len;
1905 
1906  if (hstate->cfg->response.body_limit > 0 &&
1907  (tx_ud->response_body.content_len_so_far + len) > hstate->cfg->response.body_limit)
1908  {
1909  len = hstate->cfg->response.body_limit - tx_ud->response_body.content_len_so_far;
1910  BUG_ON(len > (uint32_t)d->len);
1911  }
1912  SCLogDebug("len %u", len);
1913 
1914  HtpBodyAppendChunk(&hstate->cfg->response, &tx_ud->response_body, d->data, len);
1915 
1916  HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
1917  } else {
1918  if (tx_ud->tcflags & HTP_FILENAME_SET) {
1919  SCLogDebug("closing file that was being stored");
1920  (void)HTPFileClose(hstate, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
1921  tx_ud->tcflags &= ~HTP_FILENAME_SET;
1922  }
1923  }
1924 
1925  if (hstate->conn != NULL) {
1926  SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
1927  tx_ud->response_body.content_len_so_far,
1928  hstate->cfg->response.inspect_min_size,
1929  (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
1930  /* if we reach the inspect_min_size we'll trigger inspection,
1931  * so make sure that raw stream is also inspected. Set the
1932  * data to be used to the amount of raw bytes we've seen to
1933  * get here. */
1934  if (tx_ud->response_body.body_inspected == 0 &&
1935  tx_ud->response_body.content_len_so_far >= hstate->cfg->response.inspect_min_size) {
1936  if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
1937  (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
1938  {
1939  uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
1940 
1941  /* body still in progress, but due to min inspect size we need to inspect now */
1942  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, x);
1943  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
1944  }
1945  /* after the start of the body, disable the depth logic */
1946  } else if (tx_ud->response_body.body_inspected > 0) {
1947  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
1948  }
1949  }
1950  SCReturnInt(HTP_OK);
1951 }
1952 
1953 /**
1954  * \brief Print the stats of the HTTP requests
1955  */
1956 void HTPAtExitPrintStats(void)
1957 {
1958 #ifdef DEBUG
1959  SCEnter();
1960  SCMutexLock(&htp_state_mem_lock);
1961  SCLogDebug("http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"",
1962  htp_state_memcnt, htp_state_memuse);
1963  SCMutexUnlock(&htp_state_mem_lock);
1964  SCReturn;
1965 #endif
1966 }
1967 
1968 /** \brief Clears the HTTP server configuration memory used by HTP library */
1969 void HTPFreeConfig(void)
1970 {
1971  SCEnter();
1972 
1973  if (!AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "http") ||
1974  !AppLayerParserConfParserEnabled("tcp", "http"))
1975  {
1976  SCReturn;
1977  }
1978 
1979  HTPCfgRec *nextrec = cfglist.next;
1980  SCRadixReleaseRadixTree(cfgtree);
1981  cfgtree = NULL;
1982  htp_config_destroy(cfglist.cfg);
1983  while (nextrec != NULL) {
1984  HTPCfgRec *htprec = nextrec;
1985  nextrec = nextrec->next;
1986 
1987  htp_config_destroy(htprec->cfg);
1988  SCFree(htprec);
1989  }
1990  HTPDestroyMemcap();
1991  SCReturn;
1992 }
1993 
1994 static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
1995 {
1996  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
1997  if (htud != NULL) {
1998  htud->request_has_trailers = 1;
1999  }
2000  return HTP_OK;
2001 }
2002 
2003 static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
2004 {
2005  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2006  if (htud != NULL) {
2007  htud->response_has_trailers = 1;
2008  }
2009  return HTP_OK;
2010 }
2011 
2012 /**\internal
2013  * \brief called at start of request
2014  * Set min inspect size.
2015  */
2016 static int HTPCallbackRequestStart(htp_tx_t *tx)
2017 {
2018  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2019  if (hstate == NULL) {
2020  SCReturnInt(HTP_ERROR);
2021  }
2022 
2023  if (hstate->cfg)
2024  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2025  hstate->cfg->request.inspect_min_size);
2026  SCReturnInt(HTP_OK);
2027 }
2028 
2029 /**\internal
2030  * \brief called at start of response
2031  * Set min inspect size.
2032  */
2033 static int HTPCallbackResponseStart(htp_tx_t *tx)
2034 {
2035  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2036  if (hstate == NULL) {
2037  SCReturnInt(HTP_ERROR);
2038  }
2039 
2040  if (hstate->cfg)
2041  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
2042  hstate->cfg->response.inspect_min_size);
2043  SCReturnInt(HTP_OK);
2044 }
2045 
2046 
2047 /**
2048  * \brief callback for request to store the recent incoming request
2049  in to the recent_in_tx for the given htp state
2050  * \param connp pointer to the current connection parser which has the htp
2051  * state in it as user data
2052  */
2053 static int HTPCallbackRequest(htp_tx_t *tx)
2054 {
2055  SCEnter();
2056 
2057  if (tx == NULL) {
2058  SCReturnInt(HTP_ERROR);
2059  }
2060 
2061  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2062  if (hstate == NULL) {
2063  SCReturnInt(HTP_ERROR);
2064  }
2065 
2066  SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
2067  hstate->transaction_cnt, HTPStateGetTxCnt(hstate));
2068 
2069  SCLogDebug("HTTP request completed");
2070 
2071  HTPErrorCheckTxRequestFlags(hstate, tx);
2072 
2073  HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
2074  if (htud != NULL) {
2075  if (htud->tsflags & HTP_FILENAME_SET) {
2076  SCLogDebug("closing file that was being stored");
2077  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
2078  htud->tsflags &= ~HTP_FILENAME_SET;
2079  StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
2080  (uint32_t)hstate->conn->in_data_counter);
2081  }
2082  }
2083 
2084  hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
2085  /* request done, do raw reassembly now to inspect state and stream
2086  * at the same time. */
2087  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
2088  SCReturnInt(HTP_OK);
2089 }
2090 
2091 /**
2092  * \brief callback for response to remove the recent received requests
2093  from the recent_in_tx for the given htp state
2094  * \param connp pointer to the current connection parser which has the htp
2095  * state in it as user data
2096  */
2097 static int HTPCallbackResponse(htp_tx_t *tx)
2098 {
2099  SCEnter();
2100 
2101  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2102  if (hstate == NULL) {
2103  SCReturnInt(HTP_ERROR);
2104  }
2105 
2106  /* we have one whole transaction now */
2107  hstate->transaction_cnt++;
2108 
2109  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2110  if (htud != NULL) {
2111  if (htud->tcflags & HTP_FILENAME_SET) {
2112  SCLogDebug("closing file that was being stored");
2113  (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
2114  htud->tcflags &= ~HTP_FILENAME_SET;
2115  }
2116  }
2117 
2118  /* response done, do raw reassembly now to inspect state and stream
2119  * at the same time. */
2120  AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
2121 
2122  /* handle HTTP CONNECT */
2123  if (tx->request_method_number == HTP_M_CONNECT) {
2124  /* any 2XX status response implies that the connection will become
2125  a tunnel immediately after this packet (RFC 7230, 3.3.3). */
2126  if ((tx->response_status_number >= 200) &&
2127  (tx->response_status_number < 300) &&
2128  (hstate->transaction_cnt == 1)) {
2129  uint16_t dp = 0;
2130  if (tx->request_port_number != -1) {
2131  dp = (uint16_t)tx->request_port_number;
2132  }
2133  // both ALPROTO_HTTP and ALPROTO_TLS are normal options
2134  AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN);
2135  tx->request_progress = HTP_REQUEST_COMPLETE;
2136  tx->response_progress = HTP_RESPONSE_COMPLETE;
2137  }
2138  }
2139 
2140  hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
2141  SCReturnInt(HTP_OK);
2142 }
2143 
2144 static int HTPCallbackRequestLine(htp_tx_t *tx)
2145 {
2146  HtpTxUserData *tx_ud;
2147  bstr *request_uri_normalized;
2148  HtpState *hstate = htp_connp_get_user_data(tx->connp);
2149  const HTPCfgRec *cfg = hstate->cfg;
2150 
2151  request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
2152  if (request_uri_normalized == NULL)
2153  return HTP_OK;
2154 
2155  tx_ud = htp_tx_get_user_data(tx);
2156  if (likely(tx_ud == NULL)) {
2157  tx_ud = HTPMalloc(sizeof(*tx_ud));
2158  if (unlikely(tx_ud == NULL)) {
2159  bstr_free(request_uri_normalized);
2160  return HTP_OK;
2161  }
2162  memset(tx_ud, 0, sizeof(*tx_ud));
2163  htp_tx_set_user_data(tx, tx_ud);
2164  }
2165  if (unlikely(tx_ud->request_uri_normalized != NULL))
2166  bstr_free(tx_ud->request_uri_normalized);
2167  tx_ud->request_uri_normalized = request_uri_normalized;
2168 
2169  if (tx->flags) {
2170  HTPErrorCheckTxRequestFlags(hstate, tx);
2171  }
2172  return HTP_OK;
2173 }
2174 
2175 static int HTPCallbackDoubleDecodeUriPart(htp_tx_t *tx, bstr *part)
2176 {
2177  if (part == NULL)
2178  return HTP_OK;
2179 
2180  uint64_t flags = 0;
2181  size_t prevlen = bstr_len(part);
2182  htp_status_t res = htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, part, &flags);
2183  // shorter string means that uri was encoded
2184  if (res == HTP_OK && prevlen > bstr_len(part)) {
2185  HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2186  if (likely(htud == NULL)) {
2187  htud = HTPCalloc(1, sizeof(*htud));
2188  if (unlikely(htud == NULL))
2189  return HTP_OK;
2190  htp_tx_set_user_data(tx, htud);
2191  }
2192  HtpState *s = htp_connp_get_user_data(tx->connp);
2193  if (s == NULL)
2194  return HTP_OK;
2195  HTPSetEvent(s, htud, HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI);
2196  }
2197 
2198  return HTP_OK;
2199 }
2200 
2201 static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
2202 {
2203  if (tx->parsed_uri == NULL)
2204  return HTP_OK;
2205 
2206  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->query);
2207 }
2208 
2209 static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
2210 {
2211  if (tx->parsed_uri == NULL)
2212  return HTP_OK;
2213 
2214  return HTPCallbackDoubleDecodeUriPart(tx, tx->parsed_uri->path);
2215 }
2216 
2217 static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
2218 {
2219  void *ptmp;
2220  if (tx_data->len == 0 || tx_data->tx == NULL)
2221  return HTP_OK;
2222 
2223  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2224  if (tx_ud == NULL) {
2225  tx_ud = HTPMalloc(sizeof(*tx_ud));
2226  if (unlikely(tx_ud == NULL))
2227  return HTP_OK;
2228  memset(tx_ud, 0, sizeof(*tx_ud));
2229  htp_tx_set_user_data(tx_data->tx, tx_ud);
2230  }
2231  ptmp = HTPRealloc(tx_ud->request_headers_raw,
2232  tx_ud->request_headers_raw_len,
2233  tx_ud->request_headers_raw_len + tx_data->len);
2234  if (ptmp == NULL) {
2235  /* error: we're freeing the entire user data */
2236  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2237  HtpTxUserDataFree(hstate, tx_ud);
2238  htp_tx_set_user_data(tx_data->tx, NULL);
2239  return HTP_OK;
2240  }
2241  tx_ud->request_headers_raw = ptmp;
2242 
2243  memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
2244  tx_data->data, tx_data->len);
2245  tx_ud->request_headers_raw_len += tx_data->len;
2246 
2247  if (tx_data->tx && tx_data->tx->flags) {
2248  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2249  HTPErrorCheckTxRequestFlags(hstate, tx_data->tx);
2250  }
2251  return HTP_OK;
2252 }
2253 
2254 static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
2255 {
2256  void *ptmp;
2257  if (tx_data->len == 0 || tx_data->tx == NULL)
2258  return HTP_OK;
2259 
2260  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
2261  if (tx_ud == NULL) {
2262  tx_ud = HTPMalloc(sizeof(*tx_ud));
2263  if (unlikely(tx_ud == NULL))
2264  return HTP_OK;
2265  memset(tx_ud, 0, sizeof(*tx_ud));
2266  htp_tx_set_user_data(tx_data->tx, tx_ud);
2267  }
2268  ptmp = HTPRealloc(tx_ud->response_headers_raw,
2269  tx_ud->response_headers_raw_len,
2270  tx_ud->response_headers_raw_len + tx_data->len);
2271  if (ptmp == NULL) {
2272  /* error: we're freeing the entire user data */
2273  HtpState *hstate = htp_connp_get_user_data(tx_data->tx->connp);
2274  HtpTxUserDataFree(hstate, tx_ud);
2275  htp_tx_set_user_data(tx_data->tx, NULL);
2276  return HTP_OK;
2277  }
2278  tx_ud->response_headers_raw = ptmp;
2279 
2280  memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
2281  tx_data->data, tx_data->len);
2282  tx_ud->response_headers_raw_len += tx_data->len;
2283 
2284  return HTP_OK;
2285 }
2286 
2287 /*
2288  * We have a similar set function called HTPConfigSetDefaultsPhase1.
2289  */
2290 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
2291 {
2292  cfg_prec->uri_include_all = FALSE;
2293  cfg_prec->request.body_limit = HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT;
2294  cfg_prec->response.body_limit = HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT;
2295  cfg_prec->request.inspect_min_size = HTP_CONFIG_DEFAULT_REQUEST_INSPECT_MIN_SIZE;
2296  cfg_prec->request.inspect_window = HTP_CONFIG_DEFAULT_REQUEST_INSPECT_WINDOW;
2297  cfg_prec->response.inspect_min_size = HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_MIN_SIZE;
2298  cfg_prec->response.inspect_window = HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_WINDOW;
2299 
2300  if (!g_disable_randomness) {
2301  cfg_prec->randomize = HTP_CONFIG_DEFAULT_RANDOMIZE;
2302  } else {
2303  cfg_prec->randomize = 0;
2304  }
2305  cfg_prec->randomize_range = HTP_CONFIG_DEFAULT_RANDOMIZE_RANGE;
2306 
2307  htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2308  htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
2309  htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2310  htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
2311 
2312  htp_config_register_request_trailer(cfg_prec->cfg, HTPCallbackRequestHasTrailer);
2313  htp_config_register_response_trailer(cfg_prec->cfg, HTPCallbackResponseHasTrailer);
2314 
2315  htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
2316  htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
2317 
2318  htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
2319  htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
2320 
2321  htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
2322  htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
2323 
2324  htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
2325 
2326  /* don't convert + to space by default */
2327  htp_config_set_plusspace_decode(cfg_prec->cfg, HTP_DECODER_URLENCODED, 0);
2328 
2329  /* libhtp <= 0.5.9 doesn't use soft limit, but it's impossible to set
2330  * only the hard limit. So we set both here to the (current) htp defaults.
2331  * The reason we do this is that if the user sets the hard limit in the
2332  * config, we have to set the soft limit as well. If libhtp starts using
2333  * the soft limit in the future, we at least make sure we control what
2334  * it's value is. */
2335  htp_config_set_field_limits(cfg_prec->cfg,
2336  (size_t)HTP_CONFIG_DEFAULT_FIELD_LIMIT_SOFT,
2337  (size_t)HTP_CONFIG_DEFAULT_FIELD_LIMIT_HARD);
2338  return;
2339 }
2340 
2341 /* hack: htp random range code expects random values in range of 0-RAND_MAX,
2342  * but we can get both <0 and >RAND_MAX values from RandomGet
2343  */
2344 static int RandomGetWrap(void)
2345 {
2346  unsigned long r;
2347 
2348  do {
2349  r = RandomGet();
2350  } while(r >= ULONG_MAX - (ULONG_MAX % RAND_MAX));
2351 
2352  return r % RAND_MAX;
2353 }
2354 
2355 /*
2356  * We have this splitup so that in case double decoding has been enabled
2357  * for query and path, they would be called first on the callback queue,
2358  * before the callback set by Phase2() is called. We need this, since
2359  * the callback in Phase2() generates the normalized uri which utilizes
2360  * the query and path. */
2361 static void HTPConfigSetDefaultsPhase2(const char *name, HTPCfgRec *cfg_prec)
2362 {
2363  /* randomize inspection size if needed */
2364  if (cfg_prec->randomize) {
2365  int rdrange = cfg_prec->randomize_range;
2366 
2367  long int r = RandomGetWrap();
2368  cfg_prec->request.inspect_min_size +=
2369  (int) (cfg_prec->request.inspect_min_size *
2370  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2371 
2372  r = RandomGetWrap();
2373  cfg_prec->request.inspect_window +=
2374  (int) (cfg_prec->request.inspect_window *
2375  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2376  SCLogConfig("'%s' server has 'request-body-minimal-inspect-size' set to"
2377  " %d and 'request-body-inspect-window' set to %d after"
2378  " randomization.",
2379  name,
2380  cfg_prec->request.inspect_min_size,
2381  cfg_prec->request.inspect_window);
2382 
2383 
2384  r = RandomGetWrap();
2385  cfg_prec->response.inspect_min_size +=
2386  (int) (cfg_prec->response.inspect_min_size *
2387  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2388 
2389  r = RandomGetWrap();
2390  cfg_prec->response.inspect_window +=
2391  (int) (cfg_prec->response.inspect_window *
2392  (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
2393 
2394  SCLogConfig("'%s' server has 'response-body-minimal-inspect-size' set to"
2395  " %d and 'response-body-inspect-window' set to %d after"
2396  " randomization.",
2397  name,
2398  cfg_prec->response.inspect_min_size,
2399  cfg_prec->response.inspect_window);
2400  }
2401 
2402  htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
2403 
2404  cfg_prec->request.sbcfg.flags = 0;
2405  cfg_prec->request.sbcfg.buf_size = cfg_prec->request.inspect_window ?
2406  cfg_prec->request.inspect_window : 256;
2407  cfg_prec->request.sbcfg.buf_slide = 0;
2408  cfg_prec->request.sbcfg.Malloc = HTPMalloc;
2409  cfg_prec->request.sbcfg.Calloc = HTPCalloc;
2410  cfg_prec->request.sbcfg.Realloc = HTPRealloc;
2411  cfg_prec->request.sbcfg.Free = HTPFree;
2412 
2413  cfg_prec->response.sbcfg.flags = 0;
2414  cfg_prec->response.sbcfg.buf_size = cfg_prec->response.inspect_window ?
2415  cfg_prec->response.inspect_window : 256;
2416  cfg_prec->response.sbcfg.buf_slide = 0;
2417  cfg_prec->response.sbcfg.Malloc = HTPMalloc;
2418  cfg_prec->response.sbcfg.Calloc = HTPCalloc;
2419  cfg_prec->response.sbcfg.Realloc = HTPRealloc;
2420  cfg_prec->response.sbcfg.Free = HTPFree;
2421  return;
2422 }
2423 
2424 static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
2425  SCRadixTree *tree)
2426 {
2427  if (cfg_prec == NULL || s == NULL || tree == NULL)
2428  return;
2429 
2430  ConfNode *p = NULL;
2431 
2432  /* Default Parameters */
2433  TAILQ_FOREACH(p, &s->head, next) {
2434 
2435  if (strcasecmp("address", p->name) == 0) {
2436  ConfNode *pval;
2437  /* Addresses */
2438  TAILQ_FOREACH(pval, &p->head, next) {
2439  SCLogDebug("LIBHTP server %s: %s=%s", s->name, p->name,
2440  pval->val);
2441 
2442  /* IPV6 or IPV4? */
2443  if (strchr(pval->val, ':') != NULL) {
2444  SCLogDebug("LIBHTP adding ipv6 server %s at %s: %p",
2445  s->name, pval->val, cfg_prec->cfg);
2446  if (SCRadixAddKeyIPV6String(pval->val, tree, cfg_prec) == NULL) {
2447  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed to "
2448  "add ipv6 server %s, ignoring", pval->val);
2449  }
2450  } else {
2451  SCLogDebug("LIBHTP adding ipv4 server %s at %s: %p",
2452  s->name, pval->val, cfg_prec->cfg);
2453  if (SCRadixAddKeyIPV4String(pval->val, tree, cfg_prec) == NULL) {
2454  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP failed "
2455  "to add ipv4 server %s, ignoring",
2456  pval->val);
2457  }
2458  } /* else - if (strchr(pval->val, ':') != NULL) */
2459  } /* TAILQ_FOREACH(pval, &p->head, next) */
2460 
2461  } else if (strcasecmp("personality", p->name) == 0) {
2462  /* Personalities */
2463  int personality = HTPLookupPersonality(p->val);
2464  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2465  SCLogDebug("LIBHTP default: %s = %s", p->name, p->val);
2466 
2467  if (personality >= 0) {
2468  SCLogDebug("LIBHTP default: %s=%s (%d)", p->name, p->val,
2469  personality);
2470  if (htp_config_set_server_personality(cfg_prec->cfg, personality) == HTP_ERROR){
2471  SCLogWarning(SC_ERR_INVALID_VALUE, "LIBHTP Failed adding "
2472  "personality \"%s\", ignoring", p->val);
2473  } else {
2474  SCLogDebug("LIBHTP personality set to %s",
2475  HTPLookupPersonalityString(personality));
2476  }
2477 
2478  /* The IDS personality by default converts the path (and due to
2479  * our query string callback also the query string) to lowercase.
2480  * Signatures do not expect this, so override it. */
2481  htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
2482  } else {
2483  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
2484  "\"%s\", ignoring", p->val);
2485  continue;
2486  }
2487 
2488  } else if (strcasecmp("request-body-limit", p->name) == 0 ||
2489  strcasecmp("request_body_limit", p->name) == 0) {
2490  if (ParseSizeStringU32(p->val, &cfg_prec->request.body_limit) < 0) {
2491  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-limit "
2492  "from conf file - %s. Killing engine", p->val);
2493  exit(EXIT_FAILURE);
2494  }
2495 
2496  } else if (strcasecmp("response-body-limit", p->name) == 0) {
2497  if (ParseSizeStringU32(p->val, &cfg_prec->response.body_limit) < 0) {
2498  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-limit "
2499  "from conf file - %s. Killing engine", p->val);
2500  exit(EXIT_FAILURE);
2501  }
2502 
2503  } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
2504  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_min_size) < 0) {
2505  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
2506  "from conf file - %s. Killing engine", p->val);
2507  exit(EXIT_FAILURE);
2508  }
2509 
2510  } else if (strcasecmp("request-body-inspect-window", p->name) == 0) {
2511  if (ParseSizeStringU32(p->val, &cfg_prec->request.inspect_window) < 0) {
2512  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-inspect-window "
2513  "from conf file - %s. Killing engine", p->val);
2514  exit(EXIT_FAILURE);
2515  }
2516 
2517  } else if (strcasecmp("double-decode-query", p->name) == 0) {
2518  if (ConfValIsTrue(p->val)) {
2519  htp_config_register_request_line(cfg_prec->cfg,
2520  HTPCallbackDoubleDecodeQuery);
2521  }
2522 
2523  } else if (strcasecmp("double-decode-path", p->name) == 0) {
2524  if (ConfValIsTrue(p->val)) {
2525  htp_config_register_request_line(cfg_prec->cfg,
2526  HTPCallbackDoubleDecodePath);
2527  }
2528 
2529  } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
2530  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_min_size) < 0) {
2531  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
2532  "from conf file - %s. Killing engine", p->val);
2533  exit(EXIT_FAILURE);
2534  }
2535 
2536  } else if (strcasecmp("response-body-inspect-window", p->name) == 0) {
2537  if (ParseSizeStringU32(p->val, &cfg_prec->response.inspect_window) < 0) {
2538  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2539  "from conf file - %s. Killing engine", p->val);
2540  exit(EXIT_FAILURE);
2541  }
2542 
2543  } else if (strcasecmp("response-body-decompress-layer-limit", p->name) == 0) {
2544  uint32_t value = 2;
2545  if (ParseSizeStringU32(p->val, &value) < 0) {
2546  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-inspect-window "
2547  "from conf file - %s. Killing engine", p->val);
2548  exit(EXIT_FAILURE);
2549  }
2550 #ifdef HAVE_HTP_CONFIG_SET_RESPONSE_DECOMPRESSION_LAYER_LIMIT
2551  htp_config_set_response_decompression_layer_limit(cfg_prec->cfg, value);
2552 #else
2553  SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "can't set response-body-decompress-layer-limit "
2554  "to %u, libhtp version too old", value);
2555 #endif
2556  } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
2557  htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
2558  HTP_DECODER_URL_PATH,
2559  ConfValIsTrue(p->val));
2560  } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
2561  if (strlen(p->val) == 1) {
2562  htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
2563  HTP_DECODER_URL_PATH,
2564  p->val[0]);
2565  } else {
2566  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2567  "for libhtp param path-bestfit-replacement-char");
2568  }
2569  } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
2570  htp_config_set_convert_lowercase(cfg_prec->cfg,
2571  HTP_DECODER_URL_PATH,
2572  ConfValIsTrue(p->val));
2573  } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
2574  htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
2575  HTP_DECODER_URL_PATH,
2576  ConfValIsTrue(p->val));
2577  } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
2578  htp_config_set_nul_raw_terminates(cfg_prec->cfg,
2579  HTP_DECODER_URL_PATH,
2580  ConfValIsTrue(p->val));
2581  } else if (strcasecmp("path-separators-compress", p->name) == 0) {
2582  htp_config_set_path_separators_compress(cfg_prec->cfg,
2583  HTP_DECODER_URL_PATH,
2584  ConfValIsTrue(p->val));
2585  } else if (strcasecmp("path-separators-decode", p->name) == 0) {
2586  htp_config_set_path_separators_decode(cfg_prec->cfg,
2587  HTP_DECODER_URL_PATH,
2588  ConfValIsTrue(p->val));
2589  } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
2590  htp_config_set_u_encoding_decode(cfg_prec->cfg,
2591  HTP_DECODER_URL_PATH,
2592  ConfValIsTrue(p->val));
2593  } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
2594  enum htp_url_encoding_handling_t handling;
2595  if (strcasecmp(p->val, "preserve_percent") == 0) {
2596  handling = HTP_URL_DECODE_PRESERVE_PERCENT;
2597  } else if (strcasecmp(p->val, "remove_percent") == 0) {
2598  handling = HTP_URL_DECODE_REMOVE_PERCENT;
2599  } else if (strcasecmp(p->val, "decode_invalid") == 0) {
2600  handling = HTP_URL_DECODE_PROCESS_INVALID;
2601  } else {
2602  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
2603  "for libhtp param path-url-encoding-invalid-handling");
2604  return;
2605  }
2606  htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
2607  HTP_DECODER_URL_PATH,
2608  handling);
2609  } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
2610  htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
2611  HTP_DECODER_URL_PATH,
2612  ConfValIsTrue(p->val));
2613  } else if (strcasecmp("uri-include-all", p->name) == 0) {
2614  cfg_prec->uri_include_all = ConfValIsTrue(p->val);
2615  SCLogDebug("uri-include-all %s",
2616  cfg_prec->uri_include_all ? "enabled" : "disabled");
2617  } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
2618  htp_config_set_plusspace_decode(cfg_prec->cfg,
2619  HTP_DECODER_URLENCODED,
2620  ConfValIsTrue(p->val));
2621  } else if (strcasecmp("meta-field-limit", p->name) == 0) {
2622  uint32_t limit = 0;
2623  if (ParseSizeStringU32(p->val, &limit) < 0) {
2624  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2625  "from conf file - %s. Killing engine", p->val);
2626  exit(EXIT_FAILURE);
2627  }
2628  if (limit == 0) {
2629  SCLogError(SC_ERR_SIZE_PARSE, "Error meta-field-limit "
2630  "from conf file cannot be 0. Killing engine");
2631  exit(EXIT_FAILURE);
2632  }
2633  /* set default soft-limit with our new hard limit */
2634  htp_config_set_field_limits(cfg_prec->cfg,
2635  (size_t)HTP_CONFIG_DEFAULT_FIELD_LIMIT_SOFT,
2636  (size_t)limit);
2637  } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
2638  if (!g_disable_randomness) {
2639  cfg_prec->randomize = ConfValIsTrue(p->val);
2640  }
2641  } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
2642  uint32_t range = atoi(p->val);
2643  if (range > 100) {
2644  SCLogError(SC_ERR_SIZE_PARSE, "Invalid value for randomize"
2645  " inspection range setting from conf file - %s."
2646  " It should be inferior to 100."
2647  " Killing engine",
2648  p->val);
2649  exit(EXIT_FAILURE);
2650  }
2651  cfg_prec->randomize_range = range;
2652  } else if (strcasecmp("http-body-inline", p->name) == 0) {
2653  if (ConfValIsTrue(p->val)) {
2654  cfg_prec->http_body_inline = 1;
2655  } else if (ConfValIsFalse(p->val)) {
2656  cfg_prec->http_body_inline = 0;
2657  } else {
2658  if (strcmp("auto", p->val) != 0) {
2659  WarnInvalidConfEntry("http_body_inline", "%s", "auto");
2660  }
2661  if (EngineModeIsIPS()) {
2662  cfg_prec->http_body_inline = 1;
2663  } else {
2664  cfg_prec->http_body_inline = 0;
2665  }
2666  }
2667  } else if (strcasecmp("swf-decompression", p->name) == 0) {
2668  ConfNode *pval;
2669 
2670  TAILQ_FOREACH(pval, &p->head, next) {
2671  if (strcasecmp("enabled", pval->name) == 0) {
2672  if (ConfValIsTrue(pval->val)) {
2673  cfg_prec->swf_decompression_enabled = 1;
2674  } else if (ConfValIsFalse(pval->val)) {
2675  cfg_prec->swf_decompression_enabled = 0;
2676  } else {
2677  WarnInvalidConfEntry("swf-decompression.enabled", "%s", "no");
2678  }
2679  } else if (strcasecmp("type", pval->name) == 0) {
2680  if (strcasecmp("no", pval->val) == 0) {
2681  cfg_prec->swf_compression_type = HTTP_SWF_COMPRESSION_NONE;
2682  } else if (strcasecmp("deflate", pval->val) == 0) {
2683  cfg_prec->swf_compression_type = HTTP_SWF_COMPRESSION_ZLIB;
2684  } else if (strcasecmp("lzma", pval->val) == 0) {
2685  cfg_prec->swf_compression_type = HTTP_SWF_COMPRESSION_LZMA;
2686  } else if (strcasecmp("both", pval->val) == 0) {
2687  cfg_prec->swf_compression_type = HTTP_SWF_COMPRESSION_BOTH;
2688  } else {
2689  SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY,
2690  "Invalid entry for "
2691  "swf-decompression.type: %s - "
2692  "Killing engine", pval->val);
2693  exit(EXIT_FAILURE);
2694  }
2695  } else if (strcasecmp("compress-depth", pval->name) == 0) {
2696  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_compress_depth) < 0) {
2697  SCLogError(SC_ERR_SIZE_PARSE,
2698  "Error parsing swf-decompression.compression-depth "
2699  "from conf file - %s. Killing engine", p->val);
2700  exit(EXIT_FAILURE);
2701  }
2702  } else if (strcasecmp("decompress-depth", pval->name) == 0) {
2703  if (ParseSizeStringU32(pval->val, &cfg_prec->swf_decompress_depth) < 0) {
2704  SCLogError(SC_ERR_SIZE_PARSE,
2705  "Error parsing swf-decompression.decompression-depth "
2706  "from conf file - %s. Killing engine", p->val);
2707  exit(EXIT_FAILURE);
2708  }
2709  } else {
2710  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "Ignoring unknown param %s", pval->name);
2711  }
2712  }
2713  } else {
2714  SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
2715  "default config: %s", p->name);
2716  }
2717  } /* TAILQ_FOREACH(p, &default_config->head, next) */
2718 
2719  return;
2720 }
2721 
2722 void HTPConfigure(void)
2723 {
2724  SCEnter();
2725 
2726  cfglist.next = NULL;
2727 
2728  cfgtree = SCRadixCreateRadixTree(NULL, NULL);
2729  if (NULL == cfgtree)
2730  exit(EXIT_FAILURE);
2731 
2732  /* Default Config */
2733  cfglist.cfg = htp_config_create();
2734  if (NULL == cfglist.cfg) {
2735  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP default config");
2736  exit(EXIT_FAILURE);
2737  }
2738  SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
2739  HTPConfigSetDefaultsPhase1(&cfglist);
2740  if (ConfGetNode("app-layer.protocols.http.libhtp") == NULL) {
2741  HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
2742  cfgtree);
2743  } else {
2744  HTPConfigParseParameters(&cfglist, ConfGetNode("app-layer.protocols.http.libhtp.default-config"), cfgtree);
2745  }
2746  HTPConfigSetDefaultsPhase2("default", &cfglist);
2747 
2748  HTPParseMemcap();
2749 
2750  /* Read server config and create a parser for each IP in radix tree */
2751  ConfNode *server_config = ConfGetNode("app-layer.protocols.http.libhtp.server-config");
2752  if (server_config == NULL) {
2753  server_config = ConfGetNode("libhtp.server-config");
2754  if (server_config == NULL) {
2755  SCLogDebug("LIBHTP Configuring %p", server_config);
2756  SCReturn;
2757  }
2758  }
2759  SCLogDebug("LIBHTP Configuring %p", server_config);
2760 
2761  ConfNode *si;
2762  /* Server Nodes */
2763  TAILQ_FOREACH(si, &server_config->head, next) {
2764  /* Need the named node, not the index */
2765  ConfNode *s = TAILQ_FIRST(&si->head);
2766  if (NULL == s) {
2767  SCLogDebug("LIBHTP s NULL");
2768  continue;
2769  }
2770 
2771  SCLogDebug("LIBHTP server %s", s->name);
2772 
2773  HTPCfgRec *nextrec = cfglist.next;
2774  HTPCfgRec *htprec = SCMalloc(sizeof(HTPCfgRec));
2775  if (NULL == htprec)
2776  exit(EXIT_FAILURE);
2777  memset(htprec, 0x00, sizeof(*htprec));
2778 
2779  cfglist.next = htprec;
2780 
2781  cfglist.next->next = nextrec;
2782  cfglist.next->cfg = htp_config_create();
2783  if (NULL == cfglist.next->cfg) {
2784  SCLogError(SC_ERR_MEM_ALLOC, "Failed to create HTP server config");
2785  exit(EXIT_FAILURE);
2786  }
2787 
2788  HTPConfigSetDefaultsPhase1(htprec);
2789  HTPConfigParseParameters(htprec, s, cfgtree);
2790  HTPConfigSetDefaultsPhase2(s->name, htprec);
2791  }
2792 
2793  SCReturn;
2794 }
2795 
2796 void AppLayerHtpPrintStats(void)
2797 {
2798 #ifdef DEBUG
2799  SCMutexLock(&htp_state_mem_lock);
2800  SCLogPerf("htp memory %"PRIu64" (%"PRIu64")", htp_state_memuse, htp_state_memcnt);
2801  SCMutexUnlock(&htp_state_mem_lock);
2802 #endif
2803 }
2804 
2805 /** \internal
2806  * \brief get files callback
2807  * \param state state ptr
2808  * \param direction flow direction
2809  * \retval files files ptr
2810  */
2811 static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
2812 {
2813  if (state == NULL)
2814  return NULL;
2815 
2816  HtpState *http_state = (HtpState *)state;
2817 
2818  if (direction & STREAM_TOCLIENT) {
2819  SCReturnPtr(http_state->files_tc, "FileContainer");
2820  } else {
2821  SCReturnPtr(http_state->files_ts, "FileContainer");
2822  }
2823 }
2824 
2825 static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
2826 {
2827  if (direction & STREAM_TOSERVER)
2828  return ((htp_tx_t *)tx)->request_progress;
2829  else
2830  return ((htp_tx_t *)tx)->response_progress;
2831 }
2832 
2833 static uint64_t HTPStateGetTxCnt(void *alstate)
2834 {
2835  HtpState *http_state = (HtpState *)alstate;
2836 
2837  if (http_state != NULL && http_state->conn != NULL) {
2838  const uint64_t size = (uint64_t)htp_list_size(http_state->conn->transactions);
2839  SCLogDebug("size %"PRIu64, size);
2840  return size;
2841  } else {
2842  return 0ULL;
2843  }
2844 }
2845 
2846 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
2847 {
2848  HtpState *http_state = (HtpState *)alstate;
2849 
2850  if (http_state != NULL && http_state->conn != NULL)
2851  return htp_list_get(http_state->conn->transactions, tx_id);
2852  else
2853  return NULL;
2854 }
2855 
2856 static void HTPStateSetTxLogged(void *alstate, void *vtx, LoggerId bits)
2857 {
2858  htp_tx_t *tx = (htp_tx_t *)vtx;
2859  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2860  if (tx_ud)
2861  tx_ud->logged = bits;
2862 }
2863 
2864 static LoggerId HTPStateGetTxLogged(void *alstate, void *vtx)
2865 {
2866  htp_tx_t *tx = (htp_tx_t *)vtx;
2867  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
2868  if (tx_ud != NULL)
2869  return tx_ud->logged;
2870 
2871  return 0;
2872 }
2873 
2874 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction)
2875 {
2876  return (direction & STREAM_TOSERVER) ? HTP_REQUEST_COMPLETE : HTP_RESPONSE_COMPLETE;
2877 }
2878 
2879 static int HTPStateGetEventInfo(const char *event_name,
2880  int *event_id, AppLayerEventType *event_type)
2881 {
2882  *event_id = SCMapEnumNameToValue(event_name, http_decoder_event_table);
2883  if (*event_id == -1) {
2884  SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in "
2885  "http's enum map table.", event_name);
2886  /* this should be treated as fatal */
2887  return -1;
2888  }
2889 
2890  *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION;
2891 
2892  return 0;
2893 }
2894 
2895 static void HTPStateTruncate(void *state, uint8_t direction)
2896 {
2897  FileContainer *fc = HTPStateGetFiles(state, direction);
2898  if (fc != NULL) {
2899  FileTruncateAllOpenFiles(fc);
2900  }
2901 }
2902 
2903 static DetectEngineState *HTPGetTxDetectState(void *vtx)
2904 {
2905  htp_tx_t *tx = (htp_tx_t *)vtx;
2906  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2907  return tx_ud ? tx_ud->de_state : NULL;
2908 }
2909 
2910 static int HTPSetTxDetectState(void *vtx, DetectEngineState *s)
2911 {
2912  htp_tx_t *tx = (htp_tx_t *)vtx;
2913  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2914  if (tx_ud == NULL) {
2915  tx_ud = HTPMalloc(sizeof(*tx_ud));
2916  if (unlikely(tx_ud == NULL))
2917  return -ENOMEM;
2918  memset(tx_ud, 0, sizeof(*tx_ud));
2919  htp_tx_set_user_data(tx, tx_ud);
2920  }
2921  tx_ud->de_state = s;
2922  return 0;
2923 }
2924 
2925 static uint64_t HTPGetTxDetectFlags(void *vtx, uint8_t dir)
2926 {
2927  htp_tx_t *tx = (htp_tx_t *)vtx;
2928  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2929  if (tx_ud) {
2930  if (dir & STREAM_TOSERVER) {
2931  return tx_ud->detect_flags_ts;
2932  } else {
2933  return tx_ud->detect_flags_tc;
2934  }
2935  }
2936  return 0;
2937 }
2938 
2939 static void HTPSetTxDetectFlags(void *vtx, uint8_t dir, uint64_t detect_flags)
2940 {
2941  htp_tx_t *tx = (htp_tx_t *)vtx;
2942  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
2943  if (tx_ud == NULL) {
2944  tx_ud = HTPMalloc(sizeof(*tx_ud));
2945  if (unlikely(tx_ud == NULL))
2946  return;
2947  memset(tx_ud, 0, sizeof(*tx_ud));
2948  htp_tx_set_user_data(tx, tx_ud);
2949  }
2950  if (dir & STREAM_TOSERVER) {
2951  tx_ud->detect_flags_ts = detect_flags;
2952  } else {
2953  tx_ud->detect_flags_tc = detect_flags;
2954  }
2955  return;
2956 }
2957 
2958 static int HTPRegisterPatternsForProtocolDetection(void)
2959 {
2960  const char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS",
2961  "CONNECT", "DELETE", "PATCH", "PROPFIND", "PROPPATCH", "MKCOL",
2962  "COPY", "MOVE", "LOCK", "UNLOCK", "CHECKOUT", "UNCHECKOUT", "CHECKIN",
2963  "UPDATE", "LABEL", "REPORT", "MKWORKSPACE", "MKACTIVITY", "MERGE",
2964  "INVALID", "VERSION-CONTROL", "BASELINE-CONTROL", NULL};
2965  const char *spacings[] = { "|20|", "|09|", NULL };
2966  const char *versions[] = { "HTTP/0.9", "HTTP/1.0", "HTTP/1.1", NULL };
2967 
2968  int methods_pos;
2969  int spacings_pos;
2970  int versions_pos;
2971  int register_result;
2972  char method_buffer[32] = "";
2973 
2974  /* Loop through all the methods ands spacings and register the patterns */
2975  for (methods_pos = 0; methods[methods_pos]; methods_pos++) {
2976  for (spacings_pos = 0; spacings[spacings_pos]; spacings_pos++) {
2977 
2978  /* Combine the method name and the spacing */
2979  snprintf(method_buffer, sizeof(method_buffer), "%s%s", methods[methods_pos], spacings[spacings_pos]);
2980 
2981  /* Register the new method+spacing pattern
2982  * 3 is subtracted from the length since the spacing is hex typed as |xx|
2983  * but the pattern matching should only be one char
2984  */
2985  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
2986  ALPROTO_HTTP, method_buffer, strlen(method_buffer)-3, 0, STREAM_TOSERVER);
2987  if (register_result < 0) {
2988  return -1;
2989  }
2990  }
2991  }
2992 
2993  /* Loop through all the http verions patterns that are TO_CLIENT */
2994  for (versions_pos = 0; versions[versions_pos]; versions_pos++) {
2995  register_result = AppLayerProtoDetectPMRegisterPatternCI(IPPROTO_TCP,
2996  ALPROTO_HTTP, versions[versions_pos], strlen(versions[versions_pos]),
2997  0, STREAM_TOCLIENT);
2998  if (register_result < 0) {
2999  return -1;
3000  }
3001  }
3002 
3003  return 0;
3004 }
3005 
3006 /**
3007  * \brief Register the HTTP protocol and state handling functions to APP layer
3008  * of the engine.
3009  */
3010 void RegisterHTPParsers(void)
3011 {
3012  SCEnter();
3013 
3014  const char *proto_name = "http";
3015 
3016  /** HTTP */
3017  if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
3018  AppLayerProtoDetectRegisterProtocol(ALPROTO_HTTP, proto_name);
3019  if (HTPRegisterPatternsForProtocolDetection() < 0)
3020  return;
3021  } else {
3022  SCLogInfo("Protocol detection and parser disabled for %s protocol",
3023  proto_name);
3024  return;
3025  }
3026 
3027  if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
3028  AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateAlloc, HTPStateFree);
3029  AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTransactionFree);
3030  AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetFiles);
3031  AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetAlstateProgress);
3032  AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxCnt);
3033  AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTx);
3034  AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetTxLogged,
3035  HTPStateSetTxLogged);
3036  AppLayerParserRegisterGetStateProgressCompletionStatus(ALPROTO_HTTP,
3037  HTPStateGetAlstateProgressCompletionStatus);
3038  AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPGetEvents);
3039  AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_HTTP, HTPStateGetEventInfo);
3040 
3041  AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP, HTPStateTruncate);
3042  AppLayerParserRegisterDetectStateFuncs(IPPROTO_TCP, ALPROTO_HTTP,
3043  HTPGetTxDetectState, HTPSetTxDetectState);
3044  AppLayerParserRegisterDetectFlagsFuncs(IPPROTO_TCP, ALPROTO_HTTP,
3045  HTPGetTxDetectFlags, HTPSetTxDetectFlags);
3046 
3047  AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_HTTP, STREAM_TOSERVER,
3048  HTPHandleRequestData);
3049  AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_HTTP, STREAM_TOCLIENT,
3050  HTPHandleResponseData);
3051  SC_ATOMIC_INIT(htp_config_flags);
3052  AppLayerParserRegisterParserAcceptableDataDirection(IPPROTO_TCP,
3053  ALPROTO_HTTP, STREAM_TOSERVER|STREAM_TOCLIENT);
3054  HTPConfigure();
3055  } else {
3056  SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
3057  "still on.", proto_name);
3058  }
3059 #ifdef UNITTESTS
3060  AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_HTTP, HTPParserRegisterTests);
3061 #endif
3062 
3063  SCReturn;
3064 }
3065 
3066 #ifdef UNITTESTS
3067 static HTPCfgRec cfglist_backup;
3068 
3069 void HtpConfigCreateBackup(void)
3070 {
3071  cfglist_backup = cfglist;
3072 
3073  return;
3074 }
3075 
3076 void HtpConfigRestoreBackup(void)
3077 {
3078  cfglist = cfglist_backup;
3079 
3080  return;
3081 }
3082 
3083 /** \test Test case where chunks are sent in smaller chunks and check the
3084  * response of the parser from HTP library. */
3085 static int HTPParserTest01(void)
3086 {
3087  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3088  " Data is c0oL!";
3089  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3090 
3091  TcpSession ssn;
3092  memset(&ssn, 0, sizeof(ssn));
3093 
3094  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3095  FAIL_IF_NULL(alp_tctx);
3096 
3097  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3098  FAIL_IF_NULL(f);
3099  f->protoctx = &ssn;
3100  f->proto = IPPROTO_TCP;
3101  f->alproto = ALPROTO_HTTP;
3102 
3103  StreamTcpInitConfig(TRUE);
3104 
3105  uint32_t u;
3106  for (u = 0; u < httplen1; u++) {
3107  uint8_t flags = 0;
3108 
3109  if (u == 0)
3110  flags = STREAM_TOSERVER|STREAM_START;
3111  else if (u == (httplen1 - 1))
3112  flags = STREAM_TOSERVER|STREAM_EOF;
3113  else
3114  flags = STREAM_TOSERVER;
3115 
3116  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3117  &httpbuf1[u], 1);
3118  FAIL_IF(r != 0);
3119  }
3120 
3121  HtpState *htp_state = f->alstate;
3122  FAIL_IF_NULL(htp_state);
3123 
3124  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3125  FAIL_IF_NULL(tx);
3126 
3127  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3128  FAIL_IF_NULL(h);
3129 
3130  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3131  FAIL_IF(tx->request_method_number != HTP_M_POST);
3132  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3133 
3134  AppLayerParserThreadCtxFree(alp_tctx);
3135  StreamTcpFreeConfig(TRUE);
3136  UTHFreeFlow(f);
3137  PASS;
3138 }
3139 
3140 /** \test Test folding in 1 read case */
3141 static int HTPParserTest01b(void)
3142 {
3143  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3144  " Data is c0oL!";
3145  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3146 
3147  TcpSession ssn;
3148  memset(&ssn, 0, sizeof(ssn));
3149 
3150  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3151  FAIL_IF_NULL(alp_tctx);
3152 
3153  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3154  FAIL_IF_NULL(f);
3155  f->protoctx = &ssn;
3156  f->proto = IPPROTO_TCP;
3157  f->alproto = ALPROTO_HTTP;
3158 
3159  StreamTcpInitConfig(TRUE);
3160 
3161  uint8_t flags =STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3162  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3163  httpbuf1, httplen1);
3164  FAIL_IF(r != 0);
3165 
3166  HtpState *htp_state = f->alstate;
3167  FAIL_IF_NULL(htp_state);
3168 
3169  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3170  FAIL_IF_NULL(tx);
3171 
3172  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3173  FAIL_IF_NULL(h);
3174 
3175  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3176  FAIL_IF(tx->request_method_number != HTP_M_POST);
3177  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3178 
3179  AppLayerParserThreadCtxFree(alp_tctx);
3180  StreamTcpFreeConfig(TRUE);
3181  UTHFreeFlow(f);
3182  PASS;
3183 }
3184 
3185 /** \test Test folding in 1byte per read case */
3186 static int HTPParserTest01c(void)
3187 {
3188  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent:\r\n Victor/1.0\r\n\r\nPost"
3189  " Data is c0oL!";
3190  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3191 
3192  TcpSession ssn;
3193  memset(&ssn, 0, sizeof(ssn));
3194 
3195  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3196  FAIL_IF_NULL(alp_tctx);
3197 
3198  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3199  FAIL_IF_NULL(f);
3200  f->protoctx = &ssn;
3201  f->proto = IPPROTO_TCP;
3202  f->alproto = ALPROTO_HTTP;
3203 
3204  StreamTcpInitConfig(TRUE);
3205 
3206  uint32_t u;
3207  for (u = 0; u < httplen1; u++) {
3208  uint8_t flags = 0;
3209 
3210  if (u == 0)
3211  flags = STREAM_TOSERVER|STREAM_START;
3212  else if (u == (httplen1 - 1))
3213  flags = STREAM_TOSERVER|STREAM_EOF;
3214  else
3215  flags = STREAM_TOSERVER;
3216 
3217  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3218  &httpbuf1[u], 1);
3219  FAIL_IF(r != 0);
3220  }
3221 
3222  HtpState *htp_state = f->alstate;
3223  FAIL_IF_NULL(htp_state);
3224 
3225  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3226  FAIL_IF_NULL(tx);
3227 
3228  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3229  FAIL_IF_NULL(h);
3230 
3231  FAIL_IF(strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0"));
3232  FAIL_IF(tx->request_method_number != HTP_M_POST);
3233  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_0);
3234 
3235  AppLayerParserThreadCtxFree(alp_tctx);
3236  StreamTcpFreeConfig(TRUE);
3237  UTHFreeFlow(f);
3238  PASS;
3239 }
3240 
3241 /** \test Test case where chunks are sent in smaller chunks and check the
3242  * response of the parser from HTP library. */
3243 static int HTPParserTest01a(void)
3244 {
3245  int result = 0;
3246  Flow *f = NULL;
3247  uint8_t httpbuf1[] = " POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
3248  " Data is c0oL!";
3249  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3250  TcpSession ssn;
3251  HtpState *htp_state = NULL;
3252  int r = 0;
3253  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3254 
3255  memset(&ssn, 0, sizeof(ssn));
3256 
3257  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3258  if (f == NULL)
3259  goto end;
3260  f->protoctx = &ssn;
3261  f->proto = IPPROTO_TCP;
3262  f->alproto = ALPROTO_HTTP;
3263 
3264  StreamTcpInitConfig(TRUE);
3265 
3266  uint32_t u;
3267  for (u = 0; u < httplen1; u++) {
3268  uint8_t flags = 0;
3269 
3270  if (u == 0)
3271  flags = STREAM_TOSERVER|STREAM_START;
3272  else if (u == (httplen1 - 1))
3273  flags = STREAM_TOSERVER|STREAM_EOF;
3274  else
3275  flags = STREAM_TOSERVER;
3276 
3277  FLOWLOCK_WRLOCK(f);
3278  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3279  &httpbuf1[u], 1);
3280  if (r != 0) {
3281  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3282  " 0: ", u, r);
3283  FLOWLOCK_UNLOCK(f);
3284  goto end;
3285  }
3286  FLOWLOCK_UNLOCK(f);
3287  }
3288 
3289  htp_state = f->alstate;
3290  if (htp_state == NULL) {
3291  printf("no http state: ");
3292  goto end;
3293  }
3294 
3295  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3296  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3297  if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
3298  || tx->request_method_number != HTP_M_POST ||
3299  tx->request_protocol_number != HTP_PROTOCOL_1_0)
3300  {
3301  printf("expected header value: Victor/1.0 and got %s: and expected"
3302  " method: POST and got %s, expected protocol number HTTP/1.0"
3303  " and got: %s \n", bstr_util_strdup_to_c(h->value),
3304  bstr_util_strdup_to_c(tx->request_method),
3305  bstr_util_strdup_to_c(tx->request_protocol));
3306  goto end;
3307  }
3308  result = 1;
3309 end:
3310  if (alp_tctx != NULL)
3311  AppLayerParserThreadCtxFree(alp_tctx);
3312  StreamTcpFreeConfig(TRUE);
3313  UTHFreeFlow(f);
3314  return result;
3315 }
3316 
3317 /** \test See how it deals with an incomplete request. */
3318 static int HTPParserTest02(void)
3319 {
3320  int result = 0;
3321  Flow *f = NULL;
3322  uint8_t httpbuf1[] = "POST";
3323  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3324  TcpSession ssn;
3325  HtpState *http_state = NULL;
3326  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3327 
3328  memset(&ssn, 0, sizeof(ssn));
3329 
3330  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3331  if (f == NULL)
3332  goto end;
3333  f->protoctx = &ssn;
3334  f->proto = IPPROTO_TCP;
3335  f->alproto = ALPROTO_HTTP;
3336 
3337  StreamTcpInitConfig(TRUE);
3338 
3339  FLOWLOCK_WRLOCK(f);
3340  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3341  STREAM_TOSERVER | STREAM_START | STREAM_EOF,
3342  httpbuf1,
3343  httplen1);
3344  if (r != 0) {
3345  printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
3346  FLOWLOCK_UNLOCK(f);
3347  goto end;
3348  }
3349  FLOWLOCK_UNLOCK(f);
3350 
3351  http_state = f->alstate;
3352  if (http_state == NULL) {
3353  printf("no http state: ");
3354  goto end;
3355  }
3356 
3357  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3358  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3359  if ((tx->request_method) != NULL || h != NULL)
3360  {
3361  printf("expected method NULL, got %s \n", bstr_util_strdup_to_c(tx->request_method));
3362  goto end;
3363  }
3364  result = 1;
3365 end:
3366  if (alp_tctx != NULL)
3367  AppLayerParserThreadCtxFree(alp_tctx);
3368  StreamTcpFreeConfig(TRUE);
3369  UTHFreeFlow(f);
3370  return result;
3371 }
3372 
3373 /** \test Test case where method is invalid and data is sent in smaller chunks
3374  * and check the response of the parser from HTP library. */
3375 static int HTPParserTest03(void)
3376 {
3377  int result = 0;
3378  Flow *f = NULL;
3379  uint8_t httpbuf1[] = "HELLO / HTTP/1.0\r\n";
3380  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3381  TcpSession ssn;
3382  HtpState *htp_state = NULL;
3383  int r = 0;
3384  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3385 
3386  memset(&ssn, 0, sizeof(ssn));
3387 
3388  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3389  if (f == NULL)
3390  goto end;
3391  f->protoctx = &ssn;
3392  f->proto = IPPROTO_TCP;
3393  f->alproto = ALPROTO_HTTP;
3394 
3395  StreamTcpInitConfig(TRUE);
3396 
3397  uint32_t u;
3398  for (u = 0; u < httplen1; u++) {
3399  uint8_t flags = 0;
3400 
3401  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
3402  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
3403  else flags = STREAM_TOSERVER;
3404 
3405  FLOWLOCK_WRLOCK(f);
3406  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3407  &httpbuf1[u], 1);
3408  if (r != 0) {
3409  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3410  " 0: ", u, r);
3411  FLOWLOCK_UNLOCK(f);
3412  goto end;
3413  }
3414  FLOWLOCK_UNLOCK(f);
3415  }
3416  htp_state = f->alstate;
3417  if (htp_state == NULL) {
3418  printf("no http state: ");
3419  goto end;
3420  }
3421 
3422  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3423 
3424  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3425  if (tx->request_method_number != HTP_M_UNKNOWN ||
3426  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
3427  {
3428  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3429  "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3430  bstr_util_strdup_to_c(tx->request_protocol));
3431  goto end;
3432  }
3433  result = 1;
3434 end:
3435  if (alp_tctx != NULL)
3436  AppLayerParserThreadCtxFree(alp_tctx);
3437  StreamTcpFreeConfig(TRUE);
3438  UTHFreeFlow(f);
3439  return result;
3440 }
3441 
3442 /** \test Test case where invalid data is sent and check the response of the
3443  * parser from HTP library. */
3444 static int HTPParserTest04(void)
3445 {
3446  int result = 0;
3447  Flow *f = NULL;
3448  HtpState *htp_state = NULL;
3449  uint8_t httpbuf1[] = "World!\r\n";
3450  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3451  TcpSession ssn;
3452  int r = 0;
3453  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3454 
3455  memset(&ssn, 0, sizeof(ssn));
3456 
3457  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3458  if (f == NULL)
3459  goto end;
3460  f->protoctx = &ssn;
3461  f->proto = IPPROTO_TCP;
3462  f->alproto = ALPROTO_HTTP;
3463 
3464  StreamTcpInitConfig(TRUE);
3465 
3466  FLOWLOCK_WRLOCK(f);
3467  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3468  STREAM_TOSERVER | STREAM_START | STREAM_EOF,
3469  httpbuf1,
3470  httplen1);
3471  if (r != 0) {
3472  FLOWLOCK_UNLOCK(f);
3473  goto end;
3474  }
3475  FLOWLOCK_UNLOCK(f);
3476 
3477  htp_state = f->alstate;
3478  if (htp_state == NULL) {
3479  printf("no http state: ");
3480  goto end;
3481  }
3482 
3483  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3484  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3485  if (tx->request_method_number != HTP_M_UNKNOWN ||
3486  h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
3487  {
3488  printf("expected method M_UNKNOWN and got %s: , expected protocol "
3489  "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
3490  bstr_util_strdup_to_c(tx->request_protocol));
3491  goto end;
3492  }
3493  result = 1;
3494 end:
3495  if (alp_tctx != NULL)
3496  AppLayerParserThreadCtxFree(alp_tctx);
3497  StreamTcpFreeConfig(TRUE);
3498  UTHFreeFlow(f);
3499  return result;
3500 }
3501 
3502 /** \test Test both sides of a http stream mixed up to see if the HTP parser
3503  * properly parsed them and also keeps them separated. */
3504 static int HTPParserTest05(void)
3505 {
3506  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\nContent-Length: 17\r\n\r\n";
3507  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3508  uint8_t httpbuf2[] = "Post D";
3509  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3510  uint8_t httpbuf3[] = "ata is c0oL!";
3511  uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
3512 
3513  uint8_t httpbuf4[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
3514  uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
3515  uint8_t httpbuf5[] = "post R";
3516  uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
3517  uint8_t httpbuf6[] = "esults are tha bomb!";
3518  uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */
3519 
3520  TcpSession ssn;
3521  memset(&ssn, 0, sizeof(ssn));
3522 
3523  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3524  FAIL_IF_NULL(alp_tctx);
3525 
3526  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3527  FAIL_IF_NULL(f);
3528  f->protoctx = &ssn;
3529  f->proto = IPPROTO_TCP;
3530  f->alproto = ALPROTO_HTTP;
3531 
3532  StreamTcpInitConfig(TRUE);
3533 
3534  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3535  STREAM_TOSERVER | STREAM_START, httpbuf1,
3536  httplen1);
3537  FAIL_IF(r != 0);
3538 
3539  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3540  STREAM_TOCLIENT | STREAM_START, httpbuf4,
3541  httplen4);
3542  FAIL_IF(r != 0);
3543 
3544  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOCLIENT,
3545  httpbuf5, httplen5);
3546  FAIL_IF(r != 0);
3547 
3548  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, STREAM_TOSERVER,
3549  httpbuf2, httplen2);
3550  FAIL_IF(r != 0);
3551 
3552  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3553  STREAM_TOSERVER | STREAM_EOF, httpbuf3, httplen3);
3554  FAIL_IF(r != 0);
3555 
3556  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3557  STREAM_TOCLIENT | STREAM_EOF, httpbuf6, httplen6);
3558  FAIL_IF(r != 0);
3559 
3560  HtpState *http_state = f->alstate;
3561  FAIL_IF_NULL(http_state);
3562 
3563  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3564  FAIL_IF_NULL(tx);
3565  FAIL_IF_NOT(tx->request_method_number == HTP_M_POST);
3566  FAIL_IF_NOT(tx->request_protocol_number == HTP_PROTOCOL_1_0);
3567 
3568  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3569  FAIL_IF_NULL(h);
3570 
3571  FAIL_IF_NOT(tx->response_status_number == 200);
3572 
3573  AppLayerParserThreadCtxFree(alp_tctx);
3574  StreamTcpFreeConfig(TRUE);
3575  UTHFreeFlow(f);
3576  PASS;
3577 }
3578 
3579 /** \test Test proper chunked encoded response body
3580  */
3581 static int HTPParserTest06(void)
3582 {
3583  uint8_t httpbuf1[] = "GET /ld/index.php?id=412784631&cid=0064&version=4&"
3584  "name=try HTTP/1.1\r\nAccept: */*\r\nUser-Agent: "
3585  "LD-agent\r\nHost: 209.205.196.16\r\n\r\n";
3586  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3587  uint8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nDate: Sat, 03 Oct 2009 10:16:02 "
3588  "GMT\r\n"
3589  "Server: Apache/1.3.37 (Unix) mod_ssl/2.8.28 "
3590  "OpenSSL/0.9.7a PHP/4.4.7 mod_perl/1.29 "
3591  "FrontPage/5.0.2.2510\r\n"
3592  "X-Powered-By: PHP/4.4.7\r\nTransfer-Encoding: "
3593  "chunked\r\n"
3594  "Content-Type: text/html\r\n\r\n"
3595  "580\r\n"
3596  "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
3597  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
3598  "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
3599  "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
3600  "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
3601  "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
3602  "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
3603  "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
3604  "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
3605  "aG9uZTM9DQpsb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps"
3606  "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw"
3607  "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9"
3608  "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N"
3609  "Cm1vbnRoX2xpbWl0PQ0KW2dyb3VwM10NCnBob25lMT0NCmxvZ2lu"
3610  "MT0NCnBhc3N3b3JkMT0NCnBob25lMj0NCmxvZ2luMj0NCnBhc3N3"
3611  "b3JkMj0NCnBob25lMz0NCmxvZ2luMz0NCnBhc3N3b3JkMz0NCnBo"
3612  "b25lND0NCmxvZ2luND0NCnBhc3N3b3JkND0NCnBob25lNT0NCmxv"
3613  "Z2luNT0NCnBhc3N3b3JkNT0NCnBob25lNj0NCmxvZ2luNj0NCnBh"
3614  "c3N3b3JkNj0NCmNhbGxfdGltZTE9DQpjYWxsX3RpbWUyPQ0KZGF5"
3615  "X2xpbWl0PQ0KbW9udGhfbGltaXQ9DQpbZ3JvdXA0XQ0KcGhvbmUx"
3616  "PQ0KbG9naW4xPQ0KcGFzc3dvcmQxPQ0KcGhvbmUyPQ0KbG9naW4y"
3617  "PQ0KcGFzc3dvcmQyPQ0KcGhvbmUzPQ0KbG9naW4zPQ0KcGFzc3dv"
3618  "cmQzPQ0KcGhvbmU0PQ0KbG9naW40PQ0KcGFzc3dvcmQ0PQ0KcGhv"
3619  "bmU1PQ0KbG9naW41PQ0KcGFzc3dvcmQ1PQ0KcGhvbmU2PQ0KbG9n"
3620  "aW42PQ0KcGFzc3dvcmQ2PQ0KY2FsbF90aW1lMT0NCmNhbGxfdGlt"
3621  "ZTI9DQpkYXlfbGltaXQ9DQptb250aF9saW1pdD0NCltmaWxlc10N"
3622  "Cmxpbms9aHR0cDovLzIwOS4yMDUuMTk2LjE2L2xkL2dldGJvdC5w"
3623  "aHA=\r\n0\r\n\r\n";
3624  uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
3625  TcpSession ssn;
3626 
3627  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3628  FAIL_IF_NULL(alp_tctx);
3629 
3630  memset(&ssn, 0, sizeof(ssn));
3631 
3632  Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3633  FAIL_IF_NULL(f);
3634  f->protoctx = &ssn;
3635  f->proto = IPPROTO_TCP;
3636  f->alproto = ALPROTO_HTTP;
3637 
3638  StreamTcpInitConfig(TRUE);
3639 
3640  int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3641  STREAM_TOSERVER | STREAM_START, httpbuf1,
3642  httplen1);
3643  FAIL_IF(r != 0);
3644  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
3645  STREAM_TOCLIENT | STREAM_START, httpbuf2,
3646  httplen2);
3647  FAIL_IF(r != 0);
3648 
3649  HtpState *http_state = f->alstate;
3650  FAIL_IF_NULL(http_state);
3651 
3652  htp_tx_t *tx = HTPStateGetTx(http_state, 0);
3653  FAIL_IF_NULL(tx);
3654 
3655  FAIL_IF(tx->request_method_number != HTP_M_GET);
3656  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3657 
3658  FAIL_IF(tx->response_status_number != 200);
3659  FAIL_IF(tx->request_protocol_number != HTP_PROTOCOL_1_1);
3660 
3661  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3662  FAIL_IF_NULL(h);
3663 
3664  AppLayerParserThreadCtxFree(alp_tctx);
3665  StreamTcpFreeConfig(TRUE);
3666  UTHFreeFlow(f);
3667  PASS;
3668 }
3669 
3670 /** \test
3671  */
3672 static int HTPParserTest07(void)
3673 {
3674  int result = 0;
3675  Flow *f = NULL;
3676  uint8_t httpbuf1[] = "GET /awstats.pl?/migratemigrate%20=%20| HTTP/1.0\r\n\r\n";
3677  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3678  TcpSession ssn;
3679  HtpState *htp_state = NULL;
3680  int r = 0;
3681  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3682 
3683  memset(&ssn, 0, sizeof(ssn));
3684 
3685  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3686  if (f == NULL)
3687  goto end;
3688  f->protoctx = &ssn;
3689  f->proto = IPPROTO_TCP;
3690  f->alproto = ALPROTO_HTTP;
3691 
3692  StreamTcpInitConfig(TRUE);
3693 
3694  uint32_t u;
3695  for (u = 0; u < httplen1; u++) {
3696  uint8_t flags = 0;
3697 
3698  if (u == 0)
3699  flags = STREAM_TOSERVER|STREAM_START;
3700  else if (u == (httplen1 - 1))
3701  flags = STREAM_TOSERVER|STREAM_EOF;
3702  else
3703  flags = STREAM_TOSERVER;
3704 
3705  FLOWLOCK_WRLOCK(f);
3706  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3707  &httpbuf1[u], 1);
3708  if (r != 0) {
3709  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3710  " 0: ", u, r);
3711  FLOWLOCK_UNLOCK(f);
3712  goto end;
3713  }
3714  FLOWLOCK_UNLOCK(f);
3715  }
3716 
3717  htp_state = f->alstate;
3718  if (htp_state == NULL) {
3719  printf("no http state: ");
3720  goto end;
3721  }
3722 
3723  uint8_t ref[] = "/awstats.pl?/migratemigrate = |";
3724  size_t reflen = sizeof(ref) - 1;
3725 
3726  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3727  if (tx == NULL)
3728  goto end;
3729  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3730  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3731  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
3732  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
3733  (uintmax_t)reflen,
3734  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
3735  goto end;
3736  }
3737 
3738  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
3739  bstr_len(tx_ud->request_uri_normalized)) != 0)
3740  {
3741  printf("normalized uri \"");
3742  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
3743  printf("\" != \"");
3744  PrintRawUriFp(stdout, ref, reflen);
3745  printf("\": ");
3746  goto end;
3747  }
3748  }
3749 
3750  result = 1;
3751 end:
3752  if (alp_tctx != NULL)
3753  AppLayerParserThreadCtxFree(alp_tctx);
3754  StreamTcpFreeConfig(TRUE);
3755  UTHFreeFlow(f);
3756  return result;
3757 }
3758 
3759 #include "conf-yaml-loader.h"
3760 
3761 /** \test Abort
3762  */
3763 static int HTPParserTest08(void)
3764 {
3765  int result = 0;
3766  Flow *f = NULL;
3767  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3768  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3769  TcpSession ssn;
3770  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3771 
3772  char input[] = "\
3773 %YAML 1.1\n\
3774 ---\n\
3775 libhtp:\n\
3776 \n\
3777  default-config:\n\
3778  personality: IDS\n\
3779 ";
3780 
3781  ConfCreateContextBackup();
3782  ConfInit();
3783  HtpConfigCreateBackup();
3784 
3785  ConfYamlLoadString(input, strlen(input));
3786  HTPConfigure();
3787 
3788  HtpState *htp_state = NULL;
3789  int r = 0;
3790  memset(&ssn, 0, sizeof(ssn));
3791 
3792  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3793  if (f == NULL)
3794  goto end;
3795  f->protoctx = &ssn;
3796  f->proto = IPPROTO_TCP;
3797  f->alproto = ALPROTO_HTTP;
3798 
3799  StreamTcpInitConfig(TRUE);
3800 
3801  uint8_t flags = 0;
3802  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3803 
3804  FLOWLOCK_WRLOCK(f);
3805  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
3806  httplen1);
3807  if (r != 0) {
3808  printf("toserver chunk returned %" PRId32 ", expected"
3809  " 0: ", r);
3810  result = 0;
3811  FLOWLOCK_UNLOCK(f);
3812  goto end;
3813  }
3814  FLOWLOCK_UNLOCK(f);
3815 
3816  htp_state = f->alstate;
3817  if (htp_state == NULL) {
3818  printf("no http state: ");
3819  result = 0;
3820  goto end;
3821  }
3822 
3823  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3824  if (tx == NULL)
3825  goto end;
3826  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3827  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3828  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3829  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3830  bstr_len(tx_ud->request_uri_normalized));
3831  }
3832 
3833  result = 1;
3834 end:
3835  if (alp_tctx != NULL)
3836  AppLayerParserThreadCtxFree(alp_tctx);
3837  StreamTcpFreeConfig(TRUE);
3838  HTPFreeConfig();
3839  ConfDeInit();
3840  ConfRestoreContextBackup();
3841  HtpConfigRestoreBackup();
3842  UTHFreeFlow(f);
3843  return result;
3844 }
3845 
3846 /** \test Abort
3847  */
3848 static int HTPParserTest09(void)
3849 {
3850  int result = 0;
3851  Flow *f = NULL;
3852  uint8_t httpbuf1[] = "GET /secondhouse/image/js/\%ce\%de\%ce\%fd_RentCity.js?v=2011.05.02 HTTP/1.0\r\n\r\n";
3853  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3854  TcpSession ssn;
3855  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3856 
3857  char input[] = "\
3858 %YAML 1.1\n\
3859 ---\n\
3860 libhtp:\n\
3861 \n\
3862  default-config:\n\
3863  personality: Apache_2_2\n\
3864 ";
3865 
3866  ConfCreateContextBackup();
3867  ConfInit();
3868  HtpConfigCreateBackup();
3869 
3870  ConfYamlLoadString(input, strlen(input));
3871  HTPConfigure();
3872 
3873  HtpState *htp_state = NULL;
3874  int r = 0;
3875 
3876  memset(&ssn, 0, sizeof(ssn));
3877 
3878  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3879  if (f == NULL)
3880  goto end;
3881  f->protoctx = &ssn;
3882  f->proto = IPPROTO_TCP;
3883  f->alproto = ALPROTO_HTTP;
3884 
3885  StreamTcpInitConfig(TRUE);
3886 
3887  uint8_t flags = 0;
3888  flags = STREAM_TOSERVER|STREAM_START|STREAM_EOF;
3889 
3890  FLOWLOCK_WRLOCK(f);
3891  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags, httpbuf1,
3892  httplen1);
3893  if (r != 0) {
3894  printf("toserver chunk returned %" PRId32 ", expected"
3895  " 0: ", r);
3896  FLOWLOCK_UNLOCK(f);
3897  goto end;
3898  }
3899  FLOWLOCK_UNLOCK(f);
3900 
3901  htp_state = f->alstate;
3902  if (htp_state == NULL) {
3903  printf("no http state: ");
3904  goto end;
3905  }
3906 
3907  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3908  if (tx == NULL)
3909  goto end;
3910  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
3911  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
3912  //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
3913  PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
3914  bstr_len(tx_ud->request_uri_normalized));
3915  }
3916 
3917  result = 1;
3918 end:
3919  if (alp_tctx != NULL)
3920  AppLayerParserThreadCtxFree(alp_tctx);
3921  StreamTcpFreeConfig(TRUE);
3922  HTPFreeConfig();
3923  ConfDeInit();
3924  ConfRestoreContextBackup();
3925  HtpConfigRestoreBackup();
3926  UTHFreeFlow(f);
3927  return result;
3928 }
3929 
3930 /** \test Host:www.google.com <- missing space between name:value (rfc violation)
3931  */
3932 static int HTPParserTest10(void)
3933 {
3934  int result = 0;
3935  Flow *f = NULL;
3936  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\r\n\r\n";
3937  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3938  TcpSession ssn;
3939  HtpState *htp_state = NULL;
3940  int r = 0;
3941  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
3942 
3943  memset(&ssn, 0, sizeof(ssn));
3944 
3945  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
3946  if (f == NULL)
3947  goto end;
3948  f->protoctx = &ssn;
3949  f->proto = IPPROTO_TCP;
3950  f->alproto = ALPROTO_HTTP;
3951 
3952  StreamTcpInitConfig(TRUE);
3953 
3954  uint32_t u;
3955  for (u = 0; u < httplen1; u++) {
3956  uint8_t flags = 0;
3957 
3958  if (u == 0)
3959  flags = STREAM_TOSERVER|STREAM_START;
3960  else if (u == (httplen1 - 1))
3961  flags = STREAM_TOSERVER|STREAM_EOF;
3962  else
3963  flags = STREAM_TOSERVER;
3964 
3965  FLOWLOCK_WRLOCK(f);
3966  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
3967  &httpbuf1[u], 1);
3968  if (r != 0) {
3969  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
3970  " 0: ", u, r);
3971  FLOWLOCK_UNLOCK(f);
3972  goto end;
3973  }
3974  FLOWLOCK_UNLOCK(f);
3975  }
3976 
3977  htp_state = f->alstate;
3978  if (htp_state == NULL) {
3979  printf("no http state: ");
3980  goto end;
3981  }
3982 
3983  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
3984  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
3985  if (h == NULL) {
3986  goto end;
3987  }
3988 
3989  char *name = bstr_util_strdup_to_c(h->name);
3990  if (name == NULL) {
3991  goto end;
3992  }
3993 
3994  if (strcmp(name, "Host") != 0) {
3995  printf("header name not \"Host\", instead \"%s\": ", name);
3996  free(name);
3997  goto end;
3998  }
3999  free(name);
4000 
4001  char *value = bstr_util_strdup_to_c(h->value);
4002  if (value == NULL) {
4003  goto end;
4004  }
4005 
4006  if (strcmp(value, "www.google.com") != 0) {
4007  printf("header value not \"www.google.com\", instead \"%s\": ", value);
4008  free(value);
4009  goto end;
4010  }
4011  free(value);
4012 
4013  result = 1;
4014 end:
4015  if (alp_tctx != NULL)
4016  AppLayerParserThreadCtxFree(alp_tctx);
4017  StreamTcpFreeConfig(TRUE);
4018  UTHFreeFlow(f);
4019  return result;
4020 }
4021 
4022 /** \test double encoding in path
4023  */
4024 static int HTPParserTest11(void)
4025 {
4026  int result = 0;
4027  Flow *f = NULL;
4028  uint8_t httpbuf1[] = "GET /%2500 HTTP/1.0\r\n\r\n";
4029  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4030  TcpSession ssn;
4031  HtpState *htp_state = NULL;
4032  int r = 0;
4033  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
4034 
4035  memset(&ssn, 0, sizeof(ssn));
4036 
4037  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4038  if (f == NULL)
4039  goto end;
4040  f->protoctx = &ssn;
4041  f->proto = IPPROTO_TCP;
4042  f->alproto = ALPROTO_HTTP;
4043 
4044  StreamTcpInitConfig(TRUE);
4045 
4046  uint32_t u;
4047  for (u = 0; u < httplen1; u++) {
4048  uint8_t flags = 0;
4049 
4050  if (u == 0)
4051  flags = STREAM_TOSERVER|STREAM_START;
4052  else if (u == (httplen1 - 1))
4053  flags = STREAM_TOSERVER|STREAM_EOF;
4054  else
4055  flags = STREAM_TOSERVER;
4056 
4057  FLOWLOCK_WRLOCK(f);
4058  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4059  &httpbuf1[u], 1);
4060  if (r != 0) {
4061  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4062  " 0: ", u, r);
4063  FLOWLOCK_UNLOCK(f);
4064  goto end;
4065  }
4066  FLOWLOCK_UNLOCK(f);
4067  }
4068 
4069  htp_state = f->alstate;
4070  if (htp_state == NULL) {
4071  printf("no http state: ");
4072  goto end;
4073  }
4074 
4075  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4076  if (tx == NULL)
4077  goto end;
4078  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4079  if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4080  if (4 != bstr_len(tx_ud->request_uri_normalized)) {
4081  printf("normalized uri len should be 2, is %"PRIuMAX,
4082  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4083  goto end;
4084  }
4085 
4086  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4087  bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
4088  bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
4089  bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
4090  {
4091  printf("normalized uri \"");
4092  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4093  printf("\": ");
4094  goto end;
4095  }
4096  }
4097 
4098  result = 1;
4099 end:
4100  if (alp_tctx != NULL)
4101  AppLayerParserThreadCtxFree(alp_tctx);
4102  StreamTcpFreeConfig(TRUE);
4103  UTHFreeFlow(f);
4104  return result;
4105 }
4106 
4107 /** \test double encoding in query
4108  */
4109 static int HTPParserTest12(void)
4110 {
4111  int result = 0;
4112  Flow *f = NULL;
4113  uint8_t httpbuf1[] = "GET /?a=%2500 HTTP/1.0\r\n\r\n";
4114  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4115  TcpSession ssn;
4116  HtpState *htp_state = NULL;
4117  int r = 0;
4118  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
4119 
4120  memset(&ssn, 0, sizeof(ssn));
4121 
4122  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4123  if (f == NULL)
4124  goto end;
4125  f->protoctx = &ssn;
4126  f->proto = IPPROTO_TCP;
4127  f->alproto = ALPROTO_HTTP;
4128 
4129  StreamTcpInitConfig(TRUE);
4130 
4131  uint32_t u;
4132  for (u = 0; u < httplen1; u++) {
4133  uint8_t flags = 0;
4134 
4135  if (u == 0)
4136  flags = STREAM_TOSERVER|STREAM_START;
4137  else if (u == (httplen1 - 1))
4138  flags = STREAM_TOSERVER|STREAM_EOF;
4139  else
4140  flags = STREAM_TOSERVER;
4141 
4142  FLOWLOCK_WRLOCK(f);
4143  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4144  &httpbuf1[u], 1);
4145  if (r != 0) {
4146  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4147  " 0: ", u, r);
4148  FLOWLOCK_UNLOCK(f);
4149  goto end;
4150  }
4151  FLOWLOCK_UNLOCK(f);
4152  }
4153 
4154  htp_state = f->alstate;
4155  if (htp_state == NULL) {
4156  printf("no http state: ");
4157  goto end;
4158  }
4159 
4160  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4161  if (tx == NULL)
4162  goto end;
4163  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4164  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4165  if (7 != bstr_len(tx_ud->request_uri_normalized)) {
4166  printf("normalized uri len should be 5, is %"PRIuMAX,
4167  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4168  goto end;
4169  }
4170 
4171  if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
4172  bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
4173  bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
4174  bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
4175  bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
4176  bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
4177  bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
4178  {
4179  printf("normalized uri \"");
4180  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4181  printf("\": ");
4182  goto end;
4183  }
4184  }
4185 
4186  result = 1;
4187  end:
4188  if (alp_tctx != NULL)
4189  AppLayerParserThreadCtxFree(alp_tctx);
4190  StreamTcpFreeConfig(TRUE);
4191  UTHFreeFlow(f);
4192  return result;
4193 }
4194 
4195 /** \test Host:www.google.com0dName: Value0d0a <- missing space between name:value (rfc violation)
4196  */
4197 static int HTPParserTest13(void)
4198 {
4199  int result = 0;
4200  Flow *f = NULL;
4201  uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nHost:www.google.com\rName: Value\r\n\r\n";
4202  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4203  TcpSession ssn;
4204  HtpState *htp_state = NULL;
4205  int r = 0;
4206  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
4207 
4208  memset(&ssn, 0, sizeof(ssn));
4209 
4210  f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
4211  if (f == NULL)
4212  goto end;
4213  f->protoctx = &ssn;
4214  f->proto = IPPROTO_TCP;
4215  f->alproto = ALPROTO_HTTP;
4216 
4217  StreamTcpInitConfig(TRUE);
4218 
4219  uint32_t u;
4220  for (u = 0; u < httplen1; u++) {
4221  uint8_t flags = 0;
4222 
4223  if (u == 0)
4224  flags = STREAM_TOSERVER|STREAM_START;
4225  else if (u == (httplen1 - 1))
4226  flags = STREAM_TOSERVER|STREAM_EOF;
4227  else
4228  flags = STREAM_TOSERVER;
4229 
4230  FLOWLOCK_WRLOCK(f);
4231  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4232  &httpbuf1[u], 1);
4233  if (r != 0) {
4234  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4235  " 0: ", u, r);
4236  FLOWLOCK_UNLOCK(f);
4237  goto end;
4238  }
4239  FLOWLOCK_UNLOCK(f);
4240  }
4241 
4242  htp_state = f->alstate;
4243  if (htp_state == NULL) {
4244  printf("no http state: ");
4245  goto end;
4246  }
4247 
4248  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4249  htp_header_t *h = htp_table_get_index(tx->request_headers, 0, NULL);
4250  if (h == NULL) {
4251  goto end;
4252  }
4253 
4254  char *name = bstr_util_strdup_to_c(h->name);
4255  if (name == NULL) {
4256  goto end;
4257  }
4258 
4259  if (strcmp(name, "Host") != 0) {
4260  printf("header name not \"Host\", instead \"%s\": ", name);
4261  free(name);
4262  goto end;
4263  }
4264  free(name);
4265 
4266  char *value = bstr_util_strdup_to_c(h->value);
4267  if (value == NULL) {
4268  goto end;
4269  }
4270 
4271  if (strcmp(value, "www.google.com\rName: Value") != 0) {
4272  printf("header value not \"www.google.com\", instead \"");
4273  PrintRawUriFp(stdout, (uint8_t *)value, strlen(value));
4274  printf("\": ");
4275  free(value);
4276  goto end;
4277  }
4278  free(value);
4279 
4280  result = 1;
4281 end:
4282  if (alp_tctx != NULL)
4283  AppLayerParserThreadCtxFree(alp_tctx);
4284  StreamTcpFreeConfig(TRUE);
4285  UTHFreeFlow(f);
4286  return result;
4287 }
4288 
4289 /** \test Test basic config */
4290 static int HTPParserConfigTest01(void)
4291 {
4292  int ret = 0;
4293  char input[] = "\
4294 %YAML 1.1\n\
4295 ---\n\
4296 libhtp:\n\
4297 \n\
4298  default-config:\n\
4299  personality: IDS\n\
4300 \n\
4301  server-config:\n\
4302 \n\
4303  - apache-tomcat:\n\
4304  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4305  personality: Tomcat_6_0\n\
4306 \n\
4307  - iis7:\n\
4308  address: \n\
4309  - 192.168.0.0/24\n\
4310  - 192.168.10.0/24\n\
4311  personality: IIS_7_0\n\
4312 ";
4313 
4314  ConfCreateContextBackup();
4315  ConfInit();
4316 
4317  ConfYamlLoadString(input, strlen(input));
4318 
4319  ConfNode *outputs;
4320  outputs = ConfGetNode("libhtp.default-config.personality");
4321  if (outputs == NULL) {
4322  goto end;
4323  }
4324 
4325  outputs = ConfGetNode("libhtp.server-config");
4326  if (outputs == NULL) {
4327  goto end;
4328  }
4329 
4330  ConfNode *node = TAILQ_FIRST(&outputs->head);
4331  if (node == NULL) {
4332  goto end;
4333  }
4334  if (strcmp(node->name, "0") != 0) {
4335  goto end;
4336  }
4337  node = TAILQ_FIRST(&node->head);
4338  if (node == NULL) {
4339  goto end;
4340  }
4341  if (strcmp(node->name, "apache-tomcat") != 0) {
4342  goto end;
4343  }
4344 
4345  int i = 0;
4346  ConfNode *n;
4347 
4348  ConfNode *node2 = ConfNodeLookupChild(node, "personality");
4349  if (node2 == NULL) {
4350  goto end;
4351  }
4352  if (strcmp(node2->val, "Tomcat_6_0") != 0) {
4353  goto end;
4354  }
4355 
4356  node = ConfNodeLookupChild(node, "address");
4357  if (node == NULL) {
4358  goto end;
4359  }
4360  TAILQ_FOREACH(n, &node->head, next) {
4361  if (n == NULL) {
4362  goto end;
4363  }
4364 
4365  switch(i) {
4366  case 0:
4367  if (strcmp(n->name, "0") != 0) {
4368  goto end;
4369  }
4370  if (strcmp(n->val, "192.168.1.0/24") != 0) {
4371  goto end;
4372  }
4373  break;
4374  case 1:
4375  if (strcmp(n->name, "1") != 0) {
4376  goto end;
4377  }
4378  if (strcmp(n->val, "127.0.0.0/8") != 0) {
4379  goto end;
4380  }
4381  break;
4382  case 2:
4383  if (strcmp(n->name, "2") != 0) {
4384  goto end;
4385  }
4386  if (strcmp(n->val, "::1") != 0) {
4387  goto end;
4388  }
4389  break;
4390  default:
4391  goto end;
4392  }
4393  i++;
4394  }
4395 
4396  outputs = ConfGetNode("libhtp.server-config");
4397  if (outputs == NULL) {
4398  goto end;
4399  }
4400 
4401  node = TAILQ_FIRST(&outputs->head);
4402  node = TAILQ_NEXT(node, next);
4403  if (node == NULL) {
4404  goto end;
4405  }
4406  if (strcmp(node->name, "1") != 0) {
4407  goto end;
4408  }
4409  node = TAILQ_FIRST(&node->head);
4410  if (node == NULL) {
4411  goto end;
4412  }
4413  if (strcmp(node->name, "iis7") != 0) {
4414  goto end;
4415  }
4416 
4417  node2 = ConfNodeLookupChild(node, "personality");
4418  if (node2 == NULL) {
4419  goto end;
4420  }
4421  if (strcmp(node2->val, "IIS_7_0") != 0) {
4422  goto end;
4423  }
4424 
4425  node = ConfNodeLookupChild(node, "address");
4426  if (node == NULL) {
4427  goto end;
4428  }
4429 
4430  i = 0;
4431  TAILQ_FOREACH(n, &node->head, next) {
4432  if (n == NULL) {
4433  goto end;
4434  }
4435 
4436  switch(i) {
4437  case 0:
4438  if (strcmp(n->name, "0") != 0) {
4439  goto end;
4440  }
4441  if (strcmp(n->val, "192.168.0.0/24") != 0) {
4442  goto end;
4443  }
4444  break;
4445  case 1:
4446  if (strcmp(n->name, "1") != 0) {
4447  goto end;
4448  }
4449  if (strcmp(n->val, "192.168.10.0/24") != 0) {
4450  goto end;
4451  }
4452  break;
4453  default:
4454  goto end;
4455  }
4456  i++;
4457  }
4458 
4459  ret = 1;
4460 
4461 end:
4462  ConfDeInit();
4463  ConfRestoreContextBackup();
4464 
4465  return ret;
4466 }
4467 
4468 /** \test Test config builds radix correctly */
4469 static int HTPParserConfigTest02(void)
4470 {
4471  int ret = 0;
4472  char input[] = "\
4473 %YAML 1.1\n\
4474 ---\n\
4475 libhtp:\n\
4476 \n\
4477  default-config:\n\
4478  personality: IDS\n\
4479 \n\
4480  server-config:\n\
4481 \n\
4482  - apache-tomcat:\n\
4483  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4484  personality: Tomcat_6_0\n\
4485 \n\
4486  - iis7:\n\
4487  address: \n\
4488  - 192.168.0.0/24\n\
4489  - 192.168.10.0/24\n\
4490  personality: IIS_7_0\n\
4491 ";
4492 
4493  ConfCreateContextBackup();
4494  ConfInit();
4495  HtpConfigCreateBackup();
4496 
4497  ConfYamlLoadString(input, strlen(input));
4498 
4499  HTPConfigure();
4500 
4501  if (cfglist.cfg == NULL) {
4502  printf("No default config created.\n");
4503  goto end;
4504  }
4505 
4506  if (cfgtree == NULL) {
4507  printf("No config tree created.\n");
4508  goto end;
4509  }
4510 
4511  htp_cfg_t *htp = cfglist.cfg;
4512  uint8_t buf[128];
4513  const char *addr;
4514  void *user_data = NULL;
4515 
4516  addr = "192.168.10.42";
4517  if (inet_pton(AF_INET, addr, buf) == 1) {
4518  (void)SCRadixFindKeyIPV4BestMatch(buf, cfgtree, &user_data);
4519  if (user_data != NULL) {
4520  HTPCfgRec *htp_cfg_rec = user_data;
4521  htp = htp_cfg_rec->cfg;
4522  SCLogDebug("LIBHTP using config: %p", htp);
4523  }
4524  if (htp == NULL) {
4525  printf("Could not get config for: %s\n", addr);
4526  goto end;
4527  }
4528  }
4529  else {
4530  printf("Failed to parse address: %s\n", addr);
4531  goto end;
4532  }
4533 
4534  user_data = NULL;
4535  addr = "::1";
4536  if (inet_pton(AF_INET6, addr, buf) == 1) {
4537  (void)SCRadixFindKeyIPV6BestMatch(buf, cfgtree, &user_data);
4538  if (user_data != NULL) {
4539  HTPCfgRec *htp_cfg_rec = user_data;
4540  htp = htp_cfg_rec->cfg;
4541  SCLogDebug("LIBHTP using config: %p", htp);
4542  }
4543  if (htp == NULL) {
4544  printf("Could not get config for: %s\n", addr);
4545  goto end;
4546  }
4547  }
4548  else {
4549  printf("Failed to parse address: %s\n", addr);
4550  goto end;
4551  }
4552 
4553  ret = 1;
4554 
4555 end:
4556  HTPFreeConfig();
4557  ConfDeInit();
4558  ConfRestoreContextBackup();
4559  HtpConfigRestoreBackup();
4560 
4561  return ret;
4562 }
4563 
4564 /** \test Test traffic is handled by the correct htp config */
4565 static int HTPParserConfigTest03(void)
4566 {
4567  int result = 1;
4568  Flow *f = NULL;
4569  uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\nPost"
4570  " Data is c0oL!";
4571  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4572  TcpSession ssn;
4573  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
4574 
4575  HtpState *htp_state = NULL;
4576  int r = 0;
4577  char input[] = "\
4578 %YAML 1.1\n\
4579 ---\n\
4580 libhtp:\n\
4581 \n\
4582  default-config:\n\
4583  personality: IDS\n\
4584 \n\
4585  server-config:\n\
4586 \n\
4587  - apache-tomcat:\n\
4588  address: [192.168.1.0/24, 127.0.0.0/8, \"::1\"]\n\
4589  personality: Tomcat_6_0\n\
4590 \n\
4591  - iis7:\n\
4592  address: \n\
4593  - 192.168.0.0/24\n\
4594  - 192.168.10.0/24\n\
4595  personality: IIS_7_0\n\
4596 ";
4597 
4598  ConfCreateContextBackup();
4599  ConfInit();
4600  HtpConfigCreateBackup();
4601 
4602  ConfYamlLoadString(input, strlen(input));
4603 
4604  HTPConfigure();
4605 
4606  const char *addr = "192.168.10.42";
4607 
4608  memset(&ssn, 0, sizeof(ssn));
4609 
4610  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4611  if (f == NULL)
4612  goto end;
4613  f->protoctx = &ssn;
4614  f->proto = IPPROTO_TCP;
4615  f->alproto = ALPROTO_HTTP;
4616 
4617  htp_cfg_t *htp = cfglist.cfg;
4618 
4619  void *user_data = NULL;
4620  (void)SCRadixFindKeyIPV4BestMatch((uint8_t *)f->dst.addr_data32, cfgtree, &user_data);
4621  if (user_data != NULL) {
4622  HTPCfgRec *htp_cfg_rec = user_data;
4623  htp = htp_cfg_rec->cfg;
4624  SCLogDebug("LIBHTP using config: %p", htp);
4625  }
4626  if (htp == NULL) {
4627  printf("Could not get config for: %s\n", addr);
4628  goto end;
4629  }
4630 
4631  StreamTcpInitConfig(TRUE);
4632 
4633  uint32_t u;
4634  for (u = 0; u < httplen1; u++) {
4635  uint8_t flags = 0;
4636 
4637  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4638  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4639  else flags = STREAM_TOSERVER;
4640 
4641  FLOWLOCK_WRLOCK(f);
4642  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4643  &httpbuf1[u], 1);
4644  if (r != 0) {
4645  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4646  " 0: ", u, r);
4647  result = 0;
4648  FLOWLOCK_UNLOCK(f);
4649  goto end;
4650  }
4651  FLOWLOCK_UNLOCK(f);
4652  }
4653 
4654  htp_state = f->alstate;
4655  if (htp_state == NULL) {
4656  printf("no http state: ");
4657  result = 0;
4658  goto end;
4659  }
4660 
4661  if (HTPStateGetTxCnt(htp_state) != 2) {
4662  printf("HTPStateGetTxCnt(htp_state) failure\n");
4663  goto end;
4664  }
4665 
4666  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4667  if (tx == NULL)
4668  goto end;
4669  if (tx->cfg != htp) {
4670  printf("wrong HTP config (%p instead of %p - default=%p): ",
4671  tx->cfg, htp, cfglist.cfg);
4672  goto end;
4673  }
4674  tx = HTPStateGetTx(htp_state, 1);
4675  if (tx == NULL)
4676  goto end;
4677  if (tx->cfg != htp) {
4678  printf("wrong HTP config (%p instead of %p - default=%p): ",
4679  tx->cfg, htp, cfglist.cfg);
4680  goto end;
4681  }
4682 
4683 end:
4684  if (alp_tctx != NULL)
4685  AppLayerParserThreadCtxFree(alp_tctx);
4686  HTPFreeConfig();
4687  ConfDeInit();
4688  ConfRestoreContextBackup();
4689  HtpConfigRestoreBackup();
4690 
4691  StreamTcpFreeConfig(TRUE);
4692  UTHFreeFlow(f);
4693  return result;
4694 }
4695 
4696 /* disabled when we upgraded to libhtp 0.5.x */
4697 #if 0
4698 static int HTPParserConfigTest04(void)
4699 {
4700  int result = 0;
4701 
4702  char input[] = "\
4703 %YAML 1.1\n\
4704 ---\n\
4705 libhtp:\n\
4706 \n\
4707  default-config:\n\
4708  personality: IDS\n\
4709  path-control-char-handling: status_400\n\
4710  path-convert-utf8: yes\n\
4711  path-invalid-encoding-handling: remove_percent\n\
4712 \n\
4713  server-config:\n\
4714 \n\
4715  - apache-tomcat:\n\
4716  personality: Tomcat_6_0\n\
4717  path-invalid-utf8-handling: none\n\
4718  path-nul-encoded-handling: status_404\n\
4719  path-nul-raw-handling: status_400\n\
4720 \n\
4721  - iis7:\n\
4722  personality: IIS_7_0\n\
4723  path-replacement-char: o\n\
4724  path-unicode-mapping: status_400\n\
4725 ";
4726 
4727  ConfCreateContextBackup();
4728  ConfInit();
4729  HtpConfigCreateBackup();
4730 
4731  ConfYamlLoadString(input, strlen(input));
4732 
4733  HTPConfigure();
4734 
4735  HTPCfgRec *cfg_rec = &cfglist;
4736  if (cfg_rec->cfg->path_control_char_handling != STATUS_400 ||
4737  cfg_rec->cfg->path_convert_utf8 != 1 ||
4738  cfg_rec->cfg->path_invalid_encoding_handling != URL_DECODER_REMOVE_PERCENT) {
4739  printf("failed 1\n");
4740  goto end;
4741  }
4742 
4743  cfg_rec = cfg_rec->next;
4744  if (cfg_rec->cfg->bestfit_replacement_char != 'o' ||
4745  cfg_rec->cfg->path_unicode_mapping != STATUS_400) {
4746  printf("failed 2\n");
4747  goto end;
4748  }
4749 
4750  cfg_rec = cfg_rec->next;
4751  if (cfg_rec->cfg->path_invalid_utf8_handling != NONE ||
4752  cfg_rec->cfg->path_nul_encoded_handling != STATUS_404 ||
4753  cfg_rec->cfg->path_nul_raw_handling != STATUS_400) {
4754  printf("failed 3\n");
4755  goto end;
4756  }
4757 
4758  result = 1;
4759 
4760 end:
4761  HTPFreeConfig();
4762  ConfDeInit();
4763  ConfRestoreContextBackup();
4764  HtpConfigRestoreBackup();
4765 
4766  return result;
4767 }
4768 #endif
4769 
4770 /** \test Test %2f decoding in profile Apache_2_2
4771  *
4772  * %2f in path is left untouched
4773  * %2f in query string is normalized to %2F
4774  * %252f in query string is decoded/normalized to %2F
4775  */
4776 static int HTPParserDecodingTest01(void)
4777 {
4778  int result = 0;
4779  Flow *f = NULL;
4780  uint8_t httpbuf1[] =
4781  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4782  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4783  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4784  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4785  TcpSession ssn;
4786  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
4787 
4788  HtpState *htp_state = NULL;
4789  int r = 0;
4790  char input[] = "\
4791 %YAML 1.1\n\
4792 ---\n\
4793 libhtp:\n\
4794 \n\
4795  default-config:\n\
4796  personality: Apache_2\n\
4797 ";
4798 
4799  ConfCreateContextBackup();
4800  ConfInit();
4801  HtpConfigCreateBackup();
4802  ConfYamlLoadString(input, strlen(input));
4803  HTPConfigure();
4804  const char *addr = "4.3.2.1";
4805  memset(&ssn, 0, sizeof(ssn));
4806 
4807  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4808  if (f == NULL)
4809  goto end;
4810  f->protoctx = &ssn;
4811  f->proto = IPPROTO_TCP;
4812  f->alproto = ALPROTO_HTTP;
4813 
4814  StreamTcpInitConfig(TRUE);
4815 
4816  uint32_t u;
4817  for (u = 0; u < httplen1; u++) {
4818  uint8_t flags = 0;
4819 
4820  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4821  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4822  else flags = STREAM_TOSERVER;
4823 
4824  FLOWLOCK_WRLOCK(f);
4825  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4826  &httpbuf1[u], 1);
4827  if (r != 0) {
4828  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4829  " 0: ", u, r);
4830  FLOWLOCK_UNLOCK(f);
4831  goto end;
4832  }
4833  FLOWLOCK_UNLOCK(f);
4834  }
4835 
4836  htp_state = f->alstate;
4837  if (htp_state == NULL) {
4838  printf("no http state: ");
4839  goto end;
4840  }
4841 
4842  uint8_t ref1[] = "/abc%2fdef";
4843  size_t reflen = sizeof(ref1) - 1;
4844 
4845  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
4846  if (tx == NULL)
4847  goto end;
4848  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4849  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4850  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4851  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4852  (uintmax_t)reflen,
4853  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4854  goto end;
4855  }
4856 
4857  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
4858  bstr_len(tx_ud->request_uri_normalized)) != 0)
4859  {
4860  printf("normalized uri \"");
4861  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4862  printf("\" != \"");
4863  PrintRawUriFp(stdout, ref1, reflen);
4864  printf("\": ");
4865  goto end;
4866  }
4867  }
4868 
4869  uint8_t ref2[] = "/abc/def?ghi/jkl";
4870  reflen = sizeof(ref2) - 1;
4871 
4872  tx = HTPStateGetTx(htp_state, 1);
4873  if (tx == NULL)
4874  goto end;
4875  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
4876  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4877  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4878  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4879  (uintmax_t)reflen,
4880  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4881  goto end;
4882  }
4883 
4884  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
4885  bstr_len(tx_ud->request_uri_normalized)) != 0)
4886  {
4887  printf("normalized uri \"");
4888  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4889  printf("\" != \"");
4890  PrintRawUriFp(stdout, ref2, reflen);
4891  printf("\": ");
4892  goto end;
4893  }
4894  }
4895 
4896  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
4897  reflen = sizeof(ref3) - 1;
4898  tx = HTPStateGetTx(htp_state, 2);
4899  if (tx == NULL)
4900  goto end;
4901  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
4902  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
4903  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
4904  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
4905  (uintmax_t)reflen,
4906  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
4907  goto end;
4908  }
4909 
4910  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
4911  bstr_len(tx_ud->request_uri_normalized)) != 0)
4912  {
4913  printf("normalized uri \"");
4914  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
4915  printf("\" != \"");
4916  PrintRawUriFp(stdout, ref3, reflen);
4917  printf("\": ");
4918  goto end;
4919  }
4920  }
4921 
4922  result = 1;
4923 
4924 end:
4925  if (alp_tctx != NULL)
4926  AppLayerParserThreadCtxFree(alp_tctx);
4927  HTPFreeConfig();
4928  ConfDeInit();
4929  ConfRestoreContextBackup();
4930  HtpConfigRestoreBackup();
4931 
4932  StreamTcpFreeConfig(TRUE);
4933  UTHFreeFlow(f);
4934  return result;
4935 }
4936 
4937 /** \test Test %2f decoding in profile IDS
4938  *
4939  * %2f in path decoded to /
4940  * %2f in query string is decoded to /
4941  * %252f in query string is decoded to %2F
4942  */
4943 static int HTPParserDecodingTest02(void)
4944 {
4945  int result = 0;
4946  Flow *f = NULL;
4947  uint8_t httpbuf1[] =
4948  "GET /abc%2fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4949  "GET /abc/def?ghi%2fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
4950  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
4951  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
4952  TcpSession ssn;
4953  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
4954 
4955  HtpState *htp_state = NULL;
4956  int r = 0;
4957  char input[] = "\
4958 %YAML 1.1\n\
4959 ---\n\
4960 libhtp:\n\
4961 \n\
4962  default-config:\n\
4963  personality: IDS\n\
4964  double-decode-path: no\n\
4965  double-decode-query: no\n\
4966 ";
4967 
4968  ConfCreateContextBackup();
4969  ConfInit();
4970  HtpConfigCreateBackup();
4971  ConfYamlLoadString(input, strlen(input));
4972  HTPConfigure();
4973  const char *addr = "4.3.2.1";
4974  memset(&ssn, 0, sizeof(ssn));
4975 
4976  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
4977  if (f == NULL)
4978  goto end;
4979  f->protoctx = &ssn;
4980  f->proto = IPPROTO_TCP;
4981  f->alproto = ALPROTO_HTTP;
4982 
4983  StreamTcpInitConfig(TRUE);
4984 
4985  uint32_t u;
4986  for (u = 0; u < httplen1; u++) {
4987  uint8_t flags = 0;
4988 
4989  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
4990  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
4991  else flags = STREAM_TOSERVER;
4992 
4993  FLOWLOCK_WRLOCK(f);
4994  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
4995  &httpbuf1[u], 1);
4996  if (r != 0) {
4997  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
4998  " 0: ", u, r);
4999  FLOWLOCK_UNLOCK(f);
5000  goto end;
5001  }
5002  FLOWLOCK_UNLOCK(f);
5003  }
5004 
5005  htp_state = f->alstate;
5006  if (htp_state == NULL) {
5007  printf("no http state: ");
5008  goto end;
5009  }
5010 
5011  uint8_t ref1[] = "/abc/def";
5012  size_t reflen = sizeof(ref1) - 1;
5013 
5014  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5015  if (tx == NULL)
5016  goto end;
5017  HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5018  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5019  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5020  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5021  (uintmax_t)reflen,
5022  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5023  goto end;
5024  }
5025 
5026  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5027  bstr_len(tx_ud->request_uri_normalized)) != 0)
5028  {
5029  printf("normalized uri \"");
5030  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5031  printf("\" != \"");
5032  PrintRawUriFp(stdout, ref1, reflen);
5033  printf("\": ");
5034  goto end;
5035  }
5036  }
5037 
5038  uint8_t ref2[] = "/abc/def?ghi/jkl";
5039  reflen = sizeof(ref2) - 1;
5040 
5041  tx = HTPStateGetTx(htp_state, 1);
5042  if (tx == NULL)
5043  goto end;
5044  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5045  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5046  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5047  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5048  (uintmax_t)reflen,
5049  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5050  goto end;
5051  }
5052 
5053  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5054  bstr_len(tx_ud->request_uri_normalized)) != 0)
5055  {
5056  printf("normalized uri \"");
5057  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5058  printf("\" != \"");
5059  PrintRawUriFp(stdout, ref2, reflen);
5060  printf("\": ");
5061  goto end;
5062  }
5063  }
5064 
5065  uint8_t ref3[] = "/abc/def?ghi%2fjkl";
5066  reflen = sizeof(ref3) - 1;
5067  tx = HTPStateGetTx(htp_state, 2);
5068  if (tx == NULL)
5069  goto end;
5070  tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5071  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5072  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5073  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
5074  (uintmax_t)reflen,
5075  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5076  goto end;
5077  }
5078 
5079  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
5080  bstr_len(tx_ud->request_uri_normalized)) != 0)
5081  {
5082  printf("normalized uri \"");
5083  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5084  printf("\" != \"");
5085  PrintRawUriFp(stdout, ref3, reflen);
5086  printf("\": ");
5087  goto end;
5088  }
5089  }
5090 
5091  result = 1;
5092 
5093 end:
5094  if (alp_tctx != NULL)
5095  AppLayerParserThreadCtxFree(alp_tctx);
5096  HTPFreeConfig();
5097  ConfDeInit();
5098  ConfRestoreContextBackup();
5099  HtpConfigRestoreBackup();
5100 
5101  StreamTcpFreeConfig(TRUE);
5102  UTHFreeFlow(f);
5103  return result;
5104 }
5105 
5106 /** \test Test %2f decoding in profile IDS with double-decode-* options
5107  *
5108  * %252f in path decoded to /
5109  * %252f in query string is decoded to /
5110  */
5111 static int HTPParserDecodingTest03(void)
5112 {
5113  int result = 0;
5114  Flow *f = NULL;
5115  uint8_t httpbuf1[] =
5116  "GET /abc%252fdef HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n"
5117  "GET /abc/def?ghi%252fjkl HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5118  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5119  TcpSession ssn;
5120  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
5121 
5122  HtpState *htp_state = NULL;
5123  int r = 0;
5124  char input[] = "\
5125 %YAML 1.1\n\
5126 ---\n\
5127 libhtp:\n\
5128 \n\
5129  default-config:\n\
5130  personality: IDS\n\
5131  double-decode-path: yes\n\
5132  double-decode-query: yes\n\
5133 ";
5134 
5135  ConfCreateContextBackup();
5136  ConfInit();
5137  HtpConfigCreateBackup();
5138  ConfYamlLoadString(input, strlen(input));
5139  HTPConfigure();
5140  const char *addr = "4.3.2.1";
5141  memset(&ssn, 0, sizeof(ssn));
5142 
5143  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5144  if (f == NULL)
5145  goto end;
5146  f->protoctx = &ssn;
5147  f->proto = IPPROTO_TCP;
5148  f->alproto = ALPROTO_HTTP;
5149 
5150  StreamTcpInitConfig(TRUE);
5151 
5152  uint32_t u;
5153  for (u = 0; u < httplen1; u++) {
5154  uint8_t flags = 0;
5155 
5156  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5157  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5158  else flags = STREAM_TOSERVER;
5159 
5160  FLOWLOCK_WRLOCK(f);
5161  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5162  &httpbuf1[u], 1);
5163  if (r != 0) {
5164  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5165  " 0: ", u, r);
5166  FLOWLOCK_UNLOCK(f);
5167  goto end;
5168  }
5169  FLOWLOCK_UNLOCK(f);
5170  }
5171 
5172  htp_state = f->alstate;
5173  if (htp_state == NULL) {
5174  printf("no http state: ");
5175  goto end;
5176  }
5177 
5178  uint8_t ref1[] = "/abc/def";
5179  size_t reflen = sizeof(ref1) - 1;
5180 
5181  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5182  if (tx == NULL)
5183  goto end;
5184  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5185  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5186  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5187  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5188  (uintmax_t)reflen,
5189  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5190  goto end;
5191  }
5192 
5193  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5194  bstr_len(tx_ud->request_uri_normalized)) != 0)
5195  {
5196  printf("normalized uri \"");
5197  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5198  printf("\" != \"");
5199  PrintRawUriFp(stdout, ref1, reflen);
5200  printf("\": ");
5201  goto end;
5202  }
5203  }
5204 
5205  uint8_t ref2[] = "/abc/def?ghi/jkl";
5206  reflen = sizeof(ref2) - 1;
5207 
5208  tx = HTPStateGetTx(htp_state, 1);
5209  if (tx == NULL)
5210  goto end;
5211  tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
5212  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5213  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5214  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5215  (uintmax_t)reflen,
5216  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5217  goto end;
5218  }
5219 
5220  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
5221  bstr_len(tx_ud->request_uri_normalized)) != 0)
5222  {
5223  printf("normalized uri \"");
5224  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5225  printf("\" != \"");
5226  PrintRawUriFp(stdout, ref2, reflen);
5227  printf("\": ");
5228  goto end;
5229  }
5230  }
5231 
5232  result = 1;
5233 
5234 end:
5235  if (alp_tctx != NULL)
5236  AppLayerParserThreadCtxFree(alp_tctx);
5237  HTPFreeConfig();
5238  ConfDeInit();
5239  ConfRestoreContextBackup();
5240  HtpConfigRestoreBackup();
5241 
5242  StreamTcpFreeConfig(TRUE);
5243  UTHFreeFlow(f);
5244  return result;
5245 }
5246 
5247 /** \test Test http:// in query profile IDS
5248  */
5249 static int HTPParserDecodingTest04(void)
5250 {
5251  int result = 0;
5252  Flow *f = NULL;
5253  uint8_t httpbuf1[] =
5254  "GET /abc/def?a=http://www.abc.com/ HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5255  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5256  TcpSession ssn;
5257  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
5258 
5259  HtpState *htp_state = NULL;
5260  int r = 0;
5261  char input[] = "\
5262 %YAML 1.1\n\
5263 ---\n\
5264 libhtp:\n\
5265 \n\
5266  default-config:\n\
5267  personality: IDS\n\
5268  double-decode-path: yes\n\
5269  double-decode-query: yes\n\
5270 ";
5271 
5272  ConfCreateContextBackup();
5273  ConfInit();
5274  HtpConfigCreateBackup();
5275  ConfYamlLoadString(input, strlen(input));
5276  HTPConfigure();
5277  const char *addr = "4.3.2.1";
5278  memset(&ssn, 0, sizeof(ssn));
5279 
5280  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5281  if (f == NULL)
5282  goto end;
5283  f->protoctx = &ssn;
5284  f->proto = IPPROTO_TCP;
5285  f->alproto = ALPROTO_HTTP;
5286 
5287  StreamTcpInitConfig(TRUE);
5288 
5289  uint32_t u;
5290  for (u = 0; u < httplen1; u++) {
5291  uint8_t flags = 0;
5292 
5293  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5294  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5295  else flags = STREAM_TOSERVER;
5296 
5297  FLOWLOCK_WRLOCK(f);
5298  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5299  &httpbuf1[u], 1);
5300  if (r != 0) {
5301  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5302  " 0: ", u, r);
5303  FLOWLOCK_UNLOCK(f);
5304  goto end;
5305  }
5306  FLOWLOCK_UNLOCK(f);
5307  }
5308 
5309  htp_state = f->alstate;
5310  if (htp_state == NULL) {
5311  printf("no http state: ");
5312  goto end;
5313  }
5314 
5315  uint8_t ref1[] = "/abc/def?a=http://www.abc.com/";
5316  size_t reflen = sizeof(ref1) - 1;
5317 
5318  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5319  if (tx == NULL)
5320  goto end;
5321  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5322  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5323  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5324  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5325  (uintmax_t)reflen,
5326  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5327  goto end;
5328  }
5329 
5330  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5331  bstr_len(tx_ud->request_uri_normalized)) != 0)
5332  {
5333  printf("normalized uri \"");
5334  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5335  printf("\" != \"");
5336  PrintRawUriFp(stdout, ref1, reflen);
5337  printf("\": ");
5338  goto end;
5339  }
5340  }
5341 
5342  result = 1;
5343 
5344 end:
5345  if (alp_tctx != NULL)
5346  AppLayerParserThreadCtxFree(alp_tctx);
5347  HTPFreeConfig();
5348  ConfDeInit();
5349  ConfRestoreContextBackup();
5350  HtpConfigRestoreBackup();
5351 
5352  StreamTcpFreeConfig(TRUE);
5353  UTHFreeFlow(f);
5354  return result;
5355 }
5356 
5357 /** \test Test \ char in query profile IDS. Bug 739
5358  */
5359 static int HTPParserDecodingTest05(void)
5360 {
5361  int result = 0;
5362  Flow *f = NULL;
5363  uint8_t httpbuf1[] =
5364  "GET /index?id=\\\"<script>alert(document.cookie)</script> HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5365  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5366  TcpSession ssn;
5367  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
5368 
5369  HtpState *htp_state = NULL;
5370  int r = 0;
5371  char input[] = "\
5372 %YAML 1.1\n\
5373 ---\n\
5374 libhtp:\n\
5375 \n\
5376  default-config:\n\
5377  personality: IDS\n\
5378  double-decode-path: yes\n\
5379  double-decode-query: yes\n\
5380 ";
5381 
5382  ConfCreateContextBackup();
5383  ConfInit();
5384  HtpConfigCreateBackup();
5385  ConfYamlLoadString(input, strlen(input));
5386  HTPConfigure();
5387  const char *addr = "4.3.2.1";
5388  memset(&ssn, 0, sizeof(ssn));
5389 
5390  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5391  if (f == NULL)
5392  goto end;
5393  f->protoctx = &ssn;
5394  f->proto = IPPROTO_TCP;
5395  f->alproto = ALPROTO_HTTP;
5396 
5397  StreamTcpInitConfig(TRUE);
5398 
5399  uint32_t u;
5400  for (u = 0; u < httplen1; u++) {
5401  uint8_t flags = 0;
5402 
5403  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5404  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5405  else flags = STREAM_TOSERVER;
5406 
5407  FLOWLOCK_WRLOCK(f);
5408  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5409  &httpbuf1[u], 1);
5410  if (r != 0) {
5411  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5412  " 0: ", u, r);
5413  FLOWLOCK_UNLOCK(f);
5414  goto end;
5415  }
5416  FLOWLOCK_UNLOCK(f);
5417  }
5418 
5419  htp_state = f->alstate;
5420  if (htp_state == NULL) {
5421  printf("no http state: ");
5422  goto end;
5423  }
5424 
5425  uint8_t ref1[] = "/index?id=\\\"<script>alert(document.cookie)</script>";
5426  size_t reflen = sizeof(ref1) - 1;
5427 
5428  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5429  if (tx == NULL)
5430  goto end;
5431  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5432  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5433  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5434  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5435  (uintmax_t)reflen,
5436  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5437  goto end;
5438  }
5439 
5440  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5441  bstr_len(tx_ud->request_uri_normalized)) != 0)
5442  {
5443  printf("normalized uri \"");
5444  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5445  printf("\" != \"");
5446  PrintRawUriFp(stdout, ref1, reflen);
5447  printf("\": ");
5448  goto end;
5449  }
5450  }
5451 
5452  result = 1;
5453 
5454 end:
5455  if (alp_tctx != NULL)
5456  AppLayerParserThreadCtxFree(alp_tctx);
5457  HTPFreeConfig();
5458  ConfDeInit();
5459  ConfRestoreContextBackup();
5460  HtpConfigRestoreBackup();
5461 
5462  StreamTcpFreeConfig(TRUE);
5463  UTHFreeFlow(f);
5464  return result;
5465 }
5466 
5467 /** \test Test + char in query. Bug 1035
5468  */
5469 static int HTPParserDecodingTest06(void)
5470 {
5471  int result = 0;
5472  Flow *f = NULL;
5473  uint8_t httpbuf1[] =
5474  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5475  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5476  TcpSession ssn;
5477  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
5478 
5479  HtpState *htp_state = NULL;
5480  int r = 0;
5481  char input[] = "\
5482 %YAML 1.1\n\
5483 ---\n\
5484 libhtp:\n\
5485 \n\
5486  default-config:\n\
5487  personality: IDS\n\
5488  double-decode-path: yes\n\
5489  double-decode-query: yes\n\
5490 ";
5491 
5492  ConfCreateContextBackup();
5493  ConfInit();
5494  HtpConfigCreateBackup();
5495  ConfYamlLoadString(input, strlen(input));
5496  HTPConfigure();
5497  const char *addr = "4.3.2.1";
5498  memset(&ssn, 0, sizeof(ssn));
5499 
5500  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5501  if (f == NULL)
5502  goto end;
5503  f->protoctx = &ssn;
5504  f->proto = IPPROTO_TCP;
5505  f->alproto = ALPROTO_HTTP;
5506 
5507  StreamTcpInitConfig(TRUE);
5508 
5509  uint32_t u;
5510  for (u = 0; u < httplen1; u++) {
5511  uint8_t flags = 0;
5512 
5513  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5514  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5515  else flags = STREAM_TOSERVER;
5516 
5517  FLOWLOCK_WRLOCK(f);
5518  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5519  &httpbuf1[u], 1);
5520  if (r != 0) {
5521  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5522  " 0: ", u, r);
5523  FLOWLOCK_UNLOCK(f);
5524  goto end;
5525  }
5526  FLOWLOCK_UNLOCK(f);
5527  }
5528 
5529  htp_state = f->alstate;
5530  if (htp_state == NULL) {
5531  printf("no http state: ");
5532  goto end;
5533  }
5534 
5535  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port=+6000";
5536  size_t reflen = sizeof(ref1) - 1;
5537 
5538  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5539  if (tx == NULL)
5540  goto end;
5541  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5542  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5543  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5544  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5545  (uintmax_t)reflen,
5546  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5547  goto end;
5548  }
5549 
5550  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5551  bstr_len(tx_ud->request_uri_normalized)) != 0)
5552  {
5553  printf("normalized uri \"");
5554  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5555  printf("\" != \"");
5556  PrintRawUriFp(stdout, ref1, reflen);
5557  printf("\": ");
5558  goto end;
5559  }
5560  }
5561 
5562  result = 1;
5563 
5564 end:
5565  if (alp_tctx != NULL)
5566  AppLayerParserThreadCtxFree(alp_tctx);
5567  HTPFreeConfig();
5568  ConfDeInit();
5569  ConfRestoreContextBackup();
5570  HtpConfigRestoreBackup();
5571 
5572  StreamTcpFreeConfig(TRUE);
5573  UTHFreeFlow(f);
5574  return result;
5575 }
5576 
5577 /** \test Test + char in query. Bug 1035
5578  */
5579 static int HTPParserDecodingTest07(void)
5580 {
5581  int result = 0;
5582  Flow *f = NULL;
5583  uint8_t httpbuf1[] =
5584  "GET /put.php?ip=1.2.3.4&port=+6000 HTTP/1.1\r\nHost: www.domain.ltd\r\n\r\n";
5585  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5586  TcpSession ssn;
5587  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
5588 
5589  HtpState *htp_state = NULL;
5590  int r = 0;
5591  char input[] = "\
5592 %YAML 1.1\n\
5593 ---\n\
5594 libhtp:\n\
5595 \n\
5596  default-config:\n\
5597  personality: IDS\n\
5598  double-decode-path: yes\n\
5599  double-decode-query: yes\n\
5600  query-plusspace-decode: yes\n\
5601 ";
5602 
5603  ConfCreateContextBackup();
5604  ConfInit();
5605  HtpConfigCreateBackup();
5606  ConfYamlLoadString(input, strlen(input));
5607  HTPConfigure();
5608  const char *addr = "4.3.2.1";
5609  memset(&ssn, 0, sizeof(ssn));
5610 
5611  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5612  if (f == NULL)
5613  goto end;
5614  f->protoctx = &ssn;
5615  f->proto = IPPROTO_TCP;
5616  f->alproto = ALPROTO_HTTP;
5617 
5618  StreamTcpInitConfig(TRUE);
5619 
5620  uint32_t u;
5621  for (u = 0; u < httplen1; u++) {
5622  uint8_t flags = 0;
5623 
5624  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5625  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5626  else flags = STREAM_TOSERVER;
5627 
5628  FLOWLOCK_WRLOCK(f);
5629  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5630  &httpbuf1[u], 1);
5631  if (r != 0) {
5632  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5633  " 0: ", u, r);
5634  FLOWLOCK_UNLOCK(f);
5635  goto end;
5636  }
5637  FLOWLOCK_UNLOCK(f);
5638  }
5639 
5640  htp_state = f->alstate;
5641  if (htp_state == NULL) {
5642  printf("no http state: ");
5643  goto end;
5644  }
5645 
5646  uint8_t ref1[] = "/put.php?ip=1.2.3.4&port= 6000";
5647  size_t reflen = sizeof(ref1) - 1;
5648 
5649  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5650  if (tx == NULL)
5651  goto end;
5652  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5653  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5654  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5655  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5656  (uintmax_t)reflen,
5657  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5658  goto end;
5659  }
5660 
5661  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5662  bstr_len(tx_ud->request_uri_normalized)) != 0)
5663  {
5664  printf("normalized uri \"");
5665  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5666  printf("\" != \"");
5667  PrintRawUriFp(stdout, ref1, reflen);
5668  printf("\": ");
5669  goto end;
5670  }
5671  }
5672 
5673  result = 1;
5674 
5675 end:
5676  if (alp_tctx != NULL)
5677  AppLayerParserThreadCtxFree(alp_tctx);
5678  HTPFreeConfig();
5679  ConfDeInit();
5680  ConfRestoreContextBackup();
5681  HtpConfigRestoreBackup();
5682 
5683  StreamTcpFreeConfig(TRUE);
5684  UTHFreeFlow(f);
5685  return result;
5686 }
5687 
5688 /** \test Test 'proxy' URI normalization. Ticket 1008
5689  */
5690 static int HTPParserDecodingTest08(void)
5691 {
5692  int result = 0;
5693  Flow *f = NULL;
5694  uint8_t httpbuf1[] =
5695  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5696  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5697  TcpSession ssn;
5698  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
5699 
5700  HtpState *htp_state = NULL;
5701  int r = 0;
5702  char input[] = "\
5703 %YAML 1.1\n\
5704 ---\n\
5705 libhtp:\n\
5706 \n\
5707  default-config:\n\
5708  personality: IDS\n\
5709 ";
5710 
5711  ConfCreateContextBackup();
5712  ConfInit();
5713  HtpConfigCreateBackup();
5714  ConfYamlLoadString(input, strlen(input));
5715  HTPConfigure();
5716  const char *addr = "4.3.2.1";
5717  memset(&ssn, 0, sizeof(ssn));
5718 
5719  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5720  if (f == NULL)
5721  goto end;
5722  f->protoctx = &ssn;
5723  f->proto = IPPROTO_TCP;
5724  f->alproto = ALPROTO_HTTP;
5725 
5726  StreamTcpInitConfig(TRUE);
5727 
5728  uint32_t u;
5729  for (u = 0; u < httplen1; u++) {
5730  uint8_t flags = 0;
5731 
5732  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5733  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5734  else flags = STREAM_TOSERVER;
5735 
5736  FLOWLOCK_WRLOCK(f);
5737  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5738  &httpbuf1[u], 1);
5739  if (r != 0) {
5740  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5741  " 0: ", u, r);
5742  FLOWLOCK_UNLOCK(f);
5743  goto end;
5744  }
5745  FLOWLOCK_UNLOCK(f);
5746  }
5747 
5748  htp_state = f->alstate;
5749  if (htp_state == NULL) {
5750  printf("no http state: ");
5751  goto end;
5752  }
5753 
5754  uint8_t ref1[] = "/blah/";
5755  size_t reflen = sizeof(ref1) - 1;
5756 
5757  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5758  if (tx == NULL)
5759  goto end;
5760  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5761  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5762  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5763  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5764  (uintmax_t)reflen,
5765  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5766  goto end;
5767  }
5768 
5769  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5770  bstr_len(tx_ud->request_uri_normalized)) != 0)
5771  {
5772  printf("normalized uri \"");
5773  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5774  printf("\" != \"");
5775  PrintRawUriFp(stdout, ref1, reflen);
5776  printf("\": ");
5777  goto end;
5778  }
5779  }
5780 
5781  result = 1;
5782 
5783 end:
5784  if (alp_tctx != NULL)
5785  AppLayerParserThreadCtxFree(alp_tctx);
5786  HTPFreeConfig();
5787  ConfDeInit();
5788  ConfRestoreContextBackup();
5789  HtpConfigRestoreBackup();
5790 
5791  StreamTcpFreeConfig(TRUE);
5792  UTHFreeFlow(f);
5793  return result;
5794 }
5795 
5796 /** \test Test 'proxy' URI normalization. Ticket 1008
5797  */
5798 static int HTPParserDecodingTest09(void)
5799 {
5800  int result = 0;
5801  Flow *f = NULL;
5802  uint8_t httpbuf1[] =
5803  "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
5804  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5805  TcpSession ssn;
5806  AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
5807 
5808  HtpState *htp_state = NULL;
5809  int r = 0;
5810  char input[] = "\
5811 %YAML 1.1\n\
5812 ---\n\
5813 libhtp:\n\
5814 \n\
5815  default-config:\n\
5816  personality: IDS\n\
5817  uri-include-all: true\n\
5818 ";
5819 
5820  ConfCreateContextBackup();
5821  ConfInit();
5822  HtpConfigCreateBackup();
5823  ConfYamlLoadString(input, strlen(input));
5824  HTPConfigure();
5825  const char *addr = "4.3.2.1";
5826  memset(&ssn, 0, sizeof(ssn));
5827 
5828  f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
5829  if (f == NULL)
5830  goto end;
5831  f->protoctx = &ssn;
5832  f->proto = IPPROTO_TCP;
5833  f->alproto = ALPROTO_HTTP;
5834 
5835  StreamTcpInitConfig(TRUE);
5836 
5837  uint32_t u;
5838  for (u = 0; u < httplen1; u++) {
5839  uint8_t flags = 0;
5840 
5841  if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
5842  else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
5843  else flags = STREAM_TOSERVER;
5844 
5845  FLOWLOCK_WRLOCK(f);
5846  r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, flags,
5847  &httpbuf1[u], 1);
5848  if (r != 0) {
5849  printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
5850  " 0: ", u, r);
5851  FLOWLOCK_UNLOCK(f);
5852  goto end;
5853  }
5854  FLOWLOCK_UNLOCK(f);
5855  }
5856 
5857  htp_state = f->alstate;
5858  if (htp_state == NULL) {
5859  printf("no http state: ");
5860  goto end;
5861  }
5862 
5863  uint8_t ref1[] = "http://suricata-ids.org/blah/";
5864  size_t reflen = sizeof(ref1) - 1;
5865 
5866  htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
5867  if (tx == NULL)
5868  goto end;
5869  HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
5870  if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
5871  if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
5872  printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
5873  (uintmax_t)reflen,
5874  (uintmax_t)bstr_len(tx_ud->request_uri_normalized));
5875  goto end;
5876  }
5877 
5878  if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
5879  bstr_len(tx_ud->request_uri_normalized)) != 0)
5880  {
5881  printf("normalized uri \"");
5882  PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
5883  printf("\" != \"");
5884  PrintRawUriFp(stdout, ref1, reflen);
5885  printf("\": ");
5886  goto end;
5887  }
5888  }
5889 
5890  result = 1;
5891 
5892 end:
5893  if (alp_tctx != NULL)
5894  AppLayerParserThreadCtxFree(alp_tctx);
5895  HTPFreeConfig();
5896  ConfDeInit();
5897  ConfRestoreContextBackup();
5898  HtpConfigRestoreBackup();
5899 
5900  StreamTcpFreeConfig(TRUE);
5901  UTHFreeFlow(f);
5902  return result;
5903 }
5904 
5905 /** \test BG box crash -- chunks are messed up. Observed for real. */
5906 static int HTPBodyReassemblyTest01(void)
5907 {
5908  int result = 0;
5909  HtpTxUserData htud;
5910  memset(&htud, 0x00, sizeof(htud));
5911  HtpState hstate;
5912  memset(&hstate, 0x00, sizeof(hstate));
5913  Flow flow;
5914  memset(&flow, 0x00, sizeof(flow));
5915  AppLayerParserState *parser = AppLayerParserStateAlloc();
5916  htp_tx_t tx;
5917  memset(&tx, 0, sizeof(tx));
5918 
5919  hstate.f = &flow;
5920  flow.alparser = parser;
5921 
5922  uint8_t chunk1[] = "--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5923  uint8_t chunk2[] = "POST /uri HTTP/1.1\r\nHost: hostname.com\r\nKeep-Alive: 115\r\nAccept-Charset: utf-8\r\nUser-Agent: Mozilla/5.0 (X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nConnection: keep-alive\r\nContent-length: 68102\r\nReferer: http://otherhost.com\r\nAccept-Encoding: gzip\r\nContent-Type: multipart/form-data; boundary=e5a320f21416a02493a0a6f561b1c494\r\nCookie: blah\r\nAccept-Language: us\r\n\r\n--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5924 
5925  int r = HtpBodyAppendChunk(NULL, &htud.request_body, chunk1, sizeof(chunk1)-1);
5926  BUG_ON(r != 0);
5927  r = HtpBodyAppendChunk(NULL, &htud.request_body, chunk2, sizeof(chunk2)-1);
5928  BUG_ON(r != 0);
5929 
5930  const uint8_t *chunks_buffer = NULL;
5931  uint32_t chunks_buffer_len = 0;
5932 
5933  HtpRequestBodyReassemble(&htud, &chunks_buffer, &chunks_buffer_len);
5934  if (chunks_buffer == NULL) {
5935  goto end;
5936  }
5937 #ifdef PRINT
5938  printf("REASSCHUNK START: \n");
5939  PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
5940  printf("REASSCHUNK END: \n");
5941 #endif
5942 
5943  HtpRequestBodyHandleMultipart(&hstate, &htud, &tx, chunks_buffer, chunks_buffer_len);
5944 
5945  if (htud.request_body.content_len_so_far != 669) {
5946  printf("htud.request_body.content_len_so_far %"PRIu64": ", htud.request_body.content_len_so_far);
5947  goto end;
5948  }
5949 
5950  if (hstate.files_ts != NULL)
5951  goto end;
5952 
5953  result = 1;
5954 end:
5955  return result;
5956 }
5957 
5958 /** \test BG crash */
5959 static int HTPSegvTest01(void)
5960 {
5961  int result = 0;
5962  Flow *f = NULL;
5963  uint8_t httpbuf1[] = "POST /uri HTTP/1.1\r\nHost: hostname.com\r\nKeep-Alive: 115\r\nAccept-Charset: utf-8\r\nUser-Agent: Mozilla/5.0 (X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nConnection: keep-alive\r\nContent-length: 68102\r\nReferer: http://otherhost.com\r\nAccept-Encoding: gzip\r\nContent-Type: multipart/form-data; boundary=e5a320f21416a02493a0a6f561b1c494\r\nCookie: blah\r\nAccept-Language: us\r\n\r\n--e5a320f21416a02493a0a6f561b1c494\r\nContent-Disposition: form-data; name=\"uploadfile\"; filename=\"D2GUef.jpg\"\r";
5964  uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
5965  char i