suricata
util-decode-mime.c
Go to the documentation of this file.
1 /* Copyright (C) 2012 BAE Systems
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author David Abarbanel <david.abarbanel@baesystems.com>
22  *
23  */
24 
25 #include "suricata-common.h"
26 
27 #include "util-decode-mime.h"
28 #include "util-ip.h"
29 #include "util-spm-bs.h"
30 #include "util-unittest.h"
31 #include "util-memcmp.h"
32 #include "util-print.h"
33 
34 /* Character constants */
35 #ifndef CR
36 #define CR 13
37 #define LF 10
38 #endif
39 
40 #define CRLF "\r\n"
41 #define COLON 58
42 #define DASH 45
43 #define PRINTABLE_START 33
44 #define PRINTABLE_END 126
45 #define UC_START 65
46 #define UC_END 90
47 #define LC_START 97
48 #define LC_END 122
49 #define UC_LC_DIFF 32
50 #define EOL_LEN 2
51 
52 /* Base-64 constants */
53 #define BASE64_STR "Base64"
54 
55 /* Mime Constants */
56 #define MAX_LINE_LEN 998 /* Def in RFC 2045, excluding CRLF sequence */
57 #define MAX_ENC_LINE_LEN 76 /* Def in RFC 2045, excluding CRLF sequence */
58 #define MAX_HEADER_NAME 75 /* 75 + ":" = 76 */
59 #define MAX_HEADER_VALUE 2000 /* Default - arbitrary limit */
60 #define BOUNDARY_BUF 256
61 #define CTNT_TYPE_STR "content-type"
62 #define CTNT_DISP_STR "content-disposition"
63 #define CTNT_TRAN_STR "content-transfer-encoding"
64 #define MSG_ID_STR "message-id"
65 #define BND_START_STR "boundary="
66 #define TOK_END_STR "\""
67 #define MSG_STR "message/"
68 #define MULTIPART_STR "multipart/"
69 #define QP_STR "quoted-printable"
70 #define TXT_STR "text/plain"
71 #define HTML_STR "text/html"
72 #define URL_STR "http://"
73 
74 /* Memory Usage Constants */
75 #define STACK_FREE_NODES 10
76 
77 /* Other Constants */
78 #define MAX_IP4_CHARS 15
79 #define MAX_IP6_CHARS 39
80 
81 /* Globally hold configuration data */
82 static MimeDecConfig mime_dec_config = { 1, 1, 1, 0, MAX_HEADER_VALUE };
83 
84 /* Mime Parser String translation */
85 static const char *StateFlags[] = { "NONE",
86  "HEADER_READY",
87  "HEADER_STARTED",
88  "HEADER_DONE",
89  "BODY_STARTED",
90  "BODY_DONE",
91  "BODY_END_BOUND",
92  "PARSE_DONE",
93  "PARSE_ERROR",
94  NULL };
95 
96 /* URL executable file extensions */
97 static const char *UrlExeExts[] = { ".exe",
98  ".vbs",
99  ".bin",
100  ".cmd",
101  ".bat",
102  ".jar",
103  ".js",
104  NULL };
105 
106 /**
107  * \brief Function used to print character strings that are not null-terminated
108  *
109  * \param log_level The logging level in which to print
110  * \param label A label for the string to print
111  * \param src The source string
112  * \param len The length of the string
113  *
114  * \return none
115  */
116 static void PrintChars(int log_level, const char *label, const uint8_t *src, uint32_t len)
117 {
118 #ifdef DEBUG
119  if (log_level <= sc_log_global_log_level) {
120  printf("[%s]\n", label);
121  PrintRawDataFp(stdout, (uint8_t *)src, len);
122  }
123 #endif
124 }
125 
126 /**
127  * \brief Set global config policy
128  *
129  * \param config Config policy to set
130  * \return none
131  */
133 {
134  if (config != NULL) {
135  mime_dec_config = *config;
136 
137  /* Set to default */
138  if (mime_dec_config.header_value_depth == 0) {
139  mime_dec_config.header_value_depth = MAX_HEADER_VALUE;
140  }
141  } else {
142  SCLogWarning(SC_ERR_MISSING_CONFIG_PARAM, "Invalid null configuration parameters");
143  }
144 }
145 
146 /**
147  * \brief Get global config policy
148  *
149  * \return config data structure
150  */
152 {
153  return &mime_dec_config;
154 }
155 
156 /**
157  * \brief Follow the 'next' pointers to the leaf
158  *
159  * \param node The root entity
160  *
161  * \return Pointer to leaf on 'next' side
162  *
163  */
164 static MimeDecEntity *findLastSibling(MimeDecEntity *node)
165 {
166  if (node == NULL)
167  return NULL;
168  while(node->next != NULL)
169  node = node->next;
170  return node;
171 }
172 
173 /**
174  * \brief Frees a mime entity tree
175  *
176  * \param entity The root entity
177  *
178  * \return none
179  *
180  */
182 {
183  if (entity == NULL)
184  return;
185  MimeDecEntity *lastSibling = findLastSibling(entity);
186  while (entity != NULL)
187  {
188  /**
189  * Move child to next
190  * Transform tree into list
191  */
192  if (entity->child != NULL)
193  {
194  lastSibling->next = entity->child;
195  lastSibling = findLastSibling(lastSibling);
196  }
197 
198  /**
199  * Move to next element
200  */
201  MimeDecEntity *old = entity;
202  entity = entity->next;
203 
205  MimeDecFreeUrl(old->url_list);
206  SCFree(old->filename);
207 
208  SCFree(old);
209  }
210 }
211 
212 /**
213  * \brief Iteratively frees a header field entry list
214  *
215  * \param field The header field
216  *
217  * \return none
218  *
219  */
221 {
222  MimeDecField *temp, *curr;
223 
224  if (field != NULL) {
225 
226  curr = field;
227  while (curr != NULL) {
228  temp = curr;
229  curr = curr->next;
230 
231  /* Free contents of node */
232  SCFree(temp->name);
233  SCFree(temp->value);
234 
235  /* Now free node data */
236  SCFree(temp);
237  }
238  }
239 }
240 
241 /**
242  * \brief Iteratively frees a URL entry list
243  *
244  * \param url The url entry
245  *
246  * \return none
247  *
248  */
250 {
251  MimeDecUrl *temp, *curr;
252 
253  if (url != NULL) {
254 
255  curr = url;
256  while (curr != NULL) {
257  temp = curr;
258  curr = curr->next;
259 
260  /* Now free node data */
261  SCFree(temp->url);
262  SCFree(temp);
263  }
264  }
265 }
266 
267 /**
268  * \brief Creates and adds a header field entry to an entity
269  *
270  * The entity is optional. If NULL is specified, than a new stand-alone field
271  * is created.
272  *
273  * \param entity The parent entity
274  *
275  * \return The field object, or NULL if the operation fails
276  *
277  */
279 {
280  MimeDecField *node = SCMalloc(sizeof(MimeDecField));
281  if (unlikely(node == NULL)) {
282  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
283  return NULL;
284  }
285  memset(node, 0x00, sizeof(MimeDecField));
286 
287  /* If list is empty, then set as head of list */
288  if (entity->field_list == NULL) {
289  entity->field_list = node;
290  } else {
291  /* Otherwise add to beginning of list since these are out-of-order in
292  * the message */
293  node->next = entity->field_list;
294  entity->field_list = node;
295  }
296 
297  return node;
298 }
299 
300 
301 /**
302  * \brief Searches for header fields with the specified name
303  *
304  * \param entity The entity to search
305  * \param name The header name (lowercase)
306  *
307  * \return number of items found
308  *
309  */
310 int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
311 {
312  MimeDecField *curr = entity->field_list;
313  int found = 0;
314 
315  while (curr != NULL) {
316  /* name is stored lowercase */
317  if (strlen(name) == curr->name_len) {
318  if (SCMemcmp(curr->name, name, curr->name_len) == 0) {
319  if (DataCallback(curr->value, curr->value_len, data))
320  found++;
321  }
322  }
323  curr = curr->next;
324  }
325 
326  return found;
327 }
328 
329 /**
330  * \brief Searches for a header field with the specified name
331  *
332  * \param entity The entity to search
333  * \param name The header name (lowercase)
334  *
335  * \return The field object, or NULL if not found
336  *
337  */
338 MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name) {
339  MimeDecField *curr = entity->field_list;
340 
341  while (curr != NULL) {
342  /* name is stored lowercase */
343  if (strlen(name) == curr->name_len) {
344  if (SCMemcmp(curr->name, name, curr->name_len) == 0) {
345  break;
346  }
347  }
348  curr = curr->next;
349  }
350 
351  return curr;
352 }
353 
354 /**
355  * \brief Creates and adds a URL entry to the specified entity
356  *
357  * The entity is optional and if NULL is specified, then a new list will be created.
358  *
359  * \param entity The entity
360  *
361  * \return URL entry or NULL if the operation fails
362  *
363  */
364 static MimeDecUrl * MimeDecAddUrl(MimeDecEntity *entity, uint8_t *url, uint32_t url_len, uint8_t flags)
365 {
366  MimeDecUrl *node = SCMalloc(sizeof(MimeDecUrl));
367  if (unlikely(node == NULL)) {
368  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
369  return NULL;
370  }
371  memset(node, 0x00, sizeof(MimeDecUrl));
372 
373  node->url = url;
374  node->url_len = url_len;
375  node->url_flags = flags;
376 
377  /* If list is empty, then set as head of list */
378  if (entity->url_list == NULL) {
379  entity->url_list = node;
380  } else {
381  /* Otherwise add to beginning of list since these are out-of-order in
382  * the message */
383  node->next = entity->url_list;
384  entity->url_list = node;
385  }
386 
387  return node;
388 }
389 
390 /**
391  * \brief Creates and adds a child entity to the specified parent entity
392  *
393  * \param parent The parent entity
394  *
395  * \return The child entity, or NULL if the operation fails
396  *
397  */
399 {
400  MimeDecEntity *curr, *node = SCMalloc(sizeof(MimeDecEntity));
401  if (unlikely(node == NULL)) {
402  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
403  return NULL;
404  }
405  memset(node, 0x00, sizeof(MimeDecEntity));
406 
407  /* If parent is NULL then just return the new pointer */
408  if (parent != NULL) {
409  if (parent->child == NULL) {
410  parent->child = node;
411  } else {
412  curr = parent->child;
413  while (curr->next != NULL) {
414  curr = curr->next;
415  }
416  curr->next = node;
417  }
418  }
419 
420  return node;
421 }
422 
423 /**
424  * \brief Creates a mime header field and fills in its values and adds it to the
425  * specified entity
426  *
427  * \param entity Entity in which to add the field
428  * \param name String containing the name
429  * \param nlen Length of the name
430  * \param value String containing the value
431  * \param vlen Length of the value
432  *
433  * \return The field or NULL if the operation fails
434  */
435 static MimeDecField * MimeDecFillField(MimeDecEntity *entity, uint8_t *name,
436  uint32_t nlen, const uint8_t *value, uint32_t vlen)
437 {
438  if (nlen == 0 && vlen == 0)
439  return NULL;
440 
441  MimeDecField *field = MimeDecAddField(entity);
442  if (unlikely(field == NULL)) {
443  return NULL;
444  }
445 
446  if (nlen > 0) {
447  /* convert to lowercase and store */
448  uint32_t u;
449  for (u = 0; u < nlen; u++)
450  name[u] = tolower(name[u]);
451 
452  field->name = (uint8_t *)name;
453  field->name_len = nlen;
454  }
455 
456  if (vlen > 0) {
457  field->value = (uint8_t *)value;
458  field->value_len = vlen;
459  }
460 
461  return field;
462 }
463 
464 /**
465  * \brief Pushes a node onto a stack and returns the new node.
466  *
467  * \param stack The top of the stack
468  *
469  * \return pointer to a new node, otherwise NULL if it fails
470  */
471 static MimeDecStackNode * PushStack(MimeDecStack *stack)
472 {
473  /* Attempt to pull from free nodes list */
474  MimeDecStackNode *node = stack->free_nodes;
475  if (node == NULL) {
476  node = SCMalloc(sizeof(MimeDecStackNode));
477  if (unlikely(node == NULL)) {
478  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
479  return NULL;
480  }
481  } else {
482  /* Move free nodes pointer over */
483  stack->free_nodes = stack->free_nodes->next;
484  stack->free_nodes_cnt--;
485  }
486  memset(node, 0x00, sizeof(MimeDecStackNode));
487 
488  /* Push to top of stack */
489  node->next = stack->top;
490  stack->top = node;
491 
492  /* Return a pointer to the top of the stack */
493  return node;
494 }
495 
496 /**
497  * \brief Pops the top node from the stack and returns the next node.
498  *
499  * \param stack The top of the stack
500  *
501  * \return pointer to the next node, otherwise NULL if no nodes remain
502  */
503 static MimeDecStackNode * PopStack(MimeDecStack *stack)
504 {
505  /* Move stack pointer to next item */
506  MimeDecStackNode *curr = stack->top;
507  if (curr != NULL) {
508  curr = curr->next;
509  }
510 
511  /* Always free alloc'd memory */
512  SCFree(stack->top->bdef);
513 
514  /* Now move head to free nodes list */
515  if (stack->free_nodes_cnt < STACK_FREE_NODES) {
516  stack->top->next = stack->free_nodes;
517  stack->free_nodes = stack->top;
518  stack->free_nodes_cnt++;
519  } else {
520  SCFree(stack->top);
521  }
522  stack->top = curr;
523 
524  /* Return a pointer to the top of the stack */
525  return curr;
526 }
527 
528 /**
529  * \brief Frees the stack along with the free-nodes list
530  *
531  * \param stack The stack pointer
532  *
533  * \return none
534  */
535 static void FreeMimeDecStack(MimeDecStack *stack)
536 {
537  MimeDecStackNode *temp, *curr;
538 
539  if (stack != NULL) {
540  /* Top of stack */
541  curr = stack->top;
542  while (curr != NULL) {
543  temp = curr;
544  curr = curr->next;
545 
546  /* Now free node */
547  SCFree(temp->bdef);
548  SCFree(temp);
549  }
550 
551  /* Free nodes */
552  curr = stack->free_nodes;
553  while (curr != NULL) {
554  temp = curr;
555  curr = curr->next;
556 
557  /* Now free node */
558  SCFree(temp);
559  }
560 
561  SCFree(stack);
562  }
563 }
564 
565 /**
566  * \brief Adds a data value to the data values linked list
567  *
568  * \param dv The head of the linked list (NULL if new list)
569  *
570  * \return pointer to a new node, otherwise NULL if it fails
571  */
572 static DataValue * AddDataValue(DataValue *dv)
573 {
574  DataValue *curr, *node = SCMalloc(sizeof(DataValue));
575  if (unlikely(node == NULL)) {
576  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
577  return NULL;
578  }
579  memset(node, 0x00, sizeof(DataValue));
580 
581  if (dv != NULL) {
582  curr = dv;
583  while (curr->next != NULL) {
584  curr = curr->next;
585  }
586 
587  curr->next = node;
588  }
589 
590  return node;
591 }
592 
593 /**
594  * \brief Frees a linked list of data values starting at the head
595  *
596  * \param dv The head of the linked list
597  *
598  * \return none
599  */
600 static void FreeDataValue(DataValue *dv)
601 {
602  DataValue *temp, *curr;
603 
604  if (dv != NULL) {
605  curr = dv;
606  while (curr != NULL) {
607  temp = curr;
608  curr = curr->next;
609 
610  /* Now free node */
611  SCFree(temp->value);
612  SCFree(temp);
613  }
614  }
615 }
616 
617 /**
618  * \brief Converts a list of data values into a single value (returns dynamically
619  * allocated memory)
620  *
621  * \param dv The head of the linked list (NULL if new list)
622  * \param len The output length of the single value
623  *
624  * \return pointer to a single value, otherwise NULL if it fails or is zero-length
625  */
626 static uint8_t * GetFullValue(DataValue *dv, uint32_t *len)
627 {
628  DataValue *curr;
629  uint32_t offset = 0;
630  uint8_t *val = NULL;
631 
632  /* First calculate total length */
633  *len = 0;
634  curr = dv;
635  while (curr != NULL) {
636  *len += curr->value_len;
637 
638 #if 0
639  /* Add CRLF except on last one */
640  if (curr->next != NULL) {
641  *len += 2;
642  }
643 #endif
644  curr = curr->next;
645  }
646 
647  /* Must have at least one character in the value */
648  if (*len > 0) {
649  val = SCCalloc(1, *len);
650  if (unlikely(val == NULL)) {
651  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
652  *len = 0;
653  return NULL;
654  }
655 
656  curr = dv;
657  while (curr != NULL) {
658  memcpy(val + offset, curr->value, curr->value_len);
659  offset += curr->value_len;
660 
661 #if 0 /* VJ unclear why this is needed ? */
662  /* Add CRLF except on last one */
663  if (curr->next != NULL) {
664  memcpy(val + offset, CRLF, 2);
665  offset += 2;
666  }
667 #endif
668  curr = curr->next;
669  }
670  }
671 
672  return val;
673 }
674 
675 /**
676  * \brief Find a string while searching up to N characters within a source
677  * buffer
678  *
679  * \param src The source string (not null-terminated)
680  * \param len The length of the source string
681  * \param find The string to find (null-terminated)
682  * \param find_len length of the 'find' string
683  *
684  * \return Pointer to the position it was found, otherwise NULL if not found
685  */
686 static inline uint8_t * FindBuffer(const uint8_t *src, uint32_t len, const uint8_t *find, uint32_t find_len)
687 {
688  /* Use utility search function */
689  return BasicSearchNocase(src, len, find, find_len);
690 }
691 
692 /**
693  * \brief Get a line (CRLF or just CR or LF) from a buffer (similar to GetToken)
694  *
695  * \param buf The input buffer (not null-terminated)
696  * \param blen The length of the input buffer
697  * \param remainPtr Pointer to remaining after tokenizing iteration
698  * \param tokLen Output token length (if non-null line)
699  *
700  * \return Pointer to line
701  */
702 static uint8_t * GetLine(uint8_t *buf, uint32_t blen, uint8_t **remainPtr,
703  uint32_t *tokLen)
704 {
705  uint32_t i;
706  uint8_t *tok;
707 
708  /* So that it can be used just like strtok_r */
709  if (buf == NULL) {
710  buf = *remainPtr;
711  } else {
712  *remainPtr = buf;
713  }
714  if (buf == NULL)
715  return NULL;
716 
717  tok = buf;
718 
719  /* length must be specified */
720  for (i = 0; i < blen && buf[i] != 0; i++) {
721 
722  /* Found delimiter */
723  if (buf[i] == CR || buf[i] == LF) {
724 
725  /* Add another if we find either CRLF or LFCR */
726  *remainPtr += (i + 1);
727  if ((i + 1 < blen) && buf[i] != buf[i + 1] &&
728  (buf[i + 1] == CR || buf[i + 1] == LF)) {
729  (*remainPtr)++;
730  }
731  break;
732  }
733  }
734 
735  /* If no delimiter found, then point to end of buffer */
736  if (buf == *remainPtr) {
737  (*remainPtr) += i;
738  }
739 
740  /* Calculate token length */
741  *tokLen = (buf + i) - tok;
742 
743  return tok;
744 }
745 
746 /**
747  * \brief Get token from buffer and return pointer to it
748  *
749  * \param buf The input buffer (not null-terminated)
750  * \param blen The length of the input buffer
751  * \param delims Character delimiters (null-terminated)
752  * \param remainPtr Pointer to remaining after tokenizing iteration
753  * \param tokLen Output token length (if non-null line)
754  *
755  * \return Pointer to token, or NULL if not found
756  */
757 static uint8_t * GetToken(uint8_t *buf, uint32_t blen, const char *delims,
758  uint8_t **remainPtr, uint32_t *tokenLen)
759 {
760  uint32_t i, j, delimFound = 0;
761  uint8_t *tok = NULL;
762 
763  /* So that it can be used just like strtok_r */
764  if (buf == NULL) {
765  buf = *remainPtr;
766  } else {
767  *remainPtr = buf;
768  }
769  if (buf == NULL)
770  return NULL;
771 
772  /* Must specify length */
773  for (i = 0; i < blen && buf[i] != 0; i++) {
774 
775  /* Look for delimiters */
776  for (j = 0; delims[j] != 0; j++) {
777  if (buf[i] == delims[j]) {
778  /* Data must be found before delimiter matters */
779  if (tok != NULL) {
780  (*remainPtr) += (i + 1);
781  }
782  delimFound = 1;
783  break;
784  }
785  }
786 
787  /* If at least one non-delimiter found, then a token is found */
788  if (tok == NULL && !delimFound) {
789  tok = buf + i;
790  } else {
791  /* Reset delimiter */
792  delimFound = 0;
793  }
794 
795  /* If delimiter found, then break out of loop */
796  if (buf != *remainPtr) {
797  break;
798  }
799  }
800 
801  /* Make sure remaining points to end of buffer if delimiters not found */
802  if (tok != NULL) {
803  if (buf == *remainPtr) {
804  (*remainPtr) += i;
805  }
806 
807  /* Calculate token length */
808  *tokenLen = (buf + i) - tok;
809  }
810 
811  return tok;
812 }
813 
814 /**
815  * \brief Stores the final MIME header value into the current entity on the
816  * stack.
817  *
818  * \param state The parser state
819  *
820  * \return MIME_DEC_OK if stored, otherwise a negative number indicating error
821  */
822 static int StoreMimeHeader(MimeDecParseState *state)
823 {
824  int ret = MIME_DEC_OK, stored = 0;
825  uint8_t *val;
826  uint32_t vlen;
827 
828  /* Lets save the most recent header */
829  if (state->hname != NULL || state->hvalue != NULL) {
830  SCLogDebug("Storing last header");
831  val = GetFullValue(state->hvalue, &vlen);
832  if (val != NULL) {
833  if (state->hname == NULL) {
834  SCLogDebug("Error: Invalid parser state - header value without"
835  " name");
836  ret = MIME_DEC_ERR_PARSE;
837  } else if (state->stack->top != NULL) {
838  /* Store each header name and value */
839  if (MimeDecFillField(state->stack->top->data, state->hname,
840  state->hlen, val, vlen) == NULL) {
841  SCLogError(SC_ERR_MEM_ALLOC, "MimeDecFillField() function failed");
842  ret = MIME_DEC_ERR_MEM;
843  } else {
844  stored = 1;
845  }
846  } else {
847  SCLogDebug("Error: Stack pointer missing");
848  ret = MIME_DEC_ERR_DATA;
849  }
850  } else if (state->hvalue != NULL) {
851  /* Memory allocation must have failed since val is NULL */
852  SCLogError(SC_ERR_MEM_ALLOC, "GetFullValue() function failed");
853  ret = MIME_DEC_ERR_MEM;
854  }
855 
856  /* Do cleanup here */
857  if (!stored) {
858  SCFree(state->hname);
859  SCFree(val);
860  }
861  state->hname = NULL;
862  FreeDataValue(state->hvalue);
863  state->hvalue = NULL;
864  state->hvlen = 0;
865  }
866 
867  return ret;
868 }
869 
870 /**
871  * \brief Function determines whether a url string points to an executable
872  * based on file extension only.
873  *
874  * \param url The url string
875  * \param len The url string length
876  *
877  * \retval 1 The url points to an EXE
878  * \retval 0 The url does NOT point to an EXE
879  */
880 static int IsExeUrl(const uint8_t *url, uint32_t len)
881 {
882  int isExeUrl = 0;
883  uint32_t i, extLen;
884  uint8_t *ext;
885 
886  /* Now check for executable extensions and if not found, cut off at first '/' */
887  for (i = 0; UrlExeExts[i] != NULL; i++) {
888  extLen = strlen(UrlExeExts[i]);
889  ext = FindBuffer(url, len, (uint8_t *)UrlExeExts[i], strlen(UrlExeExts[i]));
890  if (ext != NULL && (ext + extLen - url == (int)len || ext[extLen] == '?')) {
891  isExeUrl = 1;
892  break;
893  }
894  }
895 
896  return isExeUrl;
897 }
898 
899 /**
900  * \brief Function determines whether a host string is a numeric IP v4 address
901  *
902  * \param urlhost The host string
903  * \param len The host string length
904  *
905  * \retval 1 The host is a numeric IP
906  * \retval 0 The host is NOT a numeric IP
907  */
908 static int IsIpv4Host(const uint8_t *urlhost, uint32_t len)
909 {
910  struct sockaddr_in sa;
911  char tempIp[MAX_IP4_CHARS + 1];
912 
913  /* Cut off at '/' */
914  uint32_t i = 0;
915  for ( ; i < len && urlhost[i] != 0; i++) {
916 
917  if (urlhost[i] == '/') {
918  break;
919  }
920  }
921 
922  /* Too many chars */
923  if (i > MAX_IP4_CHARS) {
924  return 0;
925  }
926 
927  /* Create null-terminated string */
928  memcpy(tempIp, urlhost, i);
929  tempIp[i] = '\0';
930 
931  if (!IPv4AddressStringIsValid(tempIp))
932  return 0;
933 
934  return inet_pton(AF_INET, tempIp, &(sa.sin_addr));
935 }
936 
937 /**
938  * \brief Function determines whether a host string is a numeric IP v6 address
939  *
940  * \param urlhost The host string
941  * \param len The host string length
942  *
943  * \retval 1 The host is a numeric IP
944  * \retval 0 The host is NOT a numeric IP
945  */
946 static int IsIpv6Host(const uint8_t *urlhost, uint32_t len)
947 {
948  struct in6_addr in6;
949  char tempIp[MAX_IP6_CHARS + 1];
950 
951  /* Cut off at '/' */
952  uint32_t i = 0;
953  for (i = 0; i < len && urlhost[i] != 0; i++) {
954  if (urlhost[i] == '/') {
955  break;
956  }
957  }
958 
959  /* Too many chars */
960  if (i > MAX_IP6_CHARS) {
961  return 0;
962  }
963 
964  /* Create null-terminated string */
965  memcpy(tempIp, urlhost, i);
966  tempIp[i] = '\0';
967 
968  if (!IPv6AddressStringIsValid(tempIp))
969  return 0;
970 
971  return inet_pton(AF_INET6, tempIp, &in6);
972 }
973 
974 /**
975  * \brief Traverses through the list of URLs for an exact match of the specified
976  * string
977  *
978  * \param entity The MIME entity
979  * \param url The matching URL string (lowercase)
980  * \param url_len The matching URL string length
981  *
982  * \return URL object or NULL if not found
983  */
984 static MimeDecUrl *FindExistingUrl(MimeDecEntity *entity, uint8_t *url, uint32_t url_len)
985 {
986  MimeDecUrl *curr = entity->url_list;
987 
988  while (curr != NULL) {
989  if (url_len == curr->url_len) {
990  /* search url and stored url are both in
991  * lowercase, so we can do an exact match */
992  if (SCMemcmp(curr->url, url, url_len) == 0) {
993  break;
994  }
995  }
996  curr = curr->next;
997  }
998 
999  return curr;
1000 }
1001 
1002 /**
1003  * \brief This function searches a text or html line for a URL string
1004  *
1005  * URLS are generally truncated to the 'host.domain' format because
1006  * some email messages contain dozens or even hundreds of URLs with
1007  * the same host, but with only small variations in path.
1008  *
1009  * The exception is that URLs with executable file extensions are stored
1010  * with the full path. They are stored in lowercase.
1011  *
1012  * Numeric IPs, malformed numeric IPs, and URLs pointing to executables are
1013  * also flagged as URLs of interest.
1014  *
1015  * \param line the line
1016  * \param len the line length
1017  * \param state The current parser state
1018  *
1019  * \return MIME_DEC_OK on success, otherwise < 0 on failure
1020  */
1021 static int FindUrlStrings(const uint8_t *line, uint32_t len,
1022  MimeDecParseState *state)
1023 {
1024  int ret = MIME_DEC_OK;
1025  MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
1026  uint8_t *fptr, *remptr, *tok = NULL, *tempUrl;
1027  uint32_t tokLen = 0, i, tempUrlLen;
1028  uint8_t urlStrLen = 0, flags = 0;
1029 
1030  remptr = (uint8_t *)line;
1031  do {
1032  SCLogDebug("Looking for URL String starting with: %s", URL_STR);
1033 
1034  /* Check for token definition */
1035  fptr = FindBuffer(remptr, len - (remptr - line), (uint8_t *)URL_STR, strlen(URL_STR));
1036  if (fptr != NULL) {
1037 
1038  urlStrLen = strlen(URL_STR);
1039  fptr += urlStrLen; /* Start at end of start string */
1040  tok = GetToken(fptr, len - (fptr - line), " \"\'<>]\t", &remptr,
1041  &tokLen);
1042  if (tok == fptr) {
1043  SCLogDebug("Found url string");
1044 
1045  /* First copy to temp URL string */
1046  tempUrl = SCMalloc(urlStrLen + tokLen);
1047  if (unlikely(tempUrl == NULL)) {
1048  SCLogError(SC_ERR_MEM_ALLOC, "Memory allocation failed");
1049  return MIME_DEC_ERR_MEM;
1050  }
1051 
1052  PrintChars(SC_LOG_DEBUG, "RAW URL", tok, tokLen);
1053 
1054  /* Copy over to temp URL while decoding */
1055  tempUrlLen = 0;
1056  for (i = 0; i < tokLen && tok[i] != 0; i++) {
1057 
1058  // URL decoding would probably go here
1059 
1060  /* url is all lowercase */
1061  tempUrl[tempUrlLen] = tolower(tok[i]);
1062  tempUrlLen++;
1063  }
1064 
1065  /* Determine if URL points to an EXE */
1066  if (IsExeUrl(tempUrl, tempUrlLen)) {
1067  flags |= URL_IS_EXE;
1068 
1069  PrintChars(SC_LOG_DEBUG, "EXE URL", tempUrl, tempUrlLen);
1070  } else {
1071  /* Not an EXE URL */
1072  /* Cut off length at first '/' */
1073  /* If seems that BAESystems had done the following
1074  in support of PEScan. We don't want it for logging.
1075  Therefore its been removed.
1076  tok = FindString(tempUrl, tempUrlLen, "/");
1077  if (tok != NULL) {
1078  tempUrlLen = tok - tempUrl;
1079  }
1080  */
1081  }
1082 
1083  /* Make sure remaining URL exists */
1084  if (tempUrlLen > 0) {
1085  if (!(FindExistingUrl(entity, tempUrl, tempUrlLen))) {
1086  /* Now look for numeric IP */
1087  if (IsIpv4Host(tempUrl, tempUrlLen)) {
1088  flags |= URL_IS_IP4;
1089 
1090  PrintChars(SC_LOG_DEBUG, "IP URL4", tempUrl, tempUrlLen);
1091  } else if (IsIpv6Host(tempUrl, tempUrlLen)) {
1092  flags |= URL_IS_IP6;
1093 
1094  PrintChars(SC_LOG_DEBUG, "IP URL6", tempUrl, tempUrlLen);
1095  }
1096 
1097  /* Add URL list item */
1098  MimeDecAddUrl(entity, tempUrl, tempUrlLen, flags);
1099  } else {
1100  SCFree(tempUrl);
1101  }
1102  } else {
1103  SCFree(tempUrl);
1104  }
1105  }
1106  }
1107  } while (fptr != NULL);
1108 
1109  return ret;
1110 }
1111 
1112 /**
1113  * \brief This function is a pre-processor for handling decoded data chunks that
1114  * then invokes the caller's callback function for further processing
1115  *
1116  * \param chunk The decoded chunk
1117  * \param len The decoded chunk length (varies)
1118  * \param state The current parser state
1119  *
1120  * \return MIME_DEC_OK on success, otherwise < 0 on failure
1121  */
1122 static int ProcessDecodedDataChunk(const uint8_t *chunk, uint32_t len,
1123  MimeDecParseState *state)
1124 {
1125  int ret = MIME_DEC_OK;
1126  uint8_t *remainPtr, *tok;
1127  uint32_t tokLen;
1128 
1129  if ((state->stack != NULL) && (state->stack->top != NULL) &&
1130  (state->stack->top->data != NULL)) {
1131  MimeDecConfig *mdcfg = MimeDecGetConfig();
1132  if (mdcfg != NULL && mdcfg->extract_urls) {
1133  MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
1134  /* If plain text or html, then look for URLs */
1135  if (((entity->ctnt_flags & CTNT_IS_TEXT) ||
1136  (entity->ctnt_flags & CTNT_IS_MSG) ||
1137  (entity->ctnt_flags & CTNT_IS_HTML)) &&
1138  ((entity->ctnt_flags & CTNT_IS_ATTACHMENT) == 0)) {
1139 
1140  /* Remainder from previous line */
1141  if (state->linerem_len > 0) {
1142  // TODO
1143  } else {
1144  /* No remainder from previous line */
1145  /* Parse each line one by one */
1146  remainPtr = (uint8_t *)chunk;
1147  do {
1148  tok = GetLine(remainPtr, len - (remainPtr - (uint8_t *)chunk),
1149  &remainPtr, &tokLen);
1150  if (tok != remainPtr) {
1151  // DEBUG - ADDED
1152  /* If last token found without CR/LF delimiter, then save
1153  * and reconstruct with next chunk
1154  */
1155  if (tok + tokLen - (uint8_t *) chunk == (int)len) {
1156  PrintChars(SC_LOG_DEBUG, "LAST CHUNK LINE - CUTOFF",
1157  tok, tokLen);
1158  SCLogDebug("\nCHUNK CUTOFF CHARS: %u delim %u\n",
1159  tokLen, len - (uint32_t)(tok + tokLen - (uint8_t *) chunk));
1160  } else {
1161  /* Search line for URL */
1162  ret = FindUrlStrings(tok, tokLen, state);
1163  if (ret != MIME_DEC_OK) {
1164  SCLogDebug("Error: FindUrlStrings() function"
1165  " failed: %d", ret);
1166  break;
1167  }
1168  }
1169  }
1170  } while (tok != remainPtr && remainPtr - (uint8_t *) chunk < (int)len);
1171  }
1172  }
1173  }
1174 
1175  /* Now invoke callback */
1176  if (state->DataChunkProcessorFunc != NULL) {
1177  ret = state->DataChunkProcessorFunc(chunk, len, state);
1178  if (ret != MIME_DEC_OK) {
1179  SCLogDebug("Error: state->dataChunkProcessor() callback function"
1180  " failed");
1181  }
1182  }
1183  } else {
1184  SCLogDebug("Error: Stack pointer missing");
1185  ret = MIME_DEC_ERR_DATA;
1186  }
1187 
1188  /* Reset data chunk buffer */
1189  state->data_chunk_len = 0;
1190 
1191  /* Mark body / file as no longer at beginning */
1192  state->body_begin = 0;
1193 
1194  return ret;
1195 }
1196 
1197 /**
1198  * \brief Processes a remainder (line % 4 = remainder) from the previous line
1199  * such that all base64 decoding attempts are divisible by 4
1200  *
1201  * \param buf The current line
1202  * \param len The length of the line
1203  * \param state The current parser state
1204  * \param force Flag indicating whether decoding should always occur
1205  *
1206  * \return Number of bytes pulled from the current buffer
1207  */
1208 static uint8_t ProcessBase64Remainder(const uint8_t *buf, uint32_t len,
1209  MimeDecParseState *state, int force)
1210 {
1211  uint32_t ret;
1212  uint8_t remainder = 0, remdec = 0;
1213 
1214  SCLogDebug("Base64 line remainder found: %u", state->bvr_len);
1215 
1216  /* Fill in block with first few bytes of current line */
1217  remainder = B64_BLOCK - state->bvr_len;
1218  remainder = remainder < len ? remainder : len;
1219  if (remainder && buf) {
1220  memcpy(state->bvremain + state->bvr_len, buf, remainder);
1221  }
1222  state->bvr_len += remainder;
1223 
1224  /* If data chunk buffer will be full, then clear it now */
1225  if (DATA_CHUNK_SIZE - state->data_chunk_len < ASCII_BLOCK) {
1226 
1227  /* Invoke pre-processor and callback */
1228  ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len,
1229  state);
1230  if (ret != MIME_DEC_OK) {
1231  SCLogDebug("Error: ProcessDecodedDataChunk() function failed");
1232  }
1233  }
1234 
1235  /* Only decode if divisible by 4 */
1236  if (state->bvr_len == B64_BLOCK || force) {
1237  remdec = DecodeBase64(state->data_chunk + state->data_chunk_len,
1238  state->bvremain, state->bvr_len, 1);
1239  if (remdec > 0) {
1240 
1241  /* Track decoded length */
1242  state->stack->top->data->decoded_body_len += remdec;
1243 
1244  /* Update length */
1245  state->data_chunk_len += remdec;
1246 
1247  /* If data chunk buffer is now full, then clear */
1248  if (DATA_CHUNK_SIZE - state->data_chunk_len < ASCII_BLOCK) {
1249 
1250  /* Invoke pre-processor and callback */
1251  ret = ProcessDecodedDataChunk(state->data_chunk,
1252  state->data_chunk_len, state);
1253  if (ret != MIME_DEC_OK) {
1254  SCLogDebug("Error: ProcessDecodedDataChunk() function "
1255  "failed");
1256  }
1257  }
1258  } else {
1259  /* Track failed base64 */
1262  SCLogDebug("Error: DecodeBase64() function failed");
1263  PrintChars(SC_LOG_DEBUG, "Base64 failed string", state->bvremain, state->bvr_len);
1264  }
1265 
1266  /* Reset remaining */
1267  state->bvr_len = 0;
1268  }
1269 
1270  return remainder;
1271 }
1272 
1273 /**
1274  * \brief Processes a body line by base64-decoding and passing to the data chunk
1275  * processing callback function when the buffer is read
1276  *
1277  * \param buf The current line
1278  * \param len The length of the line
1279  * \param state The current parser state
1280  *
1281  * \return MIME_DEC_OK on success, otherwise < 0 on failure
1282  */
1283 static int ProcessBase64BodyLine(const uint8_t *buf, uint32_t len,
1284  MimeDecParseState *state)
1285 {
1286  int ret = MIME_DEC_OK;
1287  uint8_t rem1 = 0, rem2 = 0;
1288  uint32_t numDecoded, remaining, offset, avail, tobuf;
1289 
1290  /* Track long line */
1291  if (len > MAX_ENC_LINE_LEN) {
1294  SCLogDebug("Error: Max encoded input line length exceeded %u > %u",
1295  len, MAX_ENC_LINE_LEN);
1296  }
1297 
1298  /* First process remaining from previous line */
1299  if (state->bvr_len > 0) {
1300 
1301  SCLogDebug("Base64 line remainder found: %u", state->bvr_len);
1302 
1303  /* Process remainder and return number of bytes pulled from current buffer */
1304  rem1 = ProcessBase64Remainder(buf, len, state, 0);
1305  }
1306 
1307  /* No error and at least some more data needs to be decoded */
1308  if ((int) (len - rem1) > 0) {
1309 
1310  /* Determine whether we need to save a remainder if not divisible by 4 */
1311  rem2 = (len - rem1) % B64_BLOCK;
1312  if (rem2 > 0) {
1313 
1314  SCLogDebug("Base64 saving remainder: %u", rem2);
1315 
1316  memcpy(state->bvremain, buf + (len - rem2), rem2);
1317  state->bvr_len = rem2;
1318  }
1319 
1320  /* Process remaining in loop in case buffer fills up */
1321  remaining = len - rem1 - rem2;
1322  offset = rem1;
1323  while (remaining > 0) {
1324 
1325  /* Determine amount to add to buffer */
1326  avail = (DATA_CHUNK_SIZE - state->data_chunk_len) * B64_BLOCK / ASCII_BLOCK;
1327  tobuf = avail > remaining ? remaining : avail;
1328  while (tobuf % 4 != 0) {
1329  tobuf--;
1330  }
1331 
1332  if (tobuf < B64_BLOCK) {
1333  SCLogDebug("Error: Invalid state for decoding base-64 block");
1334  return MIME_DEC_ERR_PARSE;
1335  }
1336 
1337  SCLogDebug("Decoding: %u", len - rem1 - rem2);
1338 
1339  numDecoded = DecodeBase64(state->data_chunk + state->data_chunk_len,
1340  buf + offset, tobuf, 1);
1341  if (numDecoded > 0) {
1342 
1343  /* Track decoded length */
1344  state->stack->top->data->decoded_body_len += numDecoded;
1345 
1346  /* Update length */
1347  state->data_chunk_len += numDecoded;
1348 
1349  if ((int) (DATA_CHUNK_SIZE - state->data_chunk_len) < 0) {
1350  SCLogDebug("Error: Invalid Chunk length: %u",
1351  state->data_chunk_len);
1352  ret = MIME_DEC_ERR_PARSE;
1353  break;
1354  }
1355 
1356  /* If buffer full, then invoke callback */
1357  if (DATA_CHUNK_SIZE - state->data_chunk_len < ASCII_BLOCK) {
1358 
1359  /* Invoke pre-processor and callback */
1360  ret = ProcessDecodedDataChunk(state->data_chunk,
1361  state->data_chunk_len, state);
1362  if (ret != MIME_DEC_OK) {
1363  SCLogDebug("Error: ProcessDecodedDataChunk() "
1364  "function failed");
1365  }
1366  }
1367  } else {
1368  /* Track failed base64 */
1371  SCLogDebug("Error: DecodeBase64() function failed");
1372  PrintChars(SC_LOG_DEBUG, "Base64 failed string", buf + offset, tobuf);
1373  }
1374 
1375  /* Update counts */
1376  remaining -= tobuf;
1377  offset += tobuf;
1378  }
1379  }
1380 
1381  return ret;
1382 }
1383 
1384 /**
1385  * \brief Decoded a hex character into its equivalent byte value for
1386  * quoted-printable decoding
1387  *
1388  * \param h The hex char
1389  *
1390  * \return byte value on success, -1 if failed
1391  **/
1392 static int16_t DecodeQPChar(char h)
1393 {
1394  uint16_t res = 0;
1395 
1396  /* 0-9 */
1397  if (h >= 48 && h <= 57) {
1398  res = h - 48;
1399  } else if (h >= 65 && h <= 70) {
1400  /* A-F */
1401  res = h - 55;
1402  } else {
1403  /* Invalid */
1404  res = -1;
1405  }
1406 
1407  return res;
1408 
1409 }
1410 
1411 /**
1412  * \brief Processes a quoted-printable encoded body line by decoding and passing
1413  * to the data chunk processing callback function when the buffer is read
1414  *
1415  * \param buf The current line
1416  * \param len The length of the line
1417  * \param state The current parser state
1418  *
1419  * \return MIME_DEC_OK on success, otherwise < 0 on failure
1420  */
1421 static int ProcessQuotedPrintableBodyLine(const uint8_t *buf, uint32_t len,
1422  MimeDecParseState *state)
1423 {
1424  int ret = MIME_DEC_OK;
1425  uint32_t remaining, offset;
1426  MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
1427  uint8_t c, h1, h2, val;
1428  int16_t res;
1429 
1430  /* Track long line */
1431  if (len > MAX_ENC_LINE_LEN) {
1434  SCLogDebug("Error: Max encoded input line length exceeded %u > %u",
1435  len, MAX_ENC_LINE_LEN);
1436  }
1437 
1438  remaining = len;
1439  offset = 0;
1440  while (remaining > 0) {
1441 
1442  c = *(buf + offset);
1443 
1444  /* Copy over normal character */
1445  if (c != '=') {
1446  state->data_chunk[state->data_chunk_len] = c;
1447  state->data_chunk_len++;
1448  entity->decoded_body_len += 1;
1449 
1450  /* Add CRLF sequence if end of line */
1451  if (remaining == 1) {
1452  memcpy(state->data_chunk + state->data_chunk_len, CRLF, EOL_LEN);
1453  state->data_chunk_len += EOL_LEN;
1454  entity->decoded_body_len += EOL_LEN;
1455  }
1456  } else if (remaining > 1) {
1457  /* If last character handle as soft line break by ignoring,
1458  otherwise process as escaped '=' character */
1459 
1460  /* Not enough characters */
1461  if (remaining < 3) {
1462  entity->anomaly_flags |= ANOM_INVALID_QP;
1463  state->msg->anomaly_flags |= ANOM_INVALID_QP;
1464  SCLogDebug("Error: Quoted-printable decoding failed");
1465  } else {
1466  h1 = *(buf + offset + 1);
1467  res = DecodeQPChar(h1);
1468  if (res < 0) {
1469  entity->anomaly_flags |= ANOM_INVALID_QP;
1470  state->msg->anomaly_flags |= ANOM_INVALID_QP;
1471  SCLogDebug("Error: Quoted-printable decoding failed");
1472  } else {
1473  val = (res << 4); /* Shift result left */
1474  h2 = *(buf + offset + 2);
1475  res = DecodeQPChar(h2);
1476  if (res < 0) {
1477  entity->anomaly_flags |= ANOM_INVALID_QP;
1478  state->msg->anomaly_flags |= ANOM_INVALID_QP;
1479  SCLogDebug("Error: Quoted-printable decoding failed");
1480  } else {
1481  /* Decoding sequence succeeded */
1482  val += res;
1483 
1484  state->data_chunk[state->data_chunk_len] = val;
1485  state->data_chunk_len++;
1486  entity->decoded_body_len++;
1487 
1488  /* Add CRLF sequence if end of line */
1489  if (remaining == 3) {
1490  memcpy(state->data_chunk + state->data_chunk_len,
1491  CRLF, EOL_LEN);
1492  state->data_chunk_len += EOL_LEN;
1493  entity->decoded_body_len += EOL_LEN;
1494  }
1495 
1496  /* Account for extra 2 characters in 3-characted QP
1497  * sequence */
1498  remaining -= 2;
1499  offset += 2;
1500  }
1501  }
1502  }
1503  }
1504 
1505  /* Change by 1 */
1506  remaining--;
1507  offset++;
1508 
1509  /* If buffer full, then invoke callback */
1510  if (DATA_CHUNK_SIZE - state->data_chunk_len < EOL_LEN + 1) {
1511 
1512  /* Invoke pre-processor and callback */
1513  ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len,
1514  state);
1515  if (ret != MIME_DEC_OK) {
1516  SCLogDebug("Error: ProcessDecodedDataChunk() function "
1517  "failed");
1518  }
1519  }
1520  }
1521 
1522  return ret;
1523 }
1524 
1525 /**
1526  * \brief Processes a body line by base64-decoding (if applicable) and passing to
1527  * the data chunk processing callback function
1528  *
1529  * \param buf The current line
1530  * \param len The length of the line
1531  * \param state The current parser state
1532  *
1533  * \return MIME_DEC_OK on success, otherwise < 0 on failure
1534  */
1535 static int ProcessBodyLine(const uint8_t *buf, uint32_t len,
1536  MimeDecParseState *state)
1537 {
1538  int ret = MIME_DEC_OK;
1539  uint32_t remaining, offset, avail, tobuf;
1540  MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
1541 
1542  SCLogDebug("Processing body line");
1543 
1544  /* Track length */
1545  entity->body_len += len + 2; /* With CRLF */
1546 
1547  /* Process base-64 content if enabled */
1548  MimeDecConfig *mdcfg = MimeDecGetConfig();
1549  if (mdcfg != NULL && mdcfg->decode_base64 &&
1550  (entity->ctnt_flags & CTNT_IS_BASE64)) {
1551 
1552  ret = ProcessBase64BodyLine(buf, len, state);
1553  if (ret != MIME_DEC_OK) {
1554  SCLogDebug("Error: ProcessBase64BodyLine() function failed");
1555  }
1556  } else if (mdcfg != NULL && mdcfg->decode_quoted_printable &&
1557  (entity->ctnt_flags & CTNT_IS_QP)) {
1558  /* Process quoted-printable content if enabled */
1559  ret = ProcessQuotedPrintableBodyLine(buf, len, state);
1560  if (ret != MIME_DEC_OK) {
1561  SCLogDebug("Error: ProcessQuotedPrintableBodyLine() function "
1562  "failed");
1563  }
1564  } else {
1565  /* Process non-decoded content */
1566  remaining = len;
1567  offset = 0;
1568  while (remaining > 0) {
1569 
1570  /* Plan to add CRLF to the end of each line */
1571  avail = DATA_CHUNK_SIZE - state->data_chunk_len;
1572  tobuf = avail > remaining + EOL_LEN ? remaining : avail - EOL_LEN;
1573 
1574  /* Copy over to buffer */
1575  memcpy(state->data_chunk + state->data_chunk_len, buf + offset, tobuf);
1576  state->data_chunk_len += tobuf;
1577 
1578  /* Now always add a CRLF to the end */
1579  if (tobuf == remaining) {
1580  memcpy(state->data_chunk + state->data_chunk_len, CRLF, EOL_LEN);
1581  state->data_chunk_len += EOL_LEN;
1582  }
1583 
1584  if ((int) (DATA_CHUNK_SIZE - state->data_chunk_len) < 0) {
1585  SCLogDebug("Error: Invalid Chunk length: %u",
1586  state->data_chunk_len);
1587  ret = MIME_DEC_ERR_PARSE;
1588  break;
1589  }
1590 
1591  /* If buffer full, then invoke callback */
1592  if (DATA_CHUNK_SIZE - state->data_chunk_len < EOL_LEN + 1) {
1593 
1594  /* Invoke pre-processor and callback */
1595  ret = ProcessDecodedDataChunk(state->data_chunk,
1596  state->data_chunk_len, state);
1597  if (ret != MIME_DEC_OK) {
1598  SCLogDebug("Error: ProcessDecodedDataChunk() function "
1599  "failed");
1600  }
1601  }
1602 
1603  remaining -= tobuf;
1604  offset += tobuf;
1605  }
1606  }
1607 
1608  return ret;
1609 }
1610 
1611 /**
1612  * \brief Find the start of a header name on the current line
1613  *
1614  * \param buf The input line (not null-terminated)
1615  * \param blen The length of the input line
1616  * \param glen The output length of the header name
1617  *
1618  * \return Pointer to header name, or NULL if not found
1619  */
1620 static uint8_t * FindMimeHeaderStart(const uint8_t *buf, uint32_t blen, uint32_t *hlen)
1621 {
1622  uint32_t i, valid = 0;
1623  uint8_t *hname = NULL;
1624 
1625  /* Init */
1626  *hlen = 0;
1627 
1628  /* Look for sequence of printable characters followed by ':', or
1629  CRLF then printable characters followed by ':' */
1630  for (i = 0; i < blen && buf[i] != 0; i++) {
1631 
1632  /* If ready for printable characters and found one, then increment */
1633  if (buf[i] != COLON && buf[i] >= PRINTABLE_START &&
1634  buf[i] <= PRINTABLE_END) {
1635  valid++;
1636  } else if (valid > 0 && buf[i] == COLON) {
1637  /* If ready for printable characters, found some, and found colon
1638  * delimiter, then a match is found */
1639  hname = (uint8_t *) buf + i - valid;
1640  *hlen = valid;
1641  break;
1642  } else {
1643  /* Otherwise reset and quit */
1644  break;
1645  }
1646  }
1647 
1648  return hname;
1649 }
1650 
1651 /**
1652  * \brief Find full header name and value on the current line based on the
1653  * current state
1654  *
1655  * \param buf The current line (no CRLF)
1656  * \param blen The length of the current line
1657  * \param state The current state
1658  *
1659  * \return MIME_DEC_OK on success, otherwise < 0 on failure
1660  */
1661 static int FindMimeHeader(const uint8_t *buf, uint32_t blen,
1662  MimeDecParseState *state)
1663 {
1664  int ret = MIME_DEC_OK;
1665  uint8_t *hname, *hval = NULL;
1666  DataValue *dv;
1667  uint32_t hlen, vlen;
1668  int finish_header = 0, new_header = 0;
1669  MimeDecConfig *mdcfg = MimeDecGetConfig();
1670 
1671  /* Find first header */
1672  hname = FindMimeHeaderStart(buf, blen, &hlen);
1673  if (hname != NULL) {
1674 
1675  /* Warn and track but don't do anything yet */
1676  if (hlen > MAX_HEADER_NAME) {
1679  SCLogDebug("Error: Header name exceeds limit (%u > %u)",
1680  hlen, MAX_HEADER_NAME);
1681  }
1682 
1683  /* Value starts after 'header:' (normalize spaces) */
1684  hval = hname + hlen + 1;
1685  if (hval - buf >= (int)blen) {
1686  SCLogDebug("No Header value found");
1687  hval = NULL;
1688  } else {
1689  while (hval[0] == ' ') {
1690 
1691  /* If last character before end of bounds, set to NULL */
1692  if (hval - buf >= (int)blen - 1) {
1693  SCLogDebug("No Header value found");
1694  hval = NULL;
1695  break;
1696  }
1697 
1698  hval++;
1699  }
1700  }
1701 
1702  /* If new header found, then previous header is finished */
1703  if (state->state_flag == HEADER_STARTED) {
1704  finish_header = 1;
1705  }
1706 
1707  /* Now process new header */
1708  new_header = 1;
1709 
1710  /* Must wait for next line to determine if finished */
1711  state->state_flag = HEADER_STARTED;
1712  } else if (blen == 0) {
1713  /* Found body */
1714  /* No more headers */
1715  state->state_flag = HEADER_DONE;
1716 
1717  finish_header = 1;
1718 
1719  SCLogDebug("All Header processing finished");
1720  } else if (state->state_flag == HEADER_STARTED) {
1721  /* Found multi-line value (ie. Received header) */
1722  /* If max header value exceeded, flag it */
1723  vlen = blen;
1724  if ((mdcfg != NULL) && (state->hvlen + vlen > mdcfg->header_value_depth)) {
1725  SCLogDebug("Error: Header value of length (%u) is too long",
1726  state->hvlen + vlen);
1727  vlen = mdcfg->header_value_depth - state->hvlen;
1730  }
1731  if (vlen > 0) {
1732  dv = AddDataValue(state->hvalue);
1733  if (dv == NULL) {
1734  SCLogError(SC_ERR_MEM_ALLOC, "AddDataValue() function failed");
1735  return MIME_DEC_ERR_MEM;
1736  }
1737  if (state->hvalue == NULL) {
1738  state->hvalue = dv;
1739  }
1740 
1741  dv->value = SCMalloc(vlen);
1742  if (unlikely(dv->value == NULL)) {
1743  SCLogError(SC_ERR_MEM_ALLOC, "Memory allocation failed");
1744  return MIME_DEC_ERR_MEM;
1745  }
1746  memcpy(dv->value, buf, vlen);
1747  dv->value_len = vlen;
1748  state->hvlen += vlen;
1749  }
1750  } else {
1751  /* Likely a body without headers */
1752  SCLogDebug("No headers found");
1753 
1754  state->state_flag = BODY_STARTED;
1755 
1756  /* Flag beginning of body */
1757  state->body_begin = 1;
1758  state->body_end = 0;
1759 
1760  ret = ProcessBodyLine(buf, blen, state);
1761  if (ret != MIME_DEC_OK) {
1762  SCLogDebug("Error: ProcessBodyLine() function failed");
1763  return ret;
1764  }
1765  }
1766 
1767  /* If we need to finish a header, then do so below and then cleanup */
1768  if (finish_header) {
1769  /* Store the header value */
1770  ret = StoreMimeHeader(state);
1771  if (ret != MIME_DEC_OK) {
1772  SCLogDebug("Error: StoreMimeHeader() function failed");
1773  return ret;
1774  }
1775  }
1776 
1777  /* When next header is found, we always create a new one */
1778  if (new_header) {
1779  /* Copy name and value to state */
1780  state->hname = SCMalloc(hlen);
1781  if (unlikely(state->hname == NULL)) {
1782  SCLogError(SC_ERR_MEM_ALLOC, "Memory allocation failed");
1783  return MIME_DEC_ERR_MEM;
1784  }
1785  memcpy(state->hname, hname, hlen);
1786  state->hlen = hlen;
1787 
1788  if (state->hvalue != NULL) {
1789  SCLogDebug("Error: Parser failed due to unexpected header "
1790  "value");
1791  return MIME_DEC_ERR_DATA;
1792  }
1793 
1794  if (hval != NULL) {
1795  /* If max header value exceeded, flag it */
1796  vlen = blen - (hval - buf);
1797  if ((mdcfg != NULL) && (state->hvlen + vlen > mdcfg->header_value_depth)) {
1798  SCLogDebug("Error: Header value of length (%u) is too long",
1799  state->hvlen + vlen);
1800  vlen = mdcfg->header_value_depth - state->hvlen;
1803  }
1804 
1805  if (vlen > 0) {
1806  state->hvalue = AddDataValue(NULL);
1807  if (state->hvalue == NULL) {
1808  SCLogError(SC_ERR_MEM_ALLOC, "AddDataValue() function failed");
1809  return MIME_DEC_ERR_MEM;
1810  }
1811  state->hvalue->value = SCMalloc(vlen);
1812  if (unlikely(state->hvalue->value == NULL)) {
1813  SCLogError(SC_ERR_MEM_ALLOC, "Memory allocation failed");
1814  return MIME_DEC_ERR_MEM;
1815  }
1816  memcpy(state->hvalue->value, hval, vlen);
1817  state->hvalue->value_len = vlen;
1818  state->hvlen += vlen;
1819  }
1820  }
1821  }
1822 
1823  return ret;
1824 }
1825 
1826 /**
1827  * \brief Finds a mime header token within the specified field
1828  *
1829  * \param field The current field
1830  * \param search_start The start of the search (ie. boundary=\")
1831  * \param search_end The end of the search (ie. \")
1832  * \param tlen The output length of the token (if found)
1833  *
1834  * \return A pointer to the token if found, otherwise NULL if not found
1835  */
1836 static uint8_t * FindMimeHeaderToken(MimeDecField *field, const char *search_start,
1837  const char *search_end, uint32_t *tlen)
1838 {
1839  uint8_t *fptr, *tptr = NULL, *tok = NULL;
1840 
1841  SCLogDebug("Looking for token: %s", search_start);
1842 
1843  /* Check for token definition */
1844  fptr = FindBuffer(field->value, field->value_len, (const uint8_t *)search_start, strlen(search_start));
1845  if (fptr != NULL) {
1846  fptr += strlen(search_start); /* Start at end of start string */
1847  tok = GetToken(fptr, field->value_len - (fptr - field->value), search_end,
1848  &tptr, tlen);
1849  if (tok != NULL) {
1850  SCLogDebug("Found mime token");
1851  }
1852  }
1853 
1854  return tok;
1855 }
1856 
1857 /**
1858  * \brief Processes the current line for mime headers and also does post-processing
1859  * when all headers found
1860  *
1861  * \param buf The current line
1862  * \param len The length of the line
1863  * \param state The current parser state
1864  *
1865  * \return MIME_DEC_OK on success, otherwise < 0 on failure
1866  */
1867 static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len,
1868  MimeDecParseState *state)
1869 {
1870  int ret = MIME_DEC_OK;
1871  MimeDecField *field;
1872  uint8_t *bptr = NULL, *rptr = NULL;
1873  uint32_t blen = 0;
1874  MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
1875 
1876  /* Look for mime header in current line */
1877  ret = FindMimeHeader(buf, len, state);
1878  if (ret != MIME_DEC_OK) {
1879  SCLogDebug("Error: FindMimeHeader() function failed: %d", ret);
1880  return ret;
1881  }
1882 
1883  /* Post-processing after all headers done */
1884  if (state->state_flag == HEADER_DONE) {
1885  /* First determine encoding by looking at Content-Transfer-Encoding */
1886  field = MimeDecFindField(entity, CTNT_TRAN_STR);
1887  if (field != NULL) {
1888  /* Look for base64 */
1889  if (FindBuffer(field->value, field->value_len, (const uint8_t *)BASE64_STR, strlen(BASE64_STR))) {
1890  SCLogDebug("Base64 encoding found");
1891  entity->ctnt_flags |= CTNT_IS_BASE64;
1892  } else if (FindBuffer(field->value, field->value_len, (const uint8_t *)QP_STR, strlen(QP_STR))) {
1893  /* Look for quoted-printable */
1894  SCLogDebug("quoted-printable encoding found");
1895  entity->ctnt_flags |= CTNT_IS_QP;
1896  }
1897  }
1898 
1899  /* Check for file attachment in content disposition */
1900  field = MimeDecFindField(entity, CTNT_DISP_STR);
1901  if (field != NULL) {
1902  bptr = FindMimeHeaderToken(field, "filename=", TOK_END_STR, &blen);
1903  if (bptr != NULL) {
1904  SCLogDebug("File attachment found in disposition");
1905  entity->ctnt_flags |= CTNT_IS_ATTACHMENT;
1906 
1907  /* Copy over using dynamic memory */
1908  entity->filename = SCMalloc(blen);
1909  if (unlikely(entity->filename == NULL)) {
1910  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
1911  return MIME_DEC_ERR_MEM;
1912  }
1913  memcpy(entity->filename, bptr, blen);
1914  entity->filename_len = blen;
1915  }
1916  }
1917 
1918  /* Check for boundary, encapsulated message, and file name in Content-Type */
1919  field = MimeDecFindField(entity, CTNT_TYPE_STR);
1920  if (field != NULL) {
1921  /* Check if child entity boundary definition found */
1922  bptr = FindMimeHeaderToken(field, BND_START_STR, TOK_END_STR, &blen);
1923  if (bptr != NULL) {
1924  state->found_child = 1;
1925  entity->ctnt_flags |= CTNT_IS_MULTIPART;
1926 
1927  if (blen > (BOUNDARY_BUF - 2)) {
1929  return MIME_DEC_ERR_PARSE;
1930  }
1931 
1932  /* Store boundary in parent node */
1933  state->stack->top->bdef = SCMalloc(blen);
1934  if (unlikely(state->stack->top->bdef == NULL)) {
1935  SCLogError(SC_ERR_MEM_ALLOC, "Memory allocation failed");
1936  return MIME_DEC_ERR_MEM;
1937  }
1938  memcpy(state->stack->top->bdef, bptr, blen);
1939  state->stack->top->bdef_len = blen;
1940  }
1941 
1942  /* Look for file name (if not already found) */
1943  if (!(entity->ctnt_flags & CTNT_IS_ATTACHMENT)) {
1944  bptr = FindMimeHeaderToken(field, "name=", TOK_END_STR, &blen);
1945  if (bptr != NULL) {
1946  SCLogDebug("File attachment found");
1947  entity->ctnt_flags |= CTNT_IS_ATTACHMENT;
1948 
1949  /* Copy over using dynamic memory */
1950  entity->filename = SCMalloc(blen);
1951  if (unlikely(entity->filename == NULL)) {
1952  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
1953  return MIME_DEC_ERR_MEM;
1954  }
1955  memcpy(entity->filename, bptr, blen);
1956  entity->filename_len = blen;
1957  }
1958  }
1959 
1960  /* Pull out short-hand content type */
1961  entity->ctnt_type = GetToken(field->value, field->value_len, " \r\n;",
1962  &rptr, &entity->ctnt_type_len);
1963  if (entity->ctnt_type != NULL) {
1964  /* Check for encapsulated message */
1965  if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len,
1966  (const uint8_t *)MSG_STR, strlen(MSG_STR)))
1967  {
1968  SCLogDebug("Found encapsulated message entity");
1969 
1970  entity->ctnt_flags |= CTNT_IS_ENV;
1971 
1972  /* Create and push child to stack */
1973  MimeDecEntity *child = MimeDecAddEntity(entity);
1974  if (child == NULL)
1975  return MIME_DEC_ERR_MEM;
1976  child->ctnt_flags |= (CTNT_IS_ENCAP | CTNT_IS_MSG);
1977  PushStack(state->stack);
1978  state->stack->top->data = child;
1979 
1980  /* Mark as encapsulated child */
1981  state->stack->top->is_encap = 1;
1982 
1983  /* Ready to parse headers */
1984  state->state_flag = HEADER_READY;
1985  } else if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len,
1986  (const uint8_t *)MULTIPART_STR, strlen(MULTIPART_STR)))
1987  {
1988  /* Check for multipart */
1989  SCLogDebug("Found multipart entity");
1990  entity->ctnt_flags |= CTNT_IS_MULTIPART;
1991  } else if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len,
1992  (const uint8_t *)TXT_STR, strlen(TXT_STR)))
1993  {
1994  /* Check for plain text */
1995  SCLogDebug("Found plain text entity");
1996  entity->ctnt_flags |= CTNT_IS_TEXT;
1997  } else if (FindBuffer(entity->ctnt_type, entity->ctnt_type_len,
1998  (const uint8_t *)HTML_STR, strlen(HTML_STR)))
1999  {
2000  /* Check for html */
2001  SCLogDebug("Found html entity");
2002  entity->ctnt_flags |= CTNT_IS_HTML;
2003  }
2004  }
2005  }
2006 
2007  /* Store pointer to Message-ID */
2008  field = MimeDecFindField(entity, MSG_ID_STR);
2009  if (field != NULL) {
2010  entity->msg_id = field->value;
2011  entity->msg_id_len = field->value_len;
2012  }
2013 
2014  /* Flag beginning of body */
2015  state->body_begin = 1;
2016  state->body_end = 0;
2017  }
2018 
2019  return ret;
2020 }
2021 
2022 /**
2023  * \brief Indicates to the parser that the body of an entity has completed
2024  * processing on the previous line
2025  *
2026  * \param state The current parser state
2027  *
2028  * \return MIME_DEC_OK on success, otherwise < 0 on failure
2029  */
2030 
2031 static int ProcessBodyComplete(MimeDecParseState *state)
2032 {
2033  int ret = MIME_DEC_OK;
2034 
2035  SCLogDebug("Process body complete called");
2036 
2037  /* Mark the file as hitting the end */
2038  state->body_end = 1;
2039 
2040  if (state->bvr_len > 0) {
2041  SCLogDebug("Found (%u) remaining base64 bytes not processed",
2042  state->bvr_len);
2043 
2044  /* Process the remainder */
2045  ret = ProcessBase64Remainder(NULL, 0, state, 1);
2046  if (ret != MIME_DEC_OK) {
2047  SCLogDebug("Error: ProcessBase64BodyLine() function failed");
2048  }
2049  }
2050 
2051 #ifdef HAVE_NSS
2052  if (state->md5_ctx) {
2053  unsigned int len = 0;
2054  HASH_End(state->md5_ctx, state->md5, &len, sizeof(state->md5));
2055  }
2056 #endif
2057 
2058  /* Invoke pre-processor and callback with remaining data */
2059  ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state);
2060  if (ret != MIME_DEC_OK) {
2061  SCLogDebug("Error: ProcessDecodedDataChunk() function failed");
2062  }
2063 
2064  /* Now reset */
2065  state->body_begin = 0;
2066  state->body_end = 0;
2067 
2068  return ret;
2069 }
2070 
2071 /**
2072  * \brief When a mime boundary is found, look for end boundary and also do stack
2073  * management
2074  *
2075  * \param buf The current line
2076  * \param len The length of the line
2077  * \param bdef_len The length of the current boundary
2078  *
2079  * \return MIME_DEC_OK on success, otherwise < 0 on failure
2080  */
2081 static int ProcessMimeBoundary(const uint8_t *buf, uint32_t len, uint32_t bdef_len,
2082  MimeDecParseState *state)
2083 {
2084  int ret = MIME_DEC_OK;
2085  uint8_t *rptr;
2086  MimeDecEntity *child;
2087 
2088  SCLogDebug("PROCESSING BOUNDARY - START: %d",
2089  state->state_flag);
2090 
2091  /* If previous line was not an end boundary, then we process the body as
2092  * completed */
2093  if (state->state_flag != BODY_END_BOUND) {
2094 
2095  /* First lets complete the body */
2096  ret = ProcessBodyComplete(state);
2097  if (ret != MIME_DEC_OK) {
2098  SCLogDebug("Error: ProcessBodyComplete() function failed");
2099  return ret;
2100  }
2101  } else {
2102  /* If last line was an end boundary, then now we are ready to parse
2103  * headers again */
2104  state->state_flag = HEADER_READY;
2105  }
2106 
2107  /* Update remaining buffer */
2108  rptr = (uint8_t *) buf + bdef_len + 2;
2109 
2110  /* If entity is encapsulated and current and parent didn't define the boundary,
2111  * then pop out */
2112  if (state->stack->top->is_encap && state->stack->top->bdef_len == 0) {
2113 
2114  if (state->stack->top->next == NULL) {
2115  SCLogDebug("Error: Missing parent entity from stack");
2116  return MIME_DEC_ERR_DATA;
2117  }
2118 
2119  if (state->stack->top->next->bdef_len == 0) {
2120 
2121  SCLogDebug("POPPED ENCAPSULATED CHILD FROM STACK: %p=%p",
2122  state->stack->top, state->stack->top->data);
2123 
2124  /* If end of boundary found, pop the child off the stack */
2125  PopStack(state->stack);
2126  if (state->stack->top == NULL) {
2127  SCLogDebug("Error: Message is malformed");
2128  return MIME_DEC_ERR_DATA;
2129  }
2130  }
2131  }
2132 
2133  /* Now check for end of nested boundary */
2134  if (len - (rptr - buf) > 1 && rptr[0] == DASH && rptr[1] == DASH) {
2135  SCLogDebug("FOUND END BOUNDARY, POPPING: %p=%p",
2136  state->stack->top, state->stack->top->data);
2137 
2138  /* If end of boundary found, pop the child off the stack */
2139  PopStack(state->stack);
2140  if (state->stack->top == NULL) {
2141  SCLogDebug("Error: Message is malformed");
2142  return MIME_DEC_ERR_DATA;
2143  }
2144 
2145  /* If current is an encapsulated message with a boundary definition,
2146  * then pop him as well */
2147  if (state->stack->top->is_encap && state->stack->top->bdef_len != 0) {
2148  SCLogDebug("FOUND END BOUNDARY AND ENCAP, POPPING: %p=%p",
2149  state->stack->top, state->stack->top->data);
2150 
2151  PopStack(state->stack);
2152  if (state->stack->top == NULL) {
2153  SCLogDebug("Error: Message is malformed");
2154  return MIME_DEC_ERR_DATA;
2155  }
2156  }
2157 
2158  state->state_flag = BODY_END_BOUND;
2159  } else if (state->found_child) {
2160  /* Otherwise process new child */
2161  SCLogDebug("Child entity created");
2162 
2163  /* Create and push child to stack */
2164  child = MimeDecAddEntity(state->stack->top->data);
2165  if (child == NULL)
2166  return MIME_DEC_ERR_MEM;
2167  child->ctnt_flags |= CTNT_IS_BODYPART;
2168  PushStack(state->stack);
2169  state->stack->top->data = child;
2170 
2171  /* Reset flag */
2172  state->found_child = 0;
2173  } else {
2174  /* Otherwise process sibling */
2175  if (state->stack->top->next == NULL) {
2176  SCLogDebug("Error: Missing parent entity from stack");
2177  return MIME_DEC_ERR_DATA;
2178  }
2179 
2180  SCLogDebug("SIBLING CREATED, POPPING PARENT: %p=%p",
2181  state->stack->top, state->stack->top->data);
2182 
2183  /* First pop current to get access to parent */
2184  PopStack(state->stack);
2185  if (state->stack->top == NULL) {
2186  SCLogDebug("Error: Message is malformed");
2187  return MIME_DEC_ERR_DATA;
2188  }
2189 
2190  /* Create and push child to stack */
2191  child = MimeDecAddEntity(state->stack->top->data);
2192  if (child == NULL)
2193  return MIME_DEC_ERR_MEM;
2194  child->ctnt_flags |= CTNT_IS_BODYPART;
2195  PushStack(state->stack);
2196  state->stack->top->data = child;
2197  }
2198 
2199  /* After boundary look for headers */
2200  if (state->state_flag != BODY_END_BOUND) {
2201  state->state_flag = HEADER_READY;
2202  }
2203 
2204  SCLogDebug("PROCESSING BOUNDARY - END: %d", state->state_flag);
2205  return ret;
2206 }
2207 
2208 /**
2209  * \brief Processes the MIME Entity body based on the input line and current
2210  * state of the parser
2211  *
2212  * \param buf The current line
2213  * \param len The length of the line
2214  *
2215  * \return MIME_DEC_OK on success, otherwise < 0 on failure
2216  */
2217 static int ProcessMimeBody(const uint8_t *buf, uint32_t len,
2218  MimeDecParseState *state)
2219 {
2220  int ret = MIME_DEC_OK;
2221  uint8_t temp[BOUNDARY_BUF];
2222  uint8_t *bstart;
2223  int body_found = 0;
2224  uint32_t tlen;
2225 
2226 #ifdef HAVE_NSS
2227  if (MimeDecGetConfig()->body_md5) {
2228  if (state->body_begin == 1) {
2229  if (state->md5_ctx == NULL) {
2230  state->md5_ctx = HASH_Create(HASH_AlgMD5);
2231  HASH_Begin(state->md5_ctx);
2232  }
2233  }
2234  HASH_Update(state->md5_ctx, buf, len + state->current_line_delimiter_len);
2235  }
2236 #endif
2237 
2238  /* Ignore empty lines */
2239  if (len == 0) {
2240  return ret;
2241  }
2242 
2243  /* First look for boundary */
2244  MimeDecStackNode *node = state->stack->top;
2245  if (node == NULL) {
2246  SCLogDebug("Error: Invalid stack state");
2247  return MIME_DEC_ERR_PARSE;
2248  }
2249 
2250  /* Traverse through stack to find a boundary definition */
2251  if (state->state_flag == BODY_END_BOUND || node->bdef == NULL) {
2252 
2253  /* If not found, then use parent's boundary */
2254  node = node->next;
2255  while (node != NULL && node->bdef == NULL) {
2256  SCLogDebug("Traversing through stack for node with boundary");
2257  node = node->next;
2258  }
2259  }
2260 
2261  /* This means no boundary / parent w/boundary was found so we are in the body */
2262  if (node == NULL) {
2263  body_found = 1;
2264  } else {
2265 
2266  /* Now look for start of boundary */
2267  if (len > 1 && buf[0] == '-' && buf[1] == '-') {
2268 
2269  tlen = node->bdef_len + 2;
2270  if (tlen > BOUNDARY_BUF) {
2271  if (state->stack->top->data)
2273  return MIME_DEC_ERR_PARSE;
2274  }
2275 
2276  memcpy(temp, "--", 2);
2277  memcpy(temp + 2, node->bdef, node->bdef_len);
2278 
2279  /* Find either next boundary or end boundary */
2280  bstart = FindBuffer((const uint8_t *)buf, len, temp, tlen);
2281  if (bstart != NULL) {
2282  ret = ProcessMimeBoundary(buf, len, node->bdef_len, state);
2283  if (ret != MIME_DEC_OK) {
2284  SCLogDebug("Error: ProcessMimeBoundary() function "
2285  "failed");
2286  return ret;
2287  }
2288  } else {
2289  /* Otherwise add value to body */
2290  body_found = 1;
2291  }
2292  } else {
2293  /* Otherwise add value to body */
2294  body_found = 1;
2295  }
2296  }
2297 
2298  /* Process body line */
2299  if (body_found) {
2300  state->state_flag = BODY_STARTED;
2301 
2302  ret = ProcessBodyLine(buf, len, state);
2303  if (ret != MIME_DEC_OK) {
2304  SCLogDebug("Error: ProcessBodyLine() function failed");
2305  return ret;
2306  }
2307  }
2308 
2309  return ret;
2310 }
2311 
2313 {
2314  return StateFlags[state->state_flag];
2315 }
2316 
2317 /**
2318  * \brief Processes the MIME Entity based on the input line and current state of
2319  * the parser
2320  *
2321  * \param buf The current line
2322  * \param len The length of the line
2323  *
2324  * \return MIME_DEC_OK on success, otherwise < 0 on failure
2325  */
2326 static int ProcessMimeEntity(const uint8_t *buf, uint32_t len,
2327  MimeDecParseState *state)
2328 {
2329  int ret = MIME_DEC_OK;
2330 
2331  SCLogDebug("START FLAG: %s", StateFlags[state->state_flag]);
2332 
2333  if (state->state_flag == PARSE_ERROR) {
2334  SCLogDebug("START FLAG: PARSE_ERROR, bail");
2335  return MIME_DEC_ERR_STATE;
2336  }
2337 
2338  /* Track long line */
2339  if (len > MAX_LINE_LEN) {
2340  state->stack->top->data->anomaly_flags |= ANOM_LONG_LINE;
2341  state->msg->anomaly_flags |= ANOM_LONG_LINE;
2342  SCLogDebug("Error: Max input line length exceeded %u > %u", len,
2343  MAX_LINE_LEN);
2344  }
2345 
2346  /* Looking for headers */
2347  if (state->state_flag == HEADER_READY ||
2348  state->state_flag == HEADER_STARTED) {
2349 
2350  SCLogDebug("Processing Headers");
2351 
2352  /* Process message headers */
2353  ret = ProcessMimeHeaders(buf, len, state);
2354  if (ret != MIME_DEC_OK) {
2355  SCLogDebug("Error: ProcessMimeHeaders() function failed: %d",
2356  ret);
2357  return ret;
2358  }
2359  } else {
2360  /* Processing body */
2361  SCLogDebug("Processing Body of: %p", state->stack->top);
2362 
2363  ret = ProcessMimeBody(buf, len, state);
2364  if (ret != MIME_DEC_OK) {
2365  SCLogDebug("Error: ProcessMimeBody() function failed: %d",
2366  ret);
2367  return ret;
2368  }
2369  }
2370 
2371  SCLogDebug("END FLAG: %s", StateFlags[state->state_flag]);
2372 
2373  return ret;
2374 }
2375 
2376 /**
2377  * \brief Init the parser by allocating memory for the state and top-level entity
2378  *
2379  * \param data A caller-specified pointer to data for access within the data chunk
2380  * processor callback function
2381  * \param dcpfunc The data chunk processor callback function
2382  *
2383  * \return A pointer to the state object, or NULL if the operation fails
2384  */
2386  int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len,
2387  MimeDecParseState *state))
2388 {
2389  MimeDecParseState *state;
2390  MimeDecEntity *mimeMsg;
2391 
2392  state = SCMalloc(sizeof(MimeDecParseState));
2393  if (unlikely(state == NULL)) {
2394  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
2395  return NULL;
2396  }
2397  memset(state, 0x00, sizeof(MimeDecParseState));
2398 
2399  state->stack = SCMalloc(sizeof(MimeDecStack));
2400  if (unlikely(state->stack == NULL)) {
2401  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
2402  SCFree(state);
2403  return NULL;
2404  }
2405  memset(state->stack, 0x00, sizeof(MimeDecStack));
2406 
2407  mimeMsg = SCMalloc(sizeof(MimeDecEntity));
2408  if (unlikely(mimeMsg == NULL)) {
2409  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
2410  SCFree(state->stack);
2411  SCFree(state);
2412  return NULL;
2413  }
2414  memset(mimeMsg, 0x00, sizeof(MimeDecEntity));
2415  mimeMsg->ctnt_flags |= CTNT_IS_MSG;
2416 
2417  /* Init state */
2418  state->msg = mimeMsg;
2419  PushStack(state->stack);
2420  if (state->stack->top == NULL) {
2421  SCLogError(SC_ERR_MEM_ALLOC, "memory allocation failed");
2422  SCFree(state->stack);
2423  SCFree(state);
2424  return NULL;
2425  }
2426  state->stack->top->data = mimeMsg;
2427  state->state_flag = HEADER_READY;
2428  state->data = data;
2429  state->DataChunkProcessorFunc = DataChunkProcessorFunc;
2430 
2431  return state;
2432 }
2433 
2434 /**
2435  * \brief De-Init parser by freeing up any residual memory
2436  *
2437  * \param state The parser state
2438  *
2439  * \return none
2440  */
2442 {
2443  uint32_t cnt = 0;
2444 
2445  while (state->stack->top != NULL) {
2446  SCLogDebug("Remaining on stack: [%p]=>[%p]",
2447  state->stack->top, state->stack->top->data);
2448 
2449  PopStack(state->stack);
2450  cnt++;
2451  }
2452 
2453  if (cnt > 1) {
2455  SCLogDebug("Warning: Stack is not empty upon completion of "
2456  "processing (%u items remaining)", cnt);
2457  }
2458 
2459  SCFree(state->hname);
2460  FreeDataValue(state->hvalue);
2461  FreeMimeDecStack(state->stack);
2462 #ifdef HAVE_NSS
2463  if (state->md5_ctx)
2464  HASH_Destroy(state->md5_ctx);
2465 #endif
2466  SCFree(state);
2467 }
2468 
2469 /**
2470  * \brief Called to indicate that the last message line has been processed and
2471  * the parsing operation is complete
2472  *
2473  * This function should be called directly by the caller.
2474  *
2475  * \param state The parser state
2476  *
2477  * \return MIME_DEC_OK on success, otherwise < 0 on failure
2478  */
2480 {
2481  int ret = MIME_DEC_OK;
2482 
2483  SCLogDebug("Parsing flagged as completed");
2484 
2485  if (state->state_flag == PARSE_ERROR) {
2486  SCLogDebug("parser in error state: PARSE_ERROR");
2487  return MIME_DEC_ERR_STATE;
2488  }
2489 
2490  /* Store the header value */
2491  ret = StoreMimeHeader(state);
2492  if (ret != MIME_DEC_OK) {
2493  SCLogDebug("Error: StoreMimeHeader() function failed");
2494  return ret;
2495  }
2496 
2497  /* Lets complete the body */
2498  ret = ProcessBodyComplete(state);
2499  if (ret != MIME_DEC_OK) {
2500  SCLogDebug("Error: ProcessBodyComplete() function failed");
2501  return ret;
2502  }
2503 
2504  if (state->stack->top == NULL) {
2506  SCLogDebug("Error: Message is malformed");
2507  return MIME_DEC_ERR_DATA;
2508  }
2509 
2510  /* If encapsulated, pop off the stack */
2511  if (state->stack->top->is_encap) {
2512  PopStack(state->stack);
2513  if (state->stack->top == NULL) {
2515  SCLogDebug("Error: Message is malformed");
2516  return MIME_DEC_ERR_DATA;
2517  }
2518  }
2519 
2520  /* Look extra stack items remaining */
2521  if (state->stack->top->next != NULL) {
2523  SCLogDebug("Warning: Message has unclosed message part boundary");
2524  }
2525 
2526  state->state_flag = PARSE_DONE;
2527 
2528  return ret;
2529 }
2530 
2531 /**
2532  * \brief Parse a line of a MIME message and update the parser state
2533  *
2534  * \param line A string representing the line (w/out CRLF)
2535  * \param len The length of the line
2536  * \param delim_len The length of the line end delimiter
2537  * \param state The parser state
2538  *
2539  * \return MIME_DEC_OK on success, otherwise < 0 on failure
2540  */
2541 int MimeDecParseLine(const uint8_t *line, const uint32_t len,
2542  const uint8_t delim_len, MimeDecParseState *state)
2543 {
2544  int ret = MIME_DEC_OK;
2545 
2546  /* For debugging purposes */
2547  if (len > 0) {
2548  PrintChars(SC_LOG_DEBUG, "SMTP LINE", line, len);
2549  } else {
2550  SCLogDebug("SMTP LINE - EMPTY");
2551  }
2552 
2553  state->current_line_delimiter_len = delim_len;
2554  /* Process the entity */
2555  ret = ProcessMimeEntity(line, len, state);
2556  if (ret != MIME_DEC_OK) {
2557  state->state_flag = PARSE_ERROR;
2558  SCLogDebug("Error: ProcessMimeEntity() function failed: %d", ret);
2559  }
2560 
2561  return ret;
2562 }
2563 
2564 /**
2565  * \brief Parses an entire message when available in its entirety (wraps the
2566  * line-based parsing functions)
2567  *
2568  * \param buf Buffer pointing to the full message
2569  * \param blen Length of the buffer
2570  * \param data Caller data to be available in callback
2571  * \param dcpfunc Callback for processing each decoded body data chunk
2572  *
2573  * \return A pointer to the decoded MIME message, or NULL if the operation fails
2574  */
2575 MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data,
2576  int (*dcpfunc)(const uint8_t *chunk, uint32_t len,
2577  MimeDecParseState *state))
2578 {
2579  int ret = MIME_DEC_OK;
2580  uint8_t *remainPtr, *tok;
2581  uint32_t tokLen;
2582 
2583  MimeDecParseState *state = MimeDecInitParser(data, dcpfunc);
2584  if (state == NULL) {
2585  SCLogDebug("Error: MimeDecInitParser() function failed to create "
2586  "state");
2587  return NULL;
2588  }
2589 
2590  MimeDecEntity *msg = state->msg;
2591 
2592  /* Parse each line one by one */
2593  remainPtr = (uint8_t *) buf;
2594  uint8_t *line = NULL;
2595  do {
2596  tok = GetLine(remainPtr, blen - (remainPtr - buf), &remainPtr, &tokLen);
2597  if (tok != remainPtr) {
2598 
2599  line = tok;
2600 
2601  state->current_line_delimiter_len = (remainPtr - tok) - tokLen;
2602  /* Parse the line */
2603  ret = MimeDecParseLine(line, tokLen,
2604  (remainPtr - tok) - tokLen, state);
2605  if (ret != MIME_DEC_OK) {
2606  SCLogDebug("Error: MimeDecParseLine() function failed: %d",
2607  ret);
2608  break;
2609  }
2610  }
2611 
2612  } while (tok != remainPtr && remainPtr - buf < (int)blen);
2613 
2614  if (ret == MIME_DEC_OK) {
2615  SCLogDebug("Message parser was successful");
2616 
2617  /* Now complete message */
2618  ret = MimeDecParseComplete(state);
2619  if (ret != MIME_DEC_OK) {
2620  SCLogDebug("Error: MimeDecParseComplete() function failed");
2621  }
2622  }
2623 
2624  /* De-allocate memory for parser */
2625  MimeDecDeInitParser(state);
2626 
2627  if (ret != MIME_DEC_OK) {
2628  MimeDecFreeEntity(msg);
2629  msg = NULL;
2630  }
2631 
2632  return msg;
2633 }
2634 
2635 #ifdef AFLFUZZ_MIME
2636 static int MimeParserDataFromFileCB(const uint8_t *chunk, uint32_t len,
2637  MimeDecParseState *state)
2638 {
2639  return MIME_DEC_OK;
2640 }
2641 
2642 int MimeParserDataFromFile(char *filename)
2643 {
2644  int result = 1;
2645  uint8_t buffer[256];
2646 
2647 #ifdef AFLFUZZ_PERSISTANT_MODE
2648  while (__AFL_LOOP(1000)) {
2649  /* reset state */
2650  memset(buffer, 0, sizeof(buffer));
2651 #endif /* AFLFUZZ_PERSISTANT_MODE */
2652 
2653  FILE *fp = fopen(filename, "r");
2654  BUG_ON(fp == NULL);
2655 
2656  uint32_t line_count = 0;
2657 
2658  MimeDecParseState *state = MimeDecInitParser(&line_count,
2659  MimeParserDataFromFileCB);
2660 
2661  while (1) {
2662  int done = 0;
2663  size_t size = fread(&buffer, 1, sizeof(buffer), fp);
2664  if (size < sizeof(buffer))
2665  done = 1;
2666 
2667  (void) MimeDecParseLine(buffer, size, 1, state);
2668 
2669  if (done)
2670  break;
2671  }
2672 
2673  /* Completed */
2674  (void)MimeDecParseComplete(state);
2675 
2676  if (state->msg) {
2677  MimeDecFreeEntity(state->msg);
2678  }
2679 
2680  /* De Init parser */
2681  MimeDecDeInitParser(state);
2682 
2683  fclose(fp);
2684 
2685 #ifdef AFLFUZZ_PERSISTANT_MODE
2686  }
2687 #endif /* AFLFUZZ_PERSISTANT_MODE */
2688 
2689  result = 0;
2690  return result;
2691 }
2692 #endif /* AFLFUZZ_MIME */
2693 
2694 #ifdef UNITTESTS
2695 
2696 /* Helper body chunk callback function */
2697 static int TestDataChunkCallback(const uint8_t *chunk, uint32_t len,
2698  MimeDecParseState *state)
2699 {
2700  uint32_t *line_count = (uint32_t *) state->data;
2701 
2702  if (state->body_begin) {
2703  SCLogDebug("Body begin (len=%u)", len);
2704  }
2705 
2706  /* Add up the line counts */
2707  if (len > 0) {
2708 
2709  uint8_t *remainPtr;
2710  uint8_t *tok;
2711  uint32_t tokLen;
2712 
2713  PrintChars(SC_LOG_DEBUG, "CHUNK", chunk, len);
2714 
2715  /* Parse each line one by one */
2716  remainPtr = (uint8_t *) chunk;
2717  do {
2718  tok = GetLine(remainPtr, len - (remainPtr - (uint8_t *) chunk),
2719  &remainPtr, &tokLen);
2720  if (tok != NULL && tok != remainPtr) {
2721  (*line_count)++;
2722  }
2723 
2724  } while (tok != NULL && tok != remainPtr &&
2725  (uint32_t)(remainPtr - (uint8_t *) chunk) < len);
2726 
2727  SCLogDebug("line count (len=%u): %u", len, *line_count);
2728  }
2729 
2730  if (state->body_end) {
2731  SCLogDebug("Body end (len=%u)", len);
2732  }
2733 
2734  return MIME_DEC_OK;
2735 }
2736 
2737 /* Test simple case of line counts */
2738 static int MimeDecParseLineTest01(void)
2739 {
2740  int ret = MIME_DEC_OK;
2741 
2742  uint32_t expected_count = 3;
2743  uint32_t line_count = 0;
2744 
2745  /* Init parser */
2746  MimeDecParseState *state = MimeDecInitParser(&line_count,
2747  TestDataChunkCallback);
2748 
2749  const char *str = "From: Sender1";
2750  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2751 
2752  str = "To: Recipient1";
2753  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2754 
2755  str = "Content-Type: text/plain";
2756  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2757 
2758  str = "";
2759  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2760 
2761  str = "A simple message line 1";
2762  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2763 
2764  str = "A simple message line 2";
2765  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2766 
2767  str = "A simple message line 3";
2768  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2769 
2770  if (ret != MIME_DEC_OK) {
2771  return 0;
2772  }
2773  /* Completed */
2774  ret = MimeDecParseComplete(state);
2775  if (ret != MIME_DEC_OK) {
2776  return 0;
2777  }
2778 
2779  MimeDecEntity *msg = state->msg;
2780  if (msg->next != NULL || msg->child != NULL) {
2781  SCLogInfo("Error: Invalid sibling or child message");
2782  return 0;
2783  }
2784 
2785  MimeDecFreeEntity(msg);
2786 
2787  /* De Init parser */
2788  MimeDecDeInitParser(state);
2789 
2790  SCLogInfo("LINE COUNT FINISHED: %d", line_count);
2791 
2792  if (expected_count != line_count) {
2793  SCLogInfo("Error: Line count is invalid: expected - %d actual - %d",
2794  expected_count, line_count);
2795  return 0;
2796  }
2797 
2798  return 1;
2799 }
2800 
2801 /* Test simple case of EXE URL extraction */
2802 static int MimeDecParseLineTest02(void)
2803 {
2804  int ret = MIME_DEC_OK;
2805 
2806  uint32_t expected_count = 2;
2807  uint32_t line_count = 0;
2808 
2812 
2813  /* Init parser */
2814  MimeDecParseState *state = MimeDecInitParser(&line_count,
2815  TestDataChunkCallback);
2816 
2817  const char *str = "From: Sender1";
2818  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2819 
2820  str = "To: Recipient1";
2821  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2822 
2823  str = "Content-Type: text/plain";
2824  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2825 
2826  str = "";
2827  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2828 
2829  str = "A simple message line 1";
2830  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2831 
2832  str = "A simple message line 2 click on http://www.test.com/malware.exe?"
2833  "hahah hopefully you click this link";
2834  ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
2835 
2836  if (ret != MIME_DEC_OK) {
2837  return 0;
2838  }
2839  /* Completed */
2840  ret = MimeDecParseComplete(state);
2841  if (ret != MIME_DEC_OK) {
2842  return 0;
2843  }
2844 
2845  MimeDecEntity *msg = state->msg;
2846  if (msg->url_list == NULL || (msg->url_list != NULL &&
2847  !(msg->url_list->url_flags & URL_IS_EXE))) {
2848  SCLogInfo("Warning: Expected EXE URL not found");
2849  return 0;
2850  }
2851 
2852  MimeDecFreeEntity(msg);
2853 
2854  /* De Init parser */
2855  MimeDecDeInitParser(state);
2856 
2857  SCLogInfo("LINE COUNT FINISHED: %d", line_count);
2858 
2859  if (expected_count != line_count) {
2860  SCLogInfo("Warning: Line count is invalid: expected - %d actual - %d",
2861  expected_count, line_count);
2862  return 0;
2863  }
2864 
2865  return 1;
2866 }
2867 
2868 /* Test full message with linebreaks */
2869 static int MimeDecParseFullMsgTest01(void)
2870 {
2871  uint32_t expected_count = 3;
2872  uint32_t line_count = 0;
2873 
2874  char msg[] = "From: Sender1\r\n"
2875  "To: Recipient1\r\n"
2876  "Content-Type: text/plain\r\n"
2877  "\r\n"
2878  "Line 1\r\n"
2879  "Line 2\r\n"
2880  "Line 3\r\n";
2881 
2882  MimeDecEntity *entity = MimeDecParseFullMsg((uint8_t *)msg, strlen(msg), &line_count,
2883  TestDataChunkCallback);
2884  if (entity == NULL) {
2885  SCLogInfo("Warning: Message failed to parse");
2886  return 0;
2887  }
2888 
2889  MimeDecFreeEntity(entity);
2890 
2891  if (expected_count != line_count) {
2892  SCLogInfo("Warning: Line count is invalid: expected - %d actual - %d",
2893  expected_count, line_count);
2894  return 0;
2895  }
2896 
2897  return 1;
2898 }
2899 
2900 /* Test full message with linebreaks */
2901 static int MimeDecParseFullMsgTest02(void)
2902 {
2903  uint32_t expected_count = 3;
2904  uint32_t line_count = 0;
2905 
2906  char msg[] = "From: Sender2\r\n"
2907  "To: Recipient2\r\n"
2908  "Subject: subject2\r\n"
2909  "Content-Type: text/plain\r\n"
2910  "\r\n"
2911  "Line 1\r\n"
2912  "Line 2\r\n"
2913  "Line 3\r\n";
2914 
2915  MimeDecEntity *entity = MimeDecParseFullMsg((uint8_t *)msg, strlen(msg), &line_count,
2916  TestDataChunkCallback);
2917 
2918  if (entity == NULL) {
2919  SCLogInfo("Warning: Message failed to parse");
2920  return 0;
2921  }
2922 
2923  MimeDecField *field = MimeDecFindField(entity, "subject");
2924  if (field == NULL) {
2925  SCLogInfo("Warning: Message failed to parse");
2926  return 0;
2927  }
2928 
2929  if (field->value_len != sizeof("subject2") - 1) {
2930  SCLogInfo("Warning: failed to get subject");
2931  return 0;
2932  }
2933 
2934  if (memcmp(field->value, "subject2", field->value_len) != 0) {
2935  SCLogInfo("Warning: failed to get subject");
2936  return 0;
2937  }
2938 
2939 
2940  MimeDecFreeEntity(entity);
2941 
2942  if (expected_count != line_count) {
2943  SCLogInfo("Warning: Line count is invalid: expected - %d actual - %d",
2944  expected_count, line_count);
2945  return 0;
2946  }
2947 
2948  return 1;
2949 }
2950 
2951 static int MimeBase64DecodeTest01(void)
2952 {
2953  int ret = 0;
2954 
2955  const char *msg = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890@"
2956  "#$%^&*()-=_+,./;'[]<>?:";
2957  const char *base64msg = "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QU"
2958  "VJTVFVWV1hZWjEyMzQ1Njc4OTBAIyQlXiYqKCktPV8rLC4vOydbXTw+Pzo=";
2959 
2960  uint8_t *dst = SCMalloc(strlen(msg) + 1);
2961  if (dst == NULL)
2962  return 0;
2963 
2964  ret = DecodeBase64(dst, (const uint8_t *)base64msg, strlen(base64msg), 1);
2965 
2966  if (memcmp(dst, msg, strlen(msg)) == 0) {
2967  ret = 1;
2968  }
2969 
2970  SCFree(dst);
2971 
2972  return ret;
2973 }
2974 
2975 static int MimeIsExeURLTest01(void)
2976 {
2977  int ret = 0;
2978  const char *url1 = "http://www.google.com/";
2979  const char *url2 = "http://www.google.com/test.exe";
2980 
2981  if(IsExeUrl((const uint8_t *)url1, strlen(url1)) != 0){
2982  SCLogDebug("Debug: URL1 error");
2983  goto end;
2984  }
2985  if(IsExeUrl((const uint8_t *)url2, strlen(url2)) != 1){
2986  SCLogDebug("Debug: URL2 error");
2987  goto end;
2988  }
2989  ret = 1;
2990 
2991  end:
2992 
2993  return ret;
2994 }
2995 
2996 #define TEST(str, len, expect) { \
2997  SCLogDebug("str %s", (str)); \
2998  int r = IsIpv4Host((const uint8_t *)(str),(len)); \
2999  FAIL_IF_NOT(r == (expect)); \
3000 }
3001 static int MimeIsIpv4HostTest01(void)
3002 {
3003  TEST("192.168.1.1", 11, 1);
3004  TEST("192.168.1.1.4", 13, 0);
3005  TEST("999.168.1.1", 11, 0);
3006  TEST("1111.168.1.1", 12, 0);
3007  TEST("999.oogle.com", 14, 0);
3008  TEST("0:0:0:0:0:0:0:0", 15, 0);
3009  TEST("192.168.255.255", 15, 1);
3010  TEST("192.168.255.255/testurl.html", 28, 1);
3011  TEST("www.google.com", 14, 0);
3012  PASS;
3013 }
3014 #undef TEST
3015 
3016 #define TEST(str, len, expect) { \
3017  SCLogDebug("str %s", (str)); \
3018  int r = IsIpv6Host((const uint8_t *)(str),(len)); \
3019  FAIL_IF_NOT(r == (expect)); \
3020 }
3021 static int MimeIsIpv6HostTest01(void)
3022 {
3023  TEST("0:0:0:0:0:0:0:0", 19, 1);
3024  TEST("0000:0000:0000:0000:0000:0000:0000:0000", 39, 1);
3025  TEST("XXXX:0000:0000:0000:0000:0000:0000:0000", 39, 0);
3026  TEST("00001:0000:0000:0000:0000:0000:0000:0000", 40, 0);
3027  TEST("0:0:0:0:0:0:0:0", 19, 1);
3028  TEST("0:0:0:0:0:0:0:0:0", 20, 0);
3029  TEST("192:168:1:1:0:0:0:0", 19, 1);
3030  TEST("999.oogle.com", 14, 0);
3031  TEST("192.168.255.255", 15, 0);
3032  TEST("192.168.255.255/testurl.html", 28, 0);
3033  TEST("www.google.com", 14, 0);
3034  PASS;
3035 }
3036 #undef TEST
3037 
3038 #endif /* UNITTESTS */
3039 
3041 {
3042 #ifdef UNITTESTS
3043  UtRegisterTest("MimeDecParseLineTest01", MimeDecParseLineTest01);
3044  UtRegisterTest("MimeDecParseLineTest02", MimeDecParseLineTest02);
3045  UtRegisterTest("MimeDecParseFullMsgTest01", MimeDecParseFullMsgTest01);
3046  UtRegisterTest("MimeDecParseFullMsgTest02", MimeDecParseFullMsgTest02);
3047  UtRegisterTest("MimeBase64DecodeTest01", MimeBase64DecodeTest01);
3048  UtRegisterTest("MimeIsExeURLTest01", MimeIsExeURLTest01);
3049  UtRegisterTest("MimeIsIpv4HostTest01", MimeIsIpv4HostTest01);
3050  UtRegisterTest("MimeIsIpv6HostTest01", MimeIsIpv6HostTest01);
3051 #endif /* UNITTESTS */
3052 }
uint32_t value_len
uint16_t flags
#define SCMemcmp(a, b, c)
Definition: util-memcmp.h:369
#define ANOM_LONG_HEADER_VALUE
#define SCLogDebug(...)
Definition: util-debug.h:335
SCLogLevel sc_log_global_log_level
Holds the global log level. Is the same as sc_log_config->log_level.
Definition: util-debug.c:95
void MimeDecFreeEntity(MimeDecEntity *entity)
Frees a mime entity tree.
struct MimeDecUrl * next
MimeDecField * field_list
Structure for containing configuration options.
#define MAX_LINE_LEN
#define CTNT_DISP_STR
MimeDecConfig * MimeDecGetConfig(void)
Get global config policy.
#define MAX_HEADER_NAME
#define URL_IS_IP6
#define BUG_ON(x)
uint64_t value
#define DASH
struct MimeDecField * next
#define ANOM_LONG_BOUNDARY
#define DATA_CHUNK_SIZE
#define PASS
Pass the test.
#define unlikely(expr)
Definition: util-optimize.h:35
uint8_t * value
uint32_t decoded_body_len
uint8_t * url
#define MAX_IP6_CHARS
uint8_t current_line_delimiter_len
MimeDecStack * stack
void MimeDecSetConfig(MimeDecConfig *config)
Set global config policy.
bool IPv4AddressStringIsValid(const char *str)
determine if a string is a valid ipv4 address
Definition: util-ip.c:33
uint64_t offset
#define MAX_HEADER_VALUE
uint32_t url_len
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
#define COLON
uint16_t src
#define ANOM_INVALID_QP
uint32_t ctnt_type_len
#define QP_STR
#define MAX_IP4_CHARS
#define CTNT_IS_QP
MimeDecField * MimeDecAddField(MimeDecEntity *entity)
Creates and adds a header field entry to an entity.
MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data, int(*dcpfunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Parses an entire message when available in its entirety (wraps the line-based parsing functions) ...
#define EOL_LEN
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
#define CTNT_TRAN_STR
#define ANOM_LONG_ENC_LINE
uint8_t * ctnt_type
#define ASCII_BLOCK
Definition: util-base64.h:48
#define CTNT_IS_ENV
#define HTML_STR
This represents the MIME Entity (or also top level message) in a child-sibling tree.
#define str(s)
uint16_t dst
#define SCCalloc(nm, a)
Definition: util-mem.h:253
uint8_t * BasicSearchNocase(const uint8_t *haystack, uint32_t haystack_len, const uint8_t *needle, uint16_t needle_len)
Basic search case less.
Definition: util-spm-bs.c:102
#define PARSE_DONE
Structure contains a list of value and lengths for robust data processing.
uint32_t value_len
#define BASE64_STR
void MimeDecDeInitParser(MimeDecParseState *state)
De-Init parser by freeing up any residual memory.
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
struct MimeDecEntity * child
struct MimeDecStackNode * next
uint32_t free_nodes_cnt
void UtRegisterTest(const char *name, int(*TestFn)(void))
Register unit test.
#define BODY_STARTED
#define BOUNDARY_BUF
int MimeDecParseComplete(MimeDecParseState *state)
Called to indicate that the last message line has been processed and the parsing operation is complet...
#define CTNT_IS_MULTIPART
#define CR
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
#define CTNT_IS_MSG
#define HEADER_STARTED
struct Asn1Generic_ * data
uint32_t url_flags
MimeDecUrl * url_list
void MimeDecFreeUrl(MimeDecUrl *url)
Iteratively frees a URL entry list.
#define CTNT_IS_ATTACHMENT
bool IPv6AddressStringIsValid(const char *str)
determine if a string is a valid ipv6 address
Definition: util-ip.c:80
MimeDecStackNode * free_nodes
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
Structure contains boundary and entity for the current node (entity) in the stack.
#define URL_IS_IP4
#define ANOM_LONG_LINE
#define URL_STR
void MimeDecRegisterTests(void)
#define ANOM_LONG_HEADER_NAME
#define SCMalloc(a)
Definition: util-mem.h:222
MimeDecStackNode * top
int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state)
Parse a line of a MIME message and update the parser state.
#define PRINTABLE_START
#define LF
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent)
Creates and adds a child entity to the specified parent entity.
void PrintRawDataFp(FILE *fp, const uint8_t *buf, uint32_t buflen)
Definition: util-print.c:141
#define SCFree(a)
Definition: util-mem.h:322
PoolThreadReserved res
#define PARSE_ERROR
#define TOK_END_STR
MimeDecEntity * msg
#define STACK_FREE_NODES
#define PRINTABLE_END
MimeDecParseState * MimeDecInitParser(void *data, int(*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Init the parser by allocating memory for the state and top-level entity.
#define CTNT_IS_TEXT
#define CTNT_IS_HTML
struct DataValue * next
uint32_t anomaly_flags
#define ANOM_INVALID_BASE64
uint32_t header_value_depth
#define TEST(str, len, expect)
#define ANOM_MALFORMED_MSG
void MimeDecFreeField(MimeDecField *field)
Iteratively frees a header field entry list.
#define CTNT_IS_BODYPART
Structure holds the top of the stack along with some free reusable nodes.
#define TXT_STR
#define MULTIPART_STR
Structure contains the current state of the MIME parser.
uint8_t * filename
uint32_t strlen
const char * msg
#define CTNT_TYPE_STR
uint8_t len
#define BODY_END_BOUND
uint32_t filename_len
#define HEADER_DONE
int(* DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, struct MimeDecParseState *state)
#define HEADER_READY
This represents a header field name and associated value.
#define CTNT_IS_BASE64
This represents a URL value node in a linked list.
#define MSG_ID_STR
uint32_t DecodeBase64(uint8_t *dest, const uint8_t *src, uint32_t len, int strict)
Decodes a base64-encoded string buffer into an ascii-encoded byte buffer.
Definition: util-base64.c:91
uint8_t bvremain[B64_BLOCK]
#define URL_IS_EXE
#define B64_BLOCK
Definition: util-base64.h:49
struct MimeDecEntity * next
#define MSG_STR
#define MAX_ENC_LINE_LEN
#define CRLF
#define CTNT_IS_ENCAP
#define BND_START_STR
uint8_t data_chunk[DATA_CHUNK_SIZE]
MimeDecEntity * data