suricata
util-decode-mime.h
Go to the documentation of this file.
1 /* Copyright (C) 2012 BAE Systems
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author David Abarbanel <david.abarbanel@baesystems.com>
22  *
23  */
24 
25 #ifndef MIME_DECODE_H_
26 #define MIME_DECODE_H_
27 
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <stdint.h>
31 
32 #include "suricata.h"
33 #include "util-base64.h"
34 #include "util-debug.h"
35 
36 /* Content Flags */
37 #define CTNT_IS_MSG 1
38 #define CTNT_IS_ENV 2
39 #define CTNT_IS_ENCAP 4
40 #define CTNT_IS_BODYPART 8
41 #define CTNT_IS_MULTIPART 16
42 #define CTNT_IS_ATTACHMENT 32
43 #define CTNT_IS_BASE64 64
44 #define CTNT_IS_QP 128
45 #define CTNT_IS_TEXT 256
46 #define CTNT_IS_HTML 512
47 
48 /* URL Flags */
49 #define URL_IS_IP4 1
50 #define URL_IS_IP6 2
51 #define URL_IS_EXE 4
52 
53 /* Anomaly Flags */
54 #define ANOM_INVALID_BASE64 1 /* invalid base64 chars */
55 #define ANOM_INVALID_QP 2 /* invalid quoted-printable chars */
56 #define ANOM_LONG_HEADER_NAME 4 /* header is abnormally long */
57 #define ANOM_LONG_HEADER_VALUE 8 /* header value is abnormally long
58  * (includes multi-line) */
59 #define ANOM_LONG_LINE 16 /* Lines that exceed 998 octets */
60 #define ANOM_LONG_ENC_LINE 32 /* Lines that exceed 76 octets */
61 #define ANOM_MALFORMED_MSG 64 /* Misc msg format errors found */
62 #define ANOM_LONG_BOUNDARY 128 /* Boundary too long */
63 #define ANOM_LONG_FILENAME 256 /* filename truncated */
64 
65 /* Publicly exposed size constants */
66 #define DATA_CHUNK_SIZE 3072 /* Should be divisible by 3 */
67 #define LINEREM_SIZE 256
68 
69 /* Mime Parser Constants */
70 #define HEADER_READY 0x01
71 #define HEADER_STARTED 0x02
72 #define HEADER_DONE 0x03
73 #define BODY_STARTED 0x04
74 #define BODY_DONE 0x05
75 #define BODY_END_BOUND 0x06
76 #define PARSE_DONE 0x07
77 #define PARSE_ERROR 0x08
78 
79 /**
80  * \brief Mime Decoder Error Codes
81  */
82 typedef enum MimeDecRetCode {
88  MIME_DEC_ERR_STATE = -4, /**< parser in error state */
90 
91 /**
92  * \brief Structure for containing configuration options
93  *
94  */
95 typedef struct MimeDecConfig {
96  int decode_base64; /**< Decode base64 bodies */
97  int decode_quoted_printable; /**< Decode quoted-printable bodies */
98  int extract_urls; /**< Extract and store URLs in data structure */
99  int body_md5; /**< Compute md5 sum of body */
100  uint32_t header_value_depth; /**< Depth of which to store header values
101  (Default is 2000) */
102 } MimeDecConfig;
103 
104 /**
105  * \brief This represents a header field name and associated value
106  */
107 typedef struct MimeDecField {
108  uint8_t *name; /**< Name of the header field */
109  uint32_t name_len; /**< Length of the name */
110  uint32_t value_len; /**< Length of the value */
111  uint8_t *value; /**< Value of the header field */
112  struct MimeDecField *next; /**< Pointer to next field */
113 } MimeDecField;
114 
115 /**
116  * \brief This represents a URL value node in a linked list
117  *
118  * Since HTML can sometimes contain a high number of URLs, this
119  * structure only features the URL host name/IP or those that are
120  * pointing to an executable file (see url_flags to determine which).
121  */
122 typedef struct MimeDecUrl {
123  uint8_t *url; /**< String representation of full or partial URL (lowercase) */
124  uint32_t url_len; /**< Length of the URL string */
125  uint32_t url_flags; /**< Flags indicating type of URL */
126  struct MimeDecUrl *next; /**< Pointer to next URL */
127 } MimeDecUrl;
128 
129 /**
130  * \brief This represents the MIME Entity (or also top level message) in a
131  * child-sibling tree
132  */
133 typedef struct MimeDecEntity {
134  MimeDecField *field_list; /**< Pointer to list of header fields */
135  MimeDecUrl *url_list; /**< Pointer to list of URLs */
136  uint32_t body_len; /**< Length of body (prior to any decoding) */
137  uint32_t decoded_body_len; /**< Length of body after decoding */
138  uint32_t header_flags; /**< Flags indicating header characteristics */
139  uint32_t ctnt_flags; /**< Flags indicating type of content */
140  uint32_t anomaly_flags; /**< Flags indicating an anomaly in the message */
141  uint32_t filename_len; /**< Length of file attachment name */
142  uint8_t *filename; /**< Name of file attachment */
143  uint8_t *ctnt_type; /**< Quick access pointer to short-hand content type field */
144  uint32_t ctnt_type_len; /**< Length of content type field value */
145  uint32_t msg_id_len; /**< Quick access pointer to message Id */
146  uint8_t *msg_id; /**< Quick access pointer to message Id */
147  struct MimeDecEntity *next; /**< Pointer to list of sibling entities */
148  struct MimeDecEntity *child; /**< Pointer to list of child entities */
149 } MimeDecEntity;
150 
151 /**
152  * \brief Structure contains boundary and entity for the current node (entity)
153  * in the stack
154  *
155  */
156 typedef struct MimeDecStackNode {
157  MimeDecEntity *data; /**< Pointer to the entity data structure */
158  uint8_t *bdef; /**< Copy of boundary definition for child entity */
159  uint32_t bdef_len; /**< Boundary length for child entity */
160  int is_encap; /**< Flag indicating entity is encapsulated in message */
161  struct MimeDecStackNode *next; /**< Pointer to next item on the stack */
163 
164 /**
165  * \brief Structure holds the top of the stack along with some free reusable nodes
166  *
167  */
168 typedef struct MimeDecStack {
169  MimeDecStackNode *top; /**< Pointer to the top of the stack */
170  MimeDecStackNode *free_nodes; /**< Pointer to the list of free nodes */
171  uint32_t free_nodes_cnt; /**< Count of free nodes in the list */
172 } MimeDecStack;
173 
174 /**
175  * \brief Structure contains a list of value and lengths for robust data processing
176  *
177  */
178 typedef struct DataValue {
179  uint8_t *value; /**< Copy of data value */
180  uint32_t value_len; /**< Length of data value */
181  struct DataValue *next; /**< Pointer to next value in the list */
182 } DataValue;
183 
184 /**
185  * \brief Structure contains the current state of the MIME parser
186  *
187  */
188 typedef struct MimeDecParseState {
189  MimeDecEntity *msg; /**< Pointer to the top-level message entity */
190  MimeDecStack *stack; /**< Pointer to the top of the entity stack */
191  uint8_t *hname; /**< Copy of the last known header name */
192  uint32_t hlen; /**< Length of the last known header name */
193  uint32_t hvlen; /**< Total length of value list */
194  DataValue *hvalue; /**< Pointer to the incomplete header value list */
195  uint8_t linerem[LINEREM_SIZE]; /**< Remainder from previous line (for URL extraction) */
196  uint16_t linerem_len; /**< Length of remainder from previous line */
197  uint8_t bvremain[B64_BLOCK]; /**< Remainder from base64-decoded line */
198  uint8_t bvr_len; /**< Length of remainder from base64-decoded line */
199  uint8_t data_chunk[DATA_CHUNK_SIZE]; /**< Buffer holding data chunk */
200 #ifdef HAVE_NSS
201  HASHContext *md5_ctx;
202  uint8_t md5[MD5_LENGTH];
203 #endif
204  uint8_t state_flag; /**< Flag representing current state of parser */
205  uint32_t data_chunk_len; /**< Length of data chunk */
206  int found_child; /**< Flag indicating a child entity was found */
207  int body_begin; /**< Currently at beginning of body */
208  int body_end; /**< Currently at end of body */
209  uint8_t current_line_delimiter_len; /**< Length of line delimiter */
210  void *data; /**< Pointer to data specific to the caller */
211  int (*DataChunkProcessorFunc) (const uint8_t *chunk, uint32_t len,
212  struct MimeDecParseState *state); /**< Data chunk processing function callback */
214 
215 /* Config functions */
216 void MimeDecSetConfig(MimeDecConfig *config);
218 
219 /* Memory functions */
220 void MimeDecFreeEntity(MimeDecEntity *entity);
221 void MimeDecFreeField(MimeDecField *field);
222 void MimeDecFreeUrl(MimeDecUrl *url);
223 
224 /* List functions */
226 MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name);
227 int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data);
229 
230 /* Helper functions */
231 //MimeDecField * MimeDecFillField(MimeDecEntity *entity, const char *name,
232 // uint32_t nlen, const char *value, uint32_t vlen, int copy_name_value);
233 
234 /* Parser functions */
235 MimeDecParseState * MimeDecInitParser(void *data, int (*dcpfunc)(const uint8_t *chunk,
236  uint32_t len, MimeDecParseState *state));
239 int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state);
240 MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data,
241  int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state));
243 
244 /* Test functions */
245 void MimeDecRegisterTests(void);
246 
247 #endif
MIME_DEC_MORE
@ MIME_DEC_MORE
Definition: util-decode-mime.h:83
MimeDecAddEntity
MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent)
Creates and adds a child entity to the specified parent entity.
Definition: util-decode-mime.c:399
MimeDecParseState::data_chunk
uint8_t data_chunk[DATA_CHUNK_SIZE]
Definition: util-decode-mime.h:198
MimeDecEntity::ctnt_flags
uint32_t ctnt_flags
Definition: util-decode-mime.h:138
MimeDecParseState::linerem
uint8_t linerem[LINEREM_SIZE]
Definition: util-decode-mime.h:194
MimeDecEntity::child
struct MimeDecEntity * child
Definition: util-decode-mime.h:147
MimeDecInitParser
MimeDecParseState * MimeDecInitParser(void *data, int(*dcpfunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Init the parser by allocating memory for the state and top-level entity.
Definition: util-decode-mime.c:2431
len
uint8_t len
Definition: app-layer-dnp3.h:2
MIME_DEC_OK
@ MIME_DEC_OK
Definition: util-decode-mime.h:82
MIME_DEC_ERR_MEM
@ MIME_DEC_ERR_MEM
Definition: util-decode-mime.h:85
MimeDecFreeField
void MimeDecFreeField(MimeDecField *field)
Iteratively frees a header field entry list.
Definition: util-decode-mime.c:221
DataValue::next
struct DataValue * next
Definition: util-decode-mime.h:180
MimeDecEntity::ctnt_type_len
uint32_t ctnt_type_len
Definition: util-decode-mime.h:143
MimeDecConfig::header_value_depth
uint32_t header_value_depth
Definition: util-decode-mime.h:99
MimeDecField::value
uint8_t * value
Definition: util-decode-mime.h:110
MimeDecStack
Structure holds the top of the stack along with some free reusable nodes.
Definition: util-decode-mime.h:167
MimeDecParseState::data
void * data
Definition: util-decode-mime.h:209
MimeDecEntity::filename
uint8_t * filename
Definition: util-decode-mime.h:141
MimeDecParseState::bvremain
uint8_t bvremain[B64_BLOCK]
Definition: util-decode-mime.h:196
MimeDecParseComplete
int MimeDecParseComplete(MimeDecParseState *state)
Called to indicate that the last message line has been processed and the parsing operation is complet...
Definition: util-decode-mime.c:2525
MimeDecParseState::current_line_delimiter_len
uint8_t current_line_delimiter_len
Definition: util-decode-mime.h:208
MimeDecEntity::field_list
MimeDecField * field_list
Definition: util-decode-mime.h:133
MimeDecStack::top
MimeDecStackNode * top
Definition: util-decode-mime.h:168
MimeDecEntity::filename_len
uint32_t filename_len
Definition: util-decode-mime.h:140
DataValue::value_len
uint32_t value_len
Definition: util-decode-mime.h:179
MimeDecStackNode::bdef_len
uint32_t bdef_len
Definition: util-decode-mime.h:158
util-base64.h
MimeDecRetCode
MimeDecRetCode
Mime Decoder Error Codes.
Definition: util-decode-mime.h:81
MimeDecParseState::found_child
int found_child
Definition: util-decode-mime.h:205
MimeDecUrl::next
struct MimeDecUrl * next
Definition: util-decode-mime.h:125
MimeDecField
This represents a header field name and associated value.
Definition: util-decode-mime.h:106
MimeDecParseState::hvlen
uint32_t hvlen
Definition: util-decode-mime.h:192
MimeDecParseState::linerem_len
uint16_t linerem_len
Definition: util-decode-mime.h:195
MimeDecEntity::url_list
MimeDecUrl * url_list
Definition: util-decode-mime.h:134
MimeDecAddField
MimeDecField * MimeDecAddField(MimeDecEntity *entity)
Creates and adds a header field entry to an entity.
Definition: util-decode-mime.c:279
MimeDecUrl::url_len
uint32_t url_len
Definition: util-decode-mime.h:123
MimeDecConfig
struct MimeDecConfig MimeDecConfig
Structure for containing configuration options.
MimeDecField::name
uint8_t * name
Definition: util-decode-mime.h:107
MimeDecParseState::body_begin
int body_begin
Definition: util-decode-mime.h:206
MimeDecParseState
struct MimeDecParseState MimeDecParseState
Structure contains the current state of the MIME parser.
util-debug.h
MimeDecStackNode::bdef
uint8_t * bdef
Definition: util-decode-mime.h:157
MimeDecConfig::body_md5
int body_md5
Definition: util-decode-mime.h:98
MimeDecEntity::ctnt_type
uint8_t * ctnt_type
Definition: util-decode-mime.h:142
MimeDecStackNode::is_encap
int is_encap
Definition: util-decode-mime.h:159
MimeDecFindFieldsForEach
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
Definition: util-decode-mime.c:311
MimeDecParseState::stack
MimeDecStack * stack
Definition: util-decode-mime.h:189
MimeDecEntity::decoded_body_len
uint32_t decoded_body_len
Definition: util-decode-mime.h:136
DataValue
struct DataValue DataValue
Structure contains a list of value and lengths for robust data processing.
MIME_DEC_ERR_PARSE
@ MIME_DEC_ERR_PARSE
Definition: util-decode-mime.h:86
MimeDecSetConfig
void MimeDecSetConfig(MimeDecConfig *config)
Set global config policy.
Definition: util-decode-mime.c:133
MimeDecParseState::hvalue
DataValue * hvalue
Definition: util-decode-mime.h:193
MimeDecField::next
struct MimeDecField * next
Definition: util-decode-mime.h:111
LINEREM_SIZE
#define LINEREM_SIZE
Definition: util-decode-mime.h:66
MimeDecStackNode
Structure contains boundary and entity for the current node (entity) in the stack.
Definition: util-decode-mime.h:155
MimeDecRegisterTests
void MimeDecRegisterTests(void)
Definition: util-decode-mime.c:3147
MimeDecParseState::hlen
uint32_t hlen
Definition: util-decode-mime.h:191
MimeDecParseState::data_chunk_len
uint32_t data_chunk_len
Definition: util-decode-mime.h:204
MimeDecParseState::hname
uint8_t * hname
Definition: util-decode-mime.h:190
MimeDecStackNode::next
struct MimeDecStackNode * next
Definition: util-decode-mime.h:160
MimeDecUrl::url_flags
uint32_t url_flags
Definition: util-decode-mime.h:124
MimeDecConfig
Structure for containing configuration options.
Definition: util-decode-mime.h:94
MimeDecParseStateGetStatus
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
Definition: util-decode-mime.c:2358
MimeDecEntity::msg_id
uint8_t * msg_id
Definition: util-decode-mime.h:145
MimeDecStackNode
struct MimeDecStackNode MimeDecStackNode
Structure contains boundary and entity for the current node (entity) in the stack.
MimeDecParseState::msg
MimeDecEntity * msg
Definition: util-decode-mime.h:188
MimeDecFreeUrl
void MimeDecFreeUrl(MimeDecUrl *url)
Iteratively frees a URL entry list.
Definition: util-decode-mime.c:250
MimeDecField::value_len
uint32_t value_len
Definition: util-decode-mime.h:109
MimeDecEntity::body_len
uint32_t body_len
Definition: util-decode-mime.h:135
MimeDecParseFullMsg
MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data, int(*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Parses an entire message when available in its entirety (wraps the line-based parsing functions)
Definition: util-decode-mime.c:2621
MimeDecConfig::decode_base64
int decode_base64
Definition: util-decode-mime.h:95
MimeDecEntity::anomaly_flags
uint32_t anomaly_flags
Definition: util-decode-mime.h:139
MimeDecField
struct MimeDecField MimeDecField
This represents a header field name and associated value.
MIME_DEC_ERR_STATE
@ MIME_DEC_ERR_STATE
Definition: util-decode-mime.h:87
MimeDecEntity
struct MimeDecEntity MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
MimeDecParseState::body_end
int body_end
Definition: util-decode-mime.h:207
MimeDecParseState
Structure contains the current state of the MIME parser.
Definition: util-decode-mime.h:187
MimeDecGetConfig
MimeDecConfig * MimeDecGetConfig(void)
Get global config policy.
Definition: util-decode-mime.c:152
MimeDecDeInitParser
void MimeDecDeInitParser(MimeDecParseState *state)
De-Init parser by freeing up any residual memory.
Definition: util-decode-mime.c:2487
DATA_CHUNK_SIZE
#define DATA_CHUNK_SIZE
Definition: util-decode-mime.h:65
MimeDecConfig::decode_quoted_printable
int decode_quoted_printable
Definition: util-decode-mime.h:96
MimeDecUrl
This represents a URL value node in a linked list.
Definition: util-decode-mime.h:121
MimeDecStack::free_nodes_cnt
uint32_t free_nodes_cnt
Definition: util-decode-mime.h:170
MimeDecFindField
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
Definition: util-decode-mime.c:339
MimeDecParseState::bvr_len
uint8_t bvr_len
Definition: util-decode-mime.h:197
DataValue
Structure contains a list of value and lengths for robust data processing.
Definition: util-decode-mime.h:177
suricata.h
B64_BLOCK
#define B64_BLOCK
Definition: util-base64.h:49
MimeDecEntity::msg_id_len
uint32_t msg_id_len
Definition: util-decode-mime.h:144
MimeDecEntity::next
struct MimeDecEntity * next
Definition: util-decode-mime.h:146
MimeDecField::name_len
uint32_t name_len
Definition: util-decode-mime.h:108
MimeDecStack::free_nodes
MimeDecStackNode * free_nodes
Definition: util-decode-mime.h:169
MimeDecStack
struct MimeDecStack MimeDecStack
Structure holds the top of the stack along with some free reusable nodes.
MimeDecEntity::header_flags
uint32_t header_flags
Definition: util-decode-mime.h:137
MimeDecFreeEntity
void MimeDecFreeEntity(MimeDecEntity *entity)
Frees a mime entity tree.
Definition: util-decode-mime.c:182
MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
Definition: util-decode-mime.h:132
MimeDecParseState::state_flag
uint8_t state_flag
Definition: util-decode-mime.h:203
MimeDecConfig::extract_urls
int extract_urls
Definition: util-decode-mime.h:97
MIME_DEC_ERR_DATA
@ MIME_DEC_ERR_DATA
Definition: util-decode-mime.h:84
MimeDecStackNode::data
MimeDecEntity * data
Definition: util-decode-mime.h:156
MimeDecParseState::DataChunkProcessorFunc
int(* DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, struct MimeDecParseState *state)
Definition: util-decode-mime.h:210
MimeDecParseLine
int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state)
Parse a line of a MIME message and update the parser state.
Definition: util-decode-mime.c:2587
MimeDecUrl::url
uint8_t * url
Definition: util-decode-mime.h:122
MimeDecUrl
struct MimeDecUrl MimeDecUrl
This represents a URL value node in a linked list.
DataValue::value
uint8_t * value
Definition: util-decode-mime.h:178