suricata
util-decode-mime.h
Go to the documentation of this file.
1 /* Copyright (C) 2012 BAE Systems
2  * Copyright (C) 2021 Open Information Security Foundation
3  *
4  * You can copy, redistribute or modify this Program under the terms of
5  * the GNU General Public License version 2 as published by the Free
6  * Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * version 2 along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16  * 02110-1301, USA.
17  */
18 
19 /**
20  * \file
21  *
22  * \author David Abarbanel <david.abarbanel@baesystems.com>
23  *
24  */
25 
26 #ifndef MIME_DECODE_H_
27 #define MIME_DECODE_H_
28 
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <stdint.h>
32 
33 #include "util-base64.h"
34 
35 /* Content Flags */
36 #define CTNT_IS_MSG 1
37 #define CTNT_IS_ENV 2
38 #define CTNT_IS_ENCAP 4
39 #define CTNT_IS_BODYPART 8
40 #define CTNT_IS_MULTIPART 16
41 #define CTNT_IS_ATTACHMENT 32
42 #define CTNT_IS_BASE64 64
43 #define CTNT_IS_QP 128
44 #define CTNT_IS_TEXT 256
45 #define CTNT_IS_HTML 512
46 
47 /* URL Flags */
48 #define URL_IS_IP4 1
49 #define URL_IS_IP6 2
50 #define URL_IS_EXE 4
51 
52 /* Anomaly Flags */
53 #define ANOM_INVALID_BASE64 1 /* invalid base64 chars */
54 #define ANOM_INVALID_QP 2 /* invalid quoted-printable chars */
55 #define ANOM_LONG_HEADER_NAME 4 /* header is abnormally long */
56 #define ANOM_LONG_HEADER_VALUE 8 /* header value is abnormally long
57  * (includes multi-line) */
58 #define ANOM_LONG_LINE 16 /* Lines that exceed 998 octets */
59 #define ANOM_LONG_ENC_LINE 32 /* Lines that exceed 76 octets */
60 #define ANOM_MALFORMED_MSG 64 /* Misc msg format errors found */
61 #define ANOM_LONG_BOUNDARY 128 /* Boundary too long */
62 #define ANOM_LONG_FILENAME 256 /* filename truncated */
63 
64 /* Publicly exposed size constants */
65 #define DATA_CHUNK_SIZE 3072 /* Should be divisible by 3 */
66 
67 /* Mime Parser Constants */
68 #define HEADER_READY 0x01
69 #define HEADER_STARTED 0x02
70 #define HEADER_DONE 0x03
71 #define BODY_STARTED 0x04
72 #define BODY_DONE 0x05
73 #define BODY_END_BOUND 0x06
74 #define PARSE_DONE 0x07
75 #define PARSE_ERROR 0x08
76 
77 /**
78  * \brief Mime Decoder Error Codes
79  */
80 typedef enum MimeDecRetCode {
86  MIME_DEC_ERR_STATE = -4, /**< parser in error state */
89 
90 /**
91  * \brief Structure for containing configuration options
92  *
93  */
94 typedef struct MimeDecConfig {
95  bool decode_base64; /**< Decode base64 bodies */
96  bool decode_quoted_printable; /**< Decode quoted-printable bodies */
97  bool extract_urls; /**< Extract and store URLs in data structure */
98  ConfNode *extract_urls_schemes; /**< List of schemes of which to
99  extract urls */
100  bool log_url_scheme; /**< Log the scheme of extracted URLs */
101  bool body_md5; /**< Compute md5 sum of body */
102  uint32_t header_value_depth; /**< Depth of which to store header values
103  (Default is 2000) */
104 } MimeDecConfig;
105 
106 /**
107  * \brief This represents a header field name and associated value
108  */
109 typedef struct MimeDecField {
110  uint8_t *name; /**< Name of the header field */
111  uint32_t name_len; /**< Length of the name */
112  uint32_t value_len; /**< Length of the value */
113  uint8_t *value; /**< Value of the header field */
114  struct MimeDecField *next; /**< Pointer to next field */
115 } MimeDecField;
116 
117 /**
118  * \brief This represents a URL value node in a linked list
119  *
120  * Since HTML can sometimes contain a high number of URLs, this
121  * structure only features the URL host name/IP or those that are
122  * pointing to an executable file (see url_flags to determine which).
123  */
124 typedef struct MimeDecUrl {
125  uint8_t *url; /**< String representation of full or partial URL (lowercase) */
126  uint32_t url_len; /**< Length of the URL string */
127  uint32_t url_flags; /**< Flags indicating type of URL */
128  struct MimeDecUrl *next; /**< Pointer to next URL */
129 } MimeDecUrl;
130 
131 /**
132  * \brief This represents the MIME Entity (or also top level message) in a
133  * child-sibling tree
134  */
135 typedef struct MimeDecEntity {
136  MimeDecField *field_list; /**< Pointer to list of header fields */
137  MimeDecUrl *url_list; /**< Pointer to list of URLs */
138  uint32_t header_flags; /**< Flags indicating header characteristics */
139  uint32_t ctnt_flags; /**< Flags indicating type of content */
140  uint32_t anomaly_flags; /**< Flags indicating an anomaly in the message */
141  uint32_t filename_len; /**< Length of file attachment name */
142  uint8_t *filename; /**< Name of file attachment */
143  uint8_t *ctnt_type; /**< Quick access pointer to short-hand content type field */
144  uint32_t ctnt_type_len; /**< Length of content type field value */
145  uint32_t msg_id_len; /**< Quick access pointer to message Id */
146  uint8_t *msg_id; /**< Quick access pointer to message Id */
147  struct MimeDecEntity *next; /**< Pointer to list of sibling entities */
148  struct MimeDecEntity *child; /**< Pointer to list of child entities */
149 } MimeDecEntity;
150 
151 /**
152  * \brief Structure contains boundary and entity for the current node (entity)
153  * in the stack
154  *
155  */
156 typedef struct MimeDecStackNode {
157  MimeDecEntity *data; /**< Pointer to the entity data structure */
158  uint8_t *bdef; /**< Copy of boundary definition for child entity */
159  uint16_t bdef_len; /**< Boundary length for child entity */
160  bool is_encap; /**< Flag indicating entity is encapsulated in message */
161  struct MimeDecStackNode *next; /**< Pointer to next item on the stack */
163 
164 /**
165  * \brief Structure holds the top of the stack along with some free reusable nodes
166  *
167  */
168 typedef struct MimeDecStack {
169  MimeDecStackNode *top; /**< Pointer to the top of the stack */
170  MimeDecStackNode *free_nodes; /**< Pointer to the list of free nodes */
171  uint32_t free_nodes_cnt; /**< Count of free nodes in the list */
172 } MimeDecStack;
173 
174 /**
175  * \brief Structure contains a list of value and lengths for robust data processing
176  *
177  */
178 typedef struct DataValue {
179  uint8_t *value; /**< Copy of data value */
180  uint32_t value_len; /**< Length of data value */
181  struct DataValue *next; /**< Pointer to next value in the list */
182 } DataValue;
183 
184 /**
185  * \brief Structure contains the current state of the MIME parser
186  *
187  */
188 typedef struct MimeDecParseState {
189  MimeDecEntity *msg; /**< Pointer to the top-level message entity */
190  MimeDecStack *stack; /**< Pointer to the top of the entity stack */
191  uint8_t *hname; /**< Copy of the last known header name */
192  uint32_t hlen; /**< Length of the last known header name */
193  uint32_t hvlen; /**< Total length of value list */
194  DataValue *hvalue; /**< Pointer to the incomplete header value list */
195  uint8_t bvremain[B64_BLOCK]; /**< Remainder from base64-decoded line */
196  uint8_t bvr_len; /**< Length of remainder from base64-decoded line */
197  uint8_t data_chunk[DATA_CHUNK_SIZE]; /**< Buffer holding data chunk */
199  uint8_t md5[SC_MD5_LEN];
200  bool has_md5;
201  uint8_t state_flag; /**< Flag representing current state of parser */
202  uint32_t data_chunk_len; /**< Length of data chunk */
203  int found_child; /**< Flag indicating a child entity was found */
204  int body_begin; /**< Currently at beginning of body */
205  int body_end; /**< Currently at end of body */
206  uint8_t current_line_delimiter_len; /**< Length of line delimiter */
207  void *data; /**< Pointer to data specific to the caller */
208  int (*DataChunkProcessorFunc) (const uint8_t *chunk, uint32_t len,
209  struct MimeDecParseState *state); /**< Data chunk processing function callback */
211 
212 /* Config functions */
213 void MimeDecSetConfig(MimeDecConfig *config);
215 
216 /* Memory functions */
217 void MimeDecFreeEntity(MimeDecEntity *entity);
218 void MimeDecFreeField(MimeDecField *field);
219 void MimeDecFreeUrl(MimeDecUrl *url);
220 
221 /* List functions */
223 MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name);
224 int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data);
226 
227 /* Helper functions */
228 //MimeDecField * MimeDecFillField(MimeDecEntity *entity, const char *name,
229 // uint32_t nlen, const char *value, uint32_t vlen, int copy_name_value);
230 
231 /* Parser functions */
232 MimeDecParseState * MimeDecInitParser(void *data, int (*dcpfunc)(const uint8_t *chunk,
233  uint32_t len, MimeDecParseState *state));
236 int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state);
237 MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data,
238  int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state));
240 
241 /* Test functions */
242 void MimeDecRegisterTests(void);
243 
244 #endif
MIME_DEC_MORE
@ MIME_DEC_MORE
Definition: util-decode-mime.h:81
MimeDecAddEntity
MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent)
Creates and adds a child entity to the specified parent entity.
Definition: util-decode-mime.c:392
MimeDecParseState::data_chunk
uint8_t data_chunk[DATA_CHUNK_SIZE]
Definition: util-decode-mime.h:196
MimeDecEntity::ctnt_flags
uint32_t ctnt_flags
Definition: util-decode-mime.h:138
MimeDecEntity::child
struct MimeDecEntity * child
Definition: util-decode-mime.h:147
MimeDecInitParser
MimeDecParseState * MimeDecInitParser(void *data, int(*dcpfunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Init the parser by allocating memory for the state and top-level entity.
Definition: util-decode-mime.c:2407
len
uint8_t len
Definition: app-layer-dnp3.h:2
MIME_DEC_OK
@ MIME_DEC_OK
Definition: util-decode-mime.h:80
MIME_DEC_ERR_MEM
@ MIME_DEC_ERR_MEM
Definition: util-decode-mime.h:83
MimeDecFreeField
void MimeDecFreeField(MimeDecField *field)
Iteratively frees a header field entry list.
Definition: util-decode-mime.c:214
DataValue::next
struct DataValue * next
Definition: util-decode-mime.h:180
MimeDecEntity::ctnt_type_len
uint32_t ctnt_type_len
Definition: util-decode-mime.h:143
MimeDecParseState::md5_ctx
SCMd5 * md5_ctx
Definition: util-decode-mime.h:197
MimeDecConfig::header_value_depth
uint32_t header_value_depth
Definition: util-decode-mime.h:101
MimeDecField::value
uint8_t * value
Definition: util-decode-mime.h:112
MimeDecStack
Structure holds the top of the stack along with some free reusable nodes.
Definition: util-decode-mime.h:167
MimeDecParseState::data
void * data
Definition: util-decode-mime.h:206
MimeDecEntity::filename
uint8_t * filename
Definition: util-decode-mime.h:141
MimeDecParseState::bvremain
uint8_t bvremain[B64_BLOCK]
Definition: util-decode-mime.h:194
MimeDecStackNode::bdef_len
uint16_t bdef_len
Definition: util-decode-mime.h:158
MimeDecParseComplete
int MimeDecParseComplete(MimeDecParseState *state)
Called to indicate that the last message line has been processed and the parsing operation is complet...
Definition: util-decode-mime.c:2499
MimeDecParseState::current_line_delimiter_len
uint8_t current_line_delimiter_len
Definition: util-decode-mime.h:205
MimeDecEntity::field_list
MimeDecField * field_list
Definition: util-decode-mime.h:135
MimeDecStack::top
MimeDecStackNode * top
Definition: util-decode-mime.h:168
MimeDecEntity::filename_len
uint32_t filename_len
Definition: util-decode-mime.h:140
DataValue::value_len
uint32_t value_len
Definition: util-decode-mime.h:179
SC_MD5_LEN
#define SC_MD5_LEN
Definition: rust.h:30
util-base64.h
MimeDecRetCode
MimeDecRetCode
Mime Decoder Error Codes.
Definition: util-decode-mime.h:79
MimeDecParseState::found_child
int found_child
Definition: util-decode-mime.h:202
MimeDecUrl::next
struct MimeDecUrl * next
Definition: util-decode-mime.h:127
MimeDecField
This represents a header field name and associated value.
Definition: util-decode-mime.h:108
MimeDecParseState::hvlen
uint32_t hvlen
Definition: util-decode-mime.h:192
MimeDecEntity::url_list
MimeDecUrl * url_list
Definition: util-decode-mime.h:136
MIME_DEC_ERR_OVERFLOW
@ MIME_DEC_ERR_OVERFLOW
Definition: util-decode-mime.h:86
MimeDecAddField
MimeDecField * MimeDecAddField(MimeDecEntity *entity)
Creates and adds a header field entry to an entity.
Definition: util-decode-mime.c:272
MimeDecUrl::url_len
uint32_t url_len
Definition: util-decode-mime.h:125
MimeDecConfig
struct MimeDecConfig MimeDecConfig
Structure for containing configuration options.
MimeDecField::name
uint8_t * name
Definition: util-decode-mime.h:109
MimeDecParseState::body_begin
int body_begin
Definition: util-decode-mime.h:203
MimeDecParseState
struct MimeDecParseState MimeDecParseState
Structure contains the current state of the MIME parser.
MimeDecStackNode::bdef
uint8_t * bdef
Definition: util-decode-mime.h:157
MimeDecConfig::extract_urls_schemes
ConfNode * extract_urls_schemes
Definition: util-decode-mime.h:97
MimeDecEntity::ctnt_type
uint8_t * ctnt_type
Definition: util-decode-mime.h:142
MimeDecFindFieldsForEach
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
Definition: util-decode-mime.c:304
MimeDecParseState::stack
MimeDecStack * stack
Definition: util-decode-mime.h:189
DataValue
struct DataValue DataValue
Structure contains a list of value and lengths for robust data processing.
MIME_DEC_ERR_PARSE
@ MIME_DEC_ERR_PARSE
Definition: util-decode-mime.h:84
MimeDecSetConfig
void MimeDecSetConfig(MimeDecConfig *config)
Set global config policy.
Definition: util-decode-mime.c:126
MimeDecParseState::hvalue
DataValue * hvalue
Definition: util-decode-mime.h:193
MimeDecStackNode::is_encap
bool is_encap
Definition: util-decode-mime.h:159
MimeDecField::next
struct MimeDecField * next
Definition: util-decode-mime.h:113
MimeDecStackNode
Structure contains boundary and entity for the current node (entity) in the stack.
Definition: util-decode-mime.h:155
MimeDecRegisterTests
void MimeDecRegisterTests(void)
Definition: util-decode-mime.c:3576
MimeDecConfig::decode_base64
bool decode_base64
Definition: util-decode-mime.h:94
MimeDecParseState::hlen
uint32_t hlen
Definition: util-decode-mime.h:191
MimeDecParseState::data_chunk_len
uint32_t data_chunk_len
Definition: util-decode-mime.h:201
MimeDecParseState::hname
uint8_t * hname
Definition: util-decode-mime.h:190
MimeDecStackNode::next
struct MimeDecStackNode * next
Definition: util-decode-mime.h:160
MimeDecUrl::url_flags
uint32_t url_flags
Definition: util-decode-mime.h:126
MimeDecConfig::log_url_scheme
bool log_url_scheme
Definition: util-decode-mime.h:99
MimeDecConfig
Structure for containing configuration options.
Definition: util-decode-mime.h:93
MimeDecParseStateGetStatus
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
Definition: util-decode-mime.c:2334
MimeDecEntity::msg_id
uint8_t * msg_id
Definition: util-decode-mime.h:145
MimeDecStackNode
struct MimeDecStackNode MimeDecStackNode
Structure contains boundary and entity for the current node (entity) in the stack.
MimeDecParseState::msg
MimeDecEntity * msg
Definition: util-decode-mime.h:188
MimeDecFreeUrl
void MimeDecFreeUrl(MimeDecUrl *url)
Iteratively frees a URL entry list.
Definition: util-decode-mime.c:243
MimeDecField::value_len
uint32_t value_len
Definition: util-decode-mime.h:111
MimeDecParseState::has_md5
bool has_md5
Definition: util-decode-mime.h:199
MimeDecParseFullMsg
MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data, int(*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Parses an entire message when available in its entirety (wraps the line-based parsing functions)
Definition: util-decode-mime.c:2601
MimeDecEntity::anomaly_flags
uint32_t anomaly_flags
Definition: util-decode-mime.h:139
MimeDecField
struct MimeDecField MimeDecField
This represents a header field name and associated value.
MIME_DEC_ERR_STATE
@ MIME_DEC_ERR_STATE
Definition: util-decode-mime.h:85
MimeDecConfig::decode_quoted_printable
bool decode_quoted_printable
Definition: util-decode-mime.h:95
MimeDecEntity
struct MimeDecEntity MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
MimeDecConfig::extract_urls
bool extract_urls
Definition: util-decode-mime.h:96
MimeDecParseState::body_end
int body_end
Definition: util-decode-mime.h:204
MimeDecParseState
Structure contains the current state of the MIME parser.
Definition: util-decode-mime.h:187
MimeDecGetConfig
MimeDecConfig * MimeDecGetConfig(void)
Get global config policy.
Definition: util-decode-mime.c:145
MimeDecDeInitParser
void MimeDecDeInitParser(MimeDecParseState *state)
De-Init parser by freeing up any residual memory.
Definition: util-decode-mime.c:2463
DATA_CHUNK_SIZE
#define DATA_CHUNK_SIZE
Definition: util-decode-mime.h:64
ConfNode_
Definition: conf.h:32
MimeDecUrl
This represents a URL value node in a linked list.
Definition: util-decode-mime.h:123
MimeDecStack::free_nodes_cnt
uint32_t free_nodes_cnt
Definition: util-decode-mime.h:170
MimeDecFindField
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
Definition: util-decode-mime.c:332
MimeDecParseState::bvr_len
uint8_t bvr_len
Definition: util-decode-mime.h:195
DataValue
Structure contains a list of value and lengths for robust data processing.
Definition: util-decode-mime.h:177
B64_BLOCK
#define B64_BLOCK
Definition: util-base64.h:32
MimeDecEntity::msg_id_len
uint32_t msg_id_len
Definition: util-decode-mime.h:144
MimeDecEntity::next
struct MimeDecEntity * next
Definition: util-decode-mime.h:146
MimeDecField::name_len
uint32_t name_len
Definition: util-decode-mime.h:110
MimeDecStack::free_nodes
MimeDecStackNode * free_nodes
Definition: util-decode-mime.h:169
MimeDecStack
struct MimeDecStack MimeDecStack
Structure holds the top of the stack along with some free reusable nodes.
MimeDecEntity::header_flags
uint32_t header_flags
Definition: util-decode-mime.h:137
MimeDecFreeEntity
void MimeDecFreeEntity(MimeDecEntity *entity)
Frees a mime entity tree.
Definition: util-decode-mime.c:175
MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
Definition: util-decode-mime.h:134
MimeDecParseState::state_flag
uint8_t state_flag
Definition: util-decode-mime.h:200
MimeDecConfig::body_md5
bool body_md5
Definition: util-decode-mime.h:100
SCMd5
struct SCMd5 SCMd5
Definition: util-file.h:42
MIME_DEC_ERR_DATA
@ MIME_DEC_ERR_DATA
Definition: util-decode-mime.h:82
MimeDecStackNode::data
MimeDecEntity * data
Definition: util-decode-mime.h:156
MimeDecParseState::md5
uint8_t md5[SC_MD5_LEN]
Definition: util-decode-mime.h:198
MimeDecParseState::DataChunkProcessorFunc
int(* DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, struct MimeDecParseState *state)
Definition: util-decode-mime.h:207
MimeDecParseLine
int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state)
Parse a line of a MIME message and update the parser state.
Definition: util-decode-mime.c:2567
MimeDecUrl::url
uint8_t * url
Definition: util-decode-mime.h:124
MimeDecUrl
struct MimeDecUrl MimeDecUrl
This represents a URL value node in a linked list.
DataValue::value
uint8_t * value
Definition: util-decode-mime.h:178