suricata
util-decode-mime.h
Go to the documentation of this file.
1 /* Copyright (C) 2012 BAE Systems
2  * Copyright (C) 2021 Open Information Security Foundation
3  *
4  * You can copy, redistribute or modify this Program under the terms of
5  * the GNU General Public License version 2 as published by the Free
6  * Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * version 2 along with this program; if not, write to the Free Software
15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16  * 02110-1301, USA.
17  */
18 
19 /**
20  * \file
21  *
22  * \author David Abarbanel <david.abarbanel@baesystems.com>
23  *
24  */
25 
26 #ifndef MIME_DECODE_H_
27 #define MIME_DECODE_H_
28 
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <stdint.h>
32 
33 #include "suricata.h"
34 #include "util-base64.h"
35 #include "util-debug.h"
36 
37 /* Content Flags */
38 #define CTNT_IS_MSG 1
39 #define CTNT_IS_ENV 2
40 #define CTNT_IS_ENCAP 4
41 #define CTNT_IS_BODYPART 8
42 #define CTNT_IS_MULTIPART 16
43 #define CTNT_IS_ATTACHMENT 32
44 #define CTNT_IS_BASE64 64
45 #define CTNT_IS_QP 128
46 #define CTNT_IS_TEXT 256
47 #define CTNT_IS_HTML 512
48 
49 /* URL Flags */
50 #define URL_IS_IP4 1
51 #define URL_IS_IP6 2
52 #define URL_IS_EXE 4
53 
54 /* Anomaly Flags */
55 #define ANOM_INVALID_BASE64 1 /* invalid base64 chars */
56 #define ANOM_INVALID_QP 2 /* invalid quoted-printable chars */
57 #define ANOM_LONG_HEADER_NAME 4 /* header is abnormally long */
58 #define ANOM_LONG_HEADER_VALUE 8 /* header value is abnormally long
59  * (includes multi-line) */
60 #define ANOM_LONG_LINE 16 /* Lines that exceed 998 octets */
61 #define ANOM_LONG_ENC_LINE 32 /* Lines that exceed 76 octets */
62 #define ANOM_MALFORMED_MSG 64 /* Misc msg format errors found */
63 #define ANOM_LONG_BOUNDARY 128 /* Boundary too long */
64 #define ANOM_LONG_FILENAME 256 /* filename truncated */
65 
66 /* Publicly exposed size constants */
67 #define DATA_CHUNK_SIZE 3072 /* Should be divisible by 3 */
68 #define LINEREM_SIZE 256
69 
70 /* Mime Parser Constants */
71 #define HEADER_READY 0x01
72 #define HEADER_STARTED 0x02
73 #define HEADER_DONE 0x03
74 #define BODY_STARTED 0x04
75 #define BODY_DONE 0x05
76 #define BODY_END_BOUND 0x06
77 #define PARSE_DONE 0x07
78 #define PARSE_ERROR 0x08
79 
80 /**
81  * \brief Mime Decoder Error Codes
82  */
83 typedef enum MimeDecRetCode {
89  MIME_DEC_ERR_STATE = -4, /**< parser in error state */
92 
93 /**
94  * \brief Structure for containing configuration options
95  *
96  */
97 typedef struct MimeDecConfig {
98  bool decode_base64; /**< Decode base64 bodies */
99  bool decode_quoted_printable; /**< Decode quoted-printable bodies */
100  bool extract_urls; /**< Extract and store URLs in data structure */
101  ConfNode *extract_urls_schemes; /**< List of schemes of which to
102  extract urls */
103  bool log_url_scheme; /**< Log the scheme of extracted URLs */
104  bool body_md5; /**< Compute md5 sum of body */
105  uint32_t header_value_depth; /**< Depth of which to store header values
106  (Default is 2000) */
107 } MimeDecConfig;
108 
109 /**
110  * \brief This represents a header field name and associated value
111  */
112 typedef struct MimeDecField {
113  uint8_t *name; /**< Name of the header field */
114  uint32_t name_len; /**< Length of the name */
115  uint32_t value_len; /**< Length of the value */
116  uint8_t *value; /**< Value of the header field */
117  struct MimeDecField *next; /**< Pointer to next field */
118 } MimeDecField;
119 
120 /**
121  * \brief This represents a URL value node in a linked list
122  *
123  * Since HTML can sometimes contain a high number of URLs, this
124  * structure only features the URL host name/IP or those that are
125  * pointing to an executable file (see url_flags to determine which).
126  */
127 typedef struct MimeDecUrl {
128  uint8_t *url; /**< String representation of full or partial URL (lowercase) */
129  uint32_t url_len; /**< Length of the URL string */
130  uint32_t url_flags; /**< Flags indicating type of URL */
131  struct MimeDecUrl *next; /**< Pointer to next URL */
132 } MimeDecUrl;
133 
134 /**
135  * \brief This represents the MIME Entity (or also top level message) in a
136  * child-sibling tree
137  */
138 typedef struct MimeDecEntity {
139  MimeDecField *field_list; /**< Pointer to list of header fields */
140  MimeDecUrl *url_list; /**< Pointer to list of URLs */
141  uint32_t body_len; /**< Length of body (prior to any decoding) */
142  uint32_t decoded_body_len; /**< Length of body after decoding */
143  uint32_t header_flags; /**< Flags indicating header characteristics */
144  uint32_t ctnt_flags; /**< Flags indicating type of content */
145  uint32_t anomaly_flags; /**< Flags indicating an anomaly in the message */
146  uint32_t filename_len; /**< Length of file attachment name */
147  uint8_t *filename; /**< Name of file attachment */
148  uint8_t *ctnt_type; /**< Quick access pointer to short-hand content type field */
149  uint32_t ctnt_type_len; /**< Length of content type field value */
150  uint32_t msg_id_len; /**< Quick access pointer to message Id */
151  uint8_t *msg_id; /**< Quick access pointer to message Id */
152  struct MimeDecEntity *next; /**< Pointer to list of sibling entities */
153  struct MimeDecEntity *child; /**< Pointer to list of child entities */
154 } MimeDecEntity;
155 
156 /**
157  * \brief Structure contains boundary and entity for the current node (entity)
158  * in the stack
159  *
160  */
161 typedef struct MimeDecStackNode {
162  MimeDecEntity *data; /**< Pointer to the entity data structure */
163  uint8_t *bdef; /**< Copy of boundary definition for child entity */
164  uint16_t bdef_len; /**< Boundary length for child entity */
165  bool is_encap; /**< Flag indicating entity is encapsulated in message */
166  struct MimeDecStackNode *next; /**< Pointer to next item on the stack */
168 
169 /**
170  * \brief Structure holds the top of the stack along with some free reusable nodes
171  *
172  */
173 typedef struct MimeDecStack {
174  MimeDecStackNode *top; /**< Pointer to the top of the stack */
175  MimeDecStackNode *free_nodes; /**< Pointer to the list of free nodes */
176  uint32_t free_nodes_cnt; /**< Count of free nodes in the list */
177 } MimeDecStack;
178 
179 /**
180  * \brief Structure contains a list of value and lengths for robust data processing
181  *
182  */
183 typedef struct DataValue {
184  uint8_t *value; /**< Copy of data value */
185  uint32_t value_len; /**< Length of data value */
186  struct DataValue *next; /**< Pointer to next value in the list */
187 } DataValue;
188 
189 /**
190  * \brief Structure contains the current state of the MIME parser
191  *
192  */
193 typedef struct MimeDecParseState {
194  MimeDecEntity *msg; /**< Pointer to the top-level message entity */
195  MimeDecStack *stack; /**< Pointer to the top of the entity stack */
196  uint8_t *hname; /**< Copy of the last known header name */
197  uint32_t hlen; /**< Length of the last known header name */
198  uint32_t hvlen; /**< Total length of value list */
199  DataValue *hvalue; /**< Pointer to the incomplete header value list */
200  uint8_t linerem[LINEREM_SIZE]; /**< Remainder from previous line (for URL extraction) */
201  uint16_t linerem_len; /**< Length of remainder from previous line */
202  uint8_t bvremain[B64_BLOCK]; /**< Remainder from base64-decoded line */
203  uint8_t bvr_len; /**< Length of remainder from base64-decoded line */
204  uint8_t data_chunk[DATA_CHUNK_SIZE]; /**< Buffer holding data chunk */
206  uint8_t md5[SC_MD5_LEN];
207  bool has_md5;
208  uint8_t state_flag; /**< Flag representing current state of parser */
209  uint32_t data_chunk_len; /**< Length of data chunk */
210  int found_child; /**< Flag indicating a child entity was found */
211  int body_begin; /**< Currently at beginning of body */
212  int body_end; /**< Currently at end of body */
213  uint8_t current_line_delimiter_len; /**< Length of line delimiter */
214  void *data; /**< Pointer to data specific to the caller */
215  int (*DataChunkProcessorFunc) (const uint8_t *chunk, uint32_t len,
216  struct MimeDecParseState *state); /**< Data chunk processing function callback */
218 
219 /* Config functions */
220 void MimeDecSetConfig(MimeDecConfig *config);
222 
223 /* Memory functions */
224 void MimeDecFreeEntity(MimeDecEntity *entity);
225 void MimeDecFreeField(MimeDecField *field);
226 void MimeDecFreeUrl(MimeDecUrl *url);
227 
228 /* List functions */
230 MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name);
231 int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data);
233 
234 /* Helper functions */
235 //MimeDecField * MimeDecFillField(MimeDecEntity *entity, const char *name,
236 // uint32_t nlen, const char *value, uint32_t vlen, int copy_name_value);
237 
238 /* Parser functions */
239 MimeDecParseState * MimeDecInitParser(void *data, int (*dcpfunc)(const uint8_t *chunk,
240  uint32_t len, MimeDecParseState *state));
243 int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state);
244 MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data,
245  int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state));
247 
248 /* Test functions */
249 void MimeDecRegisterTests(void);
250 
251 #endif
MIME_DEC_MORE
@ MIME_DEC_MORE
Definition: util-decode-mime.h:84
MimeDecAddEntity
MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent)
Creates and adds a child entity to the specified parent entity.
Definition: util-decode-mime.c:398
MimeDecParseState::data_chunk
uint8_t data_chunk[DATA_CHUNK_SIZE]
Definition: util-decode-mime.h:203
MimeDecEntity::ctnt_flags
uint32_t ctnt_flags
Definition: util-decode-mime.h:143
MimeDecParseState::linerem
uint8_t linerem[LINEREM_SIZE]
Definition: util-decode-mime.h:199
MimeDecEntity::child
struct MimeDecEntity * child
Definition: util-decode-mime.h:152
MimeDecInitParser
MimeDecParseState * MimeDecInitParser(void *data, int(*dcpfunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Init the parser by allocating memory for the state and top-level entity.
Definition: util-decode-mime.c:2385
len
uint8_t len
Definition: app-layer-dnp3.h:2
MIME_DEC_OK
@ MIME_DEC_OK
Definition: util-decode-mime.h:83
MIME_DEC_ERR_MEM
@ MIME_DEC_ERR_MEM
Definition: util-decode-mime.h:86
MimeDecFreeField
void MimeDecFreeField(MimeDecField *field)
Iteratively frees a header field entry list.
Definition: util-decode-mime.c:220
DataValue::next
struct DataValue * next
Definition: util-decode-mime.h:185
MimeDecEntity::ctnt_type_len
uint32_t ctnt_type_len
Definition: util-decode-mime.h:148
MimeDecParseState::md5_ctx
SCMd5 * md5_ctx
Definition: util-decode-mime.h:204
MimeDecConfig::header_value_depth
uint32_t header_value_depth
Definition: util-decode-mime.h:104
MimeDecField::value
uint8_t * value
Definition: util-decode-mime.h:115
MimeDecStack
Structure holds the top of the stack along with some free reusable nodes.
Definition: util-decode-mime.h:172
MimeDecParseState::data
void * data
Definition: util-decode-mime.h:213
MimeDecEntity::filename
uint8_t * filename
Definition: util-decode-mime.h:146
MimeDecParseState::bvremain
uint8_t bvremain[B64_BLOCK]
Definition: util-decode-mime.h:201
MimeDecStackNode::bdef_len
uint16_t bdef_len
Definition: util-decode-mime.h:163
MimeDecParseComplete
int MimeDecParseComplete(MimeDecParseState *state)
Called to indicate that the last message line has been processed and the parsing operation is complet...
Definition: util-decode-mime.c:2477
MimeDecParseState::current_line_delimiter_len
uint8_t current_line_delimiter_len
Definition: util-decode-mime.h:212
MimeDecEntity::field_list
MimeDecField * field_list
Definition: util-decode-mime.h:138
MimeDecStack::top
MimeDecStackNode * top
Definition: util-decode-mime.h:173
MimeDecEntity::filename_len
uint32_t filename_len
Definition: util-decode-mime.h:145
DataValue::value_len
uint32_t value_len
Definition: util-decode-mime.h:184
SC_MD5_LEN
#define SC_MD5_LEN
Definition: rust.h:29
util-base64.h
MimeDecRetCode
MimeDecRetCode
Mime Decoder Error Codes.
Definition: util-decode-mime.h:82
MimeDecParseState::found_child
int found_child
Definition: util-decode-mime.h:209
MimeDecUrl::next
struct MimeDecUrl * next
Definition: util-decode-mime.h:130
MimeDecField
This represents a header field name and associated value.
Definition: util-decode-mime.h:111
MimeDecParseState::hvlen
uint32_t hvlen
Definition: util-decode-mime.h:197
MimeDecParseState::linerem_len
uint16_t linerem_len
Definition: util-decode-mime.h:200
MimeDecEntity::url_list
MimeDecUrl * url_list
Definition: util-decode-mime.h:139
MIME_DEC_ERR_OVERFLOW
@ MIME_DEC_ERR_OVERFLOW
Definition: util-decode-mime.h:89
MimeDecAddField
MimeDecField * MimeDecAddField(MimeDecEntity *entity)
Creates and adds a header field entry to an entity.
Definition: util-decode-mime.c:278
MimeDecUrl::url_len
uint32_t url_len
Definition: util-decode-mime.h:128
MimeDecConfig
struct MimeDecConfig MimeDecConfig
Structure for containing configuration options.
MimeDecField::name
uint8_t * name
Definition: util-decode-mime.h:112
MimeDecParseState::body_begin
int body_begin
Definition: util-decode-mime.h:210
MimeDecParseState
struct MimeDecParseState MimeDecParseState
Structure contains the current state of the MIME parser.
util-debug.h
MimeDecStackNode::bdef
uint8_t * bdef
Definition: util-decode-mime.h:162
MimeDecConfig::extract_urls_schemes
ConfNode * extract_urls_schemes
Definition: util-decode-mime.h:100
MimeDecEntity::ctnt_type
uint8_t * ctnt_type
Definition: util-decode-mime.h:147
MimeDecFindFieldsForEach
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
Definition: util-decode-mime.c:310
MimeDecParseState::stack
MimeDecStack * stack
Definition: util-decode-mime.h:194
MimeDecEntity::decoded_body_len
uint32_t decoded_body_len
Definition: util-decode-mime.h:141
DataValue
struct DataValue DataValue
Structure contains a list of value and lengths for robust data processing.
MIME_DEC_ERR_PARSE
@ MIME_DEC_ERR_PARSE
Definition: util-decode-mime.h:87
MimeDecSetConfig
void MimeDecSetConfig(MimeDecConfig *config)
Set global config policy.
Definition: util-decode-mime.c:132
MimeDecParseState::hvalue
DataValue * hvalue
Definition: util-decode-mime.h:198
MimeDecStackNode::is_encap
bool is_encap
Definition: util-decode-mime.h:164
MimeDecField::next
struct MimeDecField * next
Definition: util-decode-mime.h:116
LINEREM_SIZE
#define LINEREM_SIZE
Definition: util-decode-mime.h:67
MimeDecStackNode
Structure contains boundary and entity for the current node (entity) in the stack.
Definition: util-decode-mime.h:160
MimeDecRegisterTests
void MimeDecRegisterTests(void)
Definition: util-decode-mime.c:3389
MimeDecConfig::decode_base64
bool decode_base64
Definition: util-decode-mime.h:97
MimeDecParseState::hlen
uint32_t hlen
Definition: util-decode-mime.h:196
MimeDecParseState::data_chunk_len
uint32_t data_chunk_len
Definition: util-decode-mime.h:208
MimeDecParseState::hname
uint8_t * hname
Definition: util-decode-mime.h:195
MimeDecStackNode::next
struct MimeDecStackNode * next
Definition: util-decode-mime.h:165
MimeDecUrl::url_flags
uint32_t url_flags
Definition: util-decode-mime.h:129
MimeDecConfig::log_url_scheme
bool log_url_scheme
Definition: util-decode-mime.h:102
MimeDecConfig
Structure for containing configuration options.
Definition: util-decode-mime.h:96
MimeDecParseStateGetStatus
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
Definition: util-decode-mime.c:2312
MimeDecEntity::msg_id
uint8_t * msg_id
Definition: util-decode-mime.h:150
MimeDecStackNode
struct MimeDecStackNode MimeDecStackNode
Structure contains boundary and entity for the current node (entity) in the stack.
MimeDecParseState::msg
MimeDecEntity * msg
Definition: util-decode-mime.h:193
MimeDecFreeUrl
void MimeDecFreeUrl(MimeDecUrl *url)
Iteratively frees a URL entry list.
Definition: util-decode-mime.c:249
MimeDecField::value_len
uint32_t value_len
Definition: util-decode-mime.h:114
MimeDecEntity::body_len
uint32_t body_len
Definition: util-decode-mime.h:140
MimeDecParseState::has_md5
bool has_md5
Definition: util-decode-mime.h:206
MimeDecParseFullMsg
MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data, int(*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state))
Parses an entire message when available in its entirety (wraps the line-based parsing functions)
Definition: util-decode-mime.c:2579
MimeDecEntity::anomaly_flags
uint32_t anomaly_flags
Definition: util-decode-mime.h:144
MimeDecField
struct MimeDecField MimeDecField
This represents a header field name and associated value.
MIME_DEC_ERR_STATE
@ MIME_DEC_ERR_STATE
Definition: util-decode-mime.h:88
MimeDecConfig::decode_quoted_printable
bool decode_quoted_printable
Definition: util-decode-mime.h:98
MimeDecEntity
struct MimeDecEntity MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
MimeDecConfig::extract_urls
bool extract_urls
Definition: util-decode-mime.h:99
MimeDecParseState::body_end
int body_end
Definition: util-decode-mime.h:211
MimeDecParseState
Structure contains the current state of the MIME parser.
Definition: util-decode-mime.h:192
MimeDecGetConfig
MimeDecConfig * MimeDecGetConfig(void)
Get global config policy.
Definition: util-decode-mime.c:151
MimeDecDeInitParser
void MimeDecDeInitParser(MimeDecParseState *state)
De-Init parser by freeing up any residual memory.
Definition: util-decode-mime.c:2441
DATA_CHUNK_SIZE
#define DATA_CHUNK_SIZE
Definition: util-decode-mime.h:66
ConfNode_
Definition: conf.h:32
MimeDecUrl
This represents a URL value node in a linked list.
Definition: util-decode-mime.h:126
MimeDecStack::free_nodes_cnt
uint32_t free_nodes_cnt
Definition: util-decode-mime.h:175
MimeDecFindField
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
Definition: util-decode-mime.c:338
MimeDecParseState::bvr_len
uint8_t bvr_len
Definition: util-decode-mime.h:202
DataValue
Structure contains a list of value and lengths for robust data processing.
Definition: util-decode-mime.h:182
suricata.h
B64_BLOCK
#define B64_BLOCK
Definition: util-base64.h:49
MimeDecEntity::msg_id_len
uint32_t msg_id_len
Definition: util-decode-mime.h:149
MimeDecEntity::next
struct MimeDecEntity * next
Definition: util-decode-mime.h:151
MimeDecField::name_len
uint32_t name_len
Definition: util-decode-mime.h:113
MimeDecStack::free_nodes
MimeDecStackNode * free_nodes
Definition: util-decode-mime.h:174
MimeDecStack
struct MimeDecStack MimeDecStack
Structure holds the top of the stack along with some free reusable nodes.
MimeDecEntity::header_flags
uint32_t header_flags
Definition: util-decode-mime.h:142
MimeDecFreeEntity
void MimeDecFreeEntity(MimeDecEntity *entity)
Frees a mime entity tree.
Definition: util-decode-mime.c:181
MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
Definition: util-decode-mime.h:137
MimeDecParseState::state_flag
uint8_t state_flag
Definition: util-decode-mime.h:207
MimeDecConfig::body_md5
bool body_md5
Definition: util-decode-mime.h:103
SCMd5
struct SCMd5 SCMd5
Definition: util-file.h:42
MIME_DEC_ERR_DATA
@ MIME_DEC_ERR_DATA
Definition: util-decode-mime.h:85
MimeDecStackNode::data
MimeDecEntity * data
Definition: util-decode-mime.h:161
MimeDecParseState::md5
uint8_t md5[SC_MD5_LEN]
Definition: util-decode-mime.h:205
MimeDecParseState::DataChunkProcessorFunc
int(* DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, struct MimeDecParseState *state)
Definition: util-decode-mime.h:214
MimeDecParseLine
int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state)
Parse a line of a MIME message and update the parser state.
Definition: util-decode-mime.c:2545
MimeDecUrl::url
uint8_t * url
Definition: util-decode-mime.h:127
MimeDecUrl
struct MimeDecUrl MimeDecUrl
This represents a URL value node in a linked list.
DataValue::value
uint8_t * value
Definition: util-decode-mime.h:183