suricata
output-json-email-common.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  * \author Eric Leblond <eric@regit.org>
23  *
24  * Implements json common email logging portion of the engine.
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 #include "tm-threads-common.h"
37 
38 #include "util-print.h"
39 #include "util-unittest.h"
40 
41 #include "util-debug.h"
42 #include "app-layer-parser.h"
43 #include "output.h"
44 #include "app-layer-smtp.h"
45 #include "app-layer.h"
46 #include "util-privs.h"
47 #include "util-buffer.h"
48 #include "util-byte.h"
49 
50 #include "util-logopenfile.h"
51 
52 #include "output-json.h"
54 
55 #define LOG_EMAIL_DEFAULT 0
56 #define LOG_EMAIL_EXTENDED (1<<0)
57 #define LOG_EMAIL_ARRAY (1<<1) /* require array handling */
58 #define LOG_EMAIL_COMMA (1<<2) /* require array handling */
59 #define LOG_EMAIL_BODY_MD5 (1<<3)
60 #define LOG_EMAIL_SUBJECT_MD5 (1<<4)
61 
62 struct {
63  const char *config_field;
64  const char *email_field;
65  uint32_t flags;
66 } email_fields[] = {
67  { "reply_to", "reply-to", LOG_EMAIL_DEFAULT },
68  { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED },
69  { "message_id", "message-id", LOG_EMAIL_EXTENDED },
70  { "subject", "subject", LOG_EMAIL_EXTENDED },
71  { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED },
72  { "user_agent", "user-agent", LOG_EMAIL_EXTENDED },
73  { "received", "received", LOG_EMAIL_ARRAY },
74  { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT },
75  { "in_reply_to", "in-reply-to", LOG_EMAIL_DEFAULT },
76  { "references", "references", LOG_EMAIL_DEFAULT },
77  { "importance", "importance", LOG_EMAIL_DEFAULT },
78  { "priority", "priority", LOG_EMAIL_DEFAULT },
79  { "sensitivity", "sensitivity", LOG_EMAIL_DEFAULT },
80  { "organization", "organization", LOG_EMAIL_DEFAULT },
81  { "content_md5", "content-md5", LOG_EMAIL_DEFAULT },
82  { "date", "date", LOG_EMAIL_DEFAULT },
83  { NULL, NULL, LOG_EMAIL_DEFAULT},
84 };
85 
86 static inline char *SkipWhiteSpaceTill(char *p, char *savep)
87 {
88  char *sp = p;
89  if (unlikely(p == NULL)) {
90  return NULL;
91  }
92  while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) {
93  sp++;
94  }
95  return sp;
96 }
97 
98 static bool EveEmailJsonArrayFromCommaList(JsonBuilder *js, const uint8_t *val, size_t len)
99 {
100  char *savep = NULL;
101  char *p;
102  char *sp;
103  char *to_line = BytesToString((uint8_t *)val, len);
104  if (likely(to_line != NULL)) {
105  p = strtok_r(to_line, ",", &savep);
106  if (p == NULL) {
107  SCFree(to_line);
108  return false;
109  }
110  sp = SkipWhiteSpaceTill(p, savep);
111  jb_append_string(js, sp);
112  while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
113  sp = SkipWhiteSpaceTill(p, savep);
114  jb_append_string(js, sp);
115  }
116  } else {
117  return false;
118  }
119  SCFree(to_line);
120  return true;
121 }
122 
123 static void EveEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
124 {
125  if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) {
126  MimeDecEntity *entity = tx->msg_tail;
127  if (entity == NULL) {
128  return;
129  }
130  MimeDecField *field = MimeDecFindField(entity, "subject");
131  if (field != NULL) {
132  char smd5[SC_MD5_HEX_LEN + 1];
133  SCMd5HashBufferToHex((uint8_t *)field->value, field->value_len, smd5, sizeof(smd5));
134  jb_set_string(js, "subject_md5", smd5);
135  }
136  }
137 
138  if (email_ctx->flags & LOG_EMAIL_BODY_MD5) {
139  MimeDecParseState *mime_state = tx->mime_state;
140  if (mime_state && mime_state->has_md5 && (mime_state->state_flag == PARSE_DONE)) {
141  size_t x;
142  int i;
143  char s[256];
144  for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) {
145  i += snprintf(s + i, 255 - i, "%02x", mime_state->md5[x]);
146  }
147  jb_set_string(js, "body_md5", s);
148  }
149  }
150 }
151 
152 static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
153 {
154  JsonBuilder *ajs = data;
155 
156  if (ajs == NULL)
157  return 0;
158  char *value = BytesToString((uint8_t *)val, len);
159  jb_append_string(ajs, value);
160  SCFree(value);
161  return 1;
162 }
163 
164 static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
165 {
166  int f = 0;
167  JsonBuilderMark mark = { 0, 0, 0 };
168  MimeDecField *field;
169  MimeDecEntity *entity = tx->msg_tail;
170  if (entity == NULL) {
171  return;
172  }
173 
174  while(email_fields[f].config_field) {
175  if (((email_ctx->fields & (1ULL<<f)) != 0)
176  ||
177  ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
178  ) {
180  jb_get_mark(js, &mark);
181  jb_open_array(js, email_fields[f].config_field);
182  int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, js);
183  if (found > 0) {
184  jb_close(js);
185  } else {
186  jb_restore_mark(js, &mark);
187  }
188  } else if (email_fields[f].flags & LOG_EMAIL_COMMA) {
189  field = MimeDecFindField(entity, email_fields[f].email_field);
190  if (field) {
191  jb_get_mark(js, &mark);
192  jb_open_array(js, email_fields[f].config_field);
193  if (EveEmailJsonArrayFromCommaList(js, field->value, field->value_len)) {
194  jb_close(js);
195  } else {
196  jb_restore_mark(js, &mark);
197  }
198  }
199  } else {
200  field = MimeDecFindField(entity, email_fields[f].email_field);
201  if (field != NULL) {
202  char *s = BytesToString((uint8_t *)field->value,
203  (size_t)field->value_len);
204  if (likely(s != NULL)) {
205  jb_set_string(js, email_fields[f].config_field, s);
206  SCFree(s);
207  }
208  }
209  }
210 
211  }
212  f++;
213  }
214 }
215 
216 /* JSON format logging */
217 static bool EveEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id, JsonBuilder *sjs)
218 {
219  SMTPState *smtp_state;
220  MimeDecParseState *mime_state;
221  MimeDecEntity *entity;
222  JsonBuilderMark mark = { 0, 0, 0 };
223 
224  /* check if we have SMTP state or not */
226  switch (proto) {
227  case ALPROTO_SMTP:
228  smtp_state = (SMTPState *)state;
229  if (smtp_state == NULL) {
230  SCLogDebug("no smtp state, so no request logging");
231  jb_free(sjs);
232  SCReturnPtr(NULL, "JsonBuilder");
233  }
234  SMTPTransaction *tx = vtx;
235  mime_state = tx->mime_state;
236  entity = tx->msg_tail;
237  SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0);
238  break;
239  default:
240  /* don't know how we got here */
241  SCReturnBool(false);
242  }
243  if ((mime_state != NULL)) {
244  if (entity == NULL) {
245  SCReturnBool(false);
246  }
247 
248  jb_set_string(sjs, "status", MimeDecParseStateGetStatus(mime_state));
249 
250  MimeDecField *field;
251 
252  /* From: */
253  field = MimeDecFindField(entity, "from");
254  if (field != NULL) {
255  char *s = BytesToString((uint8_t *)field->value,
256  (size_t)field->value_len);
257  if (likely(s != NULL)) {
258  //printf("From: \"%s\"\n", s);
259  char * sp = SkipWhiteSpaceTill(s, s + strlen(s));
260  jb_set_string(sjs, "from", sp);
261  SCFree(s);
262  }
263  }
264 
265  /* To: */
266  field = MimeDecFindField(entity, "to");
267  if (field != NULL) {
268  jb_get_mark(sjs, &mark);
269  jb_open_array(sjs, "to");
270  if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
271  jb_close(sjs);
272  } else {
273  jb_restore_mark(sjs, &mark);
274  }
275  }
276 
277  /* Cc: */
278  field = MimeDecFindField(entity, "cc");
279  if (field != NULL) {
280  jb_get_mark(sjs, &mark);
281  jb_open_array(sjs, "cc");
282  if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
283  jb_close(sjs);
284  } else {
285  jb_restore_mark(sjs, &mark);
286  }
287  }
288 
289  if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) {
290  SCReturnBool(false);
291  }
292 
293  entity = (MimeDecEntity *)mime_state->stack->top->data;
294  int attch_cnt = 0;
295  int url_cnt = 0;
296  JsonBuilder *js_attch = jb_new_array();
297  JsonBuilder *js_url = jb_new_array();
298  if (entity->url_list != NULL) {
299  MimeDecUrl *url;
300  for (url = entity->url_list; url != NULL; url = url->next) {
301  char *s = BytesToString((uint8_t *)url->url,
302  (size_t)url->url_len);
303  if (s != NULL) {
304  jb_append_string(js_url, s);
305  SCFree(s);
306  url_cnt += 1;
307  }
308  }
309  }
310  for (entity = entity->child; entity != NULL; entity = entity->next) {
311  if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
312 
313  char *s = BytesToString((uint8_t *)entity->filename,
314  (size_t)entity->filename_len);
315  jb_append_string(js_attch, s);
316  SCFree(s);
317  attch_cnt += 1;
318  }
319  if (entity->url_list != NULL) {
320  MimeDecUrl *url;
321  for (url = entity->url_list; url != NULL; url = url->next) {
322  char *s = BytesToString((uint8_t *)url->url,
323  (size_t)url->url_len);
324  if (s != NULL) {
325  jb_append_string(js_url, s);
326  SCFree(s);
327  url_cnt += 1;
328  }
329  }
330  }
331  }
332  if (attch_cnt > 0) {
333  jb_close(js_attch);
334  jb_set_object(sjs, "attachment", js_attch);
335  }
336  jb_free(js_attch);
337  if (url_cnt > 0) {
338  jb_close(js_url);
339  jb_set_object(sjs, "url", js_url);
340  }
341  jb_free(js_url);
342  SCReturnBool(true);
343  }
344 
345  SCReturnBool(false);
346 }
347 
348 /* JSON format logging */
349 TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
350 {
351  OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx;
352  SMTPTransaction *tx = (SMTPTransaction *) vtx;
353  JsonBuilderMark mark = { 0, 0, 0 };
354 
355  jb_get_mark(js, &mark);
356  jb_open_object(js, "email");
357  if (!EveEmailLogJsonData(f, state, vtx, tx_id, js)) {
358  jb_restore_mark(js, &mark);
360  }
361 
362  if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0))
363  EveEmailLogJSONCustom(email_ctx, js, tx);
364 
365  if (!g_disable_hashing) {
366  EveEmailLogJSONMd5(email_ctx, js, tx);
367  }
368 
369  jb_close(js);
371 }
372 
373 bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
374 {
375  SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
376  if (smtp_state) {
377  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id);
378  if (tx) {
379  return EveEmailLogJsonData(f, smtp_state, tx, tx_id, js);
380  }
381  }
382 
383  return false;
384 }
385 
387 {
388  if (conf) {
389  const char *extended = ConfNodeLookupChildValue(conf, "extended");
390 
391  if (extended != NULL) {
392  if (ConfValIsTrue(extended)) {
393  email_ctx->flags = LOG_EMAIL_EXTENDED;
394  }
395  }
396 
397  email_ctx->fields = 0;
398  ConfNode *custom;
399  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
400  ConfNode *field;
401  TAILQ_FOREACH(field, &custom->head, next) {
402  if (field != NULL) {
403  int f = 0;
404  while(email_fields[f].config_field) {
405  if ((strcmp(email_fields[f].config_field,
406  field->val) == 0) ||
407  (strcasecmp(email_fields[f].email_field,
408  field->val) == 0))
409  {
410  email_ctx->fields |= (1ULL<<f);
411  break;
412  }
413  f++;
414  }
415  }
416  }
417  }
418 
419  email_ctx->flags = 0;
420  ConfNode *md5_conf;
421  if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) {
422  ConfNode *field;
423  TAILQ_FOREACH(field, &md5_conf->head, next) {
424  if (field != NULL) {
425  if (strcmp("body", field->val) == 0) {
426  SCLogInfo("Going to log the md5 sum of email body");
427  email_ctx->flags |= LOG_EMAIL_BODY_MD5;
428  }
429  if (strcmp("subject", field->val) == 0) {
430  SCLogInfo("Going to log the md5 sum of email subject");
431  email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5;
432  }
433  }
434  }
435  }
436  }
437  return;
438 }
util-byte.h
MimeDecEntity::ctnt_flags
uint32_t ctnt_flags
Definition: util-decode-mime.h:143
tm-threads.h
SMTPState_
Definition: app-layer-smtp.h:106
MimeDecEntity::child
struct MimeDecEntity * child
Definition: util-decode-mime.h:152
len
uint8_t len
Definition: app-layer-dnp3.h:2
BytesToString
char * BytesToString(const uint8_t *bytes, size_t nbytes)
Turn byte array into string.
Definition: util-byte.c:41
LOG_EMAIL_COMMA
#define LOG_EMAIL_COMMA
Definition: output-json-email-common.c:58
SMTPTransaction_::msg_tail
MimeDecEntity * msg_tail
Definition: app-layer-smtp.h:80
ConfNode_::val
char * val
Definition: conf.h:34
JsonEmailLogThread_
Definition: output-json-email-common.h:33
MimeDecField::value
uint8_t * value
Definition: util-decode-mime.h:115
EveEmailLogJson
TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
Definition: output-json-email-common.c:349
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
LOG_EMAIL_EXTENDED
#define LOG_EMAIL_EXTENDED
Definition: output-json-email-common.c:56
PARSE_DONE
#define PARSE_DONE
Definition: util-decode-mime.h:76
MimeDecEntity::filename
uint8_t * filename
Definition: util-decode-mime.h:146
SMTPTransaction_::mime_state
MimeDecParseState * mime_state
Definition: app-layer-smtp.h:82
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:80
threads.h
Flow_
Flow data structure.
Definition: flow.h:353
MimeDecStack::top
MimeDecStackNode * top
Definition: util-decode-mime.h:173
MimeDecEntity::filename_len
uint32_t filename_len
Definition: util-decode-mime.h:145
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:253
tm-threads-common.h
util-privs.h
email_field
const char * email_field
Definition: output-json-email-common.c:64
MimeDecUrl::next
struct MimeDecUrl * next
Definition: util-decode-mime.h:130
MimeDecField
This represents a header field name and associated value.
Definition: util-decode-mime.h:111
flags
uint32_t flags
Definition: output-json-email-common.c:65
proto
uint8_t proto
Definition: decode-template.h:0
MimeDecEntity::url_list
MimeDecUrl * url_list
Definition: util-decode-mime.h:139
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:83
MimeDecUrl::url_len
uint32_t url_len
Definition: util-decode-mime.h:128
util-unittest.h
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:565
FlowGetAppState
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1125
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:82
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:318
email_fields
struct @130 email_fields[]
util-debug.h
OutputJsonEmailCtx_::fields
uint64_t fields
Definition: output-json-email-common.h:29
output-json.h
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
MimeDecParseState::stack
MimeDecStack * stack
Definition: util-decode-mime.h:194
util-print.h
detect.h
pkt-var.h
output-json-email-common.h
app-layer-parser.h
OutputEmailInitConf
void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx)
Definition: output-json-email-common.c:386
Packet_
Definition: decode.h:427
SMTPTransaction_
Definition: app-layer-smtp.h:70
conf.h
TmEcode
TmEcode
Definition: tm-threads-common.h:81
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1116
EveEmailAddMetadata
bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
Definition: output-json-email-common.c:373
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:814
LOG_EMAIL_DEFAULT
#define LOG_EMAIL_DEFAULT
Definition: output-json-email-common.c:55
MimeDecField::value_len
uint32_t value_len
Definition: util-decode-mime.h:114
MimeDecParseState::has_md5
bool has_md5
Definition: util-decode-mime.h:206
config_field
const char * config_field
Definition: output-json-email-common.c:63
suricata-common.h
SC_MD5_HEX_LEN
#define SC_MD5_HEX_LEN
Definition: rust.h:34
LOG_EMAIL_ARRAY
#define LOG_EMAIL_ARRAY
Definition: output-json-email-common.c:57
threadvars.h
FlowGetAppProtocol
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1120
MimeDecParseState
Structure contains the current state of the MIME parser.
Definition: util-decode-mime.h:192
MimeDecParseStateGetStatus
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
Definition: util-decode-mime.c:2312
MimeDecFindField
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
Definition: util-decode-mime.c:338
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
MimeDecUrl
This represents a URL value node in a linked list.
Definition: util-decode-mime.h:126
util-buffer.h
MimeDecFindFieldsForEach
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
Definition: util-decode-mime.c:310
MimeDecEntity::next
struct MimeDecEntity * next
Definition: util-decode-mime.h:151
LOG_EMAIL_SUBJECT_MD5
#define LOG_EMAIL_SUBJECT_MD5
Definition: output-json-email-common.c:60
likely
#define likely(expr)
Definition: util-optimize.h:32
CTNT_IS_ATTACHMENT
#define CTNT_IS_ATTACHMENT
Definition: util-decode-mime.h:43
app-layer-smtp.h
JsonEmailLogThread_::emaillog_ctx
OutputJsonEmailCtx * emaillog_ctx
Definition: output-json-email-common.h:34
OutputJsonEmailCtx_::flags
uint32_t flags
Definition: output-json-email-common.h:28
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
Definition: util-decode-mime.h:137
OutputJsonEmailCtx_
Definition: output-json-email-common.h:27
MimeDecParseState::state_flag
uint8_t state_flag
Definition: util-decode-mime.h:207
LOG_EMAIL_BODY_MD5
#define LOG_EMAIL_BODY_MD5
Definition: output-json-email-common.c:59
debug.h
output.h
MimeDecStackNode::data
MimeDecEntity * data
Definition: util-decode-mime.h:161
MimeDecParseState::md5
uint8_t md5[SC_MD5_LEN]
Definition: util-decode-mime.h:205
app-layer.h
MimeDecUrl::url
uint8_t * url
Definition: util-decode-mime.h:127
g_disable_hashing
bool g_disable_hashing
Definition: suricata.c:240
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842