suricata
output-json-email-common.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  * \author Eric Leblond <eric@regit.org>
23  *
24  * Implements json common email logging portion of the engine.
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 #include "tm-threads-common.h"
37 
38 #include "util-print.h"
39 #include "util-unittest.h"
40 
41 #include "util-debug.h"
42 #include "app-layer-parser.h"
43 #include "output.h"
44 #include "app-layer-smtp.h"
45 #include "app-layer.h"
46 #include "util-privs.h"
47 #include "util-buffer.h"
48 #include "util-byte.h"
49 
50 #include "util-logopenfile.h"
51 #include "util-crypt.h"
52 
53 #include "output-json.h"
55 
56 #define LOG_EMAIL_DEFAULT 0
57 #define LOG_EMAIL_EXTENDED (1<<0)
58 #define LOG_EMAIL_ARRAY (1<<1) /* require array handling */
59 #define LOG_EMAIL_COMMA (1<<2) /* require array handling */
60 #define LOG_EMAIL_BODY_MD5 (1<<3)
61 #define LOG_EMAIL_SUBJECT_MD5 (1<<4)
62 
63 struct {
64  const char *config_field;
65  const char *email_field;
66  uint32_t flags;
67 } email_fields[] = {
68  { "reply_to", "reply-to", LOG_EMAIL_DEFAULT },
69  { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED },
70  { "message_id", "message-id", LOG_EMAIL_EXTENDED },
71  { "subject", "subject", LOG_EMAIL_EXTENDED },
72  { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED },
73  { "user_agent", "user-agent", LOG_EMAIL_EXTENDED },
74  { "received", "received", LOG_EMAIL_ARRAY },
75  { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT },
76  { "in_reply_to", "in-reply-to", LOG_EMAIL_DEFAULT },
77  { "references", "references", LOG_EMAIL_DEFAULT },
78  { "importance", "importance", LOG_EMAIL_DEFAULT },
79  { "priority", "priority", LOG_EMAIL_DEFAULT },
80  { "sensitivity", "sensitivity", LOG_EMAIL_DEFAULT },
81  { "organization", "organization", LOG_EMAIL_DEFAULT },
82  { "content_md5", "content-md5", LOG_EMAIL_DEFAULT },
83  { "date", "date", LOG_EMAIL_DEFAULT },
84  { NULL, NULL, LOG_EMAIL_DEFAULT},
85 };
86 
87 static inline char *SkipWhiteSpaceTill(char *p, char *savep)
88 {
89  char *sp = p;
90  if (unlikely(p == NULL)) {
91  return NULL;
92  }
93  while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) {
94  sp++;
95  }
96  return sp;
97 }
98 
99 static bool EveEmailJsonArrayFromCommaList(JsonBuilder *js, const uint8_t *val, size_t len)
100 {
101  char *savep = NULL;
102  char *p;
103  char *sp;
104  char *to_line = BytesToString((uint8_t *)val, len);
105  if (likely(to_line != NULL)) {
106  p = strtok_r(to_line, ",", &savep);
107  if (p == NULL) {
108  SCFree(to_line);
109  return false;
110  }
111  sp = SkipWhiteSpaceTill(p, savep);
112  jb_append_string(js, sp);
113  while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
114  sp = SkipWhiteSpaceTill(p, savep);
115  jb_append_string(js, sp);
116  }
117  } else {
118  return false;
119  }
120  SCFree(to_line);
121  return true;
122 }
123 
124 static void EveEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
125 {
126  if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) {
127  MimeDecEntity *entity = tx->msg_tail;
128  if (entity == NULL) {
129  return;
130  }
131  MimeDecField *field = MimeDecFindField(entity, "subject");
132  if (field != NULL) {
133  char smd5[SC_MD5_HEX_LEN + 1];
134  SCMd5HashBufferToHex((uint8_t *)field->value, field->value_len, smd5, sizeof(smd5));
135  jb_set_string(js, "subject_md5", smd5);
136  }
137  }
138 
139  if (email_ctx->flags & LOG_EMAIL_BODY_MD5) {
140  MimeDecParseState *mime_state = tx->mime_state;
141  if (mime_state && mime_state->has_md5 && (mime_state->state_flag == PARSE_DONE)) {
142  size_t x;
143  int i;
144  char s[256];
145  for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) {
146  i += snprintf(s + i, 255 - i, "%02x", mime_state->md5[x]);
147  }
148  jb_set_string(js, "body_md5", s);
149  }
150  }
151 }
152 
153 static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
154 {
155  JsonBuilder *ajs = data;
156 
157  if (ajs == NULL)
158  return 0;
159  char *value = BytesToString((uint8_t *)val, len);
160  jb_append_string(ajs, value);
161  SCFree(value);
162  return 1;
163 }
164 
165 static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
166 {
167  int f = 0;
168  JsonBuilderMark mark = { 0, 0, 0 };
169  MimeDecField *field;
170  MimeDecEntity *entity = tx->msg_tail;
171  if (entity == NULL) {
172  return;
173  }
174 
175  while(email_fields[f].config_field) {
176  if (((email_ctx->fields & (1ULL<<f)) != 0)
177  ||
178  ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
179  ) {
181  jb_get_mark(js, &mark);
182  jb_open_array(js, email_fields[f].config_field);
183  int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, js);
184  if (found > 0) {
185  jb_close(js);
186  } else {
187  jb_restore_mark(js, &mark);
188  }
189  } else if (email_fields[f].flags & LOG_EMAIL_COMMA) {
190  field = MimeDecFindField(entity, email_fields[f].email_field);
191  if (field) {
192  jb_get_mark(js, &mark);
193  jb_open_array(js, email_fields[f].config_field);
194  if (EveEmailJsonArrayFromCommaList(js, field->value, field->value_len)) {
195  jb_close(js);
196  } else {
197  jb_restore_mark(js, &mark);
198  }
199  }
200  } else {
201  field = MimeDecFindField(entity, email_fields[f].email_field);
202  if (field != NULL) {
203  char *s = BytesToString((uint8_t *)field->value,
204  (size_t)field->value_len);
205  if (likely(s != NULL)) {
206  jb_set_string(js, email_fields[f].config_field, s);
207  SCFree(s);
208  }
209  }
210  }
211 
212  }
213  f++;
214  }
215 }
216 
217 /* JSON format logging */
218 static bool EveEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id, JsonBuilder *sjs)
219 {
220  SMTPState *smtp_state;
221  MimeDecParseState *mime_state;
222  MimeDecEntity *entity;
223  JsonBuilderMark mark = { 0, 0, 0 };
224 
225  /* check if we have SMTP state or not */
227  switch (proto) {
228  case ALPROTO_SMTP:
229  smtp_state = (SMTPState *)state;
230  if (smtp_state == NULL) {
231  SCLogDebug("no smtp state, so no request logging");
232  jb_free(sjs);
233  SCReturnPtr(NULL, "JsonBuilder");
234  }
235  SMTPTransaction *tx = vtx;
236  mime_state = tx->mime_state;
237  entity = tx->msg_tail;
238  SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0);
239  break;
240  default:
241  /* don't know how we got here */
242  SCReturnBool(false);
243  }
244  if ((mime_state != NULL)) {
245  if (entity == NULL) {
246  SCReturnBool(false);
247  }
248 
249  jb_set_string(sjs, "status", MimeDecParseStateGetStatus(mime_state));
250 
251  MimeDecField *field;
252 
253  /* From: */
254  field = MimeDecFindField(entity, "from");
255  if (field != NULL) {
256  char *s = BytesToString((uint8_t *)field->value,
257  (size_t)field->value_len);
258  if (likely(s != NULL)) {
259  //printf("From: \"%s\"\n", s);
260  char * sp = SkipWhiteSpaceTill(s, s + strlen(s));
261  jb_set_string(sjs, "from", sp);
262  SCFree(s);
263  }
264  }
265 
266  /* To: */
267  field = MimeDecFindField(entity, "to");
268  if (field != NULL) {
269  jb_get_mark(sjs, &mark);
270  jb_open_array(sjs, "to");
271  if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
272  jb_close(sjs);
273  } else {
274  jb_restore_mark(sjs, &mark);
275  }
276  }
277 
278  /* Cc: */
279  field = MimeDecFindField(entity, "cc");
280  if (field != NULL) {
281  jb_get_mark(sjs, &mark);
282  jb_open_array(sjs, "cc");
283  if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
284  jb_close(sjs);
285  } else {
286  jb_restore_mark(sjs, &mark);
287  }
288  }
289 
290  if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) {
291  SCReturnBool(false);
292  }
293 
294  entity = (MimeDecEntity *)mime_state->stack->top->data;
295  int attch_cnt = 0;
296  int url_cnt = 0;
297  JsonBuilder *js_attch = jb_new_array();
298  JsonBuilder *js_url = jb_new_array();
299  if (entity->url_list != NULL) {
300  MimeDecUrl *url;
301  for (url = entity->url_list; url != NULL; url = url->next) {
302  char *s = BytesToString((uint8_t *)url->url,
303  (size_t)url->url_len);
304  if (s != NULL) {
305  jb_append_string(js_url, s);
306  SCFree(s);
307  url_cnt += 1;
308  }
309  }
310  }
311  for (entity = entity->child; entity != NULL; entity = entity->next) {
312  if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
313 
314  char *s = BytesToString((uint8_t *)entity->filename,
315  (size_t)entity->filename_len);
316  jb_append_string(js_attch, s);
317  SCFree(s);
318  attch_cnt += 1;
319  }
320  if (entity->url_list != NULL) {
321  MimeDecUrl *url;
322  for (url = entity->url_list; url != NULL; url = url->next) {
323  char *s = BytesToString((uint8_t *)url->url,
324  (size_t)url->url_len);
325  if (s != NULL) {
326  jb_append_string(js_url, s);
327  SCFree(s);
328  url_cnt += 1;
329  }
330  }
331  }
332  }
333  if (attch_cnt > 0) {
334  jb_close(js_attch);
335  jb_set_object(sjs, "attachment", js_attch);
336  }
337  jb_free(js_attch);
338  if (url_cnt > 0) {
339  jb_close(js_url);
340  jb_set_object(sjs, "url", js_url);
341  }
342  jb_free(js_url);
343  SCReturnBool(true);
344  }
345 
346  SCReturnBool(false);
347 }
348 
349 /* JSON format logging */
350 TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
351 {
352  OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx;
353  SMTPTransaction *tx = (SMTPTransaction *) vtx;
354  JsonBuilderMark mark = { 0, 0, 0 };
355 
356  jb_get_mark(js, &mark);
357  jb_open_object(js, "email");
358  if (!EveEmailLogJsonData(f, state, vtx, tx_id, js)) {
359  jb_restore_mark(js, &mark);
361  }
362 
363  if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0))
364  EveEmailLogJSONCustom(email_ctx, js, tx);
365 
366  if (!g_disable_hashing) {
367  EveEmailLogJSONMd5(email_ctx, js, tx);
368  }
369 
370  jb_close(js);
372 }
373 
374 bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
375 {
376  SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
377  if (smtp_state) {
378  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id);
379  if (tx) {
380  return EveEmailLogJsonData(f, smtp_state, tx, tx_id, js);
381  }
382  }
383 
384  return false;
385 }
386 
388 {
389  if (conf) {
390  const char *extended = ConfNodeLookupChildValue(conf, "extended");
391 
392  if (extended != NULL) {
393  if (ConfValIsTrue(extended)) {
394  email_ctx->flags = LOG_EMAIL_EXTENDED;
395  }
396  }
397 
398  email_ctx->fields = 0;
399  ConfNode *custom;
400  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
401  ConfNode *field;
402  TAILQ_FOREACH(field, &custom->head, next) {
403  if (field != NULL) {
404  int f = 0;
405  while(email_fields[f].config_field) {
406  if ((strcmp(email_fields[f].config_field,
407  field->val) == 0) ||
408  (strcasecmp(email_fields[f].email_field,
409  field->val) == 0))
410  {
411  email_ctx->fields |= (1ULL<<f);
412  break;
413  }
414  f++;
415  }
416  }
417  }
418  }
419 
420  email_ctx->flags = 0;
421  ConfNode *md5_conf;
422  if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) {
423  ConfNode *field;
424  TAILQ_FOREACH(field, &md5_conf->head, next) {
425  if (field != NULL) {
426  if (strcmp("body", field->val) == 0) {
427  SCLogInfo("Going to log the md5 sum of email body");
428  email_ctx->flags |= LOG_EMAIL_BODY_MD5;
429  }
430  if (strcmp("subject", field->val) == 0) {
431  SCLogInfo("Going to log the md5 sum of email subject");
432  email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5;
433  }
434  }
435  }
436  }
437  }
438  return;
439 }
util-byte.h
MimeDecEntity::ctnt_flags
uint32_t ctnt_flags
Definition: util-decode-mime.h:138
tm-threads.h
SMTPState_
Definition: app-layer-smtp.h:104
MimeDecEntity::child
struct MimeDecEntity * child
Definition: util-decode-mime.h:147
len
uint8_t len
Definition: app-layer-dnp3.h:2
BytesToString
char * BytesToString(const uint8_t *bytes, size_t nbytes)
Turn byte array into string.
Definition: util-byte.c:41
LOG_EMAIL_COMMA
#define LOG_EMAIL_COMMA
Definition: output-json-email-common.c:59
SMTPTransaction_::msg_tail
MimeDecEntity * msg_tail
Definition: app-layer-smtp.h:78
ConfNode_::val
char * val
Definition: conf.h:34
JsonEmailLogThread_
Definition: output-json-email-common.h:33
MimeDecField::value
uint8_t * value
Definition: util-decode-mime.h:110
EveEmailLogJson
TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
Definition: output-json-email-common.c:350
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
LOG_EMAIL_EXTENDED
#define LOG_EMAIL_EXTENDED
Definition: output-json-email-common.c:57
PARSE_DONE
#define PARSE_DONE
Definition: util-decode-mime.h:75
MimeDecEntity::filename
uint8_t * filename
Definition: util-decode-mime.h:141
SMTPTransaction_::mime_state
MimeDecParseState * mime_state
Definition: app-layer-smtp.h:80
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
email_fields
struct @126 email_fields[]
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:77
threads.h
Flow_
Flow data structure.
Definition: flow.h:353
MimeDecStack::top
MimeDecStackNode * top
Definition: util-decode-mime.h:168
MimeDecEntity::filename_len
uint32_t filename_len
Definition: util-decode-mime.h:140
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:253
tm-threads-common.h
util-privs.h
email_field
const char * email_field
Definition: output-json-email-common.c:65
MimeDecUrl::next
struct MimeDecUrl * next
Definition: util-decode-mime.h:125
MimeDecField
This represents a header field name and associated value.
Definition: util-decode-mime.h:106
flags
uint32_t flags
Definition: output-json-email-common.c:66
proto
uint8_t proto
Definition: decode-template.h:0
MimeDecEntity::url_list
MimeDecUrl * url_list
Definition: util-decode-mime.h:134
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:81
MimeDecUrl::url_len
uint32_t url_len
Definition: util-decode-mime.h:123
util-unittest.h
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:565
FlowGetAppState
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1109
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:80
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:318
util-debug.h
OutputJsonEmailCtx_::fields
uint64_t fields
Definition: output-json-email-common.h:29
output-json.h
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
MimeDecParseState::stack
MimeDecStack * stack
Definition: util-decode-mime.h:189
util-print.h
util-crypt.h
detect.h
pkt-var.h
output-json-email-common.h
app-layer-parser.h
OutputEmailInitConf
void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx)
Definition: output-json-email-common.c:387
Packet_
Definition: decode.h:414
SMTPTransaction_
Definition: app-layer-smtp.h:68
conf.h
TmEcode
TmEcode
Definition: tm-threads-common.h:79
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1062
EveEmailAddMetadata
bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
Definition: output-json-email-common.c:374
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:814
LOG_EMAIL_DEFAULT
#define LOG_EMAIL_DEFAULT
Definition: output-json-email-common.c:56
MimeDecField::value_len
uint32_t value_len
Definition: util-decode-mime.h:109
MimeDecParseState::has_md5
bool has_md5
Definition: util-decode-mime.h:201
config_field
const char * config_field
Definition: output-json-email-common.c:64
suricata-common.h
SC_MD5_HEX_LEN
#define SC_MD5_HEX_LEN
Definition: rust.h:34
LOG_EMAIL_ARRAY
#define LOG_EMAIL_ARRAY
Definition: output-json-email-common.c:58
threadvars.h
FlowGetAppProtocol
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1104
MimeDecParseState
Structure contains the current state of the MIME parser.
Definition: util-decode-mime.h:187
MimeDecParseStateGetStatus
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
Definition: util-decode-mime.c:2352
MimeDecFindField
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
Definition: util-decode-mime.c:341
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
MimeDecUrl
This represents a URL value node in a linked list.
Definition: util-decode-mime.h:121
util-buffer.h
MimeDecFindFieldsForEach
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
Definition: util-decode-mime.c:313
MimeDecEntity::next
struct MimeDecEntity * next
Definition: util-decode-mime.h:146
LOG_EMAIL_SUBJECT_MD5
#define LOG_EMAIL_SUBJECT_MD5
Definition: output-json-email-common.c:61
likely
#define likely(expr)
Definition: util-optimize.h:32
CTNT_IS_ATTACHMENT
#define CTNT_IS_ATTACHMENT
Definition: util-decode-mime.h:42
app-layer-smtp.h
JsonEmailLogThread_::emaillog_ctx
OutputJsonEmailCtx * emaillog_ctx
Definition: output-json-email-common.h:34
OutputJsonEmailCtx_::flags
uint32_t flags
Definition: output-json-email-common.h:28
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
Definition: util-decode-mime.h:132
OutputJsonEmailCtx_
Definition: output-json-email-common.h:27
MimeDecParseState::state_flag
uint8_t state_flag
Definition: util-decode-mime.h:202
LOG_EMAIL_BODY_MD5
#define LOG_EMAIL_BODY_MD5
Definition: output-json-email-common.c:60
debug.h
output.h
MimeDecStackNode::data
MimeDecEntity * data
Definition: util-decode-mime.h:156
MimeDecParseState::md5
uint8_t md5[SC_MD5_LEN]
Definition: util-decode-mime.h:200
app-layer.h
MimeDecUrl::url
uint8_t * url
Definition: util-decode-mime.h:122
g_disable_hashing
bool g_disable_hashing
Definition: suricata.c:236
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842