suricata
output-json-email-common.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  * \author Eric Leblond <eric@regit.org>
23  *
24  * Implements json common email logging portion of the engine.
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 #include "tm-threads-common.h"
37 
38 #include "util-print.h"
39 #include "util-unittest.h"
40 
41 #include "util-debug.h"
42 #include "app-layer-parser.h"
43 #include "output.h"
44 #include "app-layer-smtp.h"
45 #include "app-layer.h"
46 #include "util-privs.h"
47 #include "util-buffer.h"
48 #include "util-byte.h"
49 
50 #include "util-logopenfile.h"
51 #include "util-crypt.h"
52 
53 #include "output-json.h"
55 
56 #define LOG_EMAIL_DEFAULT 0
57 #define LOG_EMAIL_EXTENDED (1<<0)
58 #define LOG_EMAIL_ARRAY (1<<1) /* require array handling */
59 #define LOG_EMAIL_COMMA (1<<2) /* require array handling */
60 #define LOG_EMAIL_BODY_MD5 (1<<3)
61 #define LOG_EMAIL_SUBJECT_MD5 (1<<4)
62 
63 struct {
64  const char *config_field;
65  const char *email_field;
66  uint32_t flags;
67 } email_fields[] = {
68  { "reply_to", "reply-to", LOG_EMAIL_DEFAULT },
69  { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED },
70  { "message_id", "message-id", LOG_EMAIL_EXTENDED },
71  { "subject", "subject", LOG_EMAIL_EXTENDED },
72  { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED },
73  { "user_agent", "user-agent", LOG_EMAIL_EXTENDED },
74  { "received", "received", LOG_EMAIL_ARRAY },
75  { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT },
76  { "in_reply_to", "in-reply-to", LOG_EMAIL_DEFAULT },
77  { "references", "references", LOG_EMAIL_DEFAULT },
78  { "importance", "importance", LOG_EMAIL_DEFAULT },
79  { "priority", "priority", LOG_EMAIL_DEFAULT },
80  { "sensitivity", "sensitivity", LOG_EMAIL_DEFAULT },
81  { "organization", "organization", LOG_EMAIL_DEFAULT },
82  { "content_md5", "content-md5", LOG_EMAIL_DEFAULT },
83  { "date", "date", LOG_EMAIL_DEFAULT },
84  { NULL, NULL, LOG_EMAIL_DEFAULT},
85 };
86 
87 static inline char *SkipWhiteSpaceTill(char *p, char *savep)
88 {
89  char *sp = p;
90  if (unlikely(p == NULL)) {
91  return NULL;
92  }
93  while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) {
94  sp++;
95  }
96  return sp;
97 }
98 
99 static bool EveEmailJsonArrayFromCommaList(JsonBuilder *js, const uint8_t *val, size_t len)
100 {
101  char *savep = NULL;
102  char *p;
103  char *sp;
104  char *to_line = BytesToString((uint8_t *)val, len);
105  if (likely(to_line != NULL)) {
106  p = strtok_r(to_line, ",", &savep);
107  if (p == NULL) {
108  SCFree(to_line);
109  return false;
110  }
111  sp = SkipWhiteSpaceTill(p, savep);
112  jb_append_string(js, sp);
113  while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
114  sp = SkipWhiteSpaceTill(p, savep);
115  jb_append_string(js, sp);
116  }
117  } else {
118  return false;
119  }
120  SCFree(to_line);
121  return true;
122 }
123 
124 #ifdef HAVE_NSS
125 static void EveEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
126 {
127  if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) {
128  MimeDecField *field;
129  MimeDecEntity *entity = tx->msg_tail;
130  if (entity == NULL) {
131  return;
132  }
133  field = MimeDecFindField(entity, "subject");
134  if (field != NULL) {
135  unsigned char md5[MD5_LENGTH];
136  char smd5[256];
137  char *value = BytesToString((uint8_t *)field->value , field->value_len);
138  if (value) {
139  size_t i,x;
140  HASH_HashBuf(HASH_AlgMD5, md5, (unsigned char *)value, strlen(value));
141  for (i = 0, x = 0; x < sizeof(md5); x++) {
142  i += snprintf(smd5 + i, 255 - i, "%02x", md5[x]);
143  }
144  jb_set_string(js, "subject_md5", smd5);
145  SCFree(value);
146  }
147  }
148  }
149 
150  if (email_ctx->flags & LOG_EMAIL_BODY_MD5) {
151  MimeDecParseState *mime_state = tx->mime_state;
152  if (mime_state && mime_state->md5_ctx && (mime_state->state_flag == PARSE_DONE)) {
153  size_t x;
154  int i;
155  char s[256];
156  if (likely(s != NULL)) {
157  for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) {
158  i += snprintf(s + i, 255-i, "%02x", mime_state->md5[x]);
159  }
160  jb_set_string(js, "body_md5", s);
161  }
162  }
163  }
164 }
165 #endif
166 
167 static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
168 {
169  JsonBuilder *ajs = data;
170 
171  if (ajs == NULL)
172  return 0;
173  char *value = BytesToString((uint8_t *)val, len);
174  jb_append_string(ajs, value);
175  SCFree(value);
176  return 1;
177 }
178 
179 static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
180 {
181  int f = 0;
182  JsonBuilderMark mark = { 0, 0, 0 };
183  MimeDecField *field;
184  MimeDecEntity *entity = tx->msg_tail;
185  if (entity == NULL) {
186  return;
187  }
188 
189  while(email_fields[f].config_field) {
190  if (((email_ctx->fields & (1ULL<<f)) != 0)
191  ||
192  ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
193  ) {
195  jb_get_mark(js, &mark);
196  jb_open_array(js, email_fields[f].config_field);
197  int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, js);
198  if (found > 0) {
199  jb_close(js);
200  } else {
201  jb_restore_mark(js, &mark);
202  }
203  } else if (email_fields[f].flags & LOG_EMAIL_COMMA) {
204  field = MimeDecFindField(entity, email_fields[f].email_field);
205  if (field) {
206  jb_get_mark(js, &mark);
207  jb_open_array(js, email_fields[f].config_field);
208  if (EveEmailJsonArrayFromCommaList(js, field->value, field->value_len)) {
209  jb_close(js);
210  } else {
211  jb_restore_mark(js, &mark);
212  }
213  }
214  } else {
215  field = MimeDecFindField(entity, email_fields[f].email_field);
216  if (field != NULL) {
217  char *s = BytesToString((uint8_t *)field->value,
218  (size_t)field->value_len);
219  if (likely(s != NULL)) {
220  jb_set_string(js, email_fields[f].config_field, s);
221  SCFree(s);
222  }
223  }
224  }
225 
226  }
227  f++;
228  }
229 }
230 
231 /* JSON format logging */
232 static bool EveEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id, JsonBuilder *sjs)
233 {
234  SMTPState *smtp_state;
235  MimeDecParseState *mime_state;
236  MimeDecEntity *entity;
237  JsonBuilderMark mark = { 0, 0, 0 };
238 
239  /* check if we have SMTP state or not */
241  switch (proto) {
242  case ALPROTO_SMTP:
243  smtp_state = (SMTPState *)state;
244  if (smtp_state == NULL) {
245  SCLogDebug("no smtp state, so no request logging");
246  jb_free(sjs);
247  SCReturnPtr(NULL, "JsonBuilder");
248  }
249  SMTPTransaction *tx = vtx;
250  mime_state = tx->mime_state;
251  entity = tx->msg_tail;
252  SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0);
253  break;
254  default:
255  /* don't know how we got here */
256  SCReturnBool(false);
257  }
258  if ((mime_state != NULL)) {
259  if (entity == NULL) {
260  SCReturnBool(false);
261  }
262 
263  jb_set_string(sjs, "status", MimeDecParseStateGetStatus(mime_state));
264 
265  MimeDecField *field;
266 
267  /* From: */
268  field = MimeDecFindField(entity, "from");
269  if (field != NULL) {
270  char *s = BytesToString((uint8_t *)field->value,
271  (size_t)field->value_len);
272  if (likely(s != NULL)) {
273  //printf("From: \"%s\"\n", s);
274  char * sp = SkipWhiteSpaceTill(s, s + strlen(s));
275  jb_set_string(sjs, "from", sp);
276  SCFree(s);
277  }
278  }
279 
280  /* To: */
281  field = MimeDecFindField(entity, "to");
282  if (field != NULL) {
283  jb_get_mark(sjs, &mark);
284  jb_open_array(sjs, "to");
285  if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
286  jb_close(sjs);
287  } else {
288  jb_restore_mark(sjs, &mark);
289  }
290  }
291 
292  /* Cc: */
293  field = MimeDecFindField(entity, "cc");
294  if (field != NULL) {
295  jb_get_mark(sjs, &mark);
296  jb_open_array(sjs, "cc");
297  if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
298  jb_close(sjs);
299  } else {
300  jb_restore_mark(sjs, &mark);
301  }
302  }
303 
304  if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) {
305  SCReturnBool(false);
306  }
307 
308  entity = (MimeDecEntity *)mime_state->stack->top->data;
309  int attch_cnt = 0;
310  int url_cnt = 0;
311  JsonBuilder *js_attch = jb_new_array();
312  JsonBuilder *js_url = jb_new_array();
313  if (entity->url_list != NULL) {
314  MimeDecUrl *url;
315  for (url = entity->url_list; url != NULL; url = url->next) {
316  char *s = BytesToString((uint8_t *)url->url,
317  (size_t)url->url_len);
318  if (s != NULL) {
319  jb_append_string(js_url, s);
320  SCFree(s);
321  url_cnt += 1;
322  }
323  }
324  }
325  for (entity = entity->child; entity != NULL; entity = entity->next) {
326  if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
327 
328  char *s = BytesToString((uint8_t *)entity->filename,
329  (size_t)entity->filename_len);
330  jb_append_string(js_attch, s);
331  SCFree(s);
332  attch_cnt += 1;
333  }
334  if (entity->url_list != NULL) {
335  MimeDecUrl *url;
336  for (url = entity->url_list; url != NULL; url = url->next) {
337  char *s = BytesToString((uint8_t *)url->url,
338  (size_t)url->url_len);
339  if (s != NULL) {
340  jb_append_string(js_url, s);
341  SCFree(s);
342  url_cnt += 1;
343  }
344  }
345  }
346  }
347  if (attch_cnt > 0) {
348  jb_close(js_attch);
349  jb_set_object(sjs, "attachment", js_attch);
350  }
351  jb_free(js_attch);
352  if (url_cnt > 0) {
353  jb_close(js_url);
354  jb_set_object(sjs, "url", js_url);
355  }
356  jb_free(js_url);
357  SCReturnBool(true);
358  }
359 
360  SCReturnBool(false);
361 }
362 
363 /* JSON format logging */
364 TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
365 {
366  OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx;
367  SMTPTransaction *tx = (SMTPTransaction *) vtx;
368  JsonBuilderMark mark = { 0, 0, 0 };
369 
370  jb_get_mark(js, &mark);
371  jb_open_object(js, "email");
372  if (!EveEmailLogJsonData(f, state, vtx, tx_id, js)) {
373  jb_restore_mark(js, &mark);
375  }
376 
377  if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0))
378  EveEmailLogJSONCustom(email_ctx, js, tx);
379 
380 #ifdef HAVE_NSS
381  EveEmailLogJSONMd5(email_ctx, js, tx);
382 #endif
383 
384  jb_close(js);
386 }
387 
388 bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
389 {
390  SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
391  if (smtp_state) {
392  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id);
393  if (tx) {
394  return EveEmailLogJsonData(f, smtp_state, tx, tx_id, js);
395  }
396  }
397 
398  return false;
399 }
400 
402 {
403  if (conf) {
404  const char *extended = ConfNodeLookupChildValue(conf, "extended");
405 
406  if (extended != NULL) {
407  if (ConfValIsTrue(extended)) {
408  email_ctx->flags = LOG_EMAIL_EXTENDED;
409  }
410  }
411 
412  email_ctx->fields = 0;
413  ConfNode *custom;
414  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
415  ConfNode *field;
416  TAILQ_FOREACH(field, &custom->head, next) {
417  if (field != NULL) {
418  int f = 0;
419  while(email_fields[f].config_field) {
420  if ((strcmp(email_fields[f].config_field,
421  field->val) == 0) ||
422  (strcasecmp(email_fields[f].email_field,
423  field->val) == 0))
424  {
425  email_ctx->fields |= (1ULL<<f);
426  break;
427  }
428  f++;
429  }
430  }
431  }
432  }
433 
434  email_ctx->flags = 0;
435  ConfNode *md5_conf;
436  if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) {
437  ConfNode *field;
438  TAILQ_FOREACH(field, &md5_conf->head, next) {
439  if (field != NULL) {
440  if (strcmp("body", field->val) == 0) {
441  SCLogInfo("Going to log the md5 sum of email body");
442  email_ctx->flags |= LOG_EMAIL_BODY_MD5;
443  }
444  if (strcmp("subject", field->val) == 0) {
445  SCLogInfo("Going to log the md5 sum of email subject");
446  email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5;
447  }
448  }
449  }
450  }
451  }
452  return;
453 }
util-byte.h
MimeDecEntity::ctnt_flags
uint32_t ctnt_flags
Definition: util-decode-mime.h:138
tm-threads.h
SMTPState_
Definition: app-layer-smtp.h:109
MimeDecEntity::child
struct MimeDecEntity * child
Definition: util-decode-mime.h:147
len
uint8_t len
Definition: app-layer-dnp3.h:2
BytesToString
char * BytesToString(const uint8_t *bytes, size_t nbytes)
Turn byte array into string.
Definition: util-byte.c:41
LOG_EMAIL_COMMA
#define LOG_EMAIL_COMMA
Definition: output-json-email-common.c:59
SMTPTransaction_::msg_tail
MimeDecEntity * msg_tail
Definition: app-layer-smtp.h:80
ConfNode_::val
char * val
Definition: conf.h:34
JsonEmailLogThread_
Definition: output-json-email-common.h:34
MimeDecField::value
uint8_t * value
Definition: util-decode-mime.h:110
EveEmailLogJson
TmEcode EveEmailLogJson(JsonEmailLogThread *aft, JsonBuilder *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
Definition: output-json-email-common.c:364
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
LOG_EMAIL_EXTENDED
#define LOG_EMAIL_EXTENDED
Definition: output-json-email-common.c:57
PARSE_DONE
#define PARSE_DONE
Definition: util-decode-mime.h:75
MimeDecEntity::filename
uint8_t * filename
Definition: util-decode-mime.h:141
SMTPTransaction_::mime_state
MimeDecParseState * mime_state
Definition: app-layer-smtp.h:82
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:71
threads.h
Flow_
Flow data structure.
Definition: flow.h:343
MimeDecStack::top
MimeDecStackNode * top
Definition: util-decode-mime.h:168
MimeDecEntity::filename_len
uint32_t filename_len
Definition: util-decode-mime.h:140
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
tm-threads-common.h
util-privs.h
email_field
const char * email_field
Definition: output-json-email-common.c:65
MimeDecUrl::next
struct MimeDecUrl * next
Definition: util-decode-mime.h:125
MimeDecField
This represents a header field name and associated value.
Definition: util-decode-mime.h:106
flags
uint32_t flags
Definition: output-json-email-common.c:66
proto
uint8_t proto
Definition: decode-template.h:0
MimeDecEntity::url_list
MimeDecUrl * url_list
Definition: util-decode-mime.h:134
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:79
MimeDecUrl::url_len
uint32_t url_len
Definition: util-decode-mime.h:123
util-unittest.h
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
FlowGetAppState
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1080
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:78
SCReturnBool
#define SCReturnBool(x)
Definition: util-debug.h:318
util-debug.h
OutputJsonEmailCtx_::fields
uint64_t fields
Definition: output-json-email-common.h:30
output-json.h
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
MimeDecParseState::stack
MimeDecStack * stack
Definition: util-decode-mime.h:189
util-print.h
util-crypt.h
detect.h
pkt-var.h
output-json-email-common.h
app-layer-parser.h
OutputEmailInitConf
void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx)
Definition: output-json-email-common.c:401
Packet_
Definition: decode.h:411
SMTPTransaction_
Definition: app-layer-smtp.h:67
conf.h
TmEcode
TmEcode
Definition: tm-threads-common.h:77
SCReturnPtr
#define SCReturnPtr(x, type)
Definition: util-debug.h:316
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1037
EveEmailAddMetadata
bool EveEmailAddMetadata(const Flow *f, uint32_t tx_id, JsonBuilder *js)
Definition: output-json-email-common.c:388
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
LOG_EMAIL_DEFAULT
#define LOG_EMAIL_DEFAULT
Definition: output-json-email-common.c:56
MimeDecField::value_len
uint32_t value_len
Definition: util-decode-mime.h:109
email_fields
struct @128 email_fields[]
config_field
const char * config_field
Definition: output-json-email-common.c:64
suricata-common.h
LOG_EMAIL_ARRAY
#define LOG_EMAIL_ARRAY
Definition: output-json-email-common.c:58
threadvars.h
FlowGetAppProtocol
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1075
MimeDecParseState
Structure contains the current state of the MIME parser.
Definition: util-decode-mime.h:187
MimeDecParseStateGetStatus
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
Definition: util-decode-mime.c:2358
MimeDecFindField
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
Definition: util-decode-mime.c:339
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
MimeDecUrl
This represents a URL value node in a linked list.
Definition: util-decode-mime.h:121
util-buffer.h
MimeDecFindFieldsForEach
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
Definition: util-decode-mime.c:311
MimeDecEntity::next
struct MimeDecEntity * next
Definition: util-decode-mime.h:146
LOG_EMAIL_SUBJECT_MD5
#define LOG_EMAIL_SUBJECT_MD5
Definition: output-json-email-common.c:61
likely
#define likely(expr)
Definition: util-optimize.h:32
CTNT_IS_ATTACHMENT
#define CTNT_IS_ATTACHMENT
Definition: util-decode-mime.h:42
app-layer-smtp.h
JsonEmailLogThread_::emaillog_ctx
OutputJsonEmailCtx * emaillog_ctx
Definition: output-json-email-common.h:35
OutputJsonEmailCtx_::flags
uint32_t flags
Definition: output-json-email-common.h:29
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
MimeDecEntity
This represents the MIME Entity (or also top level message) in a child-sibling tree.
Definition: util-decode-mime.h:132
OutputJsonEmailCtx_
Definition: output-json-email-common.h:27
MimeDecParseState::state_flag
uint8_t state_flag
Definition: util-decode-mime.h:203
LOG_EMAIL_BODY_MD5
#define LOG_EMAIL_BODY_MD5
Definition: output-json-email-common.c:60
debug.h
output.h
MimeDecStackNode::data
MimeDecEntity * data
Definition: util-decode-mime.h:156
app-layer.h
MimeDecUrl::url
uint8_t * url
Definition: util-decode-mime.h:122
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843