suricata
output-json-email-common.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  * \author Eric Leblond <eric@regit.org>
23  *
24  * Implements json common email logging portion of the engine.
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 #include "tm-threads-common.h"
37 
38 #include "util-print.h"
39 #include "util-unittest.h"
40 
41 #include "util-debug.h"
42 #include "app-layer-parser.h"
43 #include "output.h"
44 #include "app-layer-smtp.h"
45 #include "app-layer.h"
46 #include "util-privs.h"
47 #include "util-buffer.h"
48 #include "util-byte.h"
49 
50 #include "util-logopenfile.h"
51 #include "util-crypt.h"
52 
53 #include "output-json.h"
55 
56 #ifdef HAVE_LIBJANSSON
57 
58 #define LOG_EMAIL_DEFAULT 0
59 #define LOG_EMAIL_EXTENDED (1<<0)
60 #define LOG_EMAIL_ARRAY (1<<1) /* require array handling */
61 #define LOG_EMAIL_COMMA (1<<2) /* require array handling */
62 #define LOG_EMAIL_BODY_MD5 (1<<3)
63 #define LOG_EMAIL_SUBJECT_MD5 (1<<4)
64 
65 struct {
66  const char *config_field;
67  const char *email_field;
68  uint32_t flags;
69 } email_fields[] = {
70  { "reply_to", "reply-to", LOG_EMAIL_DEFAULT },
71  { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED },
72  { "message_id", "message-id", LOG_EMAIL_EXTENDED },
73  { "subject", "subject", LOG_EMAIL_EXTENDED },
74  { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED },
75  { "user_agent", "user-agent", LOG_EMAIL_EXTENDED },
76  { "received", "received", LOG_EMAIL_ARRAY },
77  { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT },
78  { "in_reply_to", "in-reply-to", LOG_EMAIL_DEFAULT },
79  { "references", "references", LOG_EMAIL_DEFAULT },
80  { "importance", "importance", LOG_EMAIL_DEFAULT },
81  { "priority", "priority", LOG_EMAIL_DEFAULT },
82  { "sensitivity", "sensitivity", LOG_EMAIL_DEFAULT },
83  { "organization", "organization", LOG_EMAIL_DEFAULT },
84  { "content_md5", "content-md5", LOG_EMAIL_DEFAULT },
85  { "date", "date", LOG_EMAIL_DEFAULT },
86  { NULL, NULL, LOG_EMAIL_DEFAULT},
87 };
88 
89 static inline char *SkipWhiteSpaceTill(char *p, char *savep)
90 {
91  char *sp = p;
92  if (unlikely(p == NULL)) {
93  return NULL;
94  }
95  while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) {
96  sp++;
97  }
98  return sp;
99 }
100 
101 static json_t* JsonEmailJsonArrayFromCommaList(const uint8_t *val, size_t len)
102 {
103  json_t *ajs = json_array();
104  if (likely(ajs != NULL)) {
105  char *savep = NULL;
106  char *p;
107  char *sp;
108  char *to_line = BytesToString((uint8_t *)val, len);
109  if (likely(to_line != NULL)) {
110  p = strtok_r(to_line, ",", &savep);
111  if (p == NULL) {
112  json_decref(ajs);
113  SCFree(to_line);
114  return NULL;
115  }
116  sp = SkipWhiteSpaceTill(p, savep);
117  json_array_append_new(ajs, SCJsonString(sp));
118  while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
119  sp = SkipWhiteSpaceTill(p, savep);
120  json_array_append_new(ajs, SCJsonString(sp));
121  }
122  } else {
123  json_decref(ajs);
124  return NULL;
125  }
126  SCFree(to_line);
127  }
128 
129  return ajs;
130 }
131 
132 
133 #ifdef HAVE_NSS
134 static void JsonEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx)
135 {
136  if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) {
137  MimeDecField *field;
138  MimeDecEntity *entity = tx->msg_tail;
139  if (entity == NULL) {
140  return;
141  }
142  field = MimeDecFindField(entity, "subject");
143  if (field != NULL) {
144  unsigned char md5[MD5_LENGTH];
145  char smd5[256];
146  char *value = BytesToString((uint8_t *)field->value , field->value_len);
147  if (value) {
148  size_t i,x;
149  HASH_HashBuf(HASH_AlgMD5, md5, (unsigned char *)value, strlen(value));
150  for (i = 0, x = 0; x < sizeof(md5); x++) {
151  i += snprintf(smd5 + i, 255 - i, "%02x", md5[x]);
152  }
153  json_object_set_new(js, "subject_md5", json_string(smd5));
154  SCFree(value);
155  }
156  }
157  }
158 
159  if (email_ctx->flags & LOG_EMAIL_BODY_MD5) {
160  MimeDecParseState *mime_state = tx->mime_state;
161  if (mime_state && mime_state->md5_ctx && (mime_state->state_flag == PARSE_DONE)) {
162  size_t x;
163  int i;
164  char s[256];
165  if (likely(s != NULL)) {
166  for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) {
167  i += snprintf(s + i, 255-i, "%02x", mime_state->md5[x]);
168  }
169  json_object_set_new(js, "body_md5", json_string(s));
170  }
171  }
172  }
173 }
174 #endif
175 
176 static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
177 {
178  json_t *ajs = data;
179 
180  if (ajs == NULL)
181  return 0;
182  char *value = BytesToString((uint8_t *)val, len);
183  json_array_append_new(ajs, SCJsonString(value));
184  SCFree(value);
185  return 1;
186 }
187 
188 static void JsonEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx)
189 {
190  int f = 0;
191  MimeDecField *field;
192  MimeDecEntity *entity = tx->msg_tail;
193  if (entity == NULL) {
194  return;
195  }
196 
197  while(email_fields[f].config_field) {
198  if (((email_ctx->fields & (1ULL<<f)) != 0)
199  ||
200  ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
201  ) {
202  if (email_fields[f].flags & LOG_EMAIL_ARRAY) {
203  json_t *ajs = json_array();
204  if (ajs) {
205  int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, ajs);
206  if (found > 0) {
207  json_object_set_new(js, email_fields[f].config_field, ajs);
208  } else {
209  json_decref(ajs);
210  }
211  }
212  } else if (email_fields[f].flags & LOG_EMAIL_COMMA) {
213  field = MimeDecFindField(entity, email_fields[f].email_field);
214  if (field) {
215  json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
216  if (ajs) {
217  json_object_set_new(js, email_fields[f].config_field, ajs);
218  }
219  }
220  } else {
221  field = MimeDecFindField(entity, email_fields[f].email_field);
222  if (field != NULL) {
223  char *s = BytesToString((uint8_t *)field->value,
224  (size_t)field->value_len);
225  if (likely(s != NULL)) {
226  json_object_set_new(js, email_fields[f].config_field, SCJsonString(s));
227  SCFree(s);
228  }
229  }
230  }
231 
232  }
233  f++;
234  }
235 }
236 
237 /* JSON format logging */
238 static json_t *JsonEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id)
239 {
240  SMTPState *smtp_state;
241  MimeDecParseState *mime_state;
242  MimeDecEntity *entity;
243 
244  json_t *sjs = json_object();
245  if (sjs == NULL) {
246  SCReturnPtr(NULL, "json_t");
247  }
248 
249  /* check if we have SMTP state or not */
251  switch (proto) {
252  case ALPROTO_SMTP:
253  smtp_state = (SMTPState *)state;
254  if (smtp_state == NULL) {
255  SCLogDebug("no smtp state, so no request logging");
256  json_decref(sjs);
257  SCReturnPtr(NULL, "json_t");
258  }
259  SMTPTransaction *tx = vtx;
260  mime_state = tx->mime_state;
261  entity = tx->msg_tail;
262  SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0);
263  break;
264  default:
265  /* don't know how we got here */
266  json_decref(sjs);
267  SCReturnPtr(NULL, "json_t");
268  }
269  if ((mime_state != NULL)) {
270  if (entity == NULL) {
271  json_decref(sjs);
272  SCReturnPtr(NULL, "json_t");
273  }
274 
275  json_object_set_new(sjs, "status",
276  json_string(MimeDecParseStateGetStatus(mime_state)));
277 
278  MimeDecField *field;
279 
280  /* From: */
281  field = MimeDecFindField(entity, "from");
282  if (field != NULL) {
283  char *s = BytesToString((uint8_t *)field->value,
284  (size_t)field->value_len);
285  if (likely(s != NULL)) {
286  //printf("From: \"%s\"\n", s);
287  char * sp = SkipWhiteSpaceTill(s, s + strlen(s));
288  json_object_set_new(sjs, "from", SCJsonString(sp));
289  SCFree(s);
290  }
291  }
292 
293  /* To: */
294  field = MimeDecFindField(entity, "to");
295  if (field != NULL) {
296  json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
297  if (ajs) {
298  json_object_set_new(sjs, "to", ajs);
299  }
300  }
301 
302  /* Cc: */
303  field = MimeDecFindField(entity, "cc");
304  if (field != NULL) {
305  json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
306  if (ajs) {
307  json_object_set_new(sjs, "cc", ajs);
308  }
309  }
310 
311  if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) {
312  json_decref(sjs);
313  SCReturnPtr(NULL, "json_t");
314  }
315 
316  entity = (MimeDecEntity *)mime_state->stack->top->data;
317  int attch_cnt = 0;
318  int url_cnt = 0;
319  json_t *js_attch = json_array();
320  json_t *js_url = json_array();
321  if (entity->url_list != NULL) {
322  MimeDecUrl *url;
323  for (url = entity->url_list; url != NULL; url = url->next) {
324  char *s = BytesToString((uint8_t *)url->url,
325  (size_t)url->url_len);
326  if (s != NULL) {
327  json_array_append_new(js_url,
328  SCJsonString(s));
329  SCFree(s);
330  url_cnt += 1;
331  }
332  }
333  }
334  for (entity = entity->child; entity != NULL; entity = entity->next) {
335  if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
336 
337  char *s = BytesToString((uint8_t *)entity->filename,
338  (size_t)entity->filename_len);
339  json_array_append_new(js_attch,
340  SCJsonString(s));
341  SCFree(s);
342  attch_cnt += 1;
343  }
344  if (entity->url_list != NULL) {
345  MimeDecUrl *url;
346  for (url = entity->url_list; url != NULL; url = url->next) {
347  char *s = BytesToString((uint8_t *)url->url,
348  (size_t)url->url_len);
349  if (s != NULL) {
350  json_array_append_new(js_url,
351  SCJsonString(s));
352  SCFree(s);
353  url_cnt += 1;
354  }
355  }
356  }
357  }
358  if (attch_cnt > 0) {
359  json_object_set_new(sjs, "attachment", js_attch);
360  } else {
361  json_decref(js_attch);
362  }
363  if (url_cnt > 0) {
364  json_object_set_new(sjs, "url", js_url);
365  } else {
366  json_decref(js_url);
367  }
368  SCReturnPtr(sjs, "json_t");
369  }
370 
371  json_decref(sjs);
372  SCReturnPtr(NULL, "json_t");
373 }
374 
375 /* JSON format logging */
376 TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
377 {
378  json_t *sjs = JsonEmailLogJsonData(f, state, vtx, tx_id);
379  OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx;
380  SMTPTransaction *tx = (SMTPTransaction *) vtx;
381 
382  if (sjs == NULL) {
384  }
385 
386  if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0))
387  JsonEmailLogJSONCustom(email_ctx, sjs, tx);
388 
389 #ifdef HAVE_NSS
390  JsonEmailLogJSONMd5(email_ctx, sjs, tx);
391 #endif
392 
393  if (sjs) {
394  json_object_set_new(js, "email", sjs);
396  } else
398 }
399 
400 json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id)
401 {
402  SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
403  if (smtp_state) {
404  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id);
405 
406  if (tx) {
407  return JsonEmailLogJsonData(f, smtp_state, tx, tx_id);
408  }
409  }
410 
411  return NULL;
412 }
413 
414 
415 void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx)
416 {
417  if (conf) {
418  const char *extended = ConfNodeLookupChildValue(conf, "extended");
419 
420  if (extended != NULL) {
421  if (ConfValIsTrue(extended)) {
422  email_ctx->flags = LOG_EMAIL_EXTENDED;
423  }
424  }
425 
426  email_ctx->fields = 0;
427  ConfNode *custom;
428  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
429  ConfNode *field;
430  TAILQ_FOREACH(field, &custom->head, next) {
431  if (field != NULL) {
432  int f = 0;
433  while(email_fields[f].config_field) {
434  if ((strcmp(email_fields[f].config_field,
435  field->val) == 0) ||
436  (strcasecmp(email_fields[f].email_field,
437  field->val) == 0))
438  {
439  email_ctx->fields |= (1ULL<<f);
440  break;
441  }
442  f++;
443  }
444  }
445  }
446  }
447 
448  email_ctx->flags = 0;
449  ConfNode *md5_conf;
450  if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) {
451  ConfNode *field;
452  TAILQ_FOREACH(field, &md5_conf->head, next) {
453  if (field != NULL) {
454  if (strcmp("body", field->val) == 0) {
455  SCLogInfo("Going to log the md5 sum of email body");
456  email_ctx->flags |= LOG_EMAIL_BODY_MD5;
457  }
458  if (strcmp("subject", field->val) == 0) {
459  SCLogInfo("Going to log the md5 sum of email subject");
460  email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5;
461  }
462  }
463  }
464  }
465  }
466  return;
467 }
468 
469 
470 #endif
uint16_t flags
#define SCLogDebug(...)
Definition: util-debug.h:335
struct MimeDecUrl * next
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
struct HtpBodyChunk_ * next
MimeDecEntity * msg_tail
#define unlikely(expr)
Definition: util-optimize.h:35
uint8_t * url
MimeDecStack * stack
uint32_t url_len
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1060
uint16_t AppProto
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
This represents the MIME Entity (or also top level message) in a child-sibling tree.
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define PARSE_DONE
uint32_t value_len
struct MimeDecEntity * child
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
MimeDecParseState * mime_state
MimeDecUrl * url_list
char * BytesToString(const uint8_t *bytes, size_t nbytes)
Turn byte array into string.
Definition: util-byte.c:41
#define CTNT_IS_ATTACHMENT
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
Definition: conf.h:32
MimeDecStackNode * top
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:322
uint16_t tx_id
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1065
Structure contains the current state of the MIME parser.
uint8_t * filename
uint8_t len
uint32_t filename_len
#define likely(expr)
Definition: util-optimize.h:32
This represents a header field name and associated value.
This represents a URL value node in a linked list.
Flow data structure.
Definition: flow.h:325
uint32_t flags
Definition: flow.h:379
struct MimeDecEntity * next
MimeDecEntity * data