suricata
output-json-email-common.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Tom DeCanio <td@npulsetech.com>
22  * \author Eric Leblond <eric@regit.org>
23  *
24  * Implements json common email logging portion of the engine.
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threads.h"
34 #include "threadvars.h"
35 #include "tm-threads.h"
36 #include "tm-threads-common.h"
37 
38 #include "util-print.h"
39 #include "util-unittest.h"
40 
41 #include "util-debug.h"
42 #include "app-layer-parser.h"
43 #include "output.h"
44 #include "app-layer-smtp.h"
45 #include "app-layer.h"
46 #include "util-privs.h"
47 #include "util-buffer.h"
48 #include "util-byte.h"
49 
50 #include "util-logopenfile.h"
51 #include "util-crypt.h"
52 
53 #include "output-json.h"
55 
56 #define LOG_EMAIL_DEFAULT 0
57 #define LOG_EMAIL_EXTENDED (1<<0)
58 #define LOG_EMAIL_ARRAY (1<<1) /* require array handling */
59 #define LOG_EMAIL_COMMA (1<<2) /* require array handling */
60 #define LOG_EMAIL_BODY_MD5 (1<<3)
61 #define LOG_EMAIL_SUBJECT_MD5 (1<<4)
62 
63 struct {
64  const char *config_field;
65  const char *email_field;
66  uint32_t flags;
67 } email_fields[] = {
68  { "reply_to", "reply-to", LOG_EMAIL_DEFAULT },
69  { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED },
70  { "message_id", "message-id", LOG_EMAIL_EXTENDED },
71  { "subject", "subject", LOG_EMAIL_EXTENDED },
72  { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED },
73  { "user_agent", "user-agent", LOG_EMAIL_EXTENDED },
74  { "received", "received", LOG_EMAIL_ARRAY },
75  { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT },
76  { "in_reply_to", "in-reply-to", LOG_EMAIL_DEFAULT },
77  { "references", "references", LOG_EMAIL_DEFAULT },
78  { "importance", "importance", LOG_EMAIL_DEFAULT },
79  { "priority", "priority", LOG_EMAIL_DEFAULT },
80  { "sensitivity", "sensitivity", LOG_EMAIL_DEFAULT },
81  { "organization", "organization", LOG_EMAIL_DEFAULT },
82  { "content_md5", "content-md5", LOG_EMAIL_DEFAULT },
83  { "date", "date", LOG_EMAIL_DEFAULT },
84  { NULL, NULL, LOG_EMAIL_DEFAULT},
85 };
86 
87 static inline char *SkipWhiteSpaceTill(char *p, char *savep)
88 {
89  char *sp = p;
90  if (unlikely(p == NULL)) {
91  return NULL;
92  }
93  while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) {
94  sp++;
95  }
96  return sp;
97 }
98 
99 static json_t* JsonEmailJsonArrayFromCommaList(const uint8_t *val, size_t len)
100 {
101  json_t *ajs = json_array();
102  if (likely(ajs != NULL)) {
103  char *savep = NULL;
104  char *p;
105  char *sp;
106  char *to_line = BytesToString((uint8_t *)val, len);
107  if (likely(to_line != NULL)) {
108  p = strtok_r(to_line, ",", &savep);
109  if (p == NULL) {
110  json_decref(ajs);
111  SCFree(to_line);
112  return NULL;
113  }
114  sp = SkipWhiteSpaceTill(p, savep);
115  json_array_append_new(ajs, SCJsonString(sp));
116  while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
117  sp = SkipWhiteSpaceTill(p, savep);
118  json_array_append_new(ajs, SCJsonString(sp));
119  }
120  } else {
121  json_decref(ajs);
122  return NULL;
123  }
124  SCFree(to_line);
125  }
126 
127  return ajs;
128 }
129 
130 
131 #ifdef HAVE_NSS
132 static void JsonEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx)
133 {
134  if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) {
135  MimeDecField *field;
136  MimeDecEntity *entity = tx->msg_tail;
137  if (entity == NULL) {
138  return;
139  }
140  field = MimeDecFindField(entity, "subject");
141  if (field != NULL) {
142  unsigned char md5[MD5_LENGTH];
143  char smd5[256];
144  char *value = BytesToString((uint8_t *)field->value , field->value_len);
145  if (value) {
146  size_t i,x;
147  HASH_HashBuf(HASH_AlgMD5, md5, (unsigned char *)value, strlen(value));
148  for (i = 0, x = 0; x < sizeof(md5); x++) {
149  i += snprintf(smd5 + i, 255 - i, "%02x", md5[x]);
150  }
151  json_object_set_new(js, "subject_md5", json_string(smd5));
152  SCFree(value);
153  }
154  }
155  }
156 
157  if (email_ctx->flags & LOG_EMAIL_BODY_MD5) {
158  MimeDecParseState *mime_state = tx->mime_state;
159  if (mime_state && mime_state->md5_ctx && (mime_state->state_flag == PARSE_DONE)) {
160  size_t x;
161  int i;
162  char s[256];
163  if (likely(s != NULL)) {
164  for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) {
165  i += snprintf(s + i, 255-i, "%02x", mime_state->md5[x]);
166  }
167  json_object_set_new(js, "body_md5", json_string(s));
168  }
169  }
170  }
171 }
172 #endif
173 
174 static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
175 {
176  json_t *ajs = data;
177 
178  if (ajs == NULL)
179  return 0;
180  char *value = BytesToString((uint8_t *)val, len);
181  json_array_append_new(ajs, SCJsonString(value));
182  SCFree(value);
183  return 1;
184 }
185 
186 static void JsonEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx)
187 {
188  int f = 0;
189  MimeDecField *field;
190  MimeDecEntity *entity = tx->msg_tail;
191  if (entity == NULL) {
192  return;
193  }
194 
195  while(email_fields[f].config_field) {
196  if (((email_ctx->fields & (1ULL<<f)) != 0)
197  ||
198  ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
199  ) {
201  json_t *ajs = json_array();
202  if (ajs) {
203  int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, ajs);
204  if (found > 0) {
205  json_object_set_new(js, email_fields[f].config_field, ajs);
206  } else {
207  json_decref(ajs);
208  }
209  }
210  } else if (email_fields[f].flags & LOG_EMAIL_COMMA) {
211  field = MimeDecFindField(entity, email_fields[f].email_field);
212  if (field) {
213  json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
214  if (ajs) {
215  json_object_set_new(js, email_fields[f].config_field, ajs);
216  }
217  }
218  } else {
219  field = MimeDecFindField(entity, email_fields[f].email_field);
220  if (field != NULL) {
221  char *s = BytesToString((uint8_t *)field->value,
222  (size_t)field->value_len);
223  if (likely(s != NULL)) {
224  json_object_set_new(js, email_fields[f].config_field, SCJsonString(s));
225  SCFree(s);
226  }
227  }
228  }
229 
230  }
231  f++;
232  }
233 }
234 
235 /* JSON format logging */
236 static json_t *JsonEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id)
237 {
238  SMTPState *smtp_state;
239  MimeDecParseState *mime_state;
240  MimeDecEntity *entity;
241 
242  json_t *sjs = json_object();
243  if (sjs == NULL) {
244  SCReturnPtr(NULL, "json_t");
245  }
246 
247  /* check if we have SMTP state or not */
249  switch (proto) {
250  case ALPROTO_SMTP:
251  smtp_state = (SMTPState *)state;
252  if (smtp_state == NULL) {
253  SCLogDebug("no smtp state, so no request logging");
254  json_decref(sjs);
255  SCReturnPtr(NULL, "json_t");
256  }
257  SMTPTransaction *tx = vtx;
258  mime_state = tx->mime_state;
259  entity = tx->msg_tail;
260  SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0);
261  break;
262  default:
263  /* don't know how we got here */
264  json_decref(sjs);
265  SCReturnPtr(NULL, "json_t");
266  }
267  if ((mime_state != NULL)) {
268  if (entity == NULL) {
269  json_decref(sjs);
270  SCReturnPtr(NULL, "json_t");
271  }
272 
273  json_object_set_new(sjs, "status",
274  json_string(MimeDecParseStateGetStatus(mime_state)));
275 
276  MimeDecField *field;
277 
278  /* From: */
279  field = MimeDecFindField(entity, "from");
280  if (field != NULL) {
281  char *s = BytesToString((uint8_t *)field->value,
282  (size_t)field->value_len);
283  if (likely(s != NULL)) {
284  //printf("From: \"%s\"\n", s);
285  char * sp = SkipWhiteSpaceTill(s, s + strlen(s));
286  json_object_set_new(sjs, "from", SCJsonString(sp));
287  SCFree(s);
288  }
289  }
290 
291  /* To: */
292  field = MimeDecFindField(entity, "to");
293  if (field != NULL) {
294  json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
295  if (ajs) {
296  json_object_set_new(sjs, "to", ajs);
297  }
298  }
299 
300  /* Cc: */
301  field = MimeDecFindField(entity, "cc");
302  if (field != NULL) {
303  json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
304  if (ajs) {
305  json_object_set_new(sjs, "cc", ajs);
306  }
307  }
308 
309  if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) {
310  json_decref(sjs);
311  SCReturnPtr(NULL, "json_t");
312  }
313 
314  entity = (MimeDecEntity *)mime_state->stack->top->data;
315  int attch_cnt = 0;
316  int url_cnt = 0;
317  json_t *js_attch = json_array();
318  json_t *js_url = json_array();
319  if (entity->url_list != NULL) {
320  MimeDecUrl *url;
321  for (url = entity->url_list; url != NULL; url = url->next) {
322  char *s = BytesToString((uint8_t *)url->url,
323  (size_t)url->url_len);
324  if (s != NULL) {
325  json_array_append_new(js_url,
326  SCJsonString(s));
327  SCFree(s);
328  url_cnt += 1;
329  }
330  }
331  }
332  for (entity = entity->child; entity != NULL; entity = entity->next) {
333  if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
334 
335  char *s = BytesToString((uint8_t *)entity->filename,
336  (size_t)entity->filename_len);
337  json_array_append_new(js_attch,
338  SCJsonString(s));
339  SCFree(s);
340  attch_cnt += 1;
341  }
342  if (entity->url_list != NULL) {
343  MimeDecUrl *url;
344  for (url = entity->url_list; url != NULL; url = url->next) {
345  char *s = BytesToString((uint8_t *)url->url,
346  (size_t)url->url_len);
347  if (s != NULL) {
348  json_array_append_new(js_url,
349  SCJsonString(s));
350  SCFree(s);
351  url_cnt += 1;
352  }
353  }
354  }
355  }
356  if (attch_cnt > 0) {
357  json_object_set_new(sjs, "attachment", js_attch);
358  } else {
359  json_decref(js_attch);
360  }
361  if (url_cnt > 0) {
362  json_object_set_new(sjs, "url", js_url);
363  } else {
364  json_decref(js_url);
365  }
366  SCReturnPtr(sjs, "json_t");
367  }
368 
369  json_decref(sjs);
370  SCReturnPtr(NULL, "json_t");
371 }
372 
373 /* JSON format logging */
374 TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
375 {
376  json_t *sjs = JsonEmailLogJsonData(f, state, vtx, tx_id);
377  OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx;
378  SMTPTransaction *tx = (SMTPTransaction *) vtx;
379 
380  if (sjs == NULL) {
382  }
383 
384  if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0))
385  JsonEmailLogJSONCustom(email_ctx, sjs, tx);
386 
387 #ifdef HAVE_NSS
388  JsonEmailLogJSONMd5(email_ctx, sjs, tx);
389 #endif
390 
391  if (sjs) {
392  json_object_set_new(js, "email", sjs);
394  } else
396 }
397 
398 json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id)
399 {
400  SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
401  if (smtp_state) {
402  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id);
403 
404  if (tx) {
405  return JsonEmailLogJsonData(f, smtp_state, tx, tx_id);
406  }
407  }
408 
409  return NULL;
410 }
411 
412 
414 {
415  if (conf) {
416  const char *extended = ConfNodeLookupChildValue(conf, "extended");
417 
418  if (extended != NULL) {
419  if (ConfValIsTrue(extended)) {
420  email_ctx->flags = LOG_EMAIL_EXTENDED;
421  }
422  }
423 
424  email_ctx->fields = 0;
425  ConfNode *custom;
426  if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
427  ConfNode *field;
428  TAILQ_FOREACH(field, &custom->head, next) {
429  if (field != NULL) {
430  int f = 0;
431  while(email_fields[f].config_field) {
432  if ((strcmp(email_fields[f].config_field,
433  field->val) == 0) ||
434  (strcasecmp(email_fields[f].email_field,
435  field->val) == 0))
436  {
437  email_ctx->fields |= (1ULL<<f);
438  break;
439  }
440  f++;
441  }
442  }
443  }
444  }
445 
446  email_ctx->flags = 0;
447  ConfNode *md5_conf;
448  if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) {
449  ConfNode *field;
450  TAILQ_FOREACH(field, &md5_conf->head, next) {
451  if (field != NULL) {
452  if (strcmp("body", field->val) == 0) {
453  SCLogInfo("Going to log the md5 sum of email body");
454  email_ctx->flags |= LOG_EMAIL_BODY_MD5;
455  }
456  if (strcmp("subject", field->val) == 0) {
457  SCLogInfo("Going to log the md5 sum of email subject");
458  email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5;
459  }
460  }
461  }
462  }
463  }
464  return;
465 }
#define SCLogDebug(...)
Definition: util-debug.h:335
void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx)
struct MimeDecUrl * next
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
struct HtpBodyChunk_ * next
MimeDecEntity * msg_tail
#define unlikely(expr)
Definition: util-optimize.h:35
#define LOG_EMAIL_ARRAY
uint8_t * url
#define LOG_EMAIL_DEFAULT
MimeDecStack * stack
uint32_t url_len
const char * config_field
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:1063
uint16_t AppProto
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
const char * MimeDecParseStateGetStatus(MimeDecParseState *state)
This represents the MIME Entity (or also top level message) in a child-sibling tree.
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define PARSE_DONE
uint32_t value_len
#define LOG_EMAIL_SUBJECT_MD5
json_t * JsonEmailAddMetadata(const Flow *f, uint32_t tx_id)
struct @130 email_fields[]
struct MimeDecEntity * child
#define LOG_EMAIL_BODY_MD5
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int(*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
Searches for header fields with the specified name.
json_t * SCJsonString(const char *val)
Definition: output-json.c:107
MimeDecParseState * mime_state
MimeDecUrl * url_list
#define LOG_EMAIL_EXTENDED
char * BytesToString(const uint8_t *bytes, size_t nbytes)
Turn byte array into string.
Definition: util-byte.c:41
#define CTNT_IS_ATTACHMENT
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define LOG_EMAIL_COMMA
Definition: conf.h:32
OutputJsonEmailCtx * emaillog_ctx
MimeDecStackNode * top
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:322
const char * email_field
uint16_t tx_id
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
uint32_t flags
void * FlowGetAppState(const Flow *f)
Definition: flow.c:1068
TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
Structure contains the current state of the MIME parser.
uint8_t * filename
uint8_t len
uint32_t filename_len
#define likely(expr)
Definition: util-optimize.h:32
This represents a header field name and associated value.
This represents a URL value node in a linked list.
Flow data structure.
Definition: flow.h:325
struct MimeDecEntity * next
MimeDecEntity * data