suricata
log-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2013 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Log files we track.
24  *
25  */
26 
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "pkt-var.h"
31 #include "conf.h"
32 
33 #include "threadvars.h"
34 #include "tm-modules.h"
35 
36 #include "threads.h"
37 
38 #include "app-layer-parser.h"
39 
40 #include "detect-filemagic.h"
41 
42 #include "stream.h"
43 
44 #include "util-print.h"
45 #include "util-unittest.h"
46 #include "util-privs.h"
47 #include "util-debug.h"
48 #include "util-atomic.h"
49 #include "util-file.h"
50 #include "util-time.h"
51 
52 #include "output.h"
53 
54 #include "log-file.h"
55 #include "util-logopenfile.h"
56 
57 #include "app-layer-htp.h"
58 #include "app-layer-smtp.h"
59 #include "util-decode-mime.h"
60 #include "util-memcmp.h"
61 #include "stream-tcp-reassemble.h"
62 
63 #define MODULE_NAME "LogFileLog"
64 
65 #define DEFAULT_LOG_FILENAME "files-json.log"
66 
67 typedef struct LogFileLogThread_ {
69  /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
70  uint32_t file_cnt;
72 
73 static void LogFileMetaGetUri(FILE *fp, const Packet *p, const File *ff)
74 {
75  HtpState *htp_state = (HtpState *)p->flow->alstate;
76  if (htp_state != NULL) {
77  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
78  if (tx != NULL) {
79  HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
80  if (tx_ud != NULL) {
81  if (tx_ud->request_uri_normalized != NULL) {
82  PrintRawJsonFp(fp,
83  bstr_ptr(tx_ud->request_uri_normalized),
84  bstr_len(tx_ud->request_uri_normalized));
85  return;
86  }
87  }
88  }
89  }
90 
91  fprintf(fp, "<unknown>");
92 }
93 
94 static void LogFileMetaGetHost(FILE *fp, const Packet *p, const File *ff)
95 {
96  HtpState *htp_state = (HtpState *)p->flow->alstate;
97  if (htp_state != NULL) {
98  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
99  if (tx != NULL && tx->request_hostname != NULL) {
100  PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(tx->request_hostname),
101  bstr_len(tx->request_hostname));
102  return;
103  }
104  }
105 
106  fprintf(fp, "<unknown>");
107 }
108 
109 static void LogFileMetaGetReferer(FILE *fp, const Packet *p, const File *ff)
110 {
111  HtpState *htp_state = (HtpState *)p->flow->alstate;
112  if (htp_state != NULL) {
113  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
114  if (tx != NULL) {
115  htp_header_t *h = NULL;
116  h = (htp_header_t *)htp_table_get_c(tx->request_headers,
117  "Referer");
118  if (h != NULL) {
119  PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
120  bstr_len(h->value));
121  return;
122  }
123  }
124  }
125 
126  fprintf(fp, "<unknown>");
127 }
128 
129 static void LogFileMetaGetUserAgent(FILE *fp, const Packet *p, const File *ff)
130 {
131  HtpState *htp_state = (HtpState *)p->flow->alstate;
132  if (htp_state != NULL) {
133  htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
134  if (tx != NULL) {
135  htp_header_t *h = NULL;
136  h = (htp_header_t *)htp_table_get_c(tx->request_headers,
137  "User-Agent");
138  if (h != NULL) {
139  PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
140  bstr_len(h->value));
141  return;
142  }
143  }
144  }
145 
146  fprintf(fp, "<unknown>");
147 }
148 
149 static void LogFileMetaGetSmtp(FILE *fp, const Packet *p, const File *ff)
150 {
151  SMTPState *state = (SMTPState *) p->flow->alstate;
152  if (state != NULL) {
153  SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, state, ff->txid);
154  if (tx == NULL || tx->msg_tail == NULL)
155  return;
156 
157  /* Message Id */
158  if (tx->msg_tail->msg_id != NULL) {
159 
160  fprintf(fp, "\"message-id\": \"");
161  PrintRawJsonFp(fp, (uint8_t *) tx->msg_tail->msg_id,
162  (int) tx->msg_tail->msg_id_len);
163  fprintf(fp, "\", ");
164  }
165 
166  /* Sender */
167  MimeDecField *field = MimeDecFindField(tx->msg_tail, "from");
168  if (field != NULL) {
169  fprintf(fp, "\"sender\": \"");
170  PrintRawJsonFp(fp, (uint8_t *) field->value,
171  (int) field->value_len);
172  fprintf(fp, "\", ");
173  }
174  }
175 }
176 
177 /**
178  * \internal
179  * \brief Write meta data on a single line json record
180  */
181 static void LogFileWriteJsonRecord(LogFileLogThread *aft, const Packet *p, const File *ff, int ipver)
182 {
183  SCMutexLock(&aft->file_ctx->fp_mutex);
184 
185  /* As writes are done via the LogFileCtx, check for rotation here. */
186  if (aft->file_ctx->rotation_flag) {
187  aft->file_ctx->rotation_flag = 0;
188  if (SCConfLogReopen(aft->file_ctx) != 0) {
189  SCLogWarning(SC_ERR_FOPEN, "Failed to re-open log file. "
190  "Logging for this module will be disabled.");
191  }
192  }
193 
194  /* Bail early if no file pointer to write to (in the unlikely
195  * event file rotation failed. */
196  if (aft->file_ctx->fp == NULL) {
198  return;
199  }
200 
201  FILE *fp = aft->file_ctx->fp;
202  char timebuf[64];
203  AppProto alproto = FlowGetAppProtocol(p->flow);
204 
205  CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
206 
207  fprintf(fp, "{ ");
208 
209  if (ff->file_store_id > 0)
210  fprintf(fp, "\"id\": %u, ", ff->file_store_id);
211 
212  fprintf(fp, "\"timestamp\": \"");
213  PrintRawJsonFp(fp, (uint8_t *)timebuf, strlen(timebuf));
214  fprintf(fp, "\", ");
215  if (p->pcap_cnt > 0) {
216  fprintf(fp, "\"pcap_pkt_num\": %"PRIu64", ", p->pcap_cnt);
217  }
218 
219  fprintf(fp, "\"ipver\": %d, ", ipver == AF_INET ? 4 : 6);
220 
221  char srcip[46], dstip[46];
222  Port sp, dp;
223  switch (ipver) {
224  case AF_INET:
225  PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip));
226  PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip));
227  break;
228  case AF_INET6:
229  PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip));
230  PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip));
231  break;
232  default:
233  strlcpy(srcip, "<unknown>", sizeof(srcip));
234  strlcpy(dstip, "<unknown>", sizeof(dstip));
235  break;
236  }
237  sp = p->sp;
238  dp = p->dp;
239 
240  fprintf(fp, "\"srcip\": \"%s\", ", srcip);
241  fprintf(fp, "\"dstip\": \"%s\", ", dstip);
242  fprintf(fp, "\"protocol\": %" PRIu32 ", ", p->proto);
243  if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) {
244  fprintf(fp, "\"sp\": %" PRIu16 ", ", sp);
245  fprintf(fp, "\"dp\": %" PRIu16 ", ", dp);
246  }
247 
248  if (alproto == ALPROTO_HTTP) {
249  fprintf(fp, "\"http_uri\": \"");
250  LogFileMetaGetUri(fp, p, ff);
251  fprintf(fp, "\", ");
252 
253  fprintf(fp, "\"http_host\": \"");
254  LogFileMetaGetHost(fp, p, ff);
255  fprintf(fp, "\", ");
256 
257  fprintf(fp, "\"http_referer\": \"");
258  LogFileMetaGetReferer(fp, p, ff);
259  fprintf(fp, "\", ");
260 
261  fprintf(fp, "\"http_user_agent\": \"");
262  LogFileMetaGetUserAgent(fp, p, ff);
263  fprintf(fp, "\", ");
264  } else if (p->flow->alproto == ALPROTO_SMTP) {
265  /* Only applicable to SMTP */
266  LogFileMetaGetSmtp(fp, p, ff);
267  }
268 
269  fprintf(fp, "\"filename\": \"");
270  PrintRawJsonFp(fp, ff->name, ff->name_len);
271  fprintf(fp, "\", ");
272 #ifdef HAVE_MAGIC
273  fprintf(fp, "\"magic\": \"");
274  if (ff->magic) {
275  PrintRawJsonFp(fp, (uint8_t *)ff->magic, strlen(ff->magic));
276  } else {
277  fprintf(fp, "unknown");
278  }
279  fprintf(fp, "\", ");
280 #endif
281  switch (ff->state) {
282  case FILE_STATE_CLOSED:
283  fprintf(fp, "\"state\": \"CLOSED\", ");
284 #ifdef HAVE_NSS
285  if (ff->flags & FILE_MD5) {
286  fprintf(fp, "\"md5\": \"");
287  size_t x;
288  for (x = 0; x < sizeof(ff->md5); x++) {
289  fprintf(fp, "%02x", ff->md5[x]);
290  }
291  fprintf(fp, "\", ");
292  }
293  if (ff->flags & FILE_SHA1) {
294  fprintf(fp, "\"sha1\": \"");
295  size_t x;
296  for (x = 0; x < sizeof(ff->sha1); x++) {
297  fprintf(fp, "%02x", ff->sha1[x]);
298  }
299  fprintf(fp, "\", ");
300  }
301  if (ff->flags & FILE_SHA256) {
302  fprintf(fp, "\"sha256\": \"");
303  size_t x;
304  for (x = 0; x < sizeof(ff->sha256); x++) {
305  fprintf(fp, "%02x", ff->sha256[x]);
306  }
307  fprintf(fp, "\", ");
308  }
309 #endif
310  break;
312  fprintf(fp, "\"state\": \"TRUNCATED\", ");
313  break;
314  case FILE_STATE_ERROR:
315  fprintf(fp, "\"state\": \"ERROR\", ");
316  break;
317  default:
318  fprintf(fp, "\"state\": \"UNKNOWN\", ");
319  break;
320  }
321  fprintf(fp, "\"stored\": %s, ", ff->flags & FILE_STORED ? "true" : "false");
322  fprintf(fp, "\"size\": %"PRIu64" ", FileTrackedSize(ff));
323  fprintf(fp, "}\n");
324  fflush(fp);
326 }
327 
328 static int LogFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
329  const File *ff, uint8_t dir)
330 {
331  SCEnter();
332  LogFileLogThread *aft = (LogFileLogThread *)thread_data;
333  int ipver = -1;
334 
335  if (PKT_IS_IPV4(p)) {
336  ipver = AF_INET;
337  } else if (PKT_IS_IPV6(p)) {
338  ipver = AF_INET6;
339  } else {
340  return 0;
341  }
342 
343  BUG_ON(ff->flags & FILE_LOGGED);
344 
345  SCLogDebug("ff %p", ff);
346 
347  LogFileWriteJsonRecord(aft, p, ff, ipver);
348 
349  aft->file_cnt++;
350  return 0;
351 }
352 
353 static TmEcode LogFileLogThreadInit(ThreadVars *t, const void *initdata, void **data)
354 {
356  if (unlikely(aft == NULL))
357  return TM_ECODE_FAILED;
358  memset(aft, 0, sizeof(LogFileLogThread));
359 
360  if (initdata == NULL)
361  {
362  SCLogDebug("Error getting context for LogFile. \"initdata\" argument NULL");
363  SCFree(aft);
364  return TM_ECODE_FAILED;
365  }
366 
367  /* Use the Ouptut Context (file pointer and mutex) */
368  aft->file_ctx = ((OutputCtx *)initdata)->data;
369 
370  *data = (void *)aft;
371  return TM_ECODE_OK;
372 }
373 
374 static TmEcode LogFileLogThreadDeinit(ThreadVars *t, void *data)
375 {
376  LogFileLogThread *aft = (LogFileLogThread *)data;
377  if (aft == NULL) {
378  return TM_ECODE_OK;
379  }
380 
381  /* clear memory */
382  memset(aft, 0, sizeof(LogFileLogThread));
383 
384  SCFree(aft);
385  return TM_ECODE_OK;
386 }
387 
388 static void LogFileLogExitPrintStats(ThreadVars *tv, void *data)
389 {
390  LogFileLogThread *aft = (LogFileLogThread *)data;
391  if (aft == NULL) {
392  return;
393  }
394 
395  SCLogInfo("(%s) Files logged: %" PRIu32 "", tv->name, aft->file_cnt);
396 }
397 
398 /**
399  * \internal
400  *
401  * \brief deinit the log ctx and write out the waldo
402  *
403  * \param output_ctx output context to deinit
404  */
405 static void LogFileLogDeInitCtx(OutputCtx *output_ctx)
406 {
407  LogFileCtx *logfile_ctx = (LogFileCtx *)output_ctx->data;
408  LogFileFreeCtx(logfile_ctx);
409  free(output_ctx);
410 }
411 
412 /** \brief Create a new http log LogFileCtx.
413  * \param conf Pointer to ConfNode containing this loggers configuration.
414  * \return NULL if failure, LogFileCtx* to the file_ctx if succesful
415  * */
416 static OutputInitResult LogFileLogInitCtx(ConfNode *conf)
417 {
418  OutputInitResult result = { NULL, false };
419  LogFileCtx *logfile_ctx = LogFileNewCtx();
420  if (logfile_ctx == NULL) {
421  SCLogDebug("Could not create new LogFileCtx");
422  return result;
423  }
424 
425  if (SCConfLogOpenGeneric(conf, logfile_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
426  LogFileFreeCtx(logfile_ctx);
427  return result;
428  }
429 
430  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
431  if (unlikely(output_ctx == NULL))
432  return result;
433 
434  output_ctx->data = logfile_ctx;
435  output_ctx->DeInit = LogFileLogDeInitCtx;
436 
437  const char *force_filestore = ConfNodeLookupChildValue(conf, "force-filestore");
438  if (force_filestore != NULL && ConfValIsTrue(force_filestore)) {
440  SCLogInfo("forcing filestore of all files");
441  }
442 
443  const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
444  if (force_magic != NULL && ConfValIsTrue(force_magic)) {
446  SCLogInfo("forcing magic lookup for logged files");
447  }
448 
449  FileForceHashParseCfg(conf);
451 
452  result.ctx = output_ctx;
453  result.ok = true;
454  SCReturnCT(result, "OutputInitResult");
455 }
456 
458 {
460  LogFileLogInitCtx, LogFileLogger, LogFileLogThreadInit,
461  LogFileLogThreadDeinit, LogFileLogExitPrintStats);
462 
463  SCLogDebug("registered");
464 }
#define GET_IPV4_SRC_ADDR_PTR(p)
Definition: decode.h:213
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:444
void PrintRawJsonFp(FILE *fp, uint8_t *buf, uint32_t buflen)
Definition: util-print.c:71
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
MimeDecEntity * msg_tail
#define BUG_ON(x)
struct LogFileLogThread_ LogFileLogThread
#define unlikely(expr)
Definition: util-optimize.h:35
#define GET_IPV4_DST_ADDR_PTR(p)
Definition: decode.h:214
Port sp
Definition: decode.h:414
#define FILE_SHA256
Definition: util-file.h:43
void LogFileLogRegister(void)
Definition: log-file.c:457
bstr * request_uri_normalized
Port dp
Definition: decode.h:422
#define PKT_IS_IPV6(p)
Definition: decode.h:251
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name)
Searches for a header field with the specified name.
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
uint64_t pcap_cnt
Definition: decode.h:566
AppProto FlowGetAppProtocol(const Flow *f)
Definition: flow.c:992
uint16_t AppProto
#define MODULE_NAME
Definition: log-file.c:63
#define PKT_IS_IPV4(p)
Definition: decode.h:250
#define DEFAULT_LOG_FILENAME
Definition: log-file.c:65
#define SCReturnCT(x, type)
Definition: util-debug.h:351
void FileForceTrackingEnable(void)
Definition: util-file.c:150
void * alstate
Definition: flow.h:436
#define FILE_LOGGED
Definition: util-file.h:44
#define SCCalloc(nm, a)
Definition: util-mem.h:205
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define SCMutexUnlock(mut)
uint16_t flags
Definition: util-file.h:65
uint32_t value_len
uint8_t proto
Definition: decode.h:429
#define GET_IPV6_DST_ADDR(p)
Definition: decode.h:219
LogFileCtx * file_ctx
Definition: log-file.c:68
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileForceMagicEnable(void)
Definition: util-file.c:91
uint64_t txid
Definition: util-file.h:69
#define SCEnter(...)
Definition: util-debug.h:337
SCMutex fp_mutex
uint16_t name_len
Definition: util-file.h:66
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
LogFileCtx * LogFileNewCtx(void)
LogFileNewCtx() Get a new LogFileCtx.
int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *log_ctx, const char *default_filename, int rotate)
open a generic output "log file", which may be a regular file or a socket
const char * PrintInet(int af, const void *src, char *dst, socklen_t size)
Definition: util-print.c:267
void OutputRegisterFileModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FileLogger FileLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file output module.
Definition: output.c:418
uint16_t Port
Definition: decode.h:234
#define FILE_MD5
Definition: util-file.h:39
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:174
#define GET_IPV6_SRC_ADDR(p)
Definition: decode.h:218
int LogFileFreeCtx(LogFileCtx *lf_ctx)
LogFileFreeCtx() Destroy a LogFileCtx (Close the file and free memory)
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:236
#define PKT_IS_TCP(p)
Definition: decode.h:252
void * data
Definition: tm-modules.h:81
#define SCMutexLock(mut)
FileState state
Definition: util-file.h:67
uint32_t file_cnt
Definition: log-file.c:70
int SCConfLogReopen(LogFileCtx *log_ctx)
Reopen a regular log file with the side-affect of truncating it.
uint32_t file_store_id
Definition: util-file.h:72
char name[16]
Definition: threadvars.h:59
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
#define PKT_IS_UDP(p)
Definition: decode.h:253
Per thread variable structure.
Definition: threadvars.h:57
struct timeval ts
Definition: decode.h:450
AppProto alproto
application level protocol
Definition: flow.h:407
This represents a header field name and associated value.
uint8_t * name
Definition: util-file.h:75
void CreateTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:237