suricata
source-pcap-file-directory-helper.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Danny Browning <danny.browning@protectwise.com>
22  *
23  * Helper methods for directory based packet acquisition
24  */
25 
27 #include "runmode-unix-socket.h"
28 #include "util-mem.h"
29 #include "source-pcap-file.h"
30 
31 static void GetTime(struct timespec *tm);
32 static void CopyTime(struct timespec *from, struct timespec *to);
33 static int CompareTimes(struct timespec *left, struct timespec *right);
34 static TmEcode PcapRunStatus(PcapFileDirectoryVars *);
35 static TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv);
36 static TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv);
37 static int PcapDirectoryGetModifiedTime(char const * file, struct timespec * out);
38 static TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
39  PendingFile *file_to_add);
40 static TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *ptv,
41  struct timespec * older_than);
42 static TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
43  struct timespec *older_than);
44 
45 void GetTime(struct timespec *tm)
46 {
47  struct timeval now;
48  if(gettimeofday(&now, NULL) == 0) {
49  tm->tv_sec = now.tv_sec;
50  tm->tv_nsec = now.tv_usec * 1000L;
51  }
52 }
53 
54 void CopyTime(struct timespec *from, struct timespec *to)
55 {
56  to->tv_sec = from->tv_sec;
57  to->tv_nsec = from->tv_nsec;
58 }
59 
60 int CompareTimes(struct timespec *left, struct timespec *right)
61 {
62  if (left->tv_sec < right->tv_sec) {
63  return -1;
64  } else if (left->tv_sec > right->tv_sec) {
65  return 1;
66  } else {
67  if (left->tv_nsec < right->tv_nsec) {
68  return -1;
69  } else if (left->tv_nsec > right->tv_nsec) {
70  return 1;
71  } else {
72  return 0;
73  }
74  }
75 }
76 
77 /**
78  * Pcap Folder Utilities
79  */
80 TmEcode PcapRunStatus(PcapFileDirectoryVars *ptv)
81 {
84  if ( (suricata_ctl_flags & SURICATA_STOP) || done != TM_ECODE_OK) {
86  }
87  } else {
90  }
91  }
93 }
94 
96  if (pending != NULL) {
97  if (pending->filename != NULL) {
98  SCFree(pending->filename);
99  }
100  SCFree(pending);
101  }
102 }
103 
105 {
106  if (ptv != NULL) {
107  if (ptv->current_file != NULL) {
109  ptv->current_file = NULL;
110  }
111  if (ptv->directory != NULL) {
112  closedir(ptv->directory);
113  ptv->directory = NULL;
114  }
115  if (ptv->filename != NULL) {
116  SCFree(ptv->filename);
117  }
118  ptv->shared = NULL;
119  PendingFile *current_file = NULL;
120  while (!TAILQ_EMPTY(&ptv->directory_content)) {
121  current_file = TAILQ_FIRST(&ptv->directory_content);
122  TAILQ_REMOVE(&ptv->directory_content, current_file, next);
123  CleanupPendingFile(current_file);
124  }
125  SCFree(ptv);
126  }
127 }
128 
129 TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv)
130 {
131  TmEcode status = TM_ECODE_FAILED;
132 
133  if (unlikely(ptv == NULL)) {
134  SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory vars was null");
136  }
137  if (unlikely(ptv->shared == NULL)) {
138  SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory shared vars was null");
140  }
141 
143  status = UnixSocketPcapFile(status, &ptv->shared->last_processed);
144  }
145 
146  SCReturnInt(status);
147 }
148 
149 TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv)
150 {
151  TmEcode status = TM_ECODE_DONE;
152 
153  if (unlikely(ptv == NULL)) {
154  SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory vars was null");
156  }
157  if (unlikely(ptv->shared == NULL)) {
158  SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory shared vars was null");
160  }
161 
163  status = UnixSocketPcapFile(status, &ptv->shared->last_processed);
164  }
165 
166  SCReturnInt(status);
167 }
168 
169 TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
170 {
171  DIR *temp_dir = NULL;
172  TmEcode return_code = TM_ECODE_FAILED;
173 
174  temp_dir = opendir(filename);
175 
176  if (temp_dir == NULL) {//if null, our filename may just be a normal file
177  switch (errno) {
178  case EACCES:
179  SCLogError(SC_ERR_FOPEN, "%s: Permission denied", filename);
180  break;
181 
182  case EBADF:
184  "%s: Not a valid file descriptor opened for reading",
185  filename);
186  break;
187 
188  case EMFILE:
190  "%s: Per process open file descriptor limit reached",
191  filename);
192  break;
193 
194  case ENFILE:
196  "%s: System wide open file descriptor limit reached",
197  filename);
198  break;
199 
200  case ENOENT:
202  "%s: Does not exist, or name is an empty string",
203  filename);
204  break;
205  case ENOMEM:
207  "%s: Insufficient memory to complete the operation",
208  filename);
209  break;
210 
211  case ENOTDIR: //no error checking the directory, just is a plain file
212  SCLogInfo("%s: Plain file, not a directory", filename);
213  return_code = TM_ECODE_OK;
214  break;
215 
216  default:
217  SCLogError(SC_ERR_FOPEN, "%s: %" PRId32, filename, errno);
218  }
219  } else {
220  //no error, filename references a directory
221  *directory = temp_dir;
222  return_code = TM_ECODE_OK;
223  }
224 
225  return return_code;
226 }
227 
228 int PcapDirectoryGetModifiedTime(char const *file, struct timespec *out)
229 {
230 #ifdef OS_WIN32
231  struct _stat buf;
232 #else
233  struct stat buf;
234 #endif /* OS_WIN32 */
235  int ret;
236 
237  if (file == NULL)
238  return -1;
239 
240 #ifdef OS_WIN32
241  if((ret = _stat(file, &buf)) != 0)
242  return ret;
243 #else
244  if ((ret = stat(file, &buf)) != 0)
245  return ret;
246 #endif
247 
248 #ifdef OS_DARWIN
249  out->tv_sec = buf.st_mtimespec.tv_sec;
250  out->tv_nsec = buf.st_mtimespec.tv_nsec;
251 #elif OS_WIN32
252  out->tv_sec = buf.st_mtime;
253 #else
254  out->tv_sec = buf.st_mtim.tv_sec;
255  out->tv_nsec = buf.st_mtim.tv_nsec;
256 #endif
257 
258  return ret;
259 }
260 
261 TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
262  PendingFile *file_to_add
263 ) {
264  PendingFile *file_to_compare = NULL;
265  PendingFile *next_file_to_compare = NULL;
266 
267  if (unlikely(pv == NULL)) {
268  SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory vars passed");
270  }
271 
272  if (unlikely(file_to_add == NULL)) {
273  SCLogError(SC_ERR_INVALID_ARGUMENT, "File passed was null");
275  }
276 
277  if (unlikely(file_to_add->filename == NULL)) {
278  SCLogError(SC_ERR_INVALID_ARGUMENT, "File was passed with null filename");
280  }
281 
282  SCLogDebug("Inserting %s into directory buffer", file_to_add->filename);
283 
284  if (TAILQ_EMPTY(&pv->directory_content)) {
285  TAILQ_INSERT_TAIL(&pv->directory_content, file_to_add, next);
286  } else {
287  file_to_compare = TAILQ_FIRST(&pv->directory_content);
288  while(file_to_compare != NULL) {
289  if (CompareTimes(&file_to_add->modified_time, &file_to_compare->modified_time) < 0) {
290  TAILQ_INSERT_BEFORE(file_to_compare, file_to_add, next);
291  file_to_compare = NULL;
292  } else {
293  next_file_to_compare = TAILQ_NEXT(file_to_compare, next);
294  if (next_file_to_compare == NULL) {
295  TAILQ_INSERT_AFTER(&pv->directory_content, file_to_compare,
296  file_to_add, next);
297  }
298  file_to_compare = next_file_to_compare;
299  }
300  }
301  }
302 
304 }
305 
306 TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv,
307  struct timespec *older_than
308 ) {
309  if (unlikely(pv == NULL)) {
310  SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory vars passed");
312  }
313  if (unlikely(pv->filename == NULL)) {
314  SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory filename was passed");
316  }
317  struct dirent * dir = NULL;
318  PendingFile *file_to_add = NULL;
319 
320  while ((dir = readdir(pv->directory)) != NULL) {
321 #ifndef OS_WIN32
322  if (dir->d_type != DT_REG) {
323  continue;
324  }
325 #endif
326  if (strcmp(dir->d_name, ".") == 0 ||
327  strcmp(dir->d_name, "..") == 0) {
328  continue;
329  }
330 
331  char pathbuff[PATH_MAX] = {0};
332 
333  int written = 0;
334 
335  written = snprintf(pathbuff, PATH_MAX, "%s/%s", pv->filename, dir->d_name);
336 
337  if (written <= 0 || written >= PATH_MAX) {
338  SCLogError(SC_ERR_SPRINTF, "Could not write path");
339 
341  } else {
342  struct timespec temp_time;
343  memset(&temp_time, 0, sizeof(struct timespec));
344 
345  if (PcapDirectoryGetModifiedTime(pathbuff, &temp_time) == 0) {
346  SCLogDebug("%" PRIuMAX " < %" PRIuMAX "(%s) < %" PRIuMAX ")",
348  (uintmax_t)SCTimespecAsEpochMillis(&temp_time),
349  pathbuff,
350  (uintmax_t)SCTimespecAsEpochMillis(older_than));
351 
352  // Skip files outside of our time range
353  if (CompareTimes(&temp_time, &pv->shared->last_processed) <= 0) {
354  SCLogDebug("Skipping old file %s", pathbuff);
355  continue;
356  }
357  else if (CompareTimes(&temp_time, older_than) >= 0) {
358  SCLogDebug("Skipping new file %s", pathbuff);
359  continue;
360  }
361  } else {
362  SCLogDebug("Unable to get modified time on %s, skipping", pathbuff);
363  continue;
364  }
365 
366  file_to_add = SCCalloc(1, sizeof(PendingFile));
367  if (unlikely(file_to_add == NULL)) {
368  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate pending file");
369 
371  }
372 
373  file_to_add->filename = SCStrdup(pathbuff);
374  if (unlikely(file_to_add->filename == NULL)) {
375  SCLogError(SC_ERR_MEM_ALLOC, "Failed to copy filename");
376  CleanupPendingFile(file_to_add);
377 
379  }
380 
381  memset(&file_to_add->modified_time, 0, sizeof(struct timespec));
382  CopyTime(&temp_time, &file_to_add->modified_time);
383 
384  SCLogInfo("Found \"%s\" at %" PRIuMAX, file_to_add->filename,
385  (uintmax_t)SCTimespecAsEpochMillis(&file_to_add->modified_time));
386 
387  if (PcapDirectoryInsertFile(pv, file_to_add) == TM_ECODE_FAILED) {
388  SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to add file");
389  CleanupPendingFile(file_to_add);
390 
392  }
393  }
394  }
395 
397 }
398 
399 
400 TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
401  struct timespec *older_than)
402 {
403  if (PcapDirectoryPopulateBuffer(pv, older_than) == TM_ECODE_FAILED) {
404  SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to populate directory buffer");
406  }
407 
408  TmEcode status = TM_ECODE_OK;
409 
410  if (TAILQ_EMPTY(&pv->directory_content)) {
411  SCLogDebug("Directory %s has no files to process", pv->filename);
412  GetTime(older_than);
413  older_than->tv_sec = older_than->tv_sec - pv->delay;
414  rewinddir(pv->directory);
415  status = TM_ECODE_OK;
416  } else {
417  PendingFile *current_file = NULL;
418 
419  struct timespec last_time_seen;
420  memset(&last_time_seen, 0, sizeof(struct timespec));
421 
422  while (status == TM_ECODE_OK && !TAILQ_EMPTY(&pv->directory_content)) {
423  current_file = TAILQ_FIRST(&pv->directory_content);
424  TAILQ_REMOVE(&pv->directory_content, current_file, next);
425 
426  if (unlikely(current_file == NULL)) {
427  SCLogWarning(SC_ERR_PCAP_DISPATCH, "Current file was null");
428  } else if (unlikely(current_file->filename == NULL)) {
429  SCLogWarning(SC_ERR_PCAP_DISPATCH, "Current file filename was null");
430  } else {
431  SCLogDebug("Processing file %s", current_file->filename);
432 
433  PcapFileFileVars *pftv = SCMalloc(sizeof(PcapFileFileVars));
434  if (unlikely(pftv == NULL)) {
435  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate PcapFileFileVars");
437  }
438  memset(pftv, 0, sizeof(PcapFileFileVars));
439 
440  pftv->filename = SCStrdup(current_file->filename);
441  if (unlikely(pftv->filename == NULL)) {
442  SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
445  }
446  pftv->shared = pv->shared;
447 
448  if (InitPcapFile(pftv) == TM_ECODE_FAILED) {
450  "Failed to init pcap file %s, skipping",
451  current_file->filename);
452  CleanupPendingFile(current_file);
454  status = TM_ECODE_OK;
455  } else {
456  pv->current_file = pftv;
457 
458  status = PcapFileDispatch(pftv);
459 
461 
462  if (status == TM_ECODE_FAILED) {
463  CleanupPendingFile(current_file);
464  SCReturnInt(status);
465  }
466 
467  SCLogInfo("Processed file %s, processed up to %" PRIuMAX,
468  current_file->filename,
469  (uintmax_t)SCTimespecAsEpochMillis(&current_file->modified_time));
470 
471  if(CompareTimes(&current_file->modified_time, &last_time_seen) > 0) {
472  CopyTime(&current_file->modified_time, &last_time_seen);
473  }
474 
475  CleanupPendingFile(current_file);
476  pv->current_file = NULL;
477 
478  status = PcapRunStatus(pv);
479  }
480  }
481  }
482 
483  if(CompareTimes(&last_time_seen, &pv->shared->last_processed) > 0) {
484  SCLogInfo("Updating processed to %" PRIuMAX,
485  (uintmax_t)SCTimespecAsEpochMillis(&last_time_seen));
486  CopyTime(&last_time_seen, &pv->shared->last_processed);
487  status = PcapRunStatus(pv);
488  }
489  }
490  GetTime(older_than);
491  older_than->tv_sec = older_than->tv_sec - pv->delay;
492 
493  SCReturnInt(status);
494 }
495 
497 {
498  SCEnter();
499 
500  DIR *directory_check = NULL;
501 
502  struct timespec older_than;
503  memset(&older_than, 0, sizeof(struct timespec));
504  older_than.tv_sec = LONG_MAX;
505  uint32_t poll_seconds = (uint32_t)localtime(&ptv->poll_interval)->tm_sec;
506 
507  if (ptv->should_loop) {
508  GetTime(&older_than);
509  older_than.tv_sec = older_than.tv_sec - ptv->delay;
510  }
511  TmEcode status = TM_ECODE_OK;
512 
513  while (status == TM_ECODE_OK) {
514  //loop while directory is ok
515  SCLogInfo("Processing pcaps directory %s, files must be newer than %" PRIuMAX " and older than %" PRIuMAX,
516  ptv->filename, (uintmax_t)SCTimespecAsEpochMillis(&ptv->shared->last_processed),
517  (uintmax_t)SCTimespecAsEpochMillis(&older_than));
518  status = PcapDirectoryDispatchForTimeRange(ptv, &older_than);
519  if (ptv->should_loop && status == TM_ECODE_OK) {
520  sleep(poll_seconds);
521  //update our status based on suricata control flags or unix command socket
522  status = PcapRunStatus(ptv);
523  if (status == TM_ECODE_OK) {
524  SCLogDebug("Checking if directory %s still exists", ptv->filename);
525  //check directory
527  &directory_check) == TM_ECODE_FAILED) {
528  SCLogInfo("Directory %s no longer exists, stopping",
529  ptv->filename);
530  status = TM_ECODE_DONE;
531  } else if(directory_check != NULL) {
532  closedir(directory_check);
533  directory_check = NULL;
534  }
535  }
536  } else if (status == TM_ECODE_OK) { //not looping, mark done
537  SCLogDebug("Not looping, stopping directory mode");
538  status = TM_ECODE_DONE;
539  }
540  }
541 
543 
544  if (status == TM_ECODE_FAILED) {
545  SCLogError(SC_ERR_PCAP_DISPATCH, "Directory %s run mode failed", ptv->filename);
546  status = PcapDirectoryFailure(ptv);
547  } else {
548  SCLogInfo("Directory run mode complete");
549  status = PcapDirectoryDone(ptv);
550  }
551 
552  SCReturnInt(status);
553 }
554 
555 /* eof */
TmEcode InitPcapFile(PcapFileFileVars *pfv)
#define TAILQ_INSERT_BEFORE(listelm, elm, field)
Definition: queue.h:405
void CleanupPcapFileFileVars(PcapFileFileVars *pfv)
#define SCLogDebug(...)
Definition: util-debug.h:335
void CleanupPendingFile(PendingFile *pending)
#define TAILQ_FIRST(head)
Definition: queue.h:339
struct HtpBodyChunk_ * next
PcapFileSharedVars * shared
#define unlikely(expr)
Definition: util-optimize.h:35
struct timespec last_processed
TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
volatile uint8_t suricata_ctl_flags
Definition: suricata.c:201
#define SURICATA_STOP
Definition: suricata.h:95
#define SCCalloc(nm, a)
Definition: util-mem.h:205
#define TAILQ_INSERT_AFTER(head, listelm, elm, field)
Definition: queue.h:395
int RunModeUnixSocketIsActive(void)
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
#define TAILQ_REMOVE(head, elm, field)
Definition: queue.h:412
TmEcode PcapFileDispatch(PcapFileFileVars *ptv)
Main PCAP file reading Loop function.
#define SCEnter(...)
Definition: util-debug.h:337
TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
uint64_t SCTimespecAsEpochMillis(const struct timespec *ts)
Definition: util-time.c:607
#define SCReturnInt(x)
Definition: util-debug.h:341
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:385
#define SCMalloc(a)
Definition: util-mem.h:174
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:236
#define StatsSyncCountersIfSignalled(tv)
Definition: counters.h:136
#define SCStrdup(a)
Definition: util-mem.h:220
#define TAILQ_NEXT(elm, field)
Definition: queue.h:341
#define TAILQ_EMPTY(head)
Definition: queue.h:347
TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed)
void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv)