suricata
source-pcap-file-directory-helper.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Danny Browning <danny.browning@protectwise.com>
22  *
23  * Helper methods for directory based packet acquisition
24  */
25 
27 #include "suricata.h"
28 #include "runmode-unix-socket.h"
29 #include "util-mem.h"
30 #include "util-time.h"
31 #include "source-pcap-file.h"
32 
33 static void GetTime(struct timespec *tm);
34 static void CopyTime(struct timespec *from, struct timespec *to);
35 static int CompareTimes(struct timespec *left, struct timespec *right);
36 static TmEcode PcapRunStatus(PcapFileDirectoryVars *);
37 static TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv);
38 static TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv);
39 static int PcapDirectoryGetModifiedTime(char const * file, struct timespec * out);
40 static TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
41  PendingFile *file_to_add);
42 static TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *ptv,
43  struct timespec * older_than);
44 static TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
45  struct timespec *older_than);
46 
47 void GetTime(struct timespec *tm)
48 {
49  struct timeval now;
50  if(gettimeofday(&now, NULL) == 0) {
51  tm->tv_sec = now.tv_sec;
52  tm->tv_nsec = now.tv_usec * 1000L;
53  }
54 }
55 
56 void CopyTime(struct timespec *from, struct timespec *to)
57 {
58  to->tv_sec = from->tv_sec;
59  to->tv_nsec = from->tv_nsec;
60 }
61 
62 int CompareTimes(struct timespec *left, struct timespec *right)
63 {
64  if (left->tv_sec < right->tv_sec) {
65  return -1;
66  } else if (left->tv_sec > right->tv_sec) {
67  return 1;
68  } else {
69  if (left->tv_nsec < right->tv_nsec) {
70  return -1;
71  } else if (left->tv_nsec > right->tv_nsec) {
72  return 1;
73  } else {
74  return 0;
75  }
76  }
77 }
78 
79 /**
80  * Pcap Folder Utilities
81  */
82 TmEcode PcapRunStatus(PcapFileDirectoryVars *ptv)
83 {
86  if ( (suricata_ctl_flags & SURICATA_STOP) || done != TM_ECODE_OK) {
88  }
89  } else {
92  }
93  }
95 }
96 
98  if (pending != NULL) {
99  if (pending->filename != NULL) {
100  SCFree(pending->filename);
101  }
102  SCFree(pending);
103  }
104 }
105 
107 {
108  if (ptv != NULL) {
109  if (ptv->current_file != NULL) {
111  ptv->current_file = NULL;
112  }
113  if (ptv->directory != NULL) {
114  closedir(ptv->directory);
115  ptv->directory = NULL;
116  }
117  if (ptv->filename != NULL) {
118  SCFree(ptv->filename);
119  }
120  ptv->shared = NULL;
121  PendingFile *current_file = NULL;
122  while (!TAILQ_EMPTY(&ptv->directory_content)) {
123  current_file = TAILQ_FIRST(&ptv->directory_content);
124  TAILQ_REMOVE(&ptv->directory_content, current_file, next);
125  CleanupPendingFile(current_file);
126  }
127  SCFree(ptv);
128  }
129 }
130 
131 TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv)
132 {
133  TmEcode status = TM_ECODE_FAILED;
134 
135  if (unlikely(ptv == NULL)) {
136  SCLogError("Directory vars was null");
138  }
139  if (unlikely(ptv->shared == NULL)) {
140  SCLogError("Directory shared vars was null");
142  }
143 
145  status = UnixSocketPcapFile(status, &ptv->shared->last_processed);
146  }
147 
148  SCReturnInt(status);
149 }
150 
151 TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv)
152 {
153  TmEcode status = TM_ECODE_DONE;
154 
155  if (unlikely(ptv == NULL)) {
156  SCLogError("Directory vars was null");
158  }
159  if (unlikely(ptv->shared == NULL)) {
160  SCLogError("Directory shared vars was null");
162  }
163 
165  status = UnixSocketPcapFile(status, &ptv->shared->last_processed);
166  }
167 
168  SCReturnInt(status);
169 }
170 
171 TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
172 {
173  DIR *temp_dir = NULL;
174  TmEcode return_code = TM_ECODE_FAILED;
175 
176  temp_dir = opendir(filename);
177 
178  if (temp_dir == NULL) {//if null, our filename may just be a normal file
179  switch (errno) {
180  case EACCES:
181  SCLogError("%s: Permission denied", filename);
182  break;
183 
184  case EBADF:
185  SCLogError("%s: Not a valid file descriptor opened for reading", filename);
186  break;
187 
188  case EMFILE:
189  SCLogError("%s: Per process open file descriptor limit reached", filename);
190  break;
191 
192  case ENFILE:
193  SCLogError("%s: System wide open file descriptor limit reached", filename);
194  break;
195 
196  case ENOENT:
197  SCLogError("%s: Does not exist, or name is an empty string", filename);
198  break;
199  case ENOMEM:
200  SCLogError("%s: Insufficient memory to complete the operation", filename);
201  break;
202 
203  case ENOTDIR: //no error checking the directory, just is a plain file
204  SCLogDebug("%s: plain file, not a directory", filename);
205  return_code = TM_ECODE_OK;
206  break;
207 
208  default:
209  SCLogError("%s: %" PRId32, filename, errno);
210  }
211  } else {
212  //no error, filename references a directory
213  *directory = temp_dir;
214  return_code = TM_ECODE_OK;
215  }
216 
217  return return_code;
218 }
219 
220 int PcapDirectoryGetModifiedTime(char const *file, struct timespec *out)
221 {
222 #ifdef OS_WIN32
223  struct _stat buf;
224 #else
225  struct stat buf;
226 #endif /* OS_WIN32 */
227  int ret;
228 
229  if (file == NULL)
230  return -1;
231 
232 #ifdef OS_WIN32
233  if((ret = _stat(file, &buf)) != 0)
234  return ret;
235 #else
236  if ((ret = stat(file, &buf)) != 0)
237  return ret;
238 #endif
239 
240 #ifdef OS_DARWIN
241  out->tv_sec = buf.st_mtimespec.tv_sec;
242  out->tv_nsec = buf.st_mtimespec.tv_nsec;
243 #elif OS_WIN32
244  out->tv_sec = buf.st_mtime;
245 #else
246  out->tv_sec = buf.st_mtim.tv_sec;
247  out->tv_nsec = buf.st_mtim.tv_nsec;
248 #endif
249 
250  return ret;
251 }
252 
253 TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
254  PendingFile *file_to_add
255 ) {
256  PendingFile *file_to_compare = NULL;
257  PendingFile *next_file_to_compare = NULL;
258 
259  if (unlikely(pv == NULL)) {
260  SCLogError("No directory vars passed");
262  }
263 
264  if (unlikely(file_to_add == NULL)) {
265  SCLogError("File passed was null");
267  }
268 
269  if (unlikely(file_to_add->filename == NULL)) {
270  SCLogError("File was passed with null filename");
272  }
273 
274  SCLogDebug("Inserting %s into directory buffer", file_to_add->filename);
275 
276  if (TAILQ_EMPTY(&pv->directory_content)) {
277  TAILQ_INSERT_TAIL(&pv->directory_content, file_to_add, next);
278  } else {
279  file_to_compare = TAILQ_FIRST(&pv->directory_content);
280  while(file_to_compare != NULL) {
281  if (CompareTimes(&file_to_add->modified_time, &file_to_compare->modified_time) < 0) {
282  TAILQ_INSERT_BEFORE(file_to_compare, file_to_add, next);
283  file_to_compare = NULL;
284  } else {
285  next_file_to_compare = TAILQ_NEXT(file_to_compare, next);
286  if (next_file_to_compare == NULL) {
287  TAILQ_INSERT_AFTER(&pv->directory_content, file_to_compare,
288  file_to_add, next);
289  }
290  file_to_compare = next_file_to_compare;
291  }
292  }
293  }
294 
296 }
297 
298 TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv,
299  struct timespec *older_than
300 ) {
301  if (unlikely(pv == NULL)) {
302  SCLogError("No directory vars passed");
304  }
305  if (unlikely(pv->filename == NULL)) {
306  SCLogError("No directory filename was passed");
308  }
309  struct dirent * dir = NULL;
310  PendingFile *file_to_add = NULL;
311 
312  while ((dir = readdir(pv->directory)) != NULL) {
313 #ifndef OS_WIN32
314  if (dir->d_type != DT_REG) {
315  continue;
316  }
317 #endif
318  if (strcmp(dir->d_name, ".") == 0 ||
319  strcmp(dir->d_name, "..") == 0) {
320  continue;
321  }
322 
323  char pathbuff[PATH_MAX] = {0};
324 
325  int written = 0;
326 
327  written = snprintf(pathbuff, PATH_MAX, "%s/%s", pv->filename, dir->d_name);
328 
329  if (written <= 0 || written >= PATH_MAX) {
330  SCLogError("Could not write path");
331 
333  } else {
334  struct timespec temp_time;
335  memset(&temp_time, 0, sizeof(struct timespec));
336 
337  if (PcapDirectoryGetModifiedTime(pathbuff, &temp_time) == 0) {
338  SCLogDebug("%" PRIuMAX " < %" PRIuMAX "(%s) < %" PRIuMAX ")",
340  (uintmax_t)SCTimespecAsEpochMillis(&temp_time),
341  pathbuff,
342  (uintmax_t)SCTimespecAsEpochMillis(older_than));
343 
344  // Skip files outside of our time range
345  if (CompareTimes(&temp_time, &pv->shared->last_processed) <= 0) {
346  SCLogDebug("Skipping old file %s", pathbuff);
347  continue;
348  }
349  else if (CompareTimes(&temp_time, older_than) >= 0) {
350  SCLogDebug("Skipping new file %s", pathbuff);
351  continue;
352  }
353  } else {
354  SCLogDebug("Unable to get modified time on %s, skipping", pathbuff);
355  continue;
356  }
357 
358  file_to_add = SCCalloc(1, sizeof(PendingFile));
359  if (unlikely(file_to_add == NULL)) {
360  SCLogError("Failed to allocate pending file");
361 
363  }
364 
365  file_to_add->filename = SCStrdup(pathbuff);
366  if (unlikely(file_to_add->filename == NULL)) {
367  SCLogError("Failed to copy filename");
368  CleanupPendingFile(file_to_add);
369 
371  }
372 
373  memset(&file_to_add->modified_time, 0, sizeof(struct timespec));
374  CopyTime(&temp_time, &file_to_add->modified_time);
375 
376  SCLogInfo("Found \"%s\" at %" PRIuMAX, file_to_add->filename,
377  (uintmax_t)SCTimespecAsEpochMillis(&file_to_add->modified_time));
378 
379  if (PcapDirectoryInsertFile(pv, file_to_add) == TM_ECODE_FAILED) {
380  SCLogError("Failed to add file");
381  CleanupPendingFile(file_to_add);
382 
384  }
385  }
386  }
387 
389 }
390 
391 
392 TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
393  struct timespec *older_than)
394 {
395  if (PcapDirectoryPopulateBuffer(pv, older_than) == TM_ECODE_FAILED) {
396  SCLogError("Failed to populate directory buffer");
398  }
399 
400  TmEcode status = TM_ECODE_OK;
401 
402  if (TAILQ_EMPTY(&pv->directory_content)) {
403  SCLogDebug("Directory %s has no files to process", pv->filename);
404  GetTime(older_than);
405  older_than->tv_sec = older_than->tv_sec - pv->delay;
406  rewinddir(pv->directory);
407  status = TM_ECODE_OK;
408  } else {
409  PendingFile *current_file = NULL;
410 
411  struct timespec last_time_seen;
412  memset(&last_time_seen, 0, sizeof(struct timespec));
413 
414  while (status == TM_ECODE_OK && !TAILQ_EMPTY(&pv->directory_content)) {
415  current_file = TAILQ_FIRST(&pv->directory_content);
416  TAILQ_REMOVE(&pv->directory_content, current_file, next);
417 
418  if (unlikely(current_file == NULL)) {
419  SCLogWarning("Current file was null");
420  } else if (unlikely(current_file->filename == NULL)) {
421  SCLogWarning("Current file filename was null");
422  } else {
423  SCLogDebug("Processing file %s", current_file->filename);
424 
425  PcapFileFileVars *pftv = SCMalloc(sizeof(PcapFileFileVars));
426  if (unlikely(pftv == NULL)) {
427  SCLogError("Failed to allocate PcapFileFileVars");
429  }
430  memset(pftv, 0, sizeof(PcapFileFileVars));
431 
432  pftv->filename = SCStrdup(current_file->filename);
433  if (unlikely(pftv->filename == NULL)) {
434  SCLogError("Failed to allocate filename");
437  }
438  pftv->shared = pv->shared;
439 
440  if (InitPcapFile(pftv) == TM_ECODE_FAILED) {
441  SCLogWarning("Failed to init pcap file %s, skipping", current_file->filename);
442  CleanupPendingFile(current_file);
444  status = TM_ECODE_OK;
445  } else {
446  pv->current_file = pftv;
447 
448  status = PcapFileDispatch(pftv);
449 
451 
452  if (status == TM_ECODE_FAILED) {
453  CleanupPendingFile(current_file);
454  SCReturnInt(status);
455  }
456 
457  SCLogInfo("Processed file %s, processed up to %" PRIuMAX,
458  current_file->filename,
459  (uintmax_t)SCTimespecAsEpochMillis(&current_file->modified_time));
460 
461  if(CompareTimes(&current_file->modified_time, &last_time_seen) > 0) {
462  CopyTime(&current_file->modified_time, &last_time_seen);
463  }
464 
465  CleanupPendingFile(current_file);
466  pv->current_file = NULL;
467 
468  status = PcapRunStatus(pv);
469  }
470  }
471  }
472 
473  if(CompareTimes(&last_time_seen, &pv->shared->last_processed) > 0) {
474  SCLogInfo("Updating processed to %" PRIuMAX,
475  (uintmax_t)SCTimespecAsEpochMillis(&last_time_seen));
476  CopyTime(&last_time_seen, &pv->shared->last_processed);
477  status = PcapRunStatus(pv);
478  }
479  }
480  GetTime(older_than);
481  older_than->tv_sec = older_than->tv_sec - pv->delay;
482 
483  SCReturnInt(status);
484 }
485 
487 {
488  SCEnter();
489 
490  DIR *directory_check = NULL;
491 
492  struct timespec older_than;
493  memset(&older_than, 0, sizeof(struct timespec));
494  older_than.tv_sec = LONG_MAX;
495  uint32_t poll_seconds = (uint32_t)localtime(&ptv->poll_interval)->tm_sec;
496 
497  if (ptv->should_loop) {
498  GetTime(&older_than);
499  older_than.tv_sec = older_than.tv_sec - ptv->delay;
500  }
501  TmEcode status = TM_ECODE_OK;
502 
503  while (status == TM_ECODE_OK) {
504  //loop while directory is ok
505  SCLogInfo("Processing pcaps directory %s, files must be newer than %" PRIuMAX " and older than %" PRIuMAX,
506  ptv->filename, (uintmax_t)SCTimespecAsEpochMillis(&ptv->shared->last_processed),
507  (uintmax_t)SCTimespecAsEpochMillis(&older_than));
508  status = PcapDirectoryDispatchForTimeRange(ptv, &older_than);
509  if (ptv->should_loop && status == TM_ECODE_OK) {
510  sleep(poll_seconds);
511  //update our status based on suricata control flags or unix command socket
512  status = PcapRunStatus(ptv);
513  if (status == TM_ECODE_OK) {
514  SCLogDebug("Checking if directory %s still exists", ptv->filename);
515  //check directory
517  &directory_check) == TM_ECODE_FAILED) {
518  SCLogInfo("Directory %s no longer exists, stopping",
519  ptv->filename);
520  status = TM_ECODE_DONE;
521  } else if(directory_check != NULL) {
522  closedir(directory_check);
523  directory_check = NULL;
524  }
525  }
526  } else if (status == TM_ECODE_OK) { //not looping, mark done
527  SCLogDebug("Not looping, stopping directory mode");
528  status = TM_ECODE_DONE;
529  }
530  }
531 
533 
534  if (status == TM_ECODE_FAILED) {
535  SCLogError("Directory %s run mode failed", ptv->filename);
536  status = PcapDirectoryFailure(ptv);
537  } else {
538  SCLogInfo("Directory run mode complete");
539  status = PcapDirectoryDone(ptv);
540  }
541 
542  SCReturnInt(status);
543 }
544 
545 /* eof */
PcapFileDispatch
TmEcode PcapFileDispatch(PcapFileFileVars *ptv)
Main PCAP file reading Loop function.
Definition: source-pcap-file-helper.c:126
PcapFileFileVars_::filename
char * filename
Definition: source-pcap-file-helper.h:70
source-pcap-file.h
PcapFileFileVars_::shared
PcapFileSharedVars * shared
Definition: source-pcap-file-helper.h:76
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
TM_ECODE_DONE
@ TM_ECODE_DONE
Definition: tm-threads-common.h:86
TAILQ_EMPTY
#define TAILQ_EMPTY(head)
Definition: queue.h:248
SURICATA_STOP
#define SURICATA_STOP
Definition: suricata.h:89
StatsSyncCountersIfSignalled
#define StatsSyncCountersIfSignalled(tv)
Definition: counters.h:140
TAILQ_INSERT_AFTER
#define TAILQ_INSERT_AFTER(head, listelm, elm, field)
Definition: queue.h:267
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:294
PcapFileDirectoryVars_::poll_interval
time_t poll_interval
Definition: source-pcap-file-directory-helper.h:49
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
runmode-unix-socket.h
PcapDetermineDirectoryOrFile
TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
Definition: source-pcap-file-directory-helper.c:171
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
PendingFile_::modified_time
struct timespec modified_time
Definition: source-pcap-file-directory-helper.h:34
TAILQ_REMOVE
#define TAILQ_REMOVE(head, elm, field)
Definition: queue.h:312
TAILQ_FIRST
#define TAILQ_FIRST(head)
Definition: queue.h:250
PcapFileDirectoryVars_
Definition: source-pcap-file-directory-helper.h:41
SCTimespecAsEpochMillis
uint64_t SCTimespecAsEpochMillis(const struct timespec *ts)
Definition: util-time.c:639
CleanupPendingFile
void CleanupPendingFile(PendingFile *pending)
Definition: source-pcap-file-directory-helper.c:97
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
InitPcapFile
TmEcode InitPcapFile(PcapFileFileVars *pfv)
Definition: source-pcap-file-helper.c:196
util-time.h
source-pcap-file-directory-helper.h
RunModeUnixSocketIsActive
int RunModeUnixSocketIsActive(void)
Definition: runmode-unix-socket.c:1708
SCLogWarning
#define SCLogWarning(...)
Macro used to log WARNING messages.
Definition: util-debug.h:249
PcapFileFileVars_
Definition: source-pcap-file-helper.h:69
TmEcode
TmEcode
Definition: tm-threads-common.h:83
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
util-mem.h
PcapFileDirectoryVars_::directory
DIR * directory
Definition: source-pcap-file-directory-helper.h:43
TAILQ_NEXT
#define TAILQ_NEXT(elm, field)
Definition: queue.h:307
CleanupPcapFileDirectoryVars
void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv)
Definition: source-pcap-file-directory-helper.c:106
PcapDirectoryDispatch
TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
Definition: source-pcap-file-directory-helper.c:486
SCStrdup
#define SCStrdup(s)
Definition: util-mem.h:56
PcapFileDirectoryVars_::current_file
PcapFileFileVars * current_file
Definition: source-pcap-file-directory-helper.h:44
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
PcapFileSharedVars_::last_processed
struct timespec last_processed
Definition: source-pcap-file-helper.h:46
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
PcapFileDirectoryVars_::delay
time_t delay
Definition: source-pcap-file-directory-helper.h:48
PendingFile_
Definition: source-pcap-file-directory-helper.h:32
PcapFileDirectoryVars_::shared
PcapFileSharedVars * shared
Definition: source-pcap-file-directory-helper.h:53
PcapFileDirectoryVars_::should_loop
bool should_loop
Definition: source-pcap-file-directory-helper.h:45
UnixSocketPcapFile
TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed)
Definition: runmode-unix-socket.c:613
suricata.h
PcapFileDirectoryVars_::filename
char * filename
Definition: source-pcap-file-directory-helper.h:42
CleanupPcapFileFileVars
void CleanupPcapFileFileVars(PcapFileFileVars *pfv)
Definition: source-pcap-file-helper.c:39
PcapFileSharedVars_::tv
ThreadVars * tv
Definition: source-pcap-file-helper.h:50
TAILQ_INSERT_BEFORE
#define TAILQ_INSERT_BEFORE(listelm, elm, field)
Definition: queue.h:277
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
PendingFile_::filename
char * filename
Definition: source-pcap-file-directory-helper.h:33
suricata_ctl_flags
volatile uint8_t suricata_ctl_flags
Definition: suricata.c:164