suricata
util-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
23  *
24  */
25 
26 #include "suricata-common.h"
27 #include "suricata.h"
28 #include "debug.h"
29 #include "flow.h"
30 #include "stream.h"
31 #include "stream-tcp.h"
32 #include "runmodes.h"
33 #include "util-hash.h"
34 #include "util-debug.h"
35 #include "util-memcmp.h"
36 #include "util-print.h"
37 #include "app-layer-parser.h"
38 #include "util-validate.h"
39 
40 /** \brief switch to force filestore on all files
41  * regardless of the rules.
42  */
43 static int g_file_force_filestore = 0;
44 
45 /** \brief switch to force magic checks on all files
46  * regardless of the rules.
47  */
48 static int g_file_force_magic = 0;
49 
50 /** \brief switch to force md5 calculation on all files
51  * regardless of the rules.
52  */
53 static int g_file_force_md5 = 0;
54 
55 /** \brief switch to force sha1 calculation on all files
56  * regardless of the rules.
57  */
58 static int g_file_force_sha1 = 0;
59 
60 /** \brief switch to force sha256 calculation on all files
61  * regardless of the rules.
62  */
63 static int g_file_force_sha256 = 0;
64 
65 /** \brief switch to force tracking off all files
66  * regardless of the rules.
67  */
68 static int g_file_force_tracking = 0;
69 
70 /** \brief switch to use g_file_store_reassembly_depth
71  * to reassembly files
72  */
73 static int g_file_store_enable = 0;
74 
75 /** \brief stream_config.reassembly_depth equivalent
76  * for files
77  */
78 static uint32_t g_file_store_reassembly_depth = 0;
79 
80 /* prototypes */
81 static void FileFree(File *);
82 #ifdef HAVE_NSS
83 static void FileEndSha256(File *ff);
84 #endif
85 
87 {
88  g_file_force_filestore = 1;
89 }
90 
92 {
93  g_file_force_magic = 1;
94 }
95 
97 {
98  g_file_force_md5 = 1;
99 }
100 
102 {
103  g_file_force_sha1 = 1;
104 }
105 
107 {
108  g_file_force_sha256 = 1;
109 }
110 
112 {
113  return g_file_force_filestore;
114 }
115 
116 void FileReassemblyDepthEnable(uint32_t size)
117 {
118  g_file_store_enable = 1;
119  g_file_store_reassembly_depth = size;
120 }
121 
122 uint32_t FileReassemblyDepth(void)
123 {
124  if (g_file_store_enable == 1)
125  return g_file_store_reassembly_depth;
126  else
128 }
129 
130 int FileForceMagic(void)
131 {
132  return g_file_force_magic;
133 }
134 
135 int FileForceMd5(void)
136 {
137  return g_file_force_md5;
138 }
139 
140 int FileForceSha1(void)
141 {
142  return g_file_force_sha1;
143 }
144 
146 {
147  return g_file_force_sha256;
148 }
149 
151 {
152  g_file_force_tracking = 1;
153 }
154 
155 /**
156  * \brief Function to parse forced file hashing configuration.
157  */
159 {
160  BUG_ON(conf == NULL);
161 
162  ConfNode *forcehash_node = NULL;
163 
164  /* legacy option */
165  const char *force_md5 = ConfNodeLookupChildValue(conf, "force-md5");
166  if (force_md5 != NULL) {
167  SCLogWarning(SC_ERR_DEPRECATED_CONF, "deprecated 'force-md5' option "
168  "found. Please use 'force-hash: [md5]' instead");
169 
170  if (ConfValIsTrue(force_md5)) {
171 #ifdef HAVE_NSS
173  SCLogInfo("forcing md5 calculation for logged files");
174 #else
175  SCLogInfo("md5 calculation requires linking against libnss");
176 #endif
177  }
178  }
179 
180  if (conf != NULL)
181  forcehash_node = ConfNodeLookupChild(conf, "force-hash");
182 
183  if (forcehash_node != NULL) {
184  ConfNode *field = NULL;
185 
186  TAILQ_FOREACH(field, &forcehash_node->head, next) {
187  if (strcasecmp("md5", field->val) == 0) {
188 #ifdef HAVE_NSS
190  SCLogConfig("forcing md5 calculation for logged or stored files");
191 #else
192  SCLogInfo("md5 calculation requires linking against libnss");
193 #endif
194  }
195 
196  if (strcasecmp("sha1", field->val) == 0) {
197 #ifdef HAVE_NSS
199  SCLogConfig("forcing sha1 calculation for logged or stored files");
200 #else
201  SCLogInfo("sha1 calculation requires linking against libnss");
202 #endif
203  }
204 
205  if (strcasecmp("sha256", field->val) == 0) {
206 #ifdef HAVE_NSS
208  SCLogConfig("forcing sha256 calculation for logged or stored files");
209 #else
210  SCLogInfo("sha256 calculation requires linking against libnss");
211 #endif
212  }
213  }
214  }
215 }
216 
217 uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction)
218 {
219  uint16_t flags = 0;
220 
221  if (direction == STREAM_TOSERVER) {
222  if (flow->file_flags & FLOWFILE_NO_STORE_TS) {
223  flags |= FILE_NOSTORE;
224  }
225 
226  if (flow->file_flags & FLOWFILE_NO_MAGIC_TS) {
227  flags |= FILE_NOMAGIC;
228  }
229 
230  if (flow->file_flags & FLOWFILE_NO_MD5_TS) {
231  flags |= FILE_NOMD5;
232  }
233 
234  if (flow->file_flags & FLOWFILE_NO_SHA1_TS) {
235  flags |= FILE_NOSHA1;
236  }
237 
238  if (flow->file_flags & FLOWFILE_NO_SHA256_TS) {
239  flags |= FILE_NOSHA256;
240  }
241  } else {
242  if (flow->file_flags & FLOWFILE_NO_STORE_TC) {
243  flags |= FILE_NOSTORE;
244  }
245 
246  if (flow->file_flags & FLOWFILE_NO_MAGIC_TC) {
247  flags |= FILE_NOMAGIC;
248  }
249 
250  if (flow->file_flags & FLOWFILE_NO_MD5_TC) {
251  flags |= FILE_NOMD5;
252  }
253 
254  if (flow->file_flags & FLOWFILE_NO_SHA1_TC) {
255  flags |= FILE_NOSHA1;
256  }
257 
258  if (flow->file_flags & FLOWFILE_NO_SHA256_TC) {
259  flags |= FILE_NOSHA256;
260  }
261  }
262  return flags;
263 }
264 
265 static int FileMagicSize(void)
266 {
267  /** \todo make this size configurable */
268  return 512;
269 }
270 
271 /**
272  * \brief get the size of the file data
273  *
274  * This doesn't reflect how much of the file we have in memory, just the
275  * total size of filedata so far.
276  */
277 uint64_t FileDataSize(const File *file)
278 {
279  if (file != NULL && file->sb != NULL) {
280  SCLogDebug("returning %"PRIu64,
281  file->sb->stream_offset + file->sb->buf_offset);
282  return file->sb->stream_offset + file->sb->buf_offset;
283  }
284  SCLogDebug("returning 0 (default)");
285  return 0;
286 }
287 
288 /**
289  * \brief get the size of the file
290  *
291  * This doesn't reflect how much of the file we have in memory, just the
292  * total size of file so far.
293  */
294 uint64_t FileTrackedSize(const File *file)
295 {
296  if (file != NULL) {
297  return file->size;
298  }
299  return 0;
300 }
301 
302 static int FilePruneFile(File *file)
303 {
304  SCEnter();
305 #ifdef HAVE_MAGIC
306  if (!(file->flags & FILE_NOMAGIC)) {
307  /* need magic but haven't set it yet, bail out */
308  if (file->magic == NULL)
309  SCReturnInt(0);
310  else
311  SCLogDebug("file->magic %s", file->magic);
312  } else {
313  SCLogDebug("file->flags & FILE_NOMAGIC == true");
314  }
315 #endif
316  uint64_t left_edge = file->content_stored;
317  if (file->flags & FILE_NOSTORE) {
318  left_edge = FileDataSize(file);
319  }
320  if (file->flags & FILE_USE_DETECT) {
321  left_edge = MIN(left_edge, file->content_inspected);
322  }
323 
324  if (left_edge) {
325  StreamingBufferSlideToOffset(file->sb, left_edge);
326  }
327 
328  if (left_edge != FileDataSize(file)) {
329  SCReturnInt(0);
330  }
331 
332  SCLogDebug("file->state %d. Is >= FILE_STATE_CLOSED: %s", file->state, (file->state >= FILE_STATE_CLOSED) ? "yes" : "no");
333 
334  /* file is done when state is closed+, logging/storing is done (if any) */
335  if (file->state >= FILE_STATE_CLOSED &&
336  (!RunModeOutputFileEnabled() || (file->flags & FILE_LOGGED)) &&
338  {
339  SCReturnInt(1);
340  } else {
341  SCReturnInt(0);
342  }
343 }
344 
346 {
347  File *file = ffc->head;
348  File *prev = NULL;
349 
350  while (file) {
351  if (FilePruneFile(file) == 0) {
352  prev = file;
353  file = file->next;
354  continue;
355  }
356 
357  SCLogDebug("removing file %p", file);
358 
359  File *file_next = file->next;
360 
361  if (prev)
362  prev->next = file_next;
363  /* update head and tail */
364  if (file == ffc->head)
365  ffc->head = file_next;
366  if (file == ffc->tail)
367  ffc->tail = prev;
368 
369  FileFree(file);
370  file = file_next;
371  }
372 }
373 
374 /**
375  * \brief allocate a FileContainer
376  *
377  * \retval new newly allocated FileContainer
378  * \retval NULL error
379  */
381 {
382  FileContainer *new = SCMalloc(sizeof(FileContainer));
383  if (unlikely(new == NULL)) {
384  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem");
385  return NULL;
386  }
387  memset(new, 0, sizeof(FileContainer));
388  new->head = new->tail = NULL;
389  return new;
390 }
391 
392 /**
393  * \brief Recycle a FileContainer
394  *
395  * \param ffc FileContainer
396  */
398 {
399  if (ffc == NULL)
400  return;
401 
402  File *cur = ffc->head;
403  File *next = NULL;
404  for (;cur != NULL; cur = next) {
405  next = cur->next;
406  FileFree(cur);
407  }
408  ffc->head = ffc->tail = NULL;
409 }
410 
411 /**
412  * \brief Free a FileContainer
413  *
414  * \param ffc FileContainer
415  */
417 {
418  if (ffc == NULL)
419  return;
420 
421  File *ptr = ffc->head;
422  File *next = NULL;
423  for (;ptr != NULL; ptr = next) {
424  next = ptr->next;
425  FileFree(ptr);
426  }
427  ffc->head = ffc->tail = NULL;
428  SCFree(ffc);
429 }
430 
431 /**
432  * \brief Alloc a new File
433  *
434  * \param name character array containing the name (not a string)
435  * \param name_len length in bytes of the name
436  *
437  * \retval new File object or NULL on error
438  */
439 static File *FileAlloc(const uint8_t *name, uint16_t name_len)
440 {
441  File *new = SCMalloc(sizeof(File));
442  if (unlikely(new == NULL)) {
443  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem");
444  return NULL;
445  }
446  memset(new, 0, sizeof(File));
447 
448  new->name = SCMalloc(name_len);
449  if (new->name == NULL) {
450  SCFree(new);
451  return NULL;
452  }
453 
454  new->name_len = name_len;
455  memcpy(new->name, name, name_len);
456 
457  new->sid_cnt = 0;
458  new->sid_max = 8;
459  /* SCMalloc() is allowed to fail here because sid well be checked later on */
460  new->sid = SCMalloc(sizeof(uint32_t) * new->sid_max);
461  if (new->sid == NULL)
462  new->sid_max = 0;
463 
464  return new;
465 }
466 
467 static void FileFree(File *ff)
468 {
469  if (ff == NULL)
470  return;
471 
472  if (ff->name != NULL)
473  SCFree(ff->name);
474  if (ff->sid != NULL)
475  SCFree(ff->sid);
476 #ifdef HAVE_MAGIC
477  /* magic returned by libmagic is strdup'd by MagicLookup. */
478  if (ff->magic != NULL)
479  SCFree(ff->magic);
480 #endif
481  if (ff->sb != NULL) {
482  StreamingBufferFree(ff->sb);
483  }
484 
485 #ifdef HAVE_NSS
486  if (ff->md5_ctx)
487  HASH_Destroy(ff->md5_ctx);
488  if (ff->sha1_ctx)
489  HASH_Destroy(ff->sha1_ctx);
490  if (ff->sha256_ctx)
491  HASH_Destroy(ff->sha256_ctx);
492 #endif
493  SCFree(ff);
494 }
495 
497 {
498  if (ffc->head == NULL || ffc->tail == NULL) {
499  ffc->head = ffc->tail = ff;
500  } else {
501  ffc->tail->next = ff;
502  ffc->tail = ff;
503  }
504 }
505 
506 /**
507  * \brief Tag a file for storing
508  *
509  * \param ff The file to store
510  */
511 int FileStore(File *ff)
512 {
513  ff->flags |= FILE_STORE;
514  SCReturnInt(0);
515 }
516 
517 /**
518  * \brief Set the TX id for a file
519  *
520  * \param ff The file to store
521  * \param txid the tx id
522  */
523 int FileSetTx(File *ff, uint64_t txid)
524 {
525  SCLogDebug("ff %p txid %"PRIu64, ff, txid);
526  if (ff != NULL)
527  ff->txid = txid;
528  SCReturnInt(0);
529 }
530 
532 {
533  if (ffc && ffc->tail) {
534  (void)FileSetTx(ffc->tail, tx_id);
535  }
536 }
537 
538 /**
539  * \brief check if we have stored enough
540  *
541  * \param ff file
542  *
543  * \retval 0 limit not reached yet
544  * \retval 1 limit reached
545  */
546 static int FileStoreNoStoreCheck(File *ff)
547 {
548  SCEnter();
549 
550  if (ff == NULL) {
551  SCReturnInt(0);
552  }
553 
554  if (ff->flags & FILE_NOSTORE) {
555  if (ff->state == FILE_STATE_OPENED &&
556  FileDataSize(ff) >= (uint64_t)FileMagicSize())
557  {
558  SCReturnInt(1);
559  }
560  }
561 
562  SCReturnInt(0);
563 }
564 
565 static int AppendData(File *file, const uint8_t *data, uint32_t data_len)
566 {
567  if (StreamingBufferAppendNoTrack(file->sb, data, data_len) != 0) {
568  SCReturnInt(-1);
569  }
570 
571 #ifdef HAVE_NSS
572  if (file->md5_ctx) {
573  HASH_Update(file->md5_ctx, data, data_len);
574  }
575  if (file->sha1_ctx) {
576  HASH_Update(file->sha1_ctx, data, data_len);
577  }
578  if (file->sha256_ctx) {
579  HASH_Update(file->sha256_ctx, data, data_len);
580  }
581 #endif
582  SCReturnInt(0);
583 }
584 
585 /** \internal
586  * \brief Store/handle a chunk of file data in the File structure
587  *
588  * \param ff the file
589  * \param data data chunk
590  * \param data_len data chunk len
591  *
592  * \retval 0 ok
593  * \retval -1 error
594  * \retval -2 no store for this file
595  */
596 static int FileAppendDataDo(File *ff, const uint8_t *data, uint32_t data_len)
597 {
598  SCEnter();
599 #ifdef DEBUG_VALIDATION
600  BUG_ON(ff == NULL);
601 #endif
602 
603  ff->size += data_len;
604 
605  if (ff->state != FILE_STATE_OPENED) {
606  if (ff->flags & FILE_NOSTORE) {
607  SCReturnInt(-2);
608  }
609  SCReturnInt(-1);
610  }
611 
612  if (FileStoreNoStoreCheck(ff) == 1) {
613 #ifdef HAVE_NSS
614  int hash_done = 0;
615  /* no storage but forced hashing */
616  if (ff->md5_ctx) {
617  HASH_Update(ff->md5_ctx, data, data_len);
618  hash_done = 1;
619  }
620  if (ff->sha1_ctx) {
621  HASH_Update(ff->sha1_ctx, data, data_len);
622  hash_done = 1;
623  }
624  if (ff->sha256_ctx) {
625  HASH_Update(ff->sha256_ctx, data, data_len);
626  hash_done = 1;
627  }
628 
629  if (hash_done)
630  SCReturnInt(0);
631 #endif
632  if (g_file_force_tracking || (!(ff->flags & FILE_NOTRACK)))
633  SCReturnInt(0);
634 
636  SCLogDebug("flowfile state transitioned to FILE_STATE_TRUNCATED");
637  SCReturnInt(-2);
638  }
639 
640  SCLogDebug("appending %"PRIu32" bytes", data_len);
641 
642  int r = AppendData(ff, data, data_len);
643  if (r != 0) {
644  ff->state = FILE_STATE_ERROR;
645  SCReturnInt(r);
646  }
647 
648  SCReturnInt(0);
649 }
650 
651 /**
652  * \brief Store/handle a chunk of file data in the File structure
653  * The last file in the FileContainer will be used.
654  *
655  * \param ffc FileContainer used to append to
656  * \param data data chunk
657  * \param data_len data chunk len
658  *
659  * \retval 0 ok
660  * \retval -1 error
661  * \retval -2 no store for this file
662  */
663 int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
664 {
665  SCEnter();
666 
667  if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
668  SCReturnInt(-1);
669  }
670  int r = FileAppendDataDo(ffc->tail, data, data_len);
671  SCReturnInt(r);
672 }
673 
674 /**
675  * \brief Store/handle a chunk of file data in the File structure
676  * The file with 'track_id' in the FileContainer will be used.
677  *
678  * \param ffc FileContainer used to append to
679  * \param track_id id to lookup the file
680  * \param data data chunk
681  * \param data_len data chunk len
682  *
683  * \retval 0 ok
684  * \retval -1 error
685  * \retval -2 no store for this file
686  */
687 int FileAppendDataById(FileContainer *ffc, uint32_t track_id,
688  const uint8_t *data, uint32_t data_len)
689 {
690  SCEnter();
691 
692  if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
693  SCReturnInt(-1);
694  }
695  File *ff = ffc->head;
696  for ( ; ff != NULL; ff = ff->next) {
697  if (track_id == ff->file_track_id) {
698  int r = FileAppendDataDo(ff, data, data_len);
699  SCReturnInt(r);
700  }
701  }
702  SCReturnInt(-1);
703 }
704 
705 /**
706  * \brief Store/handle a chunk of file data in the File structure
707  * The file with 'track_id' in the FileContainer will be used.
708  *
709  * \param ffc FileContainer used to append to
710  * \param track_id id to lookup the file
711  * \param data data chunk
712  * \param data_len data chunk len
713  *
714  * \retval 0 ok
715  * \retval -1 error
716  * \retval -2 no store for this file
717  */
718 int FileAppendGAPById(FileContainer *ffc, uint32_t track_id,
719  const uint8_t *data, uint32_t data_len)
720 {
721  SCEnter();
722 
723  if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
724  SCReturnInt(-1);
725  }
726  File *ff = ffc->head;
727  for ( ; ff != NULL; ff = ff->next) {
728  if (track_id == ff->file_track_id) {
729  ff->flags |= FILE_HAS_GAPS;
732  SCLogDebug("FILE_HAS_GAPS set");
733 
734  int r = FileAppendDataDo(ff, data, data_len);
735  SCReturnInt(r);
736  }
737  }
738  SCReturnInt(-1);
739 }
740 
741 /**
742  * \brief Open a new File
743  *
744  * \param ffc flow container
745  * \param sbcfg buffer config
746  * \param name filename character array
747  * \param name_len filename len
748  * \param data initial data
749  * \param data_len initial data len
750  * \param flags open flags
751  *
752  * \retval ff flowfile object
753  *
754  * \note filename is not a string, so it's not nul terminated.
755  */
757  const uint8_t *name, uint16_t name_len,
758  const uint8_t *data, uint32_t data_len, uint16_t flags)
759 {
760  SCEnter();
761 
762  //PrintRawDataFp(stdout, name, name_len);
763 
764  File *ff = FileAlloc(name, name_len);
765  if (ff == NULL) {
766  SCReturnPtr(NULL, "File");
767  }
768 
769  ff->sb = StreamingBufferInit(sbcfg);
770  if (ff->sb == NULL) {
771  FileFree(ff);
772  SCReturnPtr(NULL, "File");
773  }
774  SCLogDebug("ff->sb %p", ff->sb);
775 
776  if (flags & FILE_STORE || g_file_force_filestore) {
777  FileStore(ff);
778  } else if (flags & FILE_NOSTORE) {
779  SCLogDebug("not storing this file");
780  ff->flags |= FILE_NOSTORE;
781  }
782  if (flags & FILE_NOMAGIC) {
783  SCLogDebug("not doing magic for this file");
784  ff->flags |= FILE_NOMAGIC;
785  }
786  if (flags & FILE_NOMD5) {
787  SCLogDebug("not doing md5 for this file");
788  ff->flags |= FILE_NOMD5;
789  }
790  if (flags & FILE_NOSHA1) {
791  SCLogDebug("not doing sha1 for this file");
792  ff->flags |= FILE_NOSHA1;
793  }
794  if (flags & FILE_NOSHA256) {
795  SCLogDebug("not doing sha256 for this file");
796  ff->flags |= FILE_NOSHA256;
797  }
798  if (flags & FILE_USE_DETECT) {
799  SCLogDebug("considering content_inspect tracker when pruning");
800  ff->flags |= FILE_USE_DETECT;
801  }
802 
803 #ifdef HAVE_NSS
804  if (!(ff->flags & FILE_NOMD5) || g_file_force_md5) {
805  ff->md5_ctx = HASH_Create(HASH_AlgMD5);
806  if (ff->md5_ctx != NULL) {
807  HASH_Begin(ff->md5_ctx);
808  }
809  }
810  if (!(ff->flags & FILE_NOSHA1) || g_file_force_sha1) {
811  ff->sha1_ctx = HASH_Create(HASH_AlgSHA1);
812  if (ff->sha1_ctx != NULL) {
813  HASH_Begin(ff->sha1_ctx);
814  }
815  }
816  if (!(ff->flags & FILE_NOSHA256) || g_file_force_sha256) {
817  ff->sha256_ctx = HASH_Create(HASH_AlgSHA256);
818  if (ff->sha256_ctx != NULL) {
819  HASH_Begin(ff->sha256_ctx);
820  }
821  }
822 #endif
823 
824  ff->state = FILE_STATE_OPENED;
825  SCLogDebug("flowfile state transitioned to FILE_STATE_OPENED");
826 
827  ff->fd = -1;
828 
829  FileContainerAdd(ffc, ff);
830 
831  if (data != NULL) {
832  ff->size += data_len;
833  if (AppendData(ff, data, data_len) != 0) {
834  ff->state = FILE_STATE_ERROR;
835  SCReturnPtr(NULL, "File");
836  }
837  SCLogDebug("file size is now %"PRIu64, FileTrackedSize(ff));
838  }
839 
840  SCReturnPtr(ff, "File");
841 }
842 
843 /**
844  * \retval 0 ok
845  * \retval -1 failed */
847  uint32_t track_id, const uint8_t *name, uint16_t name_len,
848  const uint8_t *data, uint32_t data_len, uint16_t flags)
849 {
850  File *ff = FileOpenFile(ffc, sbcfg, name, name_len, data, data_len, flags);
851  if (ff == NULL)
852  return -1;
853 
854  ff->file_track_id = track_id;
855  ff->flags |= FILE_USE_TRACKID;
856  return 0;
857 }
858 
859 int FileCloseFilePtr(File *ff, const uint8_t *data,
860  uint32_t data_len, uint16_t flags)
861 {
862  SCEnter();
863 
864  if (ff == NULL) {
865  SCReturnInt(-1);
866  }
867 
868  if (ff->state != FILE_STATE_OPENED) {
869  SCReturnInt(-1);
870  }
871 
872  if (data != NULL) {
873  ff->size += data_len;
874  if (ff->flags & FILE_NOSTORE) {
875 #ifdef HAVE_NSS
876  /* no storage but hashing */
877  if (ff->md5_ctx)
878  HASH_Update(ff->md5_ctx, data, data_len);
879  if (ff->sha1_ctx)
880  HASH_Update(ff->sha1_ctx, data, data_len);
881  if (ff->sha256_ctx)
882  HASH_Update(ff->sha256_ctx, data, data_len);
883 #endif
884  } else {
885  if (AppendData(ff, data, data_len) != 0) {
886  ff->state = FILE_STATE_ERROR;
887  SCReturnInt(-1);
888  }
889  }
890  }
891 
892  if ((flags & FILE_TRUNCATED) || (ff->flags & FILE_HAS_GAPS)) {
894  SCLogDebug("flowfile state transitioned to FILE_STATE_TRUNCATED");
895 
896  if (flags & FILE_NOSTORE) {
897  SCLogDebug("not storing this file");
898  ff->flags |= FILE_NOSTORE;
899  } else {
900 #ifdef HAVE_NSS
901  if (g_file_force_sha256 && ff->sha256_ctx) {
902  FileEndSha256(ff);
903  }
904 #endif
905  }
906  } else {
907  ff->state = FILE_STATE_CLOSED;
908  SCLogDebug("flowfile state transitioned to FILE_STATE_CLOSED");
909 
910 #ifdef HAVE_NSS
911  if (ff->md5_ctx) {
912  unsigned int len = 0;
913  HASH_End(ff->md5_ctx, ff->md5, &len, sizeof(ff->md5));
914  ff->flags |= FILE_MD5;
915  }
916  if (ff->sha1_ctx) {
917  unsigned int len = 0;
918  HASH_End(ff->sha1_ctx, ff->sha1, &len, sizeof(ff->sha1));
919  ff->flags |= FILE_SHA1;
920  }
921  if (ff->sha256_ctx) {
922  FileEndSha256(ff);
923  }
924 #endif
925  }
926 
927  SCReturnInt(0);
928 }
929 
930 /**
931  * \brief Close a File
932  *
933  * \param ffc the container
934  * \param data final data if any
935  * \param data_len data len if any
936  * \param flags flags
937  *
938  * \retval 0 ok
939  * \retval -1 error
940  */
941 int FileCloseFile(FileContainer *ffc, const uint8_t *data,
942  uint32_t data_len, uint16_t flags)
943 {
944  SCEnter();
945 
946  if (ffc == NULL || ffc->tail == NULL) {
947  SCReturnInt(-1);
948  }
949 
950  if (FileCloseFilePtr(ffc->tail, data, data_len, flags) == -1) {
951  SCReturnInt(-1);
952  }
953 
954  SCReturnInt(0);
955 }
956 
957 int FileCloseFileById(FileContainer *ffc, uint32_t track_id,
958  const uint8_t *data, uint32_t data_len, uint16_t flags)
959 {
960  SCEnter();
961 
962  if (ffc == NULL || ffc->tail == NULL) {
963  SCReturnInt(-1);
964  }
965 
966  File *ff = ffc->head;
967  for ( ; ff != NULL; ff = ff->next) {
968  if (track_id == ff->file_track_id) {
969  int r = FileCloseFilePtr(ff, data, data_len, flags);
970  SCReturnInt(r);
971  }
972  }
973  SCReturnInt(-1);
974 }
975 
976 /**
977  * \brief disable file storage for a flow
978  *
979  * \param f *LOCKED* flow
980  * \param direction flow direction
981  */
982 void FileDisableStoring(Flow *f, uint8_t direction)
983 {
984  File *ptr = NULL;
985 
986  SCEnter();
987 
989 
990  if (direction == STREAM_TOSERVER)
992  else
994 
995  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
996  if (ffc != NULL) {
997  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
998  /* if we're already storing, we'll continue */
999  if (!(ptr->flags & FILE_STORE)) {
1000  SCLogDebug("not storing this file");
1001  ptr->flags |= FILE_NOSTORE;
1002  }
1003  }
1004  }
1005  SCReturn;
1006 }
1007 
1008 /**
1009  * \brief disable file magic lookups for this flow
1010  *
1011  * \param f *LOCKED* flow
1012  * \param direction flow direction
1013  */
1014 void FileDisableMagic(Flow *f, uint8_t direction)
1015 {
1016  File *ptr = NULL;
1017 
1018  SCEnter();
1019 
1021 
1022  if (direction == STREAM_TOSERVER)
1024  else
1026 
1027  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1028  if (ffc != NULL) {
1029  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1030  SCLogDebug("disabling magic for file %p from direction %s",
1031  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1032  ptr->flags |= FILE_NOMAGIC;
1033  }
1034  }
1035 
1036  SCReturn;
1037 }
1038 
1039 /**
1040  * \brief disable file md5 calc for this flow
1041  *
1042  * \param f *LOCKED* flow
1043  * \param direction flow direction
1044  */
1045 void FileDisableMd5(Flow *f, uint8_t direction)
1046 {
1047  File *ptr = NULL;
1048 
1049  SCEnter();
1050 
1052 
1053  if (direction == STREAM_TOSERVER)
1055  else
1057 
1058  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1059  if (ffc != NULL) {
1060  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1061  SCLogDebug("disabling md5 for file %p from direction %s",
1062  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1063  ptr->flags |= FILE_NOMD5;
1064 
1065 #ifdef HAVE_NSS
1066  /* destroy any ctx we may have so far */
1067  if (ptr->md5_ctx != NULL) {
1068  HASH_Destroy(ptr->md5_ctx);
1069  ptr->md5_ctx = NULL;
1070  }
1071 #endif
1072  }
1073  }
1074 
1075  SCReturn;
1076 }
1077 
1078 /**
1079  * \brief disable file sha1 calc for this flow
1080  *
1081  * \param f *LOCKED* flow
1082  * \param direction flow direction
1083 */
1084 void FileDisableSha1(Flow *f, uint8_t direction)
1085 {
1086  File *ptr = NULL;
1087 
1088  SCEnter();
1089 
1091 
1092  if (direction == STREAM_TOSERVER)
1094  else
1096 
1097  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1098  if (ffc != NULL) {
1099  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1100  SCLogDebug("disabling sha1 for file %p from direction %s",
1101  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1102  ptr->flags |= FILE_NOSHA1;
1103 
1104 #ifdef HAVE_NSS
1105  /* destroy any ctx we may have so far */
1106  if (ptr->sha1_ctx != NULL) {
1107  HASH_Destroy(ptr->sha1_ctx);
1108  ptr->sha1_ctx = NULL;
1109  }
1110 #endif
1111  }
1112  }
1113 
1114  SCReturn;
1115 }
1116 
1117 /**
1118  * \brief disable file sha256 calc for this flow
1119  *
1120  * \param f *LOCKED* flow
1121  * \param direction flow direction
1122  */
1123 void FileDisableSha256(Flow *f, uint8_t direction)
1124 {
1125  File *ptr = NULL;
1126 
1127  SCEnter();
1128 
1130 
1131  if (direction == STREAM_TOSERVER)
1133  else
1135 
1136  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1137  if (ffc != NULL) {
1138  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1139  SCLogDebug("disabling sha256 for file %p from direction %s",
1140  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1141  ptr->flags |= FILE_NOSHA256;
1142 
1143 #ifdef HAVE_NSS
1144  /* destroy any ctx we may have so far */
1145  if (ptr->sha256_ctx != NULL) {
1146  HASH_Destroy(ptr->sha256_ctx);
1147  ptr->sha256_ctx = NULL;
1148  }
1149 #endif
1150  }
1151  }
1152 
1153  SCReturn;
1154 }
1155 
1156 /**
1157  * \brief disable file size tracking for this flow
1158  *
1159  * \param f *LOCKED* flow
1160  * \param direction flow direction
1161  */
1162 void FileDisableFilesize(Flow *f, uint8_t direction)
1163 {
1164  File *ptr = NULL;
1165 
1166  SCEnter();
1167 
1169 
1170  if (direction == STREAM_TOSERVER)
1172  else
1174 
1175  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1176  if (ffc != NULL) {
1177  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1178  SCLogDebug("disabling size tracking for file %p from direction %s",
1179  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1180  ptr->flags |= FILE_NOTRACK;
1181  }
1182  }
1183 
1184  SCReturn;
1185 }
1186 
1187 
1188 /**
1189  * \brief set no store flag, close file if needed
1190  *
1191  * \param ff file
1192  */
1193 static void FileDisableStoringForFile(File *ff)
1194 {
1195  SCEnter();
1196 
1197  if (ff == NULL) {
1198  SCReturn;
1199  }
1200 
1201  SCLogDebug("not storing this file");
1202  ff->flags |= FILE_NOSTORE;
1203 
1204  if (ff->state == FILE_STATE_OPENED && FileDataSize(ff) >= (uint64_t)FileMagicSize()) {
1205  if (g_file_force_md5 == 0 && g_file_force_sha1 == 0 && g_file_force_sha256 == 0
1206  && g_file_force_tracking == 0) {
1207  (void)FileCloseFilePtr(ff, NULL, 0,
1209  }
1210  }
1211 }
1212 
1213 /**
1214  * \brief disable file storing for files in a transaction
1215  *
1216  * \param f *LOCKED* flow
1217  * \param direction flow direction
1218  * \param tx_id transaction id
1219  */
1220 void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id)
1221 {
1222  File *ptr = NULL;
1223 
1225 
1226  SCEnter();
1227 
1228  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1229  if (ffc != NULL) {
1230  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1231  if (ptr->txid == tx_id) {
1232  if (ptr->flags & FILE_STORE) {
1233  /* weird, already storing -- let it continue*/
1234  SCLogDebug("file is already being stored");
1235  } else {
1236  FileDisableStoringForFile(ptr);
1237  }
1238  }
1239  }
1240  }
1241 
1242  SCReturn;
1243 }
1244 
1245 /**
1246  * \brief flag a file with id "file_id" to be stored.
1247  *
1248  * \param fc file store
1249  * \param file_id the file's id
1250  */
1251 void FileStoreFileById(FileContainer *fc, uint32_t file_id)
1252 {
1253  File *ptr = NULL;
1254 
1255  SCEnter();
1256 
1257  if (fc != NULL) {
1258  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1259  if (ptr->file_store_id == file_id) {
1260  FileStore(ptr);
1261  }
1262  }
1263  }
1264 }
1265 
1267 {
1268  File *ptr = NULL;
1269 
1270  SCEnter();
1271 
1272  if (fc != NULL) {
1273  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1274  if (ptr->txid == tx_id) {
1275  FileStore(ptr);
1276  }
1277  }
1278  }
1279 }
1280 
1282 {
1283  File *ptr = NULL;
1284 
1285  SCEnter();
1286 
1287  if (fc != NULL) {
1288  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1289  FileStore(ptr);
1290  }
1291  }
1292 }
1293 
1295 {
1296  File *ptr = NULL;
1297 
1298  SCEnter();
1299 
1300  if (fc != NULL) {
1301  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1302  if (ptr->state == FILE_STATE_OPENED) {
1303  FileCloseFilePtr(ptr, NULL, 0, FILE_TRUNCATED);
1304  }
1305  }
1306  }
1307 }
1308 
1309 /**
1310  * \brief Finish the SHA256 calculation.
1311  */
1312 #ifdef HAVE_NSS
1313 static void FileEndSha256(File *ff)
1314 {
1315  if (!(ff->flags & FILE_SHA256) && ff->sha256_ctx) {
1316  unsigned int len = 0;
1317  HASH_End(ff->sha256_ctx, ff->sha256, &len, sizeof(ff->sha256));
1318  ff->flags |= FILE_SHA256;
1319  }
1320 }
1321 #endif
uint16_t flags
void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id)
disable file storing for files in a transaction
Definition: util-file.c:1220
#define SCLogDebug(...)
Definition: util-debug.h:335
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
struct HtpBodyChunk_ * next
int FileForceSha256(void)
Definition: util-file.c:145
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:343
FileContainer * FileContainerAlloc(void)
allocate a FileContainer
Definition: util-file.c:380
#define FLOWFILE_NO_SHA256_TS
Definition: flow.h:125
int RunModeOutputFileEnabled(void)
Definition: runmodes.c:485
#define unlikely(expr)
Definition: util-optimize.h:35
#define FILE_SHA256
Definition: util-file.h:43
uint64_t size
Definition: util-file.h:91
void FileDisableFilesize(Flow *f, uint8_t direction)
disable file size tracking for this flow
Definition: util-file.c:1162
void FileDisableSha1(Flow *f, uint8_t direction)
disable file sha1 calc for this flow
Definition: util-file.c:1084
void FileDisableMd5(Flow *f, uint8_t direction)
disable file md5 calc for this flow
Definition: util-file.c:1045
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:277
#define FILE_NOTRACK
Definition: util-file.h:48
#define MIN(x, y)
File * FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg, const uint8_t *name, uint16_t name_len, const uint8_t *data, uint32_t data_len, uint16_t flags)
Open a new File.
Definition: util-file.c:756
uint64_t content_inspected
Definition: util-file.h:88
#define FILE_STORE
Definition: util-file.h:46
#define FILE_NOMAGIC
Definition: util-file.h:37
int fd
Definition: util-file.h:73
char * val
Definition: conf.h:34
void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id)
Definition: util-file.c:531
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
struct File_ * next
Definition: util-file.h:79
int FileCloseFileById(FileContainer *ffc, uint32_t track_id, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:957
void FileReassemblyDepthEnable(uint32_t size)
Definition: util-file.c:116
StreamingBuffer * sb
Definition: util-file.h:68
void FileForceSha1Enable(void)
Definition: util-file.c:101
int FileForceMd5(void)
Definition: util-file.c:135
#define FILE_NOSHA1
Definition: util-file.h:40
void FileForceTrackingEnable(void)
Definition: util-file.c:150
#define FLOWFILE_NO_MAGIC_TS
Definition: flow.h:110
uint32_t FileReassemblyDepth(void)
Definition: util-file.c:122
void * alstate
Definition: flow.h:433
#define FLOWFILE_NO_MD5_TS
Definition: flow.h:117
void StreamingBufferFree(StreamingBuffer *sb)
#define FILE_LOGGED
Definition: util-file.h:44
#define FLOWFILE_NO_MAGIC_TC
Definition: flow.h:111
#define FILE_USE_TRACKID
Definition: util-file.h:50
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
int FileStore(File *ff)
Tag a file for storing.
Definition: util-file.c:511
uint16_t flags
Definition: util-file.h:65
void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id)
Definition: util-file.c:1266
#define FLOWFILE_NO_SIZE_TS
Definition: flow.h:129
uint32_t * sid
Definition: util-file.h:93
void FileStoreFileById(FileContainer *fc, uint32_t file_id)
flag a file with id "file_id" to be stored.
Definition: util-file.c:1251
void FileDisableMagic(Flow *f, uint8_t direction)
disable file magic lookups for this flow
Definition: util-file.c:1014
void StreamingBufferSlideToOffset(StreamingBuffer *sb, uint64_t offset)
slide to absolute offset
#define FILE_HAS_GAPS
Definition: util-file.h:51
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileContainerFree(FileContainer *ffc)
Free a FileContainer.
Definition: util-file.c:416
void FileForceMagicEnable(void)
Definition: util-file.c:91
#define DEBUG_ASSERT_FLOW_LOCKED(f)
uint64_t txid
Definition: util-file.h:69
#define SCEnter(...)
Definition: util-debug.h:337
#define FLOWFILE_NO_MD5_TC
Definition: flow.h:118
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
File * head
Definition: util-file.h:99
void FileDisableStoring(Flow *f, uint8_t direction)
disable file storage for a flow
Definition: util-file.c:982
#define SCReturnInt(x)
Definition: util-debug.h:341
#define FILE_MD5
Definition: util-file.h:39
void FilePrune(FileContainer *ffc)
Definition: util-file.c:345
void FileStoreAllFiles(FileContainer *fc)
Definition: util-file.c:1281
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
uint32_t reassembly_depth
Definition: stream-tcp.h:60
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
int FileForceFilestore(void)
Definition: util-file.c:111
int FileForceMagic(void)
Definition: util-file.c:130
#define FILE_STORED
Definition: util-file.h:47
Definition: conf.h:32
#define FILE_NOSHA256
Definition: util-file.h:42
#define SCMalloc(a)
Definition: util-mem.h:166
int FileAppendGAPById(FileContainer *ffc, uint32_t track_id, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The file with &#39;track_id&#39; in the FileContainer...
Definition: util-file.c:718
#define FILE_USE_DETECT
Definition: util-file.h:49
void FileContainerAdd(FileContainer *ffc, File *ff)
Definition: util-file.c:496
void FileForceSha256Enable(void)
Definition: util-file.c:106
int RunModeOutputFiledataEnabled(void)
Definition: runmodes.c:490
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
void FileContainerRecycle(FileContainer *ffc)
Recycle a FileContainer.
Definition: util-file.c:397
#define SCFree(a)
Definition: util-mem.h:228
#define FLOWFILE_NO_SHA1_TS
Definition: flow.h:121
#define FLOWFILE_NO_STORE_TS
Definition: flow.h:114
uint16_t tx_id
#define FLOWFILE_NO_SIZE_TC
Definition: flow.h:130
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
FileState state
Definition: util-file.h:67
int FileOpenFileWithId(FileContainer *ffc, const StreamingBufferConfig *sbcfg, uint32_t track_id, const uint8_t *name, uint16_t name_len, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:846
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
int FileCloseFile(FileContainer *ffc, const uint8_t *data, uint32_t data_len, uint16_t flags)
Close a File.
Definition: util-file.c:941
int FileSetTx(File *ff, uint64_t txid)
Set the TX id for a file.
Definition: util-file.c:523
uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction)
Definition: util-file.c:217
uint32_t file_store_id
Definition: util-file.h:72
uint64_t content_stored
Definition: util-file.h:90
uint32_t file_track_id
Definition: util-file.h:70
#define FILE_TRUNCATED
Definition: util-file.h:36
int FileForceSha1(void)
Definition: util-file.c:140
int FileAppendDataById(FileContainer *ffc, uint32_t track_id, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The file with &#39;track_id&#39; in the FileContainer...
Definition: util-file.c:687
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
void FileForceMd5Enable(void)
Definition: util-file.c:96
#define SCReturn
Definition: util-debug.h:339
#define FLOWFILE_NO_SHA1_TC
Definition: flow.h:122
uint8_t len
void FileDisableSha256(Flow *f, uint8_t direction)
disable file sha256 calc for this flow
Definition: util-file.c:1123
uint16_t file_flags
Definition: flow.h:376
AppProto alproto
application level protocol
Definition: flow.h:404
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The last file in the FileContainer will be us...
Definition: util-file.c:663
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
#define FLOWFILE_NO_SHA256_TC
Definition: flow.h:126
#define FILE_NOSTORE
Definition: util-file.h:45
#define FLOWFILE_NO_STORE_TC
Definition: flow.h:115
int FileCloseFilePtr(File *ff, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:859
Flow data structure.
Definition: flow.h:324
uint8_t * name
Definition: util-file.h:75
void FileTruncateAllOpenFiles(FileContainer *fc)
Definition: util-file.c:1294
#define FILE_NOMD5
Definition: util-file.h:38