suricata
util-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
23  *
24  */
25 
26 #include "suricata-common.h"
27 #include "suricata.h"
28 #include "debug.h"
29 #include "flow.h"
30 #include "stream.h"
31 #include "stream-tcp.h"
32 #include "runmodes.h"
33 #include "util-hash.h"
34 #include "util-debug.h"
35 #include "util-memcmp.h"
36 #include "util-print.h"
37 #include "app-layer-parser.h"
38 #include "util-validate.h"
39 
40 /** \brief switch to force filestore on all files
41  * regardless of the rules.
42  */
43 static int g_file_force_filestore = 0;
44 
45 /** \brief switch to force magic checks on all files
46  * regardless of the rules.
47  */
48 static int g_file_force_magic = 0;
49 
50 /** \brief switch to force md5 calculation on all files
51  * regardless of the rules.
52  */
53 static int g_file_force_md5 = 0;
54 
55 /** \brief switch to force sha1 calculation on all files
56  * regardless of the rules.
57  */
58 static int g_file_force_sha1 = 0;
59 
60 /** \brief switch to force sha256 calculation on all files
61  * regardless of the rules.
62  */
63 static int g_file_force_sha256 = 0;
64 
65 /** \brief switch to force tracking off all files
66  * regardless of the rules.
67  */
68 static int g_file_force_tracking = 0;
69 
70 /** \brief switch to use g_file_store_reassembly_depth
71  * to reassembly files
72  */
73 static int g_file_store_enable = 0;
74 
75 /** \brief stream_config.reassembly_depth equivalent
76  * for files
77  */
78 static uint32_t g_file_store_reassembly_depth = 0;
79 
80 /* prototypes */
81 static void FileFree(File *);
82 #ifdef HAVE_NSS
83 static void FileEndSha256(File *ff);
84 #endif
85 
87 {
88  g_file_force_filestore = 1;
89 }
90 
92 {
93  g_file_force_magic = 1;
94 }
95 
97 {
98  g_file_force_md5 = 1;
99 }
100 
102 {
103  g_file_force_sha1 = 1;
104 }
105 
107 {
108  g_file_force_sha256 = 1;
109 }
110 
112 {
113  return g_file_force_filestore;
114 }
115 
116 void FileReassemblyDepthEnable(uint32_t size)
117 {
118  g_file_store_enable = 1;
119  g_file_store_reassembly_depth = size;
120 }
121 
122 uint32_t FileReassemblyDepth(void)
123 {
124  if (g_file_store_enable == 1)
125  return g_file_store_reassembly_depth;
126  else
128 }
129 
130 int FileForceMagic(void)
131 {
132  return g_file_force_magic;
133 }
134 
135 int FileForceMd5(void)
136 {
137  return g_file_force_md5;
138 }
139 
140 int FileForceSha1(void)
141 {
142  return g_file_force_sha1;
143 }
144 
146 {
147  return g_file_force_sha256;
148 }
149 
151 {
152  g_file_force_tracking = 1;
153 }
154 
155 /**
156  * \brief Function to parse forced file hashing configuration.
157  */
159 {
160  BUG_ON(conf == NULL);
161 
162  ConfNode *forcehash_node = NULL;
163 
164  /* legacy option */
165  const char *force_md5 = ConfNodeLookupChildValue(conf, "force-md5");
166  if (force_md5 != NULL) {
167  SCLogWarning(SC_ERR_DEPRECATED_CONF, "deprecated 'force-md5' option "
168  "found. Please use 'force-hash: [md5]' instead");
169 
170  if (ConfValIsTrue(force_md5)) {
171 #ifdef HAVE_NSS
173  SCLogInfo("forcing md5 calculation for logged files");
174 #else
175  SCLogInfo("md5 calculation requires linking against libnss");
176 #endif
177  }
178  }
179 
180  if (conf != NULL)
181  forcehash_node = ConfNodeLookupChild(conf, "force-hash");
182 
183  if (forcehash_node != NULL) {
184  ConfNode *field = NULL;
185 
186  TAILQ_FOREACH(field, &forcehash_node->head, next) {
187  if (strcasecmp("md5", field->val) == 0) {
188 #ifdef HAVE_NSS
190  SCLogConfig("forcing md5 calculation for logged or stored files");
191 #else
192  SCLogInfo("md5 calculation requires linking against libnss");
193 #endif
194  }
195 
196  if (strcasecmp("sha1", field->val) == 0) {
197 #ifdef HAVE_NSS
199  SCLogConfig("forcing sha1 calculation for logged or stored files");
200 #else
201  SCLogInfo("sha1 calculation requires linking against libnss");
202 #endif
203  }
204 
205  if (strcasecmp("sha256", field->val) == 0) {
206 #ifdef HAVE_NSS
208  SCLogConfig("forcing sha256 calculation for logged or stored files");
209 #else
210  SCLogInfo("sha256 calculation requires linking against libnss");
211 #endif
212  }
213  }
214  }
215 }
216 
217 uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction)
218 {
219  uint16_t flags = 0;
220 
221  if (direction == STREAM_TOSERVER) {
222  if (flow->file_flags & FLOWFILE_NO_STORE_TS) {
223  flags |= FILE_NOSTORE;
224  }
225 
226  if (flow->file_flags & FLOWFILE_NO_MAGIC_TS) {
227  flags |= FILE_NOMAGIC;
228  }
229 
230  if (flow->file_flags & FLOWFILE_NO_MD5_TS) {
231  flags |= FILE_NOMD5;
232  }
233 
234  if (flow->file_flags & FLOWFILE_NO_SHA1_TS) {
235  flags |= FILE_NOSHA1;
236  }
237 
238  if (flow->file_flags & FLOWFILE_NO_SHA256_TS) {
239  flags |= FILE_NOSHA256;
240  }
241  } else {
242  if (flow->file_flags & FLOWFILE_NO_STORE_TC) {
243  flags |= FILE_NOSTORE;
244  }
245 
246  if (flow->file_flags & FLOWFILE_NO_MAGIC_TC) {
247  flags |= FILE_NOMAGIC;
248  }
249 
250  if (flow->file_flags & FLOWFILE_NO_MD5_TC) {
251  flags |= FILE_NOMD5;
252  }
253 
254  if (flow->file_flags & FLOWFILE_NO_SHA1_TC) {
255  flags |= FILE_NOSHA1;
256  }
257 
258  if (flow->file_flags & FLOWFILE_NO_SHA256_TC) {
259  flags |= FILE_NOSHA256;
260  }
261  }
262  return flags;
263 }
264 
265 static int FileMagicSize(void)
266 {
267  /** \todo make this size configurable */
268  return 512;
269 }
270 
271 /**
272  * \brief get the size of the file data
273  *
274  * This doesn't reflect how much of the file we have in memory, just the
275  * total size of filedata so far.
276  */
277 uint64_t FileDataSize(const File *file)
278 {
279  if (file != NULL && file->sb != NULL) {
280  SCLogDebug("returning %"PRIu64,
281  file->sb->stream_offset + file->sb->buf_offset);
282  return file->sb->stream_offset + file->sb->buf_offset;
283  }
284  SCLogDebug("returning 0 (default)");
285  return 0;
286 }
287 
288 /**
289  * \brief get the size of the file
290  *
291  * This doesn't reflect how much of the file we have in memory, just the
292  * total size of file so far.
293  */
294 uint64_t FileTrackedSize(const File *file)
295 {
296  if (file != NULL) {
297  return file->size;
298  }
299  return 0;
300 }
301 
302 static int FilePruneFile(File *file)
303 {
304  SCEnter();
305 #ifdef HAVE_MAGIC
306  if (!(file->flags & FILE_NOMAGIC)) {
307  /* need magic but haven't set it yet, bail out */
308  if (file->magic == NULL)
309  SCReturnInt(0);
310  else
311  SCLogDebug("file->magic %s", file->magic);
312  } else {
313  SCLogDebug("file->flags & FILE_NOMAGIC == true");
314  }
315 #endif
316  uint64_t left_edge = file->content_stored;
317  if (file->flags & FILE_NOSTORE) {
318  left_edge = FileDataSize(file);
319  }
320  if (file->flags & FILE_USE_DETECT) {
321  left_edge = MIN(left_edge, file->content_inspected);
322  }
323 
324  if (left_edge) {
325  StreamingBufferSlideToOffset(file->sb, left_edge);
326  }
327 
328  if (left_edge != FileDataSize(file)) {
329  SCReturnInt(0);
330  }
331 
332  SCLogDebug("file->state %d. Is >= FILE_STATE_CLOSED: %s", file->state, (file->state >= FILE_STATE_CLOSED) ? "yes" : "no");
333 
334  /* file is done when state is closed+, logging/storing is done (if any) */
335  if (file->state >= FILE_STATE_CLOSED) {
336  SCReturnInt(1);
337  } else {
338  SCReturnInt(0);
339  }
340 }
341 
343 {
344  File *file = ffc->head;
345  File *prev = NULL;
346 
347  while (file) {
348  if (FilePruneFile(file) == 0) {
349  prev = file;
350  file = file->next;
351  continue;
352  }
353 
354  SCLogDebug("removing file %p", file);
355 
356  File *file_next = file->next;
357 
358  if (prev)
359  prev->next = file_next;
360  /* update head and tail */
361  if (file == ffc->head)
362  ffc->head = file_next;
363  if (file == ffc->tail)
364  ffc->tail = prev;
365 
366  FileFree(file);
367  file = file_next;
368  }
369 }
370 
371 /**
372  * \brief allocate a FileContainer
373  *
374  * \retval new newly allocated FileContainer
375  * \retval NULL error
376  */
378 {
379  FileContainer *new = SCMalloc(sizeof(FileContainer));
380  if (unlikely(new == NULL)) {
381  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem");
382  return NULL;
383  }
384  memset(new, 0, sizeof(FileContainer));
385  new->head = new->tail = NULL;
386  return new;
387 }
388 
389 /**
390  * \brief Recycle a FileContainer
391  *
392  * \param ffc FileContainer
393  */
395 {
396  if (ffc == NULL)
397  return;
398 
399  File *cur = ffc->head;
400  File *next = NULL;
401  for (;cur != NULL; cur = next) {
402  next = cur->next;
403  FileFree(cur);
404  }
405  ffc->head = ffc->tail = NULL;
406 }
407 
408 /**
409  * \brief Free a FileContainer
410  *
411  * \param ffc FileContainer
412  */
414 {
415  if (ffc == NULL)
416  return;
417 
418  File *ptr = ffc->head;
419  File *next = NULL;
420  for (;ptr != NULL; ptr = next) {
421  next = ptr->next;
422  FileFree(ptr);
423  }
424  ffc->head = ffc->tail = NULL;
425  SCFree(ffc);
426 }
427 
428 /**
429  * \brief Alloc a new File
430  *
431  * \param name character array containing the name (not a string)
432  * \param name_len length in bytes of the name
433  *
434  * \retval new File object or NULL on error
435  */
436 static File *FileAlloc(const uint8_t *name, uint16_t name_len)
437 {
438  File *new = SCMalloc(sizeof(File));
439  if (unlikely(new == NULL)) {
440  SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem");
441  return NULL;
442  }
443  memset(new, 0, sizeof(File));
444 
445  new->name = SCMalloc(name_len);
446  if (new->name == NULL) {
447  SCFree(new);
448  return NULL;
449  }
450 
451  new->name_len = name_len;
452  memcpy(new->name, name, name_len);
453 
454  new->sid_cnt = 0;
455  new->sid_max = 8;
456  /* SCMalloc() is allowed to fail here because sid well be checked later on */
457  new->sid = SCMalloc(sizeof(uint32_t) * new->sid_max);
458  if (new->sid == NULL)
459  new->sid_max = 0;
460 
461  return new;
462 }
463 
464 static void FileFree(File *ff)
465 {
466  if (ff == NULL)
467  return;
468 
469  if (ff->name != NULL)
470  SCFree(ff->name);
471  if (ff->sid != NULL)
472  SCFree(ff->sid);
473 #ifdef HAVE_MAGIC
474  /* magic returned by libmagic is strdup'd by MagicLookup. */
475  if (ff->magic != NULL)
476  SCFree(ff->magic);
477 #endif
478  if (ff->sb != NULL) {
479  StreamingBufferFree(ff->sb);
480  }
481 
482 #ifdef HAVE_NSS
483  if (ff->md5_ctx)
484  HASH_Destroy(ff->md5_ctx);
485  if (ff->sha1_ctx)
486  HASH_Destroy(ff->sha1_ctx);
487  if (ff->sha256_ctx)
488  HASH_Destroy(ff->sha256_ctx);
489 #endif
490  SCFree(ff);
491 }
492 
494 {
495  if (ffc->head == NULL || ffc->tail == NULL) {
496  ffc->head = ffc->tail = ff;
497  } else {
498  ffc->tail->next = ff;
499  ffc->tail = ff;
500  }
501 }
502 
503 /**
504  * \brief Tag a file for storing
505  *
506  * \param ff The file to store
507  */
508 int FileStore(File *ff)
509 {
510  ff->flags |= FILE_STORE;
511  SCReturnInt(0);
512 }
513 
514 /**
515  * \brief Set the TX id for a file
516  *
517  * \param ff The file to store
518  * \param txid the tx id
519  */
520 int FileSetTx(File *ff, uint64_t txid)
521 {
522  SCLogDebug("ff %p txid %"PRIu64, ff, txid);
523  if (ff != NULL)
524  ff->txid = txid;
525  SCReturnInt(0);
526 }
527 
529 {
530  if (ffc && ffc->tail) {
531  (void)FileSetTx(ffc->tail, tx_id);
532  }
533 }
534 
535 /**
536  * \brief check if we have stored enough
537  *
538  * \param ff file
539  *
540  * \retval 0 limit not reached yet
541  * \retval 1 limit reached
542  */
543 static int FileStoreNoStoreCheck(File *ff)
544 {
545  SCEnter();
546 
547  if (ff == NULL) {
548  SCReturnInt(0);
549  }
550 
551  if (ff->flags & FILE_NOSTORE) {
552  if (ff->state == FILE_STATE_OPENED &&
553  FileDataSize(ff) >= (uint64_t)FileMagicSize())
554  {
555  SCReturnInt(1);
556  }
557  }
558 
559  SCReturnInt(0);
560 }
561 
562 static int AppendData(File *file, const uint8_t *data, uint32_t data_len)
563 {
564  if (StreamingBufferAppendNoTrack(file->sb, data, data_len) != 0) {
565  SCReturnInt(-1);
566  }
567 
568 #ifdef HAVE_NSS
569  if (file->md5_ctx) {
570  HASH_Update(file->md5_ctx, data, data_len);
571  }
572  if (file->sha1_ctx) {
573  HASH_Update(file->sha1_ctx, data, data_len);
574  }
575  if (file->sha256_ctx) {
576  HASH_Update(file->sha256_ctx, data, data_len);
577  }
578 #endif
579  SCReturnInt(0);
580 }
581 
582 /** \internal
583  * \brief Store/handle a chunk of file data in the File structure
584  *
585  * \param ff the file
586  * \param data data chunk
587  * \param data_len data chunk len
588  *
589  * \retval 0 ok
590  * \retval -1 error
591  * \retval -2 no store for this file
592  */
593 static int FileAppendDataDo(File *ff, const uint8_t *data, uint32_t data_len)
594 {
595  SCEnter();
596 #ifdef DEBUG_VALIDATION
597  BUG_ON(ff == NULL);
598 #endif
599 
600  ff->size += data_len;
601 
602  if (ff->state != FILE_STATE_OPENED) {
603  if (ff->flags & FILE_NOSTORE) {
604  SCReturnInt(-2);
605  }
606  SCReturnInt(-1);
607  }
608 
609  if ((ff->flags & FILE_USE_DETECT) == 0 &&
610  FileStoreNoStoreCheck(ff) == 1) {
611 #ifdef HAVE_NSS
612  int hash_done = 0;
613  /* no storage but forced hashing */
614  if (ff->md5_ctx) {
615  HASH_Update(ff->md5_ctx, data, data_len);
616  hash_done = 1;
617  }
618  if (ff->sha1_ctx) {
619  HASH_Update(ff->sha1_ctx, data, data_len);
620  hash_done = 1;
621  }
622  if (ff->sha256_ctx) {
623  HASH_Update(ff->sha256_ctx, data, data_len);
624  hash_done = 1;
625  }
626 
627  if (hash_done)
628  SCReturnInt(0);
629 #endif
630  if (g_file_force_tracking || (!(ff->flags & FILE_NOTRACK)))
631  SCReturnInt(0);
632 
634  SCLogDebug("flowfile state transitioned to FILE_STATE_TRUNCATED");
635  SCReturnInt(-2);
636  }
637 
638  SCLogDebug("appending %"PRIu32" bytes", data_len);
639 
640  int r = AppendData(ff, data, data_len);
641  if (r != 0) {
642  ff->state = FILE_STATE_ERROR;
643  SCReturnInt(r);
644  }
645 
646  SCReturnInt(0);
647 }
648 
649 /**
650  * \brief Store/handle a chunk of file data in the File structure
651  * The last file in the FileContainer will be used.
652  *
653  * \param ffc FileContainer used to append to
654  * \param data data chunk
655  * \param data_len data chunk len
656  *
657  * \retval 0 ok
658  * \retval -1 error
659  * \retval -2 no store for this file
660  */
661 int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
662 {
663  SCEnter();
664 
665  if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
666  SCReturnInt(-1);
667  }
668  int r = FileAppendDataDo(ffc->tail, data, data_len);
669  SCReturnInt(r);
670 }
671 
672 /**
673  * \brief Store/handle a chunk of file data in the File structure
674  * The file with 'track_id' in the FileContainer will be used.
675  *
676  * \param ffc FileContainer used to append to
677  * \param track_id id to lookup the file
678  * \param data data chunk
679  * \param data_len data chunk len
680  *
681  * \retval 0 ok
682  * \retval -1 error
683  * \retval -2 no store for this file
684  */
685 int FileAppendDataById(FileContainer *ffc, uint32_t track_id,
686  const uint8_t *data, uint32_t data_len)
687 {
688  SCEnter();
689 
690  if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
691  SCReturnInt(-1);
692  }
693  File *ff = ffc->head;
694  for ( ; ff != NULL; ff = ff->next) {
695  if (track_id == ff->file_track_id) {
696  int r = FileAppendDataDo(ff, data, data_len);
697  SCReturnInt(r);
698  }
699  }
700  SCReturnInt(-1);
701 }
702 
703 /**
704  * \brief Store/handle a chunk of file data in the File structure
705  * The file with 'track_id' in the FileContainer will be used.
706  *
707  * \param ffc FileContainer used to append to
708  * \param track_id id to lookup the file
709  * \param data data chunk
710  * \param data_len data chunk len
711  *
712  * \retval 0 ok
713  * \retval -1 error
714  * \retval -2 no store for this file
715  */
716 int FileAppendGAPById(FileContainer *ffc, uint32_t track_id,
717  const uint8_t *data, uint32_t data_len)
718 {
719  SCEnter();
720 
721  if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
722  SCReturnInt(-1);
723  }
724  File *ff = ffc->head;
725  for ( ; ff != NULL; ff = ff->next) {
726  if (track_id == ff->file_track_id) {
727  ff->flags |= FILE_HAS_GAPS;
730  SCLogDebug("FILE_HAS_GAPS set");
731 
732  int r = FileAppendDataDo(ff, data, data_len);
733  SCReturnInt(r);
734  }
735  }
736  SCReturnInt(-1);
737 }
738 
739 /**
740  * \brief Sets the offset range for a file.
741  *
742  * \param ffc the container
743  * \param start start offset
744  * \param end end offset
745  *
746  * \retval 0 ok
747  * \retval -1 error
748  */
749 int FileSetRange(FileContainer *ffc, uint64_t start, uint64_t end)
750 {
751  SCEnter();
752 
753  if (ffc == NULL || ffc->tail == NULL) {
754  SCReturnInt(-1);
755  }
756  ffc->tail->start = start;
757  ffc->tail->end = end;
758  SCReturnInt(0);
759 }
760 
761 /**
762  * \brief Open a new File
763  *
764  * \param ffc flow container
765  * \param sbcfg buffer config
766  * \param name filename character array
767  * \param name_len filename len
768  * \param data initial data
769  * \param data_len initial data len
770  * \param flags open flags
771  *
772  * \retval ff flowfile object
773  *
774  * \note filename is not a string, so it's not nul terminated.
775  */
776 static File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg,
777  const uint8_t *name, uint16_t name_len,
778  const uint8_t *data, uint32_t data_len, uint16_t flags)
779 {
780  SCEnter();
781 
782  //PrintRawDataFp(stdout, name, name_len);
783 
784  File *ff = FileAlloc(name, name_len);
785  if (ff == NULL) {
786  SCReturnPtr(NULL, "File");
787  }
788 
789  ff->sb = StreamingBufferInit(sbcfg);
790  if (ff->sb == NULL) {
791  FileFree(ff);
792  SCReturnPtr(NULL, "File");
793  }
794  SCLogDebug("ff->sb %p", ff->sb);
795 
796  if (flags & FILE_STORE || g_file_force_filestore) {
797  FileStore(ff);
798  } else if (flags & FILE_NOSTORE) {
799  SCLogDebug("not storing this file");
800  ff->flags |= FILE_NOSTORE;
801  }
802  if (flags & FILE_NOMAGIC) {
803  SCLogDebug("not doing magic for this file");
804  ff->flags |= FILE_NOMAGIC;
805  }
806  if (flags & FILE_NOMD5) {
807  SCLogDebug("not doing md5 for this file");
808  ff->flags |= FILE_NOMD5;
809  }
810  if (flags & FILE_NOSHA1) {
811  SCLogDebug("not doing sha1 for this file");
812  ff->flags |= FILE_NOSHA1;
813  }
814  if (flags & FILE_NOSHA256) {
815  SCLogDebug("not doing sha256 for this file");
816  ff->flags |= FILE_NOSHA256;
817  }
818  if (flags & FILE_USE_DETECT) {
819  SCLogDebug("considering content_inspect tracker when pruning");
820  ff->flags |= FILE_USE_DETECT;
821  }
822 
823 #ifdef HAVE_NSS
824  if (!(ff->flags & FILE_NOMD5) || g_file_force_md5) {
825  ff->md5_ctx = HASH_Create(HASH_AlgMD5);
826  if (ff->md5_ctx != NULL) {
827  HASH_Begin(ff->md5_ctx);
828  }
829  }
830  if (!(ff->flags & FILE_NOSHA1) || g_file_force_sha1) {
831  ff->sha1_ctx = HASH_Create(HASH_AlgSHA1);
832  if (ff->sha1_ctx != NULL) {
833  HASH_Begin(ff->sha1_ctx);
834  }
835  }
836  if (!(ff->flags & FILE_NOSHA256) || g_file_force_sha256) {
837  ff->sha256_ctx = HASH_Create(HASH_AlgSHA256);
838  if (ff->sha256_ctx != NULL) {
839  HASH_Begin(ff->sha256_ctx);
840  }
841  }
842 #endif
843 
844  ff->state = FILE_STATE_OPENED;
845  SCLogDebug("flowfile state transitioned to FILE_STATE_OPENED");
846 
847  ff->fd = -1;
848 
849  FileContainerAdd(ffc, ff);
850 
851  if (data != NULL) {
852  ff->size += data_len;
853  if (AppendData(ff, data, data_len) != 0) {
854  ff->state = FILE_STATE_ERROR;
855  SCReturnPtr(NULL, "File");
856  }
857  SCLogDebug("file size is now %"PRIu64, FileTrackedSize(ff));
858  }
859 
860  SCReturnPtr(ff, "File");
861 }
862 
863 /**
864  * \retval 0 ok
865  * \retval -1 failed */
867  uint32_t track_id, const uint8_t *name, uint16_t name_len,
868  const uint8_t *data, uint32_t data_len, uint16_t flags)
869 {
870  File *ff = FileOpenFile(ffc, sbcfg, name, name_len, data, data_len, flags);
871  if (ff == NULL)
872  return -1;
873 
874  ff->file_track_id = track_id;
875  return 0;
876 }
877 
878 int FileCloseFilePtr(File *ff, const uint8_t *data,
879  uint32_t data_len, uint16_t flags)
880 {
881  SCEnter();
882 
883  if (ff == NULL) {
884  SCReturnInt(-1);
885  }
886 
887  if (ff->state != FILE_STATE_OPENED) {
888  SCReturnInt(-1);
889  }
890 
891  if (data != NULL) {
892  ff->size += data_len;
893  if (ff->flags & FILE_NOSTORE) {
894 #ifdef HAVE_NSS
895  /* no storage but hashing */
896  if (ff->md5_ctx)
897  HASH_Update(ff->md5_ctx, data, data_len);
898  if (ff->sha1_ctx)
899  HASH_Update(ff->sha1_ctx, data, data_len);
900  if (ff->sha256_ctx)
901  HASH_Update(ff->sha256_ctx, data, data_len);
902 #endif
903  } else {
904  if (AppendData(ff, data, data_len) != 0) {
905  ff->state = FILE_STATE_ERROR;
906  SCReturnInt(-1);
907  }
908  }
909  }
910 
911  if ((flags & FILE_TRUNCATED) || (ff->flags & FILE_HAS_GAPS)) {
913  SCLogDebug("flowfile state transitioned to FILE_STATE_TRUNCATED");
914 
915  if (flags & FILE_NOSTORE) {
916  SCLogDebug("not storing this file");
917  ff->flags |= FILE_NOSTORE;
918  } else {
919 #ifdef HAVE_NSS
920  if (g_file_force_sha256 && ff->sha256_ctx) {
921  FileEndSha256(ff);
922  }
923 #endif
924  }
925  } else {
926  ff->state = FILE_STATE_CLOSED;
927  SCLogDebug("flowfile state transitioned to FILE_STATE_CLOSED");
928 
929 #ifdef HAVE_NSS
930  if (ff->md5_ctx) {
931  unsigned int len = 0;
932  HASH_End(ff->md5_ctx, ff->md5, &len, sizeof(ff->md5));
933  ff->flags |= FILE_MD5;
934  }
935  if (ff->sha1_ctx) {
936  unsigned int len = 0;
937  HASH_End(ff->sha1_ctx, ff->sha1, &len, sizeof(ff->sha1));
938  ff->flags |= FILE_SHA1;
939  }
940  if (ff->sha256_ctx) {
941  FileEndSha256(ff);
942  }
943 #endif
944  }
945 
946  SCReturnInt(0);
947 }
948 
949 /**
950  * \brief Close a File
951  *
952  * \param ffc the container
953  * \param data final data if any
954  * \param data_len data len if any
955  * \param flags flags
956  *
957  * \retval 0 ok
958  * \retval -1 error
959  */
960 int FileCloseFile(FileContainer *ffc, const uint8_t *data,
961  uint32_t data_len, uint16_t flags)
962 {
963  SCEnter();
964 
965  if (ffc == NULL || ffc->tail == NULL) {
966  SCReturnInt(-1);
967  }
968 
969  if (FileCloseFilePtr(ffc->tail, data, data_len, flags) == -1) {
970  SCReturnInt(-1);
971  }
972 
973  SCReturnInt(0);
974 }
975 
976 int FileCloseFileById(FileContainer *ffc, uint32_t track_id,
977  const uint8_t *data, uint32_t data_len, uint16_t flags)
978 {
979  SCEnter();
980 
981  if (ffc == NULL || ffc->tail == NULL) {
982  SCReturnInt(-1);
983  }
984 
985  File *ff = ffc->head;
986  for ( ; ff != NULL; ff = ff->next) {
987  if (track_id == ff->file_track_id) {
988  int r = FileCloseFilePtr(ff, data, data_len, flags);
989  SCReturnInt(r);
990  }
991  }
992  SCReturnInt(-1);
993 }
994 
995 /**
996  * \brief disable file storage for a flow
997  *
998  * \param f *LOCKED* flow
999  * \param direction flow direction
1000  */
1001 void FileDisableStoring(Flow *f, uint8_t direction)
1002 {
1003  File *ptr = NULL;
1004 
1005  SCEnter();
1006 
1008 
1009  if (direction == STREAM_TOSERVER)
1011  else
1013 
1014  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1015  if (ffc != NULL) {
1016  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1017  /* if we're already storing, we'll continue */
1018  if (!(ptr->flags & FILE_STORE)) {
1019  SCLogDebug("not storing this file");
1020  ptr->flags |= FILE_NOSTORE;
1021  }
1022  }
1023  }
1024  SCReturn;
1025 }
1026 
1027 /**
1028  * \brief disable file magic lookups for this flow
1029  *
1030  * \param f *LOCKED* flow
1031  * \param direction flow direction
1032  */
1033 void FileDisableMagic(Flow *f, uint8_t direction)
1034 {
1035  File *ptr = NULL;
1036 
1037  SCEnter();
1038 
1040 
1041  if (direction == STREAM_TOSERVER)
1043  else
1045 
1046  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1047  if (ffc != NULL) {
1048  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1049  SCLogDebug("disabling magic for file %p from direction %s",
1050  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1051  ptr->flags |= FILE_NOMAGIC;
1052  }
1053  }
1054 
1055  SCReturn;
1056 }
1057 
1058 /**
1059  * \brief disable file md5 calc for this flow
1060  *
1061  * \param f *LOCKED* flow
1062  * \param direction flow direction
1063  */
1064 void FileDisableMd5(Flow *f, uint8_t direction)
1065 {
1066  File *ptr = NULL;
1067 
1068  SCEnter();
1069 
1071 
1072  if (direction == STREAM_TOSERVER)
1074  else
1076 
1077  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1078  if (ffc != NULL) {
1079  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1080  SCLogDebug("disabling md5 for file %p from direction %s",
1081  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1082  ptr->flags |= FILE_NOMD5;
1083 
1084 #ifdef HAVE_NSS
1085  /* destroy any ctx we may have so far */
1086  if (ptr->md5_ctx != NULL) {
1087  HASH_Destroy(ptr->md5_ctx);
1088  ptr->md5_ctx = NULL;
1089  }
1090 #endif
1091  }
1092  }
1093 
1094  SCReturn;
1095 }
1096 
1097 /**
1098  * \brief disable file sha1 calc for this flow
1099  *
1100  * \param f *LOCKED* flow
1101  * \param direction flow direction
1102 */
1103 void FileDisableSha1(Flow *f, uint8_t direction)
1104 {
1105  File *ptr = NULL;
1106 
1107  SCEnter();
1108 
1110 
1111  if (direction == STREAM_TOSERVER)
1113  else
1115 
1116  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1117  if (ffc != NULL) {
1118  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1119  SCLogDebug("disabling sha1 for file %p from direction %s",
1120  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1121  ptr->flags |= FILE_NOSHA1;
1122 
1123 #ifdef HAVE_NSS
1124  /* destroy any ctx we may have so far */
1125  if (ptr->sha1_ctx != NULL) {
1126  HASH_Destroy(ptr->sha1_ctx);
1127  ptr->sha1_ctx = NULL;
1128  }
1129 #endif
1130  }
1131  }
1132 
1133  SCReturn;
1134 }
1135 
1136 /**
1137  * \brief disable file sha256 calc for this flow
1138  *
1139  * \param f *LOCKED* flow
1140  * \param direction flow direction
1141  */
1142 void FileDisableSha256(Flow *f, uint8_t direction)
1143 {
1144  File *ptr = NULL;
1145 
1146  SCEnter();
1147 
1149 
1150  if (direction == STREAM_TOSERVER)
1152  else
1154 
1155  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1156  if (ffc != NULL) {
1157  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1158  SCLogDebug("disabling sha256 for file %p from direction %s",
1159  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1160  ptr->flags |= FILE_NOSHA256;
1161 
1162 #ifdef HAVE_NSS
1163  /* destroy any ctx we may have so far */
1164  if (ptr->sha256_ctx != NULL) {
1165  HASH_Destroy(ptr->sha256_ctx);
1166  ptr->sha256_ctx = NULL;
1167  }
1168 #endif
1169  }
1170  }
1171 
1172  SCReturn;
1173 }
1174 
1175 /**
1176  * \brief disable file size tracking for this flow
1177  *
1178  * \param f *LOCKED* flow
1179  * \param direction flow direction
1180  */
1181 void FileDisableFilesize(Flow *f, uint8_t direction)
1182 {
1183  File *ptr = NULL;
1184 
1185  SCEnter();
1186 
1188 
1189  if (direction == STREAM_TOSERVER)
1191  else
1193 
1194  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1195  if (ffc != NULL) {
1196  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1197  SCLogDebug("disabling size tracking for file %p from direction %s",
1198  ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
1199  ptr->flags |= FILE_NOTRACK;
1200  }
1201  }
1202 
1203  SCReturn;
1204 }
1205 
1206 
1207 /**
1208  * \brief set no store flag, close file if needed
1209  *
1210  * \param ff file
1211  */
1212 static void FileDisableStoringForFile(File *ff)
1213 {
1214  SCEnter();
1215 
1216  if (ff == NULL) {
1217  SCReturn;
1218  }
1219 
1220  SCLogDebug("not storing this file");
1221  ff->flags |= FILE_NOSTORE;
1222 
1223  if (ff->state == FILE_STATE_OPENED && FileDataSize(ff) >= (uint64_t)FileMagicSize()) {
1224  if (g_file_force_md5 == 0 && g_file_force_sha1 == 0 && g_file_force_sha256 == 0
1225  && g_file_force_tracking == 0) {
1226  (void)FileCloseFilePtr(ff, NULL, 0,
1228  }
1229  }
1230 }
1231 
1232 /**
1233  * \brief disable file storing for files in a transaction
1234  *
1235  * \param f *LOCKED* flow
1236  * \param direction flow direction
1237  * \param tx_id transaction id
1238  */
1239 void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id)
1240 {
1241  File *ptr = NULL;
1242 
1244 
1245  SCEnter();
1246 
1247  FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction);
1248  if (ffc != NULL) {
1249  for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
1250  if (ptr->txid == tx_id) {
1251  if (ptr->flags & FILE_STORE) {
1252  /* weird, already storing -- let it continue*/
1253  SCLogDebug("file is already being stored");
1254  } else {
1255  FileDisableStoringForFile(ptr);
1256  }
1257  }
1258  }
1259  }
1260 
1261  SCReturn;
1262 }
1263 
1264 /**
1265  * \brief flag a file with id "file_id" to be stored.
1266  *
1267  * \param fc file store
1268  * \param file_id the file's id
1269  */
1270 void FileStoreFileById(FileContainer *fc, uint32_t file_id)
1271 {
1272  File *ptr = NULL;
1273 
1274  SCEnter();
1275 
1276  if (fc != NULL) {
1277  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1278  if (ptr->file_track_id == file_id) {
1279  FileStore(ptr);
1280  }
1281  }
1282  }
1283 }
1284 
1286 {
1287  File *ptr = NULL;
1288 
1289  SCEnter();
1290 
1291  if (fc != NULL) {
1292  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1293  if (ptr->txid == tx_id) {
1294  FileStore(ptr);
1295  }
1296  }
1297  }
1298 }
1299 
1301 {
1302  File *ptr = NULL;
1303 
1304  SCEnter();
1305 
1306  if (fc != NULL) {
1307  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1308  FileStore(ptr);
1309  }
1310  }
1311 }
1312 
1314 {
1315  File *ptr = NULL;
1316 
1317  SCEnter();
1318 
1319  if (fc != NULL) {
1320  for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
1321  if (ptr->state == FILE_STATE_OPENED) {
1322  FileCloseFilePtr(ptr, NULL, 0, FILE_TRUNCATED);
1323  }
1324  }
1325  }
1326 }
1327 
1328 /**
1329  * \brief Finish the SHA256 calculation.
1330  */
1331 #ifdef HAVE_NSS
1332 static void FileEndSha256(File *ff)
1333 {
1334  if (!(ff->flags & FILE_SHA256) && ff->sha256_ctx) {
1335  unsigned int len = 0;
1336  HASH_End(ff->sha256_ctx, ff->sha256, &len, sizeof(ff->sha256));
1337  ff->flags |= FILE_SHA256;
1338  }
1339 }
1340 #endif
uint16_t flags
void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id)
disable file storing for files in a transaction
Definition: util-file.c:1239
#define SCLogDebug(...)
Definition: util-debug.h:335
StreamingBuffer * StreamingBufferInit(const StreamingBufferConfig *cfg)
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
struct HtpBodyChunk_ * next
int FileForceSha256(void)
Definition: util-file.c:145
TcpStreamCnf stream_config
Definition: stream-tcp.h:106
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:344
FileContainer * FileContainerAlloc(void)
allocate a FileContainer
Definition: util-file.c:377
#define FLOWFILE_NO_SHA256_TS
Definition: flow.h:126
#define unlikely(expr)
Definition: util-optimize.h:35
#define FILE_SHA256
Definition: util-file.h:43
uint64_t size
Definition: util-file.h:89
void FileDisableFilesize(Flow *f, uint8_t direction)
disable file size tracking for this flow
Definition: util-file.c:1181
void FileDisableSha1(Flow *f, uint8_t direction)
disable file sha1 calc for this flow
Definition: util-file.c:1103
void FileDisableMd5(Flow *f, uint8_t direction)
disable file md5 calc for this flow
Definition: util-file.c:1064
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:277
#define FILE_NOTRACK
Definition: util-file.h:48
#define MIN(x, y)
uint64_t content_inspected
Definition: util-file.h:86
#define FILE_STORE
Definition: util-file.h:46
#define FILE_NOMAGIC
Definition: util-file.h:37
int fd
Definition: util-file.h:71
char * val
Definition: conf.h:34
void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id)
Definition: util-file.c:528
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
struct File_ * next
Definition: util-file.h:77
int FileCloseFileById(FileContainer *ffc, uint32_t track_id, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:976
void FileReassemblyDepthEnable(uint32_t size)
Definition: util-file.c:116
StreamingBuffer * sb
Definition: util-file.h:67
void FileForceSha1Enable(void)
Definition: util-file.c:101
int FileForceMd5(void)
Definition: util-file.c:135
#define FILE_NOSHA1
Definition: util-file.h:40
void FileForceTrackingEnable(void)
Definition: util-file.c:150
uint64_t start
Definition: util-file.h:90
#define FLOWFILE_NO_MAGIC_TS
Definition: flow.h:111
uint32_t FileReassemblyDepth(void)
Definition: util-file.c:122
void * alstate
Definition: flow.h:438
#define FLOWFILE_NO_MD5_TS
Definition: flow.h:118
void StreamingBufferFree(StreamingBuffer *sb)
#define FLOWFILE_NO_MAGIC_TC
Definition: flow.h:112
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
int FileStore(File *ff)
Tag a file for storing.
Definition: util-file.c:508
uint16_t flags
Definition: util-file.h:64
void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id)
Definition: util-file.c:1285
#define FLOWFILE_NO_SIZE_TS
Definition: flow.h:130
uint64_t end
Definition: util-file.h:91
uint32_t * sid
Definition: util-file.h:93
void FileStoreFileById(FileContainer *fc, uint32_t file_id)
flag a file with id "file_id" to be stored.
Definition: util-file.c:1270
void FileDisableMagic(Flow *f, uint8_t direction)
disable file magic lookups for this flow
Definition: util-file.c:1033
void StreamingBufferSlideToOffset(StreamingBuffer *sb, uint64_t offset)
slide to absolute offset
#define FILE_HAS_GAPS
Definition: util-file.h:50
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
Definition: util-file.c:158
void FileContainerFree(FileContainer *ffc)
Free a FileContainer.
Definition: util-file.c:413
void FileForceMagicEnable(void)
Definition: util-file.c:91
#define DEBUG_ASSERT_FLOW_LOCKED(f)
uint64_t txid
Definition: util-file.h:68
#define SCEnter(...)
Definition: util-debug.h:337
int FileSetRange(FileContainer *ffc, uint64_t start, uint64_t end)
Sets the offset range for a file.
Definition: util-file.c:749
#define FLOWFILE_NO_MD5_TC
Definition: flow.h:119
void FileForceFilestoreEnable(void)
Definition: util-file.c:86
File * head
Definition: util-file.h:99
void FileDisableStoring(Flow *f, uint8_t direction)
disable file storage for a flow
Definition: util-file.c:1001
#define SCReturnInt(x)
Definition: util-debug.h:341
#define FILE_MD5
Definition: util-file.h:39
void FilePrune(FileContainer *ffc)
Definition: util-file.c:342
void FileStoreAllFiles(FileContainer *fc)
Definition: util-file.c:1300
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
Definition: util-debug.h:281
uint32_t reassembly_depth
Definition: stream-tcp.h:60
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:566
#define FILE_SHA1
Definition: util-file.h:41
int FileForceFilestore(void)
Definition: util-file.c:111
int FileForceMagic(void)
Definition: util-file.c:130
Definition: conf.h:32
#define FILE_NOSHA256
Definition: util-file.h:42
#define SCMalloc(a)
Definition: util-mem.h:222
int FileAppendGAPById(FileContainer *ffc, uint32_t track_id, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The file with &#39;track_id&#39; in the FileContainer...
Definition: util-file.c:716
#define FILE_USE_DETECT
Definition: util-file.h:49
void FileContainerAdd(FileContainer *ffc, File *ff)
Definition: util-file.c:493
void FileForceSha256Enable(void)
Definition: util-file.c:106
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
void FileContainerRecycle(FileContainer *ffc)
Recycle a FileContainer.
Definition: util-file.c:394
#define SCFree(a)
Definition: util-mem.h:322
#define FLOWFILE_NO_SHA1_TS
Definition: flow.h:122
#define FLOWFILE_NO_STORE_TS
Definition: flow.h:115
uint16_t tx_id
#define FLOWFILE_NO_SIZE_TC
Definition: flow.h:131
#define STREAM_TOSERVER
Definition: stream.h:31
#define SCReturnPtr(x, type)
Definition: util-debug.h:353
FileState state
Definition: util-file.h:66
int FileOpenFileWithId(FileContainer *ffc, const StreamingBufferConfig *sbcfg, uint32_t track_id, const uint8_t *name, uint16_t name_len, const uint8_t *data, uint32_t data_len, uint16_t flags)
Open a new File.
Definition: util-file.c:866
int StreamingBufferAppendNoTrack(StreamingBuffer *sb, const uint8_t *data, uint32_t data_len)
add data w/o tracking a segment
int FileCloseFile(FileContainer *ffc, const uint8_t *data, uint32_t data_len, uint16_t flags)
Close a File.
Definition: util-file.c:960
int FileSetTx(File *ff, uint64_t txid)
Set the TX id for a file.
Definition: util-file.c:520
uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction)
Definition: util-file.c:217
uint64_t content_stored
Definition: util-file.h:88
uint32_t file_track_id
Definition: util-file.h:69
#define FILE_TRUNCATED
Definition: util-file.h:36
int FileForceSha1(void)
Definition: util-file.c:140
int FileAppendDataById(FileContainer *ffc, uint32_t track_id, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The file with &#39;track_id&#39; in the FileContainer...
Definition: util-file.c:685
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:294
void FileForceMd5Enable(void)
Definition: util-file.c:96
#define SCReturn
Definition: util-debug.h:339
#define FLOWFILE_NO_SHA1_TC
Definition: flow.h:123
uint8_t len
void FileDisableSha256(Flow *f, uint8_t direction)
disable file sha256 calc for this flow
Definition: util-file.c:1142
uint16_t file_flags
Definition: flow.h:381
AppProto alproto
application level protocol
Definition: flow.h:409
FileContainer * AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction)
int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The last file in the FileContainer will be us...
Definition: util-file.c:661
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
#define FLOWFILE_NO_SHA256_TC
Definition: flow.h:127
#define FILE_NOSTORE
Definition: util-file.h:45
#define FLOWFILE_NO_STORE_TC
Definition: flow.h:116
int FileCloseFilePtr(File *ff, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:878
Flow data structure.
Definition: flow.h:325
uint8_t * name
Definition: util-file.h:73
void FileTruncateAllOpenFiles(FileContainer *fc)
Definition: util-file.c:1313
#define FILE_NOMD5
Definition: util-file.h:38