suricata
util-file.h
Go to the documentation of this file.
1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #ifndef SURICATA_UTIL_FILE_H
26 #define SURICATA_UTIL_FILE_H
27 
28 #include "conf.h"
29 #include "util-streaming-buffer.h"
30 #include "flow.h"
31 
32 /* Hack: Pulling rust.h to get the SCSha256 causes all sorts of problems with
33  * header include orders, which is something we'll have to resolve as we provide
34  * more functionality via Rust. But this lets me continue with replacing nss
35  * without fighting the headers at this time. */
36 typedef struct SCSha256 SCSha256;
37 #define SC_SHA256_LEN 32
38 
39 typedef struct SCSha1 SCSha1;
40 #define SC_SHA1_LEN 20
41 
42 typedef struct SCMd5 SCMd5;
43 #define SC_MD5_LEN 16
44 
45 #define FILE_TRUNCATED BIT_U16(0)
46 #define FILE_NOMAGIC BIT_U16(1)
47 #define FILE_NOMD5 BIT_U16(2)
48 #define FILE_MD5 BIT_U16(3)
49 #define FILE_NOSHA1 BIT_U16(4)
50 #define FILE_SHA1 BIT_U16(5)
51 #define FILE_NOSHA256 BIT_U16(6)
52 #define FILE_SHA256 BIT_U16(7)
53 #define FILE_LOGGED BIT_U16(8)
54 #define FILE_NOSTORE BIT_U16(9)
55 #define FILE_STORE BIT_U16(10)
56 #define FILE_STORED BIT_U16(11)
57 #define FILE_NOTRACK BIT_U16(12) /**< track size of file */
58 #define FILE_USE_DETECT BIT_U16(13) /**< use content_inspected tracker */
59 #define FILE_HAS_GAPS BIT_U16(15)
60 
61 // to be used instead of PATH_MAX which depends on the OS
62 #define SC_FILENAME_MAX 4096
63 
64 #define FILEDATA_CONTENT_LIMIT 100000
65 #define FILEDATA_CONTENT_INSPECT_MIN_SIZE 32768
66 #define FILEDATA_CONTENT_INSPECT_WINDOW 4096
67 
68 typedef enum FileState_ {
69  FILE_STATE_NONE = 0, /**< no state */
70  FILE_STATE_OPENED, /**< flow file is opened */
71  FILE_STATE_CLOSED, /**< flow file is completed,
72  there will be no more data. */
73  FILE_STATE_TRUNCATED, /**< flow file is not complete, but
74  there will be no more data. */
75  FILE_STATE_ERROR, /**< file is in an error state */
78 
79 typedef struct File_ {
80  uint16_t flags;
81  uint16_t name_len;
84  uint32_t file_track_id; /**< id used by protocol parser */
85  uint32_t file_store_id; /**< id used in store file name file.<id> */
86  int fd; /**< file descriptor for filestore, not
87  open if equal to -1 */
88  uint8_t *name;
89 #ifdef HAVE_MAGIC
90  char *magic;
91 #endif
92  struct File_ *next;
94  uint8_t md5[SC_MD5_LEN];
96  uint8_t sha1[SC_SHA1_LEN];
99  uint64_t content_inspected; /**< used in pruning if FILE_USE_DETECT
100  * flag is set */
101  uint64_t content_stored;
102  uint64_t size;
103  uint32_t inspect_window;
105  uint64_t start;
106  uint64_t end;
107 
108  uint32_t *sid; /* signature id of a rule that triggered the filestore event */
109  uint32_t sid_cnt;
110  uint32_t sid_max;
112 
113 typedef struct FileContainer_ {
117 
120 
122 
124 
125 /**
126  * \brief Open a new File
127  *
128  * \param ffc flow container
129  * \param sbcfg buffer config
130  * \param name filename character array
131  * \param name_len filename len
132  * \param data initial data
133  * \param data_len initial data len
134  * \param flags open flags
135  *
136  * \retval ff flowfile object
137  *
138  * \note filename is not a string, so it's not nul terminated.
139  *
140  * If flags contains the FILE_USE_DETECT bit, the pruning code will
141  * consider not just the content_stored tracker, but also content_inspected.
142  * It's the responsibility of the API user to make sure this tracker is
143  * properly updated.
144  */
146  uint32_t track_id, const uint8_t *name, uint16_t name_len,
147  const uint8_t *data, uint32_t data_len, uint16_t flags);
148 
149 /**
150  * \brief Close a File
151  *
152  * \param ffc the container
153  * \param data final data if any
154  * \param data_len data len if any
155  * \param flags flags
156  *
157  * \retval 0 ok
158  * \retval -1 error
159  */
160 int FileCloseFile(FileContainer *, const StreamingBufferConfig *sbcfg, const uint8_t *data,
161  uint32_t data_len, uint16_t flags);
162 int FileCloseFileById(FileContainer *, const StreamingBufferConfig *sbcfg, uint32_t track_id,
163  const uint8_t *data, uint32_t data_len, uint16_t flags);
164 int FileCloseFilePtr(File *ff, const StreamingBufferConfig *sbcfg, const uint8_t *data,
165  uint32_t data_len, uint16_t flags);
166 
167 /**
168  * \brief Store a chunk of file data in the flow. The open "flowfile"
169  * will be used.
170  *
171  * \param ffc the container
172  * \param data data chunk
173  * \param data_len data chunk len
174  *
175  * \retval 0 ok
176  * \retval -1 error
177  */
178 int FileAppendData(FileContainer *, const StreamingBufferConfig *sbcfg, const uint8_t *data,
179  uint32_t data_len);
180 int FileAppendDataById(FileContainer *, const StreamingBufferConfig *sbcfg, uint32_t track_id,
181  const uint8_t *data, uint32_t data_len);
182 int FileAppendGAPById(FileContainer *ffc, const StreamingBufferConfig *sbcfg, uint32_t track_id,
183  const uint8_t *data, uint32_t data_len);
184 
185 void FileSetInspectSizes(File *file, const uint32_t win, const uint32_t min);
186 
187 /**
188  * \brief Sets the offset range for a file.
189  *
190  * \param ffc the container
191  * \param start start offset
192  * \param end end offset
193  *
194  * \retval 0 ok
195  * \retval -1 error
196  */
197 int FileSetRange(FileContainer *, uint64_t start, uint64_t end);
198 
199 /**
200  * \brief Tag a file for storing
201  *
202  * \param ff The file to store
203  */
204 int FileStore(File *);
205 
206 /**
207  * \brief disable file storing for a transaction
208  *
209  * \param f flow
210  * \param direction STREAM_TOSERVER or STREAM_TOCLIENT
211  * \param tx transaction pointer
212  * \param tx_id transaction id
213  */
214 void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id);
215 
216 void FileForceFilestoreEnable(void);
217 int FileForceFilestore(void);
218 void FileReassemblyDepthEnable(uint32_t size);
219 uint32_t FileReassemblyDepth(void);
220 
221 void FileForceMagicEnable(void);
222 int FileForceMagic(void);
223 
224 void FileForceMd5Enable(void);
225 int FileForceMd5(void);
226 
227 void FileForceSha1Enable(void);
228 int FileForceSha1(void);
229 
230 void FileForceSha256Enable(void);
231 int FileForceSha256(void);
232 
233 void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction);
234 
236 
237 void FileForceTrackingEnable(void);
238 
239 void FileStoreFileById(FileContainer *fc, uint32_t);
240 
241 uint64_t FileDataSize(const File *file);
242 uint64_t FileTrackedSize(const File *file);
243 
244 uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction);
245 uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction);
246 
247 #ifdef DEBUG
248 void FilePrintFlags(const File *file);
249 #else
250 #define FilePrintFlags(file)
251 #endif
252 
253 void FilesPrune(FileContainer *fc, const StreamingBufferConfig *sbcfg, const bool trunc);
254 
255 #endif /* SURICATA_UTIL_FILE_H */
FileFlowToFlags
uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction)
Definition: util-file.c:290
FileContainerRecycle
void FileContainerRecycle(FileContainer *, const StreamingBufferConfig *cfg)
Recycle a FileContainer.
Definition: util-file.c:513
SCSha1
struct SCSha1 SCSha1
Definition: util-file.h:39
FileForceMagicEnable
void FileForceMagicEnable(void)
Definition: util-file.c:98
FileCloseFile
int FileCloseFile(FileContainer *, const StreamingBufferConfig *sbcfg, const uint8_t *data, uint32_t data_len, uint16_t flags)
Close a File.
Definition: util-file.c:1077
FileContainer_
Definition: util-file.h:113
FileState_
FileState_
Definition: util-file.h:68
FileReassemblyDepth
uint32_t FileReassemblyDepth(void)
Definition: util-file.c:133
File_::inspect_min_size
uint32_t inspect_min_size
Definition: util-file.h:104
File_::size
uint64_t size
Definition: util-file.h:102
SC_SHA256_LEN
#define SC_SHA256_LEN
Definition: util-file.h:37
FileOpenFileWithId
int FileOpenFileWithId(FileContainer *, const StreamingBufferConfig *, uint32_t track_id, const uint8_t *name, uint16_t name_len, const uint8_t *data, uint32_t data_len, uint16_t flags)
Open a new File.
Definition: util-file.c:981
FILE_STATE_OPENED
@ FILE_STATE_OPENED
Definition: util-file.h:70
Flow_
Flow data structure.
Definition: flow.h:350
File_::state
FileState state
Definition: util-file.h:82
FileStore
int FileStore(File *)
Tag a file for storing.
Definition: util-file.c:628
SC_SHA1_LEN
#define SC_SHA1_LEN
Definition: util-file.h:40
File_::file_store_id
uint32_t file_store_id
Definition: util-file.h:85
FileForceSha1Enable
void FileForceSha1Enable(void)
Definition: util-file.c:110
FileReassemblyDepthEnable
void FileReassemblyDepthEnable(uint32_t size)
Definition: util-file.c:127
FileContainer_::tail
File * tail
Definition: util-file.h:115
FileContainer
struct FileContainer_ FileContainer
FILE_STATE_TRUNCATED
@ FILE_STATE_TRUNCATED
Definition: util-file.h:73
FileForceMagic
int FileForceMagic(void)
Definition: util-file.c:141
File_::sha1
uint8_t sha1[SC_SHA1_LEN]
Definition: util-file.h:96
SCSha256
struct SCSha256 SCSha256
Definition: util-file.h:36
FileCloseFileById
int FileCloseFileById(FileContainer *, const StreamingBufferConfig *sbcfg, uint32_t track_id, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:1093
FileForceTrackingEnable
void FileForceTrackingEnable(void)
Definition: util-file.c:161
File_::sb
StreamingBuffer * sb
Definition: util-file.h:83
File_::name_len
uint16_t name_len
Definition: util-file.h:81
FileForceSha1
int FileForceSha1(void)
Definition: util-file.c:151
File_::md5
uint8_t md5[SC_MD5_LEN]
Definition: util-file.h:94
File_::file_track_id
uint32_t file_track_id
Definition: util-file.h:84
File_::end
uint64_t end
Definition: util-file.h:106
File_::fd
int fd
Definition: util-file.h:86
FileTrackedSize
uint64_t FileTrackedSize(const File *file)
get the size of the file
Definition: util-file.c:343
FileForceMd5Enable
void FileForceMd5Enable(void)
Definition: util-file.c:104
FileForceFilestore
int FileForceFilestore(void)
Definition: util-file.c:122
FileSetRange
int FileSetRange(FileContainer *, uint64_t start, uint64_t end)
Sets the offset range for a file.
Definition: util-file.c:873
File_::sid_max
uint32_t sid_max
Definition: util-file.h:110
FileContainer_::head
File * head
Definition: util-file.h:114
FileAppendGAPById
int FileAppendGAPById(FileContainer *ffc, const StreamingBufferConfig *sbcfg, uint32_t track_id, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The file with 'track_id' in the FileContainer...
Definition: util-file.c:836
File_::sha256_ctx
SCSha256 * sha256_ctx
Definition: util-file.h:97
conf.h
FILE_STATE_MAX
@ FILE_STATE_MAX
Definition: util-file.h:76
File_::name
uint8_t * name
Definition: util-file.h:88
FileSetInspectSizes
void FileSetInspectSizes(File *file, const uint32_t win, const uint32_t min)
Definition: util-file.c:857
FileAppendData
int FileAppendData(FileContainer *, const StreamingBufferConfig *sbcfg, const uint8_t *data, uint32_t data_len)
Store a chunk of file data in the flow. The open "flowfile" will be used.
Definition: util-file.c:780
File_::sid
uint32_t * sid
Definition: util-file.h:108
File_::sid_cnt
uint32_t sid_cnt
Definition: util-file.h:109
StreamingBuffer_
Definition: util-streaming-buffer.h:108
FileForceFilestoreEnable
void FileForceFilestoreEnable(void)
Definition: util-file.c:92
File_::flags
uint16_t flags
Definition: util-file.h:80
File_::content_inspected
uint64_t content_inspected
Definition: util-file.h:99
FILE_STATE_CLOSED
@ FILE_STATE_CLOSED
Definition: util-file.h:71
File_
Definition: util-file.h:79
File_::content_stored
uint64_t content_stored
Definition: util-file.h:101
flags
uint8_t flags
Definition: decode-gre.h:0
util-streaming-buffer.h
FileForceMd5
int FileForceMd5(void)
Definition: util-file.c:146
File_::next
struct File_ * next
Definition: util-file.h:92
FileContainerFree
void FileContainerFree(FileContainer *, const StreamingBufferConfig *cfg)
Free a FileContainer.
Definition: util-file.c:533
FileContainerAdd
void FileContainerAdd(FileContainer *, File *)
Definition: util-file.c:612
File_::sha256
uint8_t sha256[SC_SHA256_LEN]
Definition: util-file.h:98
File_::start
uint64_t start
Definition: util-file.h:105
FilePrintFlags
#define FilePrintFlags(file)
Definition: util-file.h:250
FileDataSize
uint64_t FileDataSize(const File *file)
get the size of the file data
Definition: util-file.c:326
StreamingBufferConfig_
Definition: util-streaming-buffer.h:65
FilesPrune
void FilesPrune(FileContainer *fc, const StreamingBufferConfig *sbcfg, const bool trunc)
Definition: util-file.c:1204
FileForceSha256Enable
void FileForceSha256Enable(void)
Definition: util-file.c:116
FileDisableStoringForTransaction
void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id)
disable file storing for a transaction
Definition: util-file.c:1154
FileFlowFlagsToFlags
uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction)
Definition: util-file.c:233
ConfNode_
Definition: conf.h:32
File
struct File_ File
FileCloseFilePtr
int FileCloseFilePtr(File *ff, const StreamingBufferConfig *sbcfg, const uint8_t *data, uint32_t data_len, uint16_t flags)
Definition: util-file.c:994
File_::md5_ctx
SCMd5 * md5_ctx
Definition: util-file.h:93
FileForceSha256
int FileForceSha256(void)
Definition: util-file.c:156
File_::inspect_window
uint32_t inspect_window
Definition: util-file.h:103
FileForceHashParseCfg
void FileForceHashParseCfg(ConfNode *)
Function to parse forced file hashing configuration.
Definition: util-file.c:170
FileContainerAlloc
FileContainer * FileContainerAlloc(void)
allocate a FileContainer
Definition: util-file.c:497
FileState
enum FileState_ FileState
FileAppendDataById
int FileAppendDataById(FileContainer *, const StreamingBufferConfig *sbcfg, uint32_t track_id, const uint8_t *data, uint32_t data_len)
Store/handle a chunk of file data in the File structure The file with 'track_id' in the FileContainer...
Definition: util-file.c:805
FileStoreFileById
void FileStoreFileById(FileContainer *fc, uint32_t)
flag a file with id "file_id" to be stored.
Definition: util-file.c:1174
SC_MD5_LEN
#define SC_MD5_LEN
Definition: util-file.h:43
flow.h
FileUpdateFlowFileFlags
void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction)
set a flow's file flags
Definition: util-file.c:1118
FILE_STATE_ERROR
@ FILE_STATE_ERROR
Definition: util-file.h:75
SCMd5
struct SCMd5 SCMd5
Definition: util-file.h:42
FILE_STATE_NONE
@ FILE_STATE_NONE
Definition: util-file.h:69
File_::sha1_ctx
SCSha1 * sha1_ctx
Definition: util-file.h:95