#include "conf.h"
#include "util-streaming-buffer.h"
#include "flow.h"

Include dependency graph for util-file.h:

This graph shows which files directly or indirectly include this file:

Data Structures
struct	File_

struct	FileContainer_

Macros
#define	SC_SHA256_LEN 32

#define	SC_SHA1_LEN 20

#define	SC_MD5_LEN 16

#define	FILE_TRUNCATED BIT_U16(0)

#define	FILE_NOMAGIC BIT_U16(1)

#define	FILE_NOMD5 BIT_U16(2)

#define	FILE_MD5 BIT_U16(3)

#define	FILE_NOSHA1 BIT_U16(4)

#define	FILE_SHA1 BIT_U16(5)

#define	FILE_NOSHA256 BIT_U16(6)

#define	FILE_SHA256 BIT_U16(7)

#define	FILE_LOGGED BIT_U16(8)

#define	FILE_NOSTORE BIT_U16(9)

#define	FILE_STORE BIT_U16(10)

#define	FILE_STORED BIT_U16(11)

#define	FILE_NOTRACK BIT_U16(12)

#define	FILE_USE_DETECT BIT_U16(13)

#define	FILE_HAS_GAPS BIT_U16(15)

#define	SC_FILENAME_MAX 4096

#define	FILEDATA_CONTENT_LIMIT 100000

#define	FILEDATA_CONTENT_INSPECT_MIN_SIZE 32768

#define	FILEDATA_CONTENT_INSPECT_WINDOW 4096

#define	FilePrintFlags(file)

Typedefs
typedef struct SCSha256	SCSha256

typedef struct SCSha1	SCSha1

typedef struct SCMd5	SCMd5

typedef enum FileState_	FileState

typedef struct File_	File

typedef struct FileContainer_	FileContainer

Enumerations
enum	FileState_ { FILE_STATE_NONE = 0, FILE_STATE_OPENED, FILE_STATE_CLOSED, FILE_STATE_TRUNCATED, FILE_STATE_ERROR, FILE_STATE_MAX }

Functions
FileContainer *	FileContainerAlloc (void)
	allocate a FileContainer More...

void	FileContainerFree (FileContainer , const StreamingBufferConfig cfg)
	Free a FileContainer. More...

void	FileContainerRecycle (FileContainer , const StreamingBufferConfig cfg)
	Recycle a FileContainer. More...

void	FileContainerAdd (FileContainer , File )

int	FileOpenFileWithId (FileContainer , const StreamingBufferConfig , uint32_t track_id, const uint8_t name, uint16_t name_len, const uint8_t data, uint32_t data_len, uint16_t flags)
	Open a new File. More...

int	FileCloseFile (FileContainer , const StreamingBufferConfig sbcfg, const uint8_t *data, uint32_t data_len, uint16_t flags)
	Close a File. More...

int	FileCloseFileById (FileContainer , const StreamingBufferConfig sbcfg, uint32_t track_id, const uint8_t *data, uint32_t data_len, uint16_t flags)

int	FileCloseFilePtr (File ff, const StreamingBufferConfig sbcfg, const uint8_t *data, uint32_t data_len, uint16_t flags)

int	FileAppendData (FileContainer , const StreamingBufferConfig sbcfg, const uint8_t *data, uint32_t data_len)
	Store a chunk of file data in the flow. The open "flowfile" will be used. More...

int	FileAppendDataById (FileContainer , const StreamingBufferConfig sbcfg, uint32_t track_id, const uint8_t *data, uint32_t data_len)
	Store/handle a chunk of file data in the File structure The file with 'track_id' in the FileContainer will be used. More...

int	FileAppendGAPById (FileContainer ffc, const StreamingBufferConfig sbcfg, uint32_t track_id, const uint8_t *data, uint32_t data_len)
	Store/handle a chunk of file data in the File structure The file with 'track_id' in the FileContainer will be used. More...

void	FileSetInspectSizes (File *file, const uint32_t win, const uint32_t min)

int	FileSetRange (FileContainer *, uint64_t start, uint64_t end)
	Sets the offset range for a file. More...

int	FileStore (File *)
	Tag a file for storing. More...

void	FileDisableStoringForTransaction (Flow f, const uint8_t direction, void tx, uint64_t tx_id)
	disable file storing for a transaction More...

void	FileForceFilestoreEnable (void)

int	FileForceFilestore (void)

void	FileReassemblyDepthEnable (uint32_t size)

uint32_t	FileReassemblyDepth (void)

void	FileForceMagicEnable (void)

int	FileForceMagic (void)

void	FileForceMd5Enable (void)

int	FileForceMd5 (void)

void	FileForceSha1Enable (void)

int	FileForceSha1 (void)

void	FileForceSha256Enable (void)

int	FileForceSha256 (void)

void	FileUpdateFlowFileFlags (Flow *f, uint16_t set_file_flags, uint8_t direction)
	set a flow's file flags More...

void	FileForceHashParseCfg (ConfNode *)
	Function to parse forced file hashing configuration. More...

void	FileForceTrackingEnable (void)

void	FileStoreFileById (FileContainer *fc, uint32_t)
	flag a file with id "file_id" to be stored. More...

uint64_t	FileDataSize (const File *file)
	get the size of the file data More...

uint64_t	FileTrackedSize (const File *file)
	get the size of the file More...

uint16_t	FileFlowFlagsToFlags (const uint16_t flow_file_flags, uint8_t direction)

uint16_t	FileFlowToFlags (const Flow *flow, uint8_t direction)

void	FilesPrune (FileContainer fc, const StreamingBufferConfig sbcfg, const bool trunc)

Detailed Description

Author: Victor Julien victo.nosp@m.r@in.nosp@m.linia.nosp@m.c.ne.nosp@m.t

Definition in file util-file.h.

Macro Definition Documentation

◆ FILE_HAS_GAPS

#define FILE_HAS_GAPS BIT_U16(15)

Definition at line 60 of file util-file.h.

◆ FILE_LOGGED

#define FILE_LOGGED BIT_U16(8)

Definition at line 54 of file util-file.h.

◆ FILE_MD5

#define FILE_MD5 BIT_U16(3)

Definition at line 49 of file util-file.h.

◆ FILE_NOMAGIC

#define FILE_NOMAGIC BIT_U16(1)

Definition at line 47 of file util-file.h.

◆ FILE_NOMD5

#define FILE_NOMD5 BIT_U16(2)

Definition at line 48 of file util-file.h.

◆ FILE_NOSHA1

#define FILE_NOSHA1 BIT_U16(4)

Definition at line 50 of file util-file.h.

◆ FILE_NOSHA256

#define FILE_NOSHA256 BIT_U16(6)

Definition at line 52 of file util-file.h.

◆ FILE_NOSTORE

#define FILE_NOSTORE BIT_U16(9)

Definition at line 55 of file util-file.h.

◆ FILE_NOTRACK

#define FILE_NOTRACK BIT_U16(12)

track size of file

Definition at line 58 of file util-file.h.

◆ FILE_SHA1

#define FILE_SHA1 BIT_U16(5)

Definition at line 51 of file util-file.h.

◆ FILE_SHA256

#define FILE_SHA256 BIT_U16(7)

Definition at line 53 of file util-file.h.

◆ FILE_STORE

#define FILE_STORE BIT_U16(10)

Definition at line 56 of file util-file.h.

◆ FILE_STORED

#define FILE_STORED BIT_U16(11)

Definition at line 57 of file util-file.h.

◆ FILE_TRUNCATED

#define FILE_TRUNCATED BIT_U16(0)

Definition at line 46 of file util-file.h.

◆ FILE_USE_DETECT

#define FILE_USE_DETECT BIT_U16(13)

use content_inspected tracker

Definition at line 59 of file util-file.h.

◆ FILEDATA_CONTENT_INSPECT_MIN_SIZE

#define FILEDATA_CONTENT_INSPECT_MIN_SIZE 32768

Definition at line 66 of file util-file.h.

◆ FILEDATA_CONTENT_INSPECT_WINDOW

#define FILEDATA_CONTENT_INSPECT_WINDOW 4096

Definition at line 67 of file util-file.h.

◆ FILEDATA_CONTENT_LIMIT

#define FILEDATA_CONTENT_LIMIT 100000

Definition at line 65 of file util-file.h.

◆ FilePrintFlags

#define FilePrintFlags ( file )

Definition at line 251 of file util-file.h.

◆ SC_FILENAME_MAX

#define SC_FILENAME_MAX 4096

Definition at line 63 of file util-file.h.

◆ SC_MD5_LEN

#define SC_MD5_LEN 16

Definition at line 44 of file util-file.h.

◆ SC_SHA1_LEN

#define SC_SHA1_LEN 20

Definition at line 41 of file util-file.h.

◆ SC_SHA256_LEN

#define SC_SHA256_LEN 32

Definition at line 38 of file util-file.h.

Typedef Documentation

◆ File

typedef struct File_ File

◆ FileContainer

typedef struct FileContainer_ FileContainer

◆ FileState

typedef enum FileState_ FileState

◆ SCMd5

typedef struct SCMd5 SCMd5

Definition at line 1 of file util-file.h.

◆ SCSha1

typedef struct SCSha1 SCSha1

Definition at line 1 of file util-file.h.

◆ SCSha256

typedef struct SCSha256 SCSha256

Definition at line 1 of file util-file.h.

Enumeration Type Documentation

◆ FileState_

enum FileState_

Enumerator
FILE_STATE_NONE	no state
FILE_STATE_OPENED	flow file is opened
FILE_STATE_CLOSED	flow file is completed, there will be no more data.
FILE_STATE_TRUNCATED	flow file is not complete, but there will be no more data.
FILE_STATE_ERROR	file is in an error state
FILE_STATE_MAX

Definition at line 68 of file util-file.h.

Function Documentation

◆ FileAppendData()

int FileAppendData	(	FileContainer *	ffc,
		const StreamingBufferConfig *	sbcfg,
		const uint8_t *	data,
		uint32_t	data_len
	)

Store a chunk of file data in the flow. The open "flowfile" will be used.

Parameters

ffc	the container
data	data chunk
data_len	data chunk len

Return values

0	ok
-1	error

Store a chunk of file data in the flow. The open "flowfile" will be used.

Parameters

ffc	FileContainer used to append to
data	data chunk
data_len	data chunk len

Return values

0	ok
-1	error
-2	no store for this file

Definition at line 783 of file util-file.c.

Referenced by HTPFileStoreChunk().

Here is the caller graph for this function:

◆ FileAppendDataById()

int FileAppendDataById	(	FileContainer *	ffc,
		const StreamingBufferConfig *	sbcfg,
		uint32_t	track_id,
		const uint8_t *	data,
		uint32_t	data_len
	)

Store/handle a chunk of file data in the File structure The file with 'track_id' in the FileContainer will be used.

Parameters

ffc	FileContainer used to append to
track_id	id to lookup the file
data	data chunk
data_len	data chunk len

Return values

0	ok
-1	error
-2	no store for this file

Definition at line 808 of file util-file.c.

◆ FileAppendGAPById()

int FileAppendGAPById	(	FileContainer *	ffc,
		const StreamingBufferConfig *	sbcfg,
		uint32_t	track_id,
		const uint8_t *	data,
		uint32_t	data_len
	)

Store/handle a chunk of file data in the File structure The file with 'track_id' in the FileContainer will be used.

Parameters

ffc	FileContainer used to append to
track_id	id to lookup the file
data	data chunk
data_len	data chunk len

Return values

0	ok
-1	error
-2	no store for this file

Definition at line 839 of file util-file.c.

◆ FileCloseFile()

int FileCloseFile	(	FileContainer *	ffc,
		const StreamingBufferConfig *	sbcfg,
		const uint8_t *	data,
		uint32_t	data_len,
		uint16_t	flags
	)

Close a File.

Parameters

ffc	the container
data	final data if any
data_len	data len if any
flags	flags

Return values

0	ok
-1	error

Definition at line 1080 of file util-file.c.

Referenced by HTPFileClose().

Here is the caller graph for this function:

◆ FileCloseFileById()

int FileCloseFileById	(	FileContainer *	,
		const StreamingBufferConfig *	sbcfg,
		uint32_t	track_id,
		const uint8_t *	data,
		uint32_t	data_len,
		uint16_t	flags
	)

Definition at line 1096 of file util-file.c.

◆ FileCloseFilePtr()

int FileCloseFilePtr	(	File *	ff,
		const StreamingBufferConfig *	sbcfg,
		const uint8_t *	data,
		uint32_t	data_len,
		uint16_t	flags
	)

Definition at line 997 of file util-file.c.

Referenced by OutputFiledataLogFfc().

Here is the caller graph for this function:

◆ FileContainerAdd()

void FileContainerAdd	(	FileContainer *	,
		File *
	)

Definition at line 612 of file util-file.c.

References FileContainer_::head, File_::next, SCLogDebug, and FileContainer_::tail.

◆ FileContainerAlloc()

FileContainer* FileContainerAlloc ( void )

allocate a FileContainer

Return values

new	newly allocated FileContainer
NULL	error

Definition at line 497 of file util-file.c.

References SCCalloc, SCLogError, and unlikely.

◆ FileContainerFree()

void FileContainerFree	(	FileContainer *	ffc,
		const StreamingBufferConfig *	cfg
	)

Free a FileContainer.

Parameters

ffc	FileContainer

Definition at line 533 of file util-file.c.

References FileContainer_::head, next, File_::next, and SCLogDebug.

◆ FileContainerRecycle()

void FileContainerRecycle	(	FileContainer *	ffc,
		const StreamingBufferConfig *	cfg
	)

Recycle a FileContainer.

Parameters

ffc	FileContainer

Definition at line 513 of file util-file.c.

References FileContainer_::head, next, File_::next, and SCLogDebug.

◆ FileDataSize()

uint64_t FileDataSize ( const File * file )

get the size of the file data

This doesn't reflect how much of the file we have in memory, just the total size of filedata so far.

Definition at line 326 of file util-file.c.

References File_::sb.

Referenced by OutputFiledataLogFfc().

Here is the caller graph for this function:

◆ FileDisableStoringForTransaction()

void FileDisableStoringForTransaction	(	Flow *	f,
		const uint8_t	direction,
		void *	tx,
		uint64_t	tx_id
	)

disable file storing for a transaction

Parameters

f	flow
direction	STREAM_TOSERVER or STREAM_TOCLIENT
tx	transaction pointer
tx_id	transaction id

disable file storing for a transaction

Parameters

f	LOCKED flow
direction	flow direction
tx_id	transaction id

Definition at line 1157 of file util-file.c.

◆ FileFlowFlagsToFlags()

uint16_t FileFlowFlagsToFlags	(	const uint16_t	flow_file_flags,
		uint8_t	direction
	)

Definition at line 233 of file util-file.c.

References DEBUG_VALIDATE_BUG_ON, FILE_NOMAGIC, FILE_NOMD5, FILE_NOSHA1, FILE_NOSHA256, FILE_NOSTORE, FILE_STORE, flags, FLOWFILE_NO_MAGIC_TC, FLOWFILE_NO_MAGIC_TS, FLOWFILE_NO_MD5_TC, FLOWFILE_NO_MD5_TS, FLOWFILE_NO_SHA1_TC, FLOWFILE_NO_SHA1_TS, FLOWFILE_NO_SHA256_TC, FLOWFILE_NO_SHA256_TS, FLOWFILE_NO_STORE_TC, FLOWFILE_NO_STORE_TS, FLOWFILE_STORE_TC, FLOWFILE_STORE_TS, and SCLogDebug.

Referenced by FileApplyTxFlags(), FileFlowToFlags(), and HTPFileOpen().

Here is the caller graph for this function:

◆ FileFlowToFlags()

uint16_t FileFlowToFlags	(	const Flow *	flow,
		uint8_t	direction
	)

Definition at line 290 of file util-file.c.

References Flow_::file_flags, and FileFlowFlagsToFlags().

Here is the call graph for this function:

◆ FileForceFilestore()

int FileForceFilestore ( void )

Definition at line 122 of file util-file.c.

◆ FileForceFilestoreEnable()

void FileForceFilestoreEnable ( void )

Definition at line 92 of file util-file.c.

◆ FileForceHashParseCfg()

void FileForceHashParseCfg ( ConfNode * )

Function to parse forced file hashing configuration.

Definition at line 170 of file util-file.c.

References BUG_ON, ConfNodeLookupChild(), ConfNodeLookupChildValue(), ConfValIsTrue(), FileForceMd5Enable(), FileForceSha1Enable(), FileForceSha256Enable(), g_disable_hashing, next, SCLogInfo, SCLogWarning, TAILQ_FOREACH, and ConfNode_::val.

Here is the call graph for this function:

◆ FileForceMagic()

int FileForceMagic ( void )

Definition at line 141 of file util-file.c.

Referenced by OutputFiledataLogFfc(), and OutputFileLogFfc().

Here is the caller graph for this function:

◆ FileForceMagicEnable()

void FileForceMagicEnable ( void )

Definition at line 98 of file util-file.c.

◆ FileForceMd5()

int FileForceMd5 ( void )

Definition at line 146 of file util-file.c.

◆ FileForceMd5Enable()

void FileForceMd5Enable ( void )

Definition at line 104 of file util-file.c.

Referenced by FileForceHashParseCfg().

Here is the caller graph for this function:

◆ FileForceSha1()

int FileForceSha1 ( void )

Definition at line 151 of file util-file.c.

◆ FileForceSha1Enable()

void FileForceSha1Enable ( void )

Definition at line 110 of file util-file.c.

Referenced by FileForceHashParseCfg().

Here is the caller graph for this function:

◆ FileForceSha256()

int FileForceSha256 ( void )

Definition at line 156 of file util-file.c.

◆ FileForceSha256Enable()

void FileForceSha256Enable ( void )

Definition at line 116 of file util-file.c.

Referenced by FileForceHashParseCfg().

Here is the caller graph for this function:

◆ FileForceTrackingEnable()

void FileForceTrackingEnable ( void )

Definition at line 161 of file util-file.c.

◆ FileOpenFileWithId()

int FileOpenFileWithId	(	FileContainer *	ffc,
		const StreamingBufferConfig *	sbcfg,
		uint32_t	track_id,
		const uint8_t *	name,
		uint16_t	name_len,
		const uint8_t *	data,
		uint32_t	data_len,
		uint16_t	flags
	)

Open a new File.

Parameters

ffc	flow container
sbcfg	buffer config
name	filename character array
name_len	filename len
data	initial data
data_len	initial data len
flags	open flags

Return values

ff	flowfile object

Note: filename is not a string, so it's not nul terminated.

If flags contains the FILE_USE_DETECT bit, the pruning code will consider not just the content_stored tracker, but also content_inspected. It's the responsibility of the API user to make sure this tracker is properly updated.

Return values

0	ok
-1	failed

Definition at line 984 of file util-file.c.

Referenced by HTPFileOpen().

Here is the caller graph for this function:

◆ FileReassemblyDepth()

uint32_t FileReassemblyDepth ( void )

Definition at line 133 of file util-file.c.

◆ FileReassemblyDepthEnable()

void FileReassemblyDepthEnable ( uint32_t size )

Definition at line 127 of file util-file.c.

◆ FileSetInspectSizes()

void FileSetInspectSizes	(	File *	file,
		const uint32_t	win,
		const uint32_t	min
	)

Definition at line 860 of file util-file.c.

References File_::inspect_min_size, and File_::inspect_window.

Referenced by HTPFileOpen().

Here is the caller graph for this function:

◆ FileSetRange()

int FileSetRange	(	FileContainer *	ffc,
		uint64_t	start,
		uint64_t	end
	)

Sets the offset range for a file.

Parameters

ffc	the container
start	start offset
end	end offset

Return values

0	ok
-1	error

Definition at line 876 of file util-file.c.

References File_::end, SCEnter, SCReturnInt, File_::start, and FileContainer_::tail.

◆ FilesPrune()

void FilesPrune	(	FileContainer *	fc,
		const StreamingBufferConfig *	sbcfg,
		const bool	trunc
	)

Definition at line 1207 of file util-file.c.

◆ FileStore()

int FileStore ( File * ff )

Tag a file for storing.

Parameters

ff	The file to store

Definition at line 628 of file util-file.c.

References FILE_STORE, File_::flags, SCLogDebug, and SCReturnInt.

Referenced by FileStoreFileById().

Here is the caller graph for this function:

◆ FileStoreFileById()

void FileStoreFileById	(	FileContainer *	fc,
		uint32_t	file_id
	)

flag a file with id "file_id" to be stored.

Parameters

fc	file store
file_id	the file's id

Definition at line 1177 of file util-file.c.

References File_::file_track_id, FileStore(), FileContainer_::head, File_::next, and SCEnter.

Here is the call graph for this function:

◆ FileTrackedSize()

uint64_t FileTrackedSize ( const File * file )

get the size of the file

This doesn't reflect how much of the file we have in memory, just the total size of file so far.

Definition at line 343 of file util-file.c.

References File_::size.

Referenced by EveFileInfo().

Here is the caller graph for this function:

◆ FileUpdateFlowFileFlags()

void FileUpdateFlowFileFlags	(	Flow *	f,
		uint16_t	set_file_flags,
		uint8_t	direction
	)

set a flow's file flags

Parameters

set_file_flags flags in both directions that are requested to set

This function will ignore the flags for the irrelevant direction and also mask the flags with the global settings.

Definition at line 1121 of file util-file.c.

References DEBUG_ASSERT_FLOW_LOCKED, FLOWFILE_NONE_TC, and SCEnter.

Data Structures

Macros

Typedefs

Enumerations

Functions

Detailed Description

Macro Definition Documentation

◆ FILE_HAS_GAPS

◆ FILE_LOGGED

◆ FILE_MD5

◆ FILE_NOMAGIC

◆ FILE_NOMD5

◆ FILE_NOSHA1

◆ FILE_NOSHA256

◆ FILE_NOSTORE

◆ FILE_NOTRACK

◆ FILE_SHA1

◆ FILE_SHA256

◆ FILE_STORE

◆ FILE_STORED

◆ FILE_TRUNCATED

◆ FILE_USE_DETECT

◆ FILEDATA_CONTENT_INSPECT_MIN_SIZE

◆ FILEDATA_CONTENT_INSPECT_WINDOW

◆ FILEDATA_CONTENT_LIMIT

◆ FilePrintFlags

◆ SC_FILENAME_MAX

◆ SC_MD5_LEN

◆ SC_SHA1_LEN

◆ SC_SHA256_LEN

Typedef Documentation

◆ File

◆ FileContainer

◆ FileState

◆ SCMd5

◆ SCSha1

◆ SCSha256

Enumeration Type Documentation

◆ FileState_

Function Documentation

◆ FileAppendData()

◆ FileAppendDataById()

◆ FileAppendGAPById()

◆ FileCloseFile()

◆ FileCloseFileById()

◆ FileCloseFilePtr()

◆ FileContainerAdd()

◆ FileContainerAlloc()

◆ FileContainerFree()

◆ FileContainerRecycle()

◆ FileDataSize()

◆ FileDisableStoringForTransaction()

◆ FileFlowFlagsToFlags()

◆ FileFlowToFlags()

◆ FileForceFilestore()

◆ FileForceFilestoreEnable()

◆ FileForceHashParseCfg()

◆ FileForceMagic()

◆ FileForceMagicEnable()

◆ FileForceMd5()

◆ FileForceMd5Enable()

◆ FileForceSha1()

◆ FileForceSha1Enable()

◆ FileForceSha256()

◆ FileForceSha256Enable()

◆ FileForceTrackingEnable()

◆ FileOpenFileWithId()

◆ FileReassemblyDepth()

◆ FileReassemblyDepthEnable()

◆ FileSetInspectSizes()

◆ FileSetRange()

◆ FilesPrune()

◆ FileStore()

◆ FileStoreFileById()

◆ FileTrackedSize()

◆ FileUpdateFlowFileFlags()