#include "suricata-common.h"
#include "suricata.h"
#include "decode.h"
#include "conf.h"
#include "threadvars.h"
#include "tm-threads.h"
#include "runmodes.h"
#include "util-random.h"
#include "util-time.h"
#include "flow.h"
#include "flow-queue.h"
#include "flow-hash.h"
#include "flow-util.h"
#include "flow-var.h"
#include "flow-private.h"
#include "flow-timeout.h"
#include "flow-manager.h"
#include "flow-storage.h"
#include "flow-bypass.h"
#include "stream-tcp-private.h"
#include "stream-tcp-reassemble.h"
#include "stream-tcp.h"
#include "util-unittest.h"
#include "util-unittest-helper.h"
#include "util-byte.h"
#include "util-misc.h"
#include "util-debug.h"
#include "util-privs.h"
#include "detect.h"
#include "detect-engine-state.h"
#include "stream.h"
#include "app-layer-parser.h"
#include "threads.h"

Include dependency graph for flow.c:

Macros
#define	FLOW_DEFAULT_EMERGENCY_RECOVERY 30

#define	FLOW_DEFAULT_HASHSIZE 65536

#define	FLOW_DEFAULT_MEMCAP (32 * 1024 * 1024) /* 32 MB */

#define	FLOW_DEFAULT_PREALLOC 10000

#define	SET_DEFAULTS(p, n, e, c, b, ne, ee, ce, be)

Functions
	SC_ATOMIC_DECLARE (unsigned int, flow_prune_idx)

	SC_ATOMIC_DECLARE (unsigned int, flow_flags)

void	FlowRegisterTests (void)
	Function to register the Flow Unitests. More...

void	FlowInitFlowProto (void)
	Function to set the default timeout, free function and flow state function for all supported flow_proto. More...

int	FlowSetProtoFreeFunc (uint8_t proto, void(Free)(void ))
	Function to set the function to get protocol specific flow state. More...

int	FlowSetMemcap (uint64_t size)
	Update memcap value. More...

uint64_t	FlowGetMemcap (void)
	Return memcap value. More...

uint64_t	FlowGetMemuse (void)

void	FlowCleanupAppLayer (Flow *f)

int	FlowUpdateSpareFlows (void)
	Make sure we have enough spare flows. More...

void	FlowSetIPOnlyFlag (Flow *f, int direction)
	Set the IPOnly scanned flag for 'direction'. More...

void	FlowSetHasAlertsFlag (Flow *f)
	Set flag to indicate that flow has alerts. More...

int	FlowHasAlerts (const Flow *f)
	Check if flow has alerts. More...

void	FlowSetChangeProtoFlag (Flow *f)
	Set flag to indicate to change proto for the flow. More...

void	FlowUnsetChangeProtoFlag (Flow *f)
	Unset flag to indicate to change proto for the flow. More...

int	FlowChangeProto (Flow *f)
	Check if change proto flag is set for flow. More...

void	FlowSwap (Flow *f)
	swap the flow's direction More...

int	FlowGetPacketDirection (const Flow f, const Packet p)
	determine the direction of the packet compared to the flow More...

void	FlowHandlePacketUpdate (Flow f, Packet p)
	Update Packet and Flow. More...

void	FlowHandlePacket (ThreadVars tv, DecodeThreadVars dtv, Packet *p)
	Entry point for packet flow handling. More...

void	FlowInitConfig (char quiet)
	initialize the configuration More...

void	FlowShutdown (void)
	shutdown the flow engine More...

int	FlowClearMemory (Flow *f, uint8_t proto_map)
	Function clear the flow memory before queueing it to spare flow queue. More...

AppProto	FlowGetAppProtocol (const Flow *f)

void *	FlowGetAppState (const Flow *f)

uint8_t	FlowGetDisruptionFlags (const Flow *f, uint8_t flags)
	get 'disruption' flags: GAP/DEPTH/PASS More...

void	FlowUpdateState (Flow *f, enum FlowState s)

Variables
int	run_mode

Detailed Description

Author: Victor Julien victo.nosp@m.r@in.nosp@m.linia.nosp@m.c.ne.nosp@m.t

Flow implementation.

Definition in file flow.c.

Macro Definition Documentation

#define FLOW_DEFAULT_EMERGENCY_RECOVERY 30

Definition at line 66 of file flow.c.

Referenced by FlowInitConfig().

#define FLOW_DEFAULT_HASHSIZE 65536

Definition at line 69 of file flow.c.

Referenced by FlowInitConfig().

#define FLOW_DEFAULT_MEMCAP (32 * 1024 * 1024) /* 32 MB */

Definition at line 71 of file flow.c.

Referenced by FlowInitConfig().

#define FLOW_DEFAULT_PREALLOC 10000

Definition at line 73 of file flow.c.

Referenced by FlowInitConfig().

#define SET_DEFAULTS	(	p,
		n,
		e,
		c,
		b,
		ne,
		ee,
		ce,
		be
	)

Value:

flow_timeouts_normal[(p)].new_timeout = (n);     \
    flow_timeouts_normal[(p)].est_timeout = (e);     \
    flow_timeouts_normal[(p)].closed_timeout = (c);  \
    flow_timeouts_normal[(p)].bypassed_timeout = (b); \
    flow_timeouts_emerg[(p)].new_timeout = (ne);     \
    flow_timeouts_emerg[(p)].est_timeout = (ee);     \
    flow_timeouts_emerg[(p)].closed_timeout = (ce); \
    flow_timeouts_emerg[(p)].bypassed_timeout = (be); \

Referenced by FlowInitFlowProto().

Function Documentation

int FlowChangeProto ( Flow * f )

Check if change proto flag is set for flow.

Parameters

f flow

Return values

1	change proto flag is set
0	change proto flag is not set

Definition at line 240 of file flow.c.

References TcpSession_::client, TcpSession_::data_first_seen_dir, Flow_::file_flags, Flow_::flags, FLOW_CHANGE_PROTO, FLOW_PROTO_DETECT_TC_DONE, FLOW_PROTO_DETECT_TS_DONE, FLOW_SGH_TOCLIENT, FLOW_SGH_TOSERVER, FLOW_TC_PE_ALPROTO_DETECT_DONE, FLOW_TC_PM_ALPROTO_DETECT_DONE, FLOW_TC_PP_ALPROTO_DETECT_DONE, FLOW_TO_DST_SEEN, FLOW_TO_SRC_SEEN, FLOW_TOCLIENT_DROP_LOGGED, FLOW_TOCLIENT_IPONLY_SET, FLOW_TOSERVER_DROP_LOGGED, FLOW_TOSERVER_IPONLY_SET, FLOW_TS_PE_ALPROTO_DETECT_DONE, FLOW_TS_PM_ALPROTO_DETECT_DONE, FLOW_TS_PP_ALPROTO_DETECT_DONE, FLOWFILE_NO_MAGIC_TC, FLOWFILE_NO_MAGIC_TS, Flow_::protoctx, TcpSession_::server, STREAM_TOCLIENT, STREAM_TOSERVER, SWAP_FLAGS, and SWAP_VARS.

Referenced by AppLayerHandleTCPData(), and SMTPParserCleanup().

Here is the caller graph for this function:

void FlowCleanupAppLayer ( Flow * f )

Definition at line 125 of file flow.c.

References Flow_::alparser, Flow_::alstate, AppLayerParserStateCleanup(), and Flow_::proto.

Referenced by AppLayerIncTxCounter().

Here is the call graph for this function:

Here is the caller graph for this function:

int FlowClearMemory	(	Flow *	f,
		uint8_t	proto_map
	)

Function clear the flow memory before queueing it to spare flow queue.

Parameters

f	pointer to the flow needed to be cleared.
proto_map	mapped value of the protocol to FLOW_PROTO's.

Definition at line 1030 of file flow.c.

References flow_freefuncs, FLOW_RECYCLE, FlowFreeStorage(), FlowProtoFreeFunc_::Freefunc, Flow_::protoctx, SCEnter, and SCReturnInt.

Referenced by FlowGetExistingFlowFromHash(), FlowShutdown(), FlowStorageRegister(), and TagTimeoutCheck().

Here is the call graph for this function:

Here is the caller graph for this function:

AppProto FlowGetAppProtocol ( const Flow * f )

Definition at line 1063 of file flow.c.

References Flow_::alproto.

Referenced by AlertJsonHeader(), JsonBuildFileInfoRecord(), SigMatchSignaturesGetSgh(), and Unified2Logger().

Here is the caller graph for this function:

void* FlowGetAppState ( const Flow * f )

Definition at line 1068 of file flow.c.

References Flow_::alstate.

Referenced by DetectEngineStateResetTxs(), DetectFilestoreRegister(), DetectTlsRegister(), HttpXFFGetIP(), HttpXFFGetIPFromTx(), JsonEmailAddMetadata(), JsonHttpAddMetadata(), JsonHttpLogJSONBodyBase64(), JsonHttpLogJSONBodyPrintable(), JsonNFSAddMetadata(), JsonNFSAddMetadataRPC(), JsonSIPAddMetadata(), JsonSMBAddMetadata(), and JsonSMTPAddMetadata().

Here is the caller graph for this function:

uint8_t FlowGetDisruptionFlags	(	const Flow *	f,
		uint8_t	flags
	)

get 'disruption' flags: GAP/DEPTH/PASS

Parameters

f	locked flow
flags	existing flags to be ammended

Return values

flags original flags + disrupt flags (if any) handle UDP

Definition at line 1080 of file flow.c.

References TcpSession_::client, TcpStream_::flags, flags, Flow_::proto, Flow_::protoctx, TcpSession_::server, STREAM_DEPTH, STREAM_GAP, STREAM_TOSERVER, STREAMTCP_STREAM_FLAG_DEPTH_REACHED, and STREAMTCP_STREAM_FLAG_GAP.

Referenced by DetectSignatureApplyActions(), OutputRegisterTxLogger(), and SigMatchSignaturesGetSgh().

Here is the caller graph for this function:

uint64_t FlowGetMemcap ( void )

Return memcap value.

Return values

memcap value

Definition at line 113 of file flow.c.

References flow_config, and SC_ATOMIC_GET.

Referenced by RunModeUnixSocketGetDefaultMode().

Here is the caller graph for this function:

uint64_t FlowGetMemuse ( void )

Definition at line 119 of file flow.c.

References SC_ATOMIC_GET.

Referenced by FlowManagerThreadSpawn(), and RunModeUnixSocketGetDefaultMode().

Here is the caller graph for this function:

int FlowGetPacketDirection	(	const Flow *	f,
		const Packet *	p
	)

determine the direction of the packet compared to the flow

Return values

0	to_server
1	to_client

Definition at line 320 of file flow.c.

References CMP_ADDR, CMP_PORT, Packet_::dp, Flow_::flags, FLOW_DIR_REVERSED, FlowGetPacketDirection(), ICMPV4_IS_ERROR_MSG, MAX, Flow_::max_ttl_toclient, Flow_::max_ttl_toserver, MIN, Flow_::min_ttl_toclient, Flow_::min_ttl_toserver, PKT_IS_ICMPV4, Packet_::proto, Flow_::sp, Packet_::sp, Flow_::src, Packet_::src, TOCLIENT, and TOSERVER.

Referenced by FlowGetPacketDirection(), FlowHandlePacketUpdate(), and StreamTcpPacket().

Here is the call graph for this function:

Here is the caller graph for this function:

void FlowHandlePacket	(	ThreadVars *	tv,
		DecodeThreadVars *	dtv,
		Packet *	p
	)

Entry point for packet flow handling.

This is called for every packet.

Parameters

tv	threadvars
dtv	decode thread vars (for flow output api thread data)
p	packet to handle flow for

Definition at line 499 of file flow.c.

References Packet_::flags, Packet_::flow, FlowGetFlowFromHash(), and PKT_HAS_FLOW.

Referenced by UTHBuildPacketOfFlows().

Here is the call graph for this function:

Here is the caller graph for this function:

void FlowHandlePacketUpdate	(	Flow *	f,
		Packet *	p
	)

Update Packet and Flow.

Updates packet and flow based on the new packet.

Parameters

f	locked flow
p	packet

Note: overwrites p::flowflags

Definition at line 398 of file flow.c.

References BypassedFlowUpdate(), COPY_TIMESTAMP, DecodeSetNoPacketInspectionFlag, DecodeSetNoPayloadInspectionFlag, EngineModeIsIPS(), Flow_::flags, Packet_::flags, FLOW_BYPASSED_TIMEOUT, FLOW_NOPACKET_INSPECTION, FLOW_NOPAYLOAD_INSPECTION, FLOW_PKT_ESTABLISHED, FLOW_PKT_TOCLIENT, FLOW_PKT_TOCLIENT_FIRST, FLOW_PKT_TOSERVER, FLOW_PKT_TOSERVER_FIRST, FLOW_PROTO_DETECT_TC_DONE, FLOW_PROTO_DETECT_TS_DONE, FLOW_STATE_ESTABLISHED, FLOW_STATE_LOCAL_BYPASSED, FLOW_TO_DST_SEEN, FLOW_TO_SRC_SEEN, Packet_::flowflags, FlowGetPacketDirection(), FlowUpdateState(), GET_PKT_LEN, IPV4_GET_IPTTL, IPV6_GET_HLIM, Flow_::lastts, Packet_::pcap_cnt, PKT_IS_IPV4, PKT_IS_IPV6, PKT_PROTO_DETECT_TC_DONE, PKT_PROTO_DETECT_TS_DONE, Flow_::proto, SC_ATOMIC_GET, SCLogDebug, Flow_::todstbytecnt, Flow_::todstpktcnt, TOSERVER, Flow_::tosrcbytecnt, Flow_::tosrcpktcnt, and Packet_::ts.

Here is the call graph for this function:

int FlowHasAlerts ( const Flow * f )

Check if flow has alerts.

Parameters

f flow

Return values

1	has alerts
0	has not alerts

Definition at line 208 of file flow.c.

References Flow_::flags, and FLOW_HAS_ALERTS.

Referenced by JsonAddFlow().

Here is the caller graph for this function:

void FlowInitConfig ( char quiet )

initialize the configuration

Warning: Not thread safe

set config values for memcap, prealloc and hash_size

Definition at line 515 of file flow.c.

References ByteExtractStringUint32(), CLS, ConfGet(), ConfGetInt(), FlowCnf_::emergency_recovery, FALSE, FBLOCK_INIT, FLOW_CHECK_MEMCAP, flow_config, FLOW_DEFAULT_EMERGENCY_RECOVERY, FLOW_DEFAULT_HASHSIZE, FLOW_DEFAULT_MEMCAP, FLOW_DEFAULT_PREALLOC, flow_hash, flow_recycle_q, flow_spare_q, FlowAlloc(), FlowEnqueue(), FlowInitFlowProto(), FlowQueueInit(), FlowStorageSize(), FlowCnf_::hash_rand, FlowCnf_::hash_size, FlowQueue_::len, ParseSizeStringU64(), FlowCnf_::prealloc, RandomGet(), SC_ATOMIC_ADD, SC_ATOMIC_GET, SC_ATOMIC_INIT, SC_ATOMIC_SET, SC_ERR_FATAL, SC_ERR_FLOW_INIT, SC_ERR_INVALID_VALUE, SC_ERR_INVALID_YAML_CONF_ENTRY, SC_ERR_SIZE_PARSE, SCLogDebug, SCLogError, SCMallocAligned, and unlikely.

Referenced by __attribute__(), DecodeICMPV6(), DecodeIPV4(), DecodeIPV6(), DecodePPP(), DecodePPPOESession(), DecodeRaw(), DecodeTCP(), DecodeVXLAN(), DetectFastPatternRegister(), DetectFragBitsRegister(), DetectFragOffsetFree(), DetectIcmpIdFree(), DetectIPProtoRemoveAllSMs(), DetectPortHashFree(), DetectReplaceFreeInternal(), DetectSetupParseRegexes(), FlowStorageRegister(), FlowUpdateState(), ICMPv4GetCounterpart(), PreRunInit(), SCSigSignatureOrderingModuleCleanup(), SigParseApplyDsizeToContent(), TagTimeoutCheck(), TmModuleFlowRecyclerRegister(), Unified2AlertInitCtx(), and UTHParseSignature().

Here is the call graph for this function:

Here is the caller graph for this function:

void FlowInitFlowProto ( void )

Function to set the default timeout, free function and flow state function for all supported flow_proto.

Definition at line 723 of file flow.c.

Referenced by FlowInitConfig(), and FlowUpdateState().

Here is the call graph for this function:

Here is the caller graph for this function:

void FlowRegisterTests ( void )

Function to register the Flow Unitests.

Definition at line 1330 of file flow.c.

References FlowMgrRegisterTests(), RegisterFlowStorageTests(), and UtRegisterTest().

Here is the call graph for this function:

void FlowSetChangeProtoFlag ( Flow * f )

Set flag to indicate to change proto for the flow.

Parameters

f flow

Definition at line 221 of file flow.c.

References Flow_::flags, and FLOW_CHANGE_PROTO.

Referenced by AppLayerRequestProtocolChange().

Here is the caller graph for this function:

void FlowSetHasAlertsFlag ( Flow * f )

Set flag to indicate that flow has alerts.

Parameters

f flow

Definition at line 197 of file flow.c.

References Flow_::flags, and FLOW_HAS_ALERTS.

Referenced by PacketAlertFinalize().

Here is the caller graph for this function:

void FlowSetIPOnlyFlag	(	Flow *	f,
		int	direction
	)

Set the IPOnly scanned flag for 'direction'.

Parameters

f	Flow to set the flag in
direction	direction to set the flag in

Definition at line 186 of file flow.c.

References Flow_::flags, FLOW_TOCLIENT_IPONLY_SET, and FLOW_TOSERVER_IPONLY_SET.

Referenced by PacketAlertFinalize(), and SigMatchSignaturesGetSgh().

Here is the caller graph for this function:

int FlowSetMemcap ( uint64_t size )

Update memcap value.

Parameters

size	new memcap value

Definition at line 98 of file flow.c.

References flow_config, SC_ATOMIC_GET, and SC_ATOMIC_SET.

Referenced by RunModeUnixSocketGetDefaultMode().

Here is the caller graph for this function:

int FlowSetProtoFreeFunc	(	uint8_t	proto,
		void()(void )	Free
	)

Function to set the function to get protocol specific flow state.

Parameters

proto	protocol of which function is needed to be set.
Free	Function pointer which will be called to free the protocol specific memory.

Definition at line 1054 of file flow.c.

References flow_freefuncs, FlowGetProtoMapping(), and FlowProtoFreeFunc_::Freefunc.

Referenced by FlowUpdateState(), and StreamTcpInitConfig().

Here is the call graph for this function:

Here is the caller graph for this function:

void FlowShutdown ( void )

shutdown the flow engine

Warning: Not thread safe

Definition at line 670 of file flow.c.

References BUG_ON, FBLOCK_DESTROY, flow_config, flow_hash, flow_recycle_q, flow_spare_q, FlowClearMemory(), FlowDequeue(), FlowFree(), FlowGetProtoMapping(), FlowQueueDestroy(), FlowCnf_::hash_size, Flow_::hnext, Flow_::proto, SC_ATOMIC_DESTROY, SC_ATOMIC_GET, SC_ATOMIC_SUB, and SCFreeAligned.

Referenced by __attribute__(), DecodeICMPV6(), DecodeIPV4(), DecodeIPV6(), DecodePPP(), DecodePPPOESession(), DecodeRaw(), DecodeTCP(), DecodeVXLAN(), DetectFastPatternRegister(), DetectFragBitsRegister(), DetectFragOffsetFree(), DetectIcmpIdFree(), DetectIPProtoRemoveAllSMs(), DetectPortHashFree(), DetectReplaceFreeInternal(), DetectSetupParseRegexes(), FlowStorageRegister(), FlowUpdateState(), ICMPv4GetCounterpart(), PostRunDeinit(), SCSigSignatureOrderingModuleCleanup(), SigParseApplyDsizeToContent(), TagTimeoutCheck(), TmModuleFlowRecyclerRegister(), Unified2AlertInitCtx(), and UTHParseSignature().

Here is the call graph for this function:

Here is the caller graph for this function:

void FlowSwap ( Flow * f )

swap the flow's direction

Note: leaves the 'header' untouched. Interpret that based on FLOW_DIR_REVERSED flag.

Warning: : only valid before applayer parsing started. This function doesn't swap anything in Flow::alparser, Flow::alstate

Definition at line 289 of file flow.c.

References Flow_::alproto_tc, Flow_::alproto_ts, Flow_::flags, FLOW_DIR_REVERSED, Flow_::max_ttl_toclient, Flow_::max_ttl_toserver, Flow_::min_ttl_toclient, Flow_::min_ttl_toserver, Flow_::probing_parser_toclient_alproto_masks, Flow_::probing_parser_toserver_alproto_masks, Flow_::proto, Flow_::sgh_toclient, Flow_::sgh_toserver, SWAP_VARS, Flow_::todstbytecnt, Flow_::todstpktcnt, Flow_::tosrcbytecnt, and Flow_::tosrcpktcnt.

Referenced by AppLayerHandleUdp(), and AppLayerIncTxCounter().

Here is the caller graph for this function:

void FlowUnsetChangeProtoFlag ( Flow * f )

Unset flag to indicate to change proto for the flow.

Parameters

f flow

Definition at line 230 of file flow.c.

References Flow_::flags, and FLOW_CHANGE_PROTO.

Referenced by AppLayerProtoDetectReset().

Here is the caller graph for this function:

int FlowUpdateSpareFlows ( void )

Make sure we have enough spare flows.

Enforce the prealloc parameter, so keep at least prealloc flows in the spare queue and free flows going over the limit.

Return values

1	if the queue was properly updated (or if it already was in good shape)
0	otherwise.

Definition at line 144 of file flow.c.

References flow_config, flow_spare_q, FlowAlloc(), FlowDequeue(), FlowEnqueue(), FlowFree(), FQLOCK_LOCK, FQLOCK_UNLOCK, FlowQueue_::len, len, FlowCnf_::prealloc, and SCEnter.

Here is the call graph for this function:

void FlowUpdateState	(	Flow *	f,
		enum FlowState	s
	)

Definition at line 1104 of file flow.c.

Referenced by FlowGetExistingFlowFromHash(), FlowGetFlowFromHash(), FlowHandlePacketUpdate(), PacketBypassCallback(), and StreamTcpFreeConfig().

Here is the call graph for this function:

Here is the caller graph for this function:

SC_ATOMIC_DECLARE	(	unsigned	int,
		flow_prune_idx
	)

atomic int that is used when freeing a flow from the hash. In this case we walk the hash to find a flow to free. This var records where we left off in the hash. Without this only the top rows of the hash are freed. This isn't just about fairness. Under severe presure, the hash rows on top would be all freed and the time to find a flow to free increased with every run.

SC_ATOMIC_DECLARE	(	unsigned	int,
		flow_flags
	)

atomic flags

Variable Documentation

int run_mode

Run mode selected

Definition at line 204 of file suricata.c.

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

Function Documentation

Variable Documentation