suricata
|
#include "suricata-common.h"
#include "suricata.h"
#include "packet.h"
#include "detect.h"
#include "flow.h"
#include "threads.h"
#include "conf.h"
#include "action-globals.h"
#include "flow-util.h"
#include "threadvars.h"
#include "tm-threads.h"
#include "util-pool.h"
#include "util-unittest.h"
#include "util-print.h"
#include "util-host-os-info.h"
#include "util-unittest-helper.h"
#include "util-byte.h"
#include "util-device.h"
#include "stream-tcp.h"
#include "stream-tcp-private.h"
#include "stream-tcp-cache.h"
#include "stream-tcp-reassemble.h"
#include "stream-tcp-inline.h"
#include "stream-tcp-list.h"
#include "stream-tcp-util.h"
#include "stream.h"
#include "util-debug.h"
#include "app-layer-protos.h"
#include "app-layer.h"
#include "app-layer-events.h"
#include "app-layer-parser.h"
#include "app-layer-frames.h"
#include "detect-engine-state.h"
#include "util-profiling.h"
#include "util-validate.h"
#include "util-exception-policy.h"
#include "tests/stream-tcp-reassemble.c"
Go to the source code of this file.
Macros | |
#define | STREAMTCP_STREAM_FLAG_FLUSH_FLAGS |
#define | SET_ISN(stream, setseq) |
#define | MISSED_START(isn) |
#define | MISSED_END |
#define | MISSED_STEP(seq, seg, seglen, buf, buflen) |
#define | MISSED_ADD_PAYLOAD(seq, seg, seglen) StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); |
Functions | |
SC_ATOMIC_DECLARE (uint64_t, ra_memuse) | |
bool | IsTcpSessionDumpingEnabled (void) |
void | EnableTcpSessionDumping (void) |
TcpSegment * | StreamTcpGetSegment (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx) |
get a segment from the pool More... | |
void | StreamTcpCreateTestPacket (uint8_t *payload, uint8_t value, uint8_t payload_len, uint8_t len) |
The Function to create the packet with given payload, which is used to test the reassembly of the engine. More... | |
void | StreamTcpReassembleInitMemuse (void) |
uint64_t | StreamTcpReassembleMemuseGlobalCounter (void) |
int | StreamTcpReassembleCheckMemcap (uint64_t size) |
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP segments. More... | |
int | StreamTcpReassembleSetMemcap (uint64_t size) |
Update memcap value. More... | |
uint64_t | StreamTcpReassembleGetMemcap (void) |
Return memcap value. More... | |
void * | StreamTcpReassembleRealloc (void *optr, size_t orig_size, size_t size) |
void | StreamTcpSegmentReturntoPool (TcpSegment *seg) |
Function to return the segment back to the pool. More... | |
void | StreamTcpReturnStreamSegments (TcpStream *stream) |
return all segments in this stream into the pool(s) More... | |
uint64_t | StreamDataRightEdge (const TcpStream *stream, const bool eof) |
uint64_t | StreamTcpGetUsable (const TcpStream *stream, const bool eof) |
void | StreamTcpDisableAppLayer (Flow *f) |
int | StreamTcpAppLayerIsDisabled (Flow *f) |
int | StreamTcpReassembleInit (bool quiet) |
void | StreamTcpReassembleFree (bool quiet) |
TcpReassemblyThreadCtx * | StreamTcpReassembleInitThreadCtx (ThreadVars *tv) |
void | StreamTcpReassembleFreeThreadCtx (TcpReassemblyThreadCtx *ra_ctx) |
bool | StreamTcpReassembleDepthReached (Packet *p) |
check if stream in pkt direction has depth reached More... | |
uint32_t | StreamDataAvailableForProtoDetect (TcpStream *stream) |
int | StreamTcpReassembleHandleSegmentHandleData (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) |
Insert a TCP packet data into the stream reassembly engine. More... | |
uint8_t | StreamNeedsReassembly (const TcpSession *ssn, uint8_t direction) |
see what if any work the TCP session still needs More... | |
int | StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir) |
Update the stream reassembly upon receiving a packet. More... | |
bool | StreamReassembleRawHasDataReady (TcpSession *ssn, Packet *p) |
does the stream engine have data to inspect? More... | |
void | StreamReassembleRawUpdateProgress (TcpSession *ssn, Packet *p, const uint64_t progress) |
update stream engine after detection More... | |
int | StreamReassembleForFrame (TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof) |
int | StreamReassembleRaw (TcpSession *ssn, const Packet *p, StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out, bool respect_inspect_depth) |
int | StreamReassembleLog (const TcpSession *ssn, const TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t progress_in, uint64_t *progress_out, const bool eof) |
int | StreamTcpReassembleHandleSegment (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) |
void | StreamTcpReassembleTriggerRawReassembly (TcpSession *ssn, int direction) |
Trigger RAW stream reassembly. More... | |
void | StreamTcpReassemblySetMinInspectDepth (TcpSession *ssn, int direction, uint32_t depth) |
int | UTHCheckGapAtPosition (TcpStream *stream, int pos, uint64_t offset, uint32_t len) |
int | UTHCheckDataAtPosition (TcpStream *stream, int pos, uint64_t offset, const char *data, uint32_t len) |
void | StreamTcpReassembleRegisterTests (void) |
The Function Register the Unit tests to test the reassembly engine for various OS policies. More... | |
Variables | |
thread_local uint64_t | t_pcapcnt = UINT64_MAX |
PoolThread * | segment_thread_pool = NULL |
Reference: Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
Definition in file stream-tcp-reassemble.c.
#define MISSED_ADD_PAYLOAD | ( | seq, | |
seg, | |||
seglen | |||
) | StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); |
Definition at line 2265 of file stream-tcp-reassemble.c.
#define MISSED_END |
Definition at line 2256 of file stream-tcp-reassemble.c.
#define MISSED_START | ( | isn | ) |
Definition at line 2242 of file stream-tcp-reassemble.c.
#define MISSED_STEP | ( | seq, | |
seg, | |||
seglen, | |||
buf, | |||
buflen | |||
) |
Definition at line 2261 of file stream-tcp-reassemble.c.
#define SET_ISN | ( | stream, | |
setseq | |||
) |
unit tests and it's support functions below
Definition at line 2199 of file stream-tcp-reassemble.c.
#define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS |
void EnableTcpSessionDumping | ( | void | ) |
Definition at line 96 of file stream-tcp-reassemble.c.
|
inline |
Definition at line 91 of file stream-tcp-reassemble.c.
SC_ATOMIC_DECLARE | ( | uint64_t | , |
ra_memuse | |||
) |
uint32_t StreamDataAvailableForProtoDetect | ( | TcpStream * | stream | ) |
Definition at line 722 of file stream-tcp-reassemble.c.
References StreamingBufferRegion_::buf_offset, DEBUG_VALIDATE_BUG_ON, StreamingBuffer_::head, RB_EMPTY, StreamingBuffer_::region, TcpStream_::sb, StreamingBuffer_::sbb_size, StreamingBuffer_::sbb_tree, and StreamingBufferRegion_::stream_offset.
uint64_t StreamDataRightEdge | ( | const TcpStream * | stream, |
const bool | eof | ||
) |
Definition at line 417 of file stream-tcp-reassemble.c.
References TcpStream_::base_seq, MIN, TcpStream_::segs_right_edge, STREAM_BASE_OFFSET, and StreamTcpInlineMode().
uint8_t StreamNeedsReassembly | ( | const TcpSession * | ssn, |
uint8_t | direction | ||
) |
see what if any work the TCP session still needs
Definition at line 977 of file stream-tcp-reassemble.c.
References TcpStream_::base_seq, TcpSession_::client, TcpStream_::flags, TcpSession_::flags, SCLogDebug, TcpStream_::segs_right_edge, TcpSession_::server, STREAM_APP_PROGRESS, STREAM_BASE_OFFSET, STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION, STREAM_RAW_PROGRESS, STREAMTCP_FLAG_APP_LAYER_DISABLED, and STREAMTCP_STREAM_FLAG_DISABLE_RAW.
Referenced by FlowNeedsReassembly().
int StreamReassembleForFrame | ( | TcpSession * | ssn, |
TcpStream * | stream, | ||
StreamReassembleRawFunc | Callback, | ||
void * | cb_data, | ||
const uint64_t | offset, | ||
const bool | eof | ||
) |
Definition at line 1897 of file stream-tcp-reassemble.c.
References SCLogDebug, and STREAM_APP_PROGRESS.
int StreamReassembleLog | ( | const TcpSession * | ssn, |
const TcpStream * | stream, | ||
StreamReassembleRawFunc | Callback, | ||
void * | cb_data, | ||
const uint64_t | progress_in, | ||
uint64_t * | progress_out, | ||
const bool | eof | ||
) |
Definition at line 1974 of file stream-tcp-reassemble.c.
References TcpStream_::flags, and STREAMTCP_STREAM_FLAG_NOREASSEMBLY.
int StreamReassembleRaw | ( | TcpSession * | ssn, |
const Packet * | p, | ||
StreamReassembleRawFunc | Callback, | ||
void * | cb_data, | ||
uint64_t * | progress_out, | ||
bool | respect_inspect_depth | ||
) |
Definition at line 1909 of file stream-tcp-reassemble.c.
References StreamTcpInlineMode().
Referenced by DetectEngineInspectStreamPayload().
bool StreamReassembleRawHasDataReady | ( | TcpSession * | ssn, |
Packet * | p | ||
) |
does the stream engine have data to inspect?
Returns true if there is data to inspect. In IDS case this is about ACK'd data in the packet's direction.
In the IPS case this is about the packet itself.
Definition at line 1512 of file stream-tcp-reassemble.c.
References TcpStream_::base_seq, TcpSession_::client, TcpStream_::flags, PKT_IS_TOSERVER, RB_EMPTY, TcpStream_::seg_tree, TcpStream_::segs_right_edge, TcpSession_::server, STREAM_BASE_OFFSET, STREAM_RAW_PROGRESS, STREAMTCP_STREAM_FLAG_DISABLE_RAW, STREAMTCP_STREAM_FLAG_NOREASSEMBLY, and StreamTcpInlineMode().
void StreamReassembleRawUpdateProgress | ( | TcpSession * | ssn, |
Packet * | p, | ||
const uint64_t | progress | ||
) |
update stream engine after detection
Tasked with progressing the 'progress' for Raw reassembly. 2 main scenario's:
Additionally, this function is tasked with disabling raw reassembly if the app-layer requested to disable it.
Definition at line 1558 of file stream-tcp-reassemble.c.
References TcpSession_::client, DEBUG_VALIDATE_BUG_ON, TcpStream_::flags, TcpSession_::flags, PKT_IS_TOSERVER, TcpStream_::raw_progress_rel, TcpSession_::server, STREAM_APP_PROGRESS, STREAM_RAW_PROGRESS, STREAMTCP_FLAG_APP_LAYER_DISABLED, and STREAMTCP_STREAM_FLAG_TRIGGER_RAW.
int StreamTcpAppLayerIsDisabled | ( | Flow * | f | ) |
f | locked flow |
Definition at line 463 of file stream-tcp-reassemble.c.
References TcpSession_::flags, Flow_::proto, Flow_::protoctx, and STREAMTCP_FLAG_APP_LAYER_DISABLED.
void StreamTcpCreateTestPacket | ( | uint8_t * | payload, |
uint8_t | value, | ||
uint8_t | payload_len, | ||
uint8_t | len | ||
) |
The Function to create the packet with given payload, which is used to test the reassembly of the engine.
payload | The variable used to store the payload contents of the current packet. |
value | The value which current payload will have for this packet |
payload_len | The length of the filed payload for current packet. |
len | Length of the payload array |
Definition at line 2212 of file stream-tcp-reassemble.c.
References len, and payload_len.
void StreamTcpDisableAppLayer | ( | Flow * | f | ) |
f | locked flow |
Definition at line 447 of file stream-tcp-reassemble.c.
References Flow_::alparser, APP_LAYER_PARSER_EOF_TC, APP_LAYER_PARSER_EOF_TS, AppLayerParserStateSetFlag(), TcpSession_::client, Flow_::protoctx, TcpSession_::server, StreamTcpDisableAppLayerReassembly, and StreamTcpSetStreamFlagAppProtoDetectionCompleted.
TcpSegment * StreamTcpGetSegment | ( | ThreadVars * | tv, |
TcpReassemblyThreadCtx * | ra_ctx | ||
) |
get a segment from the pool
seg | Segment from the pool or NULL |
Definition at line 2125 of file stream-tcp-reassemble.c.
References TcpReassemblyThreadCtx_::counter_tcp_segment_from_cache, TcpReassemblyThreadCtx_::counter_tcp_segment_from_pool, TcpReassemblyThreadCtx_::counter_tcp_segment_memcap, PoolThreadGetById(), TcpSegment::sbseg, SCLogDebug, segment_thread_pool, TcpReassemblyThreadCtx_::segment_thread_pool_id, StatsIncr(), StreamTcpThreadCacheGetSegment(), and tv.
Referenced by StreamTcpUTAddSegmentWithByte(), and StreamTcpUTAddSegmentWithPayload().
uint64_t StreamTcpGetUsable | ( | const TcpStream * | stream, |
const bool | eof | ||
) |
Definition at line 426 of file stream-tcp-reassemble.c.
Referenced by FrameJsonLogOneFrame().
int StreamTcpReassembleAppLayer | ( | ThreadVars * | tv, |
TcpReassemblyThreadCtx * | ra_ctx, | ||
TcpSession * | ssn, | ||
TcpStream * | stream, | ||
Packet * | p, | ||
enum StreamUpdateDir | app_update_dir | ||
) |
Update the stream reassembly upon receiving a packet.
For IDS mode, the stream is in the opposite direction of the packet, as the ACK-packet is ACK'ing the stream.
One of the utilities call by this function AppLayerHandleTCPData(), has a feature where it will call this very same function for the stream opposing the stream it is called with. This shouldn't cause any issues, since processing of each stream is independent of the other stream.
Definition at line 1398 of file stream-tcp-reassemble.c.
References TcpStream_::flags, TcpSession_::flags, Packet_::flags, PKT_PSEUDO_STREAM_END, RB_MIN, SCEnter, SCLogDebug, SCReturnInt, TcpStream_::seg_tree, TcpSession_::state, STREAM_APP_PROGRESS, STREAM_HAS_SEEN_DATA, STREAM_RIGHT_EDGE, STREAMTCP_FLAG_APP_LAYER_DISABLED, STREAMTCP_STREAM_FLAG_NOREASSEMBLY, and TCP_CLOSING.
int StreamTcpReassembleCheckMemcap | ( | uint64_t | size | ) |
Function to Check the reassembly memory usage counter against the allowed max memory usage for TCP segments.
size | Size of the TCP segment and its payload length memory allocated |
1 | if in bounds |
0 | if not in bounds |
Definition at line 164 of file stream-tcp-reassemble.c.
References SC_ATOMIC_GET, SCLogNotice, stream_config, t_pcapcnt, and unlikely.
Referenced by StreamTcpReassembleRealloc().
bool StreamTcpReassembleDepthReached | ( | Packet * | p | ) |
check if stream in pkt direction has depth reached
p | packet with LOCKED flow |
true | stream has depth reached |
false | stream does not have depth reached |
Definition at line 625 of file stream-tcp-reassemble.c.
References TcpSession_::client, TcpStream_::flags, Packet_::flow, FLOW_PKT_TOSERVER, Packet_::flowflags, Flow_::protoctx, TcpSession_::server, and STREAMTCP_STREAM_FLAG_DEPTH_REACHED.
void StreamTcpReassembleFree | ( | bool | quiet | ) |
Definition at line 539 of file stream-tcp-reassemble.c.
References SCMutexLock.
Referenced by StreamTcpFreeConfig().
void StreamTcpReassembleFreeThreadCtx | ( | TcpReassemblyThreadCtx * | ra_ctx | ) |
Definition at line 596 of file stream-tcp-reassemble.c.
References TcpReassemblyThreadCtx_::app_tctx, AppLayerDestroyCtxThread(), SCEnter, SCFree, SCReturn, and StreamTcpThreadCacheCleanup().
Referenced by StreamTcpThreadDeinit(), and StreamTcpUTDeinit().
uint64_t StreamTcpReassembleGetMemcap | ( | void | ) |
Return memcap value.
Definition at line 200 of file stream-tcp-reassemble.c.
References SC_ATOMIC_GET, and stream_config.
int StreamTcpReassembleHandleSegment | ( | ThreadVars * | tv, |
TcpReassemblyThreadCtx * | ra_ctx, | ||
TcpSession * | ssn, | ||
TcpStream * | stream, | ||
Packet * | p | ||
) |
Definition at line 2005 of file stream-tcp-reassemble.c.
References DEBUG_VALIDATE_BUG_ON, and SCEnter.
int StreamTcpReassembleHandleSegmentHandleData | ( | ThreadVars * | tv, |
TcpReassemblyThreadCtx * | ra_ctx, | ||
TcpSession * | ssn, | ||
TcpStream * | stream, | ||
Packet * | p | ||
) |
Insert a TCP packet data into the stream reassembly engine.
0 | good segment, as far as we checked. |
-1 | insert failure due to memcap |
If the retval is 0 the segment is inserted correctly, or overlap is handled, or it wasn't added because of reassembly depth.
Definition at line 746 of file stream-tcp-reassemble.c.
References TcpSession_::data_first_seen_dir, TcpStream_::flags, TcpSession_::flags, TcpStream_::os_policy, PKT_IS_TOSERVER, SCEnter, SCLogDebug, SCReturnInt, STREAMTCP_FLAG_APP_LAYER_DISABLED, STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED, StreamTcpSetOSPolicy(), TcpSession_::urg_offset_tc, and TcpSession_::urg_offset_ts.
Referenced by StreamTcpUTAddPayload().
int StreamTcpReassembleInit | ( | bool | quiet | ) |
Definition at line 523 of file stream-tcp-reassemble.c.
References StreamTcpReassembleInitMemuse().
void StreamTcpReassembleInitMemuse | ( | void | ) |
Definition at line 105 of file stream-tcp-reassemble.c.
References SC_ATOMIC_INIT.
Referenced by StreamTcpReassembleInit(), and UtRunTests().
TcpReassemblyThreadCtx* StreamTcpReassembleInitThreadCtx | ( | ThreadVars * | tv | ) |
Definition at line 557 of file stream-tcp-reassemble.c.
References TcpReassemblyThreadCtx_::app_tctx, AppLayerGetCtxThread(), SCCalloc, SCEnter, SCMutexLock, and unlikely.
Referenced by StreamTcpUTInit().
uint64_t StreamTcpReassembleMemuseGlobalCounter | ( | void | ) |
Definition at line 150 of file stream-tcp-reassemble.c.
References SC_ATOMIC_GET.
void* StreamTcpReassembleRealloc | ( | void * | optr, |
size_t | orig_size, | ||
size_t | size | ||
) |
Definition at line 229 of file stream-tcp-reassemble.c.
References SC_ATOMIC_GET, SC_ELIMIT, SC_ENOMEM, sc_errno, SCLogDebug, SCRealloc, stream_config, and StreamTcpReassembleCheckMemcap().
void StreamTcpReassembleRegisterTests | ( | void | ) |
The Function Register the Unit tests to test the reassembly engine for various OS policies.
Definition at line 3907 of file stream-tcp-reassemble.c.
References UtRegisterTest().
int StreamTcpReassembleSetMemcap | ( | uint64_t | size | ) |
Update memcap value.
size | new memcap value |
Definition at line 185 of file stream-tcp-reassemble.c.
References SC_ATOMIC_GET, SC_ATOMIC_SET, and stream_config.
void StreamTcpReassembleTriggerRawReassembly | ( | TcpSession * | ssn, |
int | direction | ||
) |
Trigger RAW stream reassembly.
Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream reassembly from the applayer, for example upon completion of a HTTP request.
It sets a flag in the stream so that the next Raw call will return the data.
ssn | TcpSession |
Definition at line 2161 of file stream-tcp-reassemble.c.
References BUG_ON, TcpSession_::client, TcpStream_::flags, SCLogDebug, TcpSession_::server, and STREAMTCP_STREAM_FLAG_TRIGGER_RAW.
Referenced by AppLayerParserTriggerRawStreamReassembly().
void StreamTcpReassemblySetMinInspectDepth | ( | TcpSession * | ssn, |
int | direction, | ||
uint32_t | depth | ||
) |
Definition at line 2178 of file stream-tcp-reassemble.c.
References BUG_ON, TcpSession_::client, TcpStream_::min_inspect_depth, SCLogDebug, and TcpSession_::server.
void StreamTcpReturnStreamSegments | ( | TcpStream * | stream | ) |
return all segments in this stream into the pool(s)
stream | the stream to cleanup |
Definition at line 398 of file stream-tcp-reassemble.c.
References RB_FOREACH_SAFE, RB_REMOVE, TcpStream_::seg_tree, and StreamTcpSegmentReturntoPool().
Referenced by StreamTcpPruneSession(), StreamTcpSessionPktFree(), and StreamTcpStreamCleanup().
void StreamTcpSegmentReturntoPool | ( | TcpSegment * | seg | ) |
Function to return the segment back to the pool.
seg | Segment which will be returned back to the pool. |
Definition at line 381 of file stream-tcp-reassemble.c.
References TcpSegment::pcap_hdr_storage, TcpSegmentPcapHdrStorage_::pktlen, and StreamTcpThreadCacheReturnSegment().
Referenced by StreamTcpReturnStreamSegments().
int UTHCheckDataAtPosition | ( | TcpStream * | stream, |
int | pos, | ||
uint64_t | offset, | ||
const char * | data, | ||
uint32_t | len | ||
) |
Definition at line 2292 of file stream-tcp-reassemble.c.
References cnt, len, StreamingBufferBlock::len, offset, StreamingBufferBlock::offset, RB_FOREACH, TcpStream_::sb, StreamingBuffer_::sbb_tree, and StreamingBufferSBBGetData().
int UTHCheckGapAtPosition | ( | TcpStream * | stream, |
int | pos, | ||
uint64_t | offset, | ||
uint32_t | len | ||
) |
Definition at line 2269 of file stream-tcp-reassemble.c.
References cnt, len, StreamingBufferBlock::len, offset, StreamingBufferBlock::offset, RB_FOREACH, TcpStream_::sb, and StreamingBuffer_::sbb_tree.
PoolThread* segment_thread_pool = NULL |
Definition at line 82 of file stream-tcp-reassemble.c.
Referenced by StreamTcpGetSegment(), and StreamTcpThreadCacheReturnSegment().
thread_local uint64_t t_pcapcnt = UINT64_MAX |
Definition at line 80 of file stream-tcp-reassemble.c.
Referenced by StreamTcp(), and StreamTcpReassembleCheckMemcap().