suricata
flow-timeout.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2017 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "decode.h"
27 #include "conf.h"
28 #include "threadvars.h"
29 #include "tm-threads.h"
30 #include "runmodes.h"
31 
32 #include "util-random.h"
33 #include "util-time.h"
34 
35 #include "flow.h"
36 #include "flow-queue.h"
37 #include "flow-hash.h"
38 #include "flow-util.h"
39 #include "flow-var.h"
40 #include "flow-private.h"
41 #include "flow-manager.h"
42 #include "flow-timeout.h"
43 #include "pkt-var.h"
44 #include "host.h"
45 
46 #include "stream-tcp-private.h"
47 #include "stream-tcp-reassemble.h"
48 #include "stream-tcp.h"
49 
50 #include "util-unittest.h"
51 #include "util-unittest-helper.h"
52 #include "util-byte.h"
53 
54 #include "util-debug.h"
55 #include "util-privs.h"
56 
57 #include "detect.h"
58 #include "detect-engine-state.h"
59 #include "stream.h"
60 
61 #include "app-layer-parser.h"
62 #include "app-layer.h"
63 
64 #include "util-profiling.h"
65 
66 /**
67  * \internal
68  * \brief Pseudo packet setup for flow forced reassembly.
69  *
70  * \param direction Direction of the packet. 0 indicates toserver and 1
71  * indicates toclient.
72  * \param f Pointer to the flow.
73  * \param ssn Pointer to the tcp session.
74  * \param dummy Indicates to create a dummy pseudo packet. Not all pseudo
75  * packets need to force reassembly, in which case we just
76  * set dummy ack/seq values.
77  */
78 static inline Packet *FlowForceReassemblyPseudoPacketSetup(Packet *p,
79  int direction,
80  Flow *f,
81  TcpSession *ssn)
82 {
83  p->tenant_id = f->tenant_id;
84  p->datalink = DLT_RAW;
85  p->proto = IPPROTO_TCP;
86  FlowReference(&p->flow, f);
87  p->flags |= PKT_STREAM_EST;
88  p->flags |= PKT_STREAM_EOF;
89  p->flags |= PKT_HAS_FLOW;
91 
94  }
97  }
98 
99  if (direction == 0)
101  else
104  p->payload = NULL;
105  p->payload_len = 0;
106 
107  if (FLOW_IS_IPV4(f)) {
108  if (direction == 0) {
111  p->sp = f->sp;
112  p->dp = f->dp;
113  } else {
116  p->sp = f->dp;
117  p->dp = f->sp;
118  }
119 
120  /* Check if we have enough room in direct data. We need ipv4 hdr + tcp hdr.
121  * Force an allocation if it is not the case.
122  */
123  if (GET_PKT_DIRECT_MAX_SIZE(p) < 40) {
124  if (PacketCallocExtPkt(p, 40) == -1) {
125  goto error;
126  }
127  }
128  /* set the ip header */
129  p->ip4h = (IPV4Hdr *)GET_PKT_DATA(p);
130  /* version 4 and length 20 bytes for the tcp header */
131  p->ip4h->ip_verhl = 0x45;
132  p->ip4h->ip_tos = 0;
133  p->ip4h->ip_len = htons(40);
134  p->ip4h->ip_id = 0;
135  p->ip4h->ip_off = 0;
136  p->ip4h->ip_ttl = 64;
137  p->ip4h->ip_proto = IPPROTO_TCP;
138  //p->ip4h->ip_csum =
139  if (direction == 0) {
140  p->ip4h->s_ip_src.s_addr = f->src.addr_data32[0];
141  p->ip4h->s_ip_dst.s_addr = f->dst.addr_data32[0];
142  } else {
143  p->ip4h->s_ip_src.s_addr = f->dst.addr_data32[0];
144  p->ip4h->s_ip_dst.s_addr = f->src.addr_data32[0];
145  }
146 
147  /* set the tcp header */
148  p->tcph = (TCPHdr *)((uint8_t *)GET_PKT_DATA(p) + 20);
149 
150  SET_PKT_LEN(p, 40); /* ipv4 hdr + tcp hdr */
151 
152  } else if (FLOW_IS_IPV6(f)) {
153  if (direction == 0) {
156  p->sp = f->sp;
157  p->dp = f->dp;
158  } else {
161  p->sp = f->dp;
162  p->dp = f->sp;
163  }
164 
165  /* Check if we have enough room in direct data. We need ipv6 hdr + tcp hdr.
166  * Force an allocation if it is not the case.
167  */
168  if (GET_PKT_DIRECT_MAX_SIZE(p) < 60) {
169  if (PacketCallocExtPkt(p, 60) == -1) {
170  goto error;
171  }
172  }
173  /* set the ip header */
174  p->ip6h = (IPV6Hdr *)GET_PKT_DATA(p);
175  /* version 6 */
176  p->ip6h->s_ip6_vfc = 0x60;
177  p->ip6h->s_ip6_flow = 0;
178  p->ip6h->s_ip6_nxt = IPPROTO_TCP;
179  p->ip6h->s_ip6_plen = htons(20);
180  p->ip6h->s_ip6_hlim = 64;
181  if (direction == 0) {
182  p->ip6h->s_ip6_src[0] = f->src.addr_data32[0];
183  p->ip6h->s_ip6_src[1] = f->src.addr_data32[1];
184  p->ip6h->s_ip6_src[2] = f->src.addr_data32[2];
185  p->ip6h->s_ip6_src[3] = f->src.addr_data32[3];
186  p->ip6h->s_ip6_dst[0] = f->dst.addr_data32[0];
187  p->ip6h->s_ip6_dst[1] = f->dst.addr_data32[1];
188  p->ip6h->s_ip6_dst[2] = f->dst.addr_data32[2];
189  p->ip6h->s_ip6_dst[3] = f->dst.addr_data32[3];
190  } else {
191  p->ip6h->s_ip6_src[0] = f->dst.addr_data32[0];
192  p->ip6h->s_ip6_src[1] = f->dst.addr_data32[1];
193  p->ip6h->s_ip6_src[2] = f->dst.addr_data32[2];
194  p->ip6h->s_ip6_src[3] = f->dst.addr_data32[3];
195  p->ip6h->s_ip6_dst[0] = f->src.addr_data32[0];
196  p->ip6h->s_ip6_dst[1] = f->src.addr_data32[1];
197  p->ip6h->s_ip6_dst[2] = f->src.addr_data32[2];
198  p->ip6h->s_ip6_dst[3] = f->src.addr_data32[3];
199  }
200 
201  /* set the tcp header */
202  p->tcph = (TCPHdr *)((uint8_t *)GET_PKT_DATA(p) + 40);
203 
204  SET_PKT_LEN(p, 60); /* ipv6 hdr + tcp hdr */
205  }
206 
207  p->tcph->th_offx2 = 0x50;
208  p->tcph->th_flags |= TH_ACK;
209  p->tcph->th_win = 10;
210  p->tcph->th_urp = 0;
211 
212  /* to server */
213  if (direction == 0) {
214  p->tcph->th_sport = htons(f->sp);
215  p->tcph->th_dport = htons(f->dp);
216 
217  p->tcph->th_seq = htonl(ssn->client.next_seq);
218  p->tcph->th_ack = htonl(ssn->server.last_ack);
219 
220  /* to client */
221  } else {
222  p->tcph->th_sport = htons(f->dp);
223  p->tcph->th_dport = htons(f->sp);
224 
225  p->tcph->th_seq = htonl(ssn->server.next_seq);
226  p->tcph->th_ack = htonl(ssn->client.last_ack);
227  }
228 
229  if (FLOW_IS_IPV4(f)) {
230  p->tcph->th_sum = TCPChecksum(p->ip4h->s_ip_addrs,
231  (uint16_t *)p->tcph, 20, 0);
232  /* calc ipv4 csum as we may log it and barnyard might reject
233  * a wrong checksum */
234  p->ip4h->ip_csum = IPV4Checksum((uint16_t *)p->ip4h,
235  IPV4_GET_RAW_HLEN(p->ip4h), 0);
236  } else if (FLOW_IS_IPV6(f)) {
237  p->tcph->th_sum = TCPChecksum(p->ip6h->s_ip6_addrs,
238  (uint16_t *)p->tcph, 20, 0);
239  }
240 
241  memset(&p->ts, 0, sizeof(struct timeval));
242  TimeGet(&p->ts);
243 
245 
246  return p;
247 
248 error:
249  FlowDeReference(&p->flow);
250  return NULL;
251 }
252 
253 static inline Packet *FlowForceReassemblyPseudoPacketGet(int direction,
254  Flow *f,
255  TcpSession *ssn)
256 {
257  PacketPoolWait();
259  if (p == NULL) {
260  return NULL;
261  }
262 
264 
265  return FlowForceReassemblyPseudoPacketSetup(p, direction, f, ssn);
266 }
267 
268 /**
269  * \brief Check if a flow needs forced reassembly, or any other processing
270  *
271  * \param f *LOCKED* flow
272  * \param server ptr to int that should be set to 1 or 2 if we return 1
273  * \param client ptr to int that should be set to 1 or 2 if we return 1
274  *
275  * \retval 0 no
276  * \retval 1 yes
277  */
278 int FlowForceReassemblyNeedReassembly(Flow *f, int *server, int *client)
279 {
280  if (f == NULL || f->protoctx == NULL) {
281  *server = *client = STREAM_HAS_UNPROCESSED_SEGMENTS_NONE;
282  SCReturnInt(0);
283  }
284 
285  TcpSession *ssn = (TcpSession *)f->protoctx;
286  *client = StreamNeedsReassembly(ssn, STREAM_TOSERVER);
287  *server = StreamNeedsReassembly(ssn, STREAM_TOCLIENT);
288 
289  /* if state is not fully closed we assume that we haven't fully
290  * inspected the app layer state yet */
291  if (ssn->state >= TCP_ESTABLISHED && ssn->state != TCP_CLOSED)
292  {
295  }
296 
297  /* if app layer still needs some love, push through */
298  if (f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL &&
300  {
301  const uint64_t total_txs = AppLayerParserGetTxCnt(f, f->alstate);
302 
304  {
306  }
308  {
310  }
311  }
312 
313  /* nothing to do */
314  if (*client == STREAM_HAS_UNPROCESSED_SEGMENTS_NONE &&
316  SCReturnInt(0);
317  }
318 
319  SCReturnInt(1);
320 }
321 
322 /**
323  * \internal
324  * \brief Forces reassembly for flow if it needs it.
325  *
326  * The function requires flow to be locked beforehand.
327  *
328  * \param f Pointer to the flow.
329  * \param server action required for server: 1 or 2
330  * \param client action required for client: 1 or 2
331  *
332  * \retval 0 This flow doesn't need any reassembly processing; 1 otherwise.
333  */
334 int FlowForceReassemblyForFlow(Flow *f, int server, int client)
335 {
336  Packet *p1 = NULL, *p2 = NULL;
337 
338  /* looks like we have no flows in this queue */
339  if (f == NULL || f->protoctx == NULL) {
340  return 0;
341  }
342 
343  /* Get the tcp session for the flow */
344  TcpSession *ssn = (TcpSession *)f->protoctx;
345 
346  /* The packets we use are based on what segments in what direction are
347  * unprocessed.
348  * p1 if we have client segments for reassembly purpose only. If we
349  * have no server segments p2 can be a toserver packet with dummy
350  * seq/ack, and if we have server segments p2 has to carry out reassembly
351  * for server segment as well, in which case we will also need a p3 in the
352  * toclient which is now dummy since all we need it for is detection */
353 
354  /* insert a pseudo packet in the toserver direction */
356  p1 = FlowForceReassemblyPseudoPacketGet(0, f, ssn);
357  if (p1 == NULL) {
358  goto done;
359  }
361 
363  p2 = FlowForceReassemblyPseudoPacketGet(1, f, ssn);
364  if (p2 == NULL) {
365  FlowDeReference(&p1->flow);
366  TmqhOutputPacketpool(NULL, p1);
367  goto done;
368  }
370  }
371  } else {
373  p1 = FlowForceReassemblyPseudoPacketGet(1, f, ssn);
374  if (p1 == NULL) {
375  goto done;
376  }
378  } else {
379  /* impossible */
380  BUG_ON(1);
381  }
382  }
383 
384  /* inject the packet(s) into the appropriate thread */
385  int thread_id = (int)f->thread_id;
386  Packet *packets[3] = { p1, p2 ? p2 : NULL, NULL }; /**< null terminated array of packets */
387  if (unlikely(!(TmThreadsInjectPacketsById(packets, thread_id)))) {
388  FlowDeReference(&p1->flow);
389  TmqhOutputPacketpool(NULL, p1);
390  if (p2) {
391  FlowDeReference(&p2->flow);
392  TmqhOutputPacketpool(NULL, p2);
393  }
394  }
395 
396  /* done, in case of error (no packet) we still tag flow as complete
397  * as we're probably resource stress if we couldn't get packets */
398 done:
400  return 1;
401 }
402 
403 /**
404  * \internal
405  * \brief Forces reassembly for flows that need it.
406  *
407  * When this function is called we're running in virtually dead engine,
408  * so locking the flows is not strictly required. The reasons it is still
409  * done are:
410  * - code consistency
411  * - silence complaining profilers
412  * - allow us to aggressively check using debug valdation assertions
413  * - be robust in case of future changes
414  * - locking overhead if neglectable when no other thread fights us
415  *
416  * \param q The queue to process flows from.
417  */
418 static inline void FlowForceReassemblyForHash(void)
419 {
420  for (uint32_t idx = 0; idx < flow_config.hash_size; idx++) {
421  FlowBucket *fb = &flow_hash[idx];
422 
424  FBLOCK_LOCK(fb);
425 
426  /* get the topmost flow from the QUEUE */
427  Flow *f = fb->head;
428 
429  /* we need to loop through all the flows in the queue */
430  while (f != NULL) {
432 
433  FLOWLOCK_WRLOCK(f);
434 
435  /* Get the tcp session for the flow */
436  TcpSession *ssn = (TcpSession *)f->protoctx;
437  /* \todo Also skip flows that shouldn't be inspected */
438  if (ssn == NULL) {
439  FLOWLOCK_UNLOCK(f);
440  f = f->hnext;
441  continue;
442  }
443 
444  int client_ok = 0;
445  int server_ok = 0;
446  if (FlowForceReassemblyNeedReassembly(f, &server_ok, &client_ok) == 1) {
447  FlowForceReassemblyForFlow(f, server_ok, client_ok);
448  }
449 
450  FLOWLOCK_UNLOCK(f);
451 
452  /* next flow in the queue */
453  f = f->hnext;
454  }
455  FBLOCK_UNLOCK(fb);
456  }
457  return;
458 }
459 
460 /**
461  * \brief Force reassembly for all the flows that have unprocessed segments.
462  */
464 {
465  /* Carry out flow reassembly for unattended flows */
466  FlowForceReassemblyForHash();
467  return;
468 }
469 
#define FLOW_IS_IPV4(f)
Definition: flow.h:131
#define GET_PKT_DIRECT_MAX_SIZE(p)
Definition: decode.h:226
#define FBLOCK_UNLOCK(fb)
Definition: flow-hash.h:70
#define FLOW_TIMEOUT_REASSEMBLY_DONE
Definition: flow.h:90
struct Flow_ * flow
Definition: decode.h:444
void FlowForceReassembly(void)
Force reassembly for all the flows that have unprocessed segments.
Definition: flow-timeout.c:463
struct Flow_ * hnext
Definition: flow.h:449
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:346
#define SET_PKT_LEN(p, len)
Definition: decode.h:228
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:235
Port sp
Definition: flow.h:333
#define FLOW_COPY_IPV6_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:141
int TmThreadsInjectPacketsById(Packet **packets, const int id)
Definition: tm-threads.c:2397
#define unlikely(expr)
Definition: util-optimize.h:35
Port sp
Definition: decode.h:414
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
void PacketPoolWaitForN(int n)
Wait until we have the requested amount of packets in the pool.
int FlowForceReassemblyNeedReassembly(Flow *f, int *server, int *client)
Check if a flow needs forced reassembly, or any other processing.
Definition: flow-timeout.c:278
uint32_t tenant_id
Definition: flow.h:372
Port dp
Definition: decode.h:422
#define FLOW_COPY_IPV4_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:136
Address dst
Definition: decode.h:412
#define FLOW_PKT_ESTABLISHED
Definition: flow.h:195
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:232
TCPHdr * tcph
Definition: decode.h:525
#define DecodeSetNoPacketInspectionFlag(p)
Set the No packet inspection Flag for the packet.
Definition: decode.h:992
Packet * PacketPoolGetPacket(void)
Get a new packet from the packet pool.
void * protoctx
Definition: flow.h:398
FlowConfig flow_config
Definition: flow-private.h:97
#define PKT_SET_SRC(p, src_val)
Definition: decode.h:1143
FlowAddress dst
Definition: flow.h:331
void * alstate
Definition: flow.h:436
#define FLOW_NOPACKET_INSPECTION
Definition: flow.h:58
#define DecodeSetNoPayloadInspectionFlag(p)
Set the No payload inspection Flag for the packet.
Definition: decode.h:980
IPV6Hdr * ip6h
Definition: decode.h:505
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
int datalink
Definition: decode.h:579
uint8_t proto
Definition: decode.h:429
void TimeGet(struct timeval *tv)
Definition: util-time.c:138
Data structures and function prototypes for keeping state for the detection engine.
#define FLOW_NOPAYLOAD_INSPECTION
Definition: flow.h:60
void AppLayerParserSetEOF(AppLayerParserState *pstate)
uint64_t AppLayerParserGetTransactionActive(const Flow *f, AppLayerParserState *pstate, uint8_t direction)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:438
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:193
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1102
#define SCReturnInt(x)
Definition: util-debug.h:341
int FlowForceReassemblyForFlow(Flow *f, int server, int client)
Definition: flow-timeout.c:334
IPV4Hdr * ip4h
Definition: decode.h:503
void PacketPoolWait(void)
#define IPV4_GET_RAW_HLEN(ip4h)
Definition: decode-ipv4.h:95
FlowThreadId thread_id
Definition: flow.h:424
FlowBucket * flow_hash
Definition: flow-private.h:96
#define PKT_STREAM_EOF
Definition: decode.h:1100
Port dp
Definition: flow.h:340
#define FLOW_IS_IPV6(f)
Definition: flow.h:133
int PacketCallocExtPkt(Packet *p, int datalen)
Definition: decode.c:191
#define PACKET_PROFILING_START(p)
#define STREAM_TOSERVER
Definition: stream.h:31
#define GET_PKT_DATA(p)
Definition: decode.h:224
#define PKT_HAS_FLOW
Definition: decode.h:1101
FlowAddress src
Definition: flow.h:331
uint32_t tenant_id
Definition: decode.h:599
struct timeval ts
Definition: decode.h:450
#define FLOW_PKT_TOCLIENT
Definition: flow.h:194
AppProto alproto
application level protocol
Definition: flow.h:407
uint32_t flags
Definition: decode.h:442
uint16_t payload_len
Definition: decode.h:546
Flow data structure.
Definition: flow.h:327
uint8_t * payload
Definition: decode.h:545
uint32_t flags
Definition: flow.h:377
int AppLayerParserProtocolSupportsTxs(uint8_t ipproto, AppProto alproto)
#define PKT_STREAM_EST
Definition: decode.h:1099
#define FBLOCK_LOCK(fb)
Definition: flow-hash.h:68
AppLayerParserState * alparser
Definition: flow.h:435
Address src
Definition: decode.h:411
uint32_t hash_size
Definition: flow.h:256