suricata
flow-timeout.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2017 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "decode.h"
27 #include "conf.h"
28 #include "threadvars.h"
29 #include "tm-threads.h"
30 #include "runmodes.h"
31 
32 #include "util-random.h"
33 #include "util-time.h"
34 
35 #include "flow.h"
36 #include "flow-queue.h"
37 #include "flow-hash.h"
38 #include "flow-util.h"
39 #include "flow-var.h"
40 #include "flow-private.h"
41 #include "flow-manager.h"
42 #include "flow-timeout.h"
43 #include "pkt-var.h"
44 #include "host.h"
45 
46 #include "stream-tcp-private.h"
47 #include "stream-tcp-reassemble.h"
48 #include "stream-tcp.h"
49 
50 #include "util-unittest.h"
51 #include "util-unittest-helper.h"
52 #include "util-byte.h"
53 
54 #include "util-debug.h"
55 #include "util-privs.h"
56 
57 #include "detect.h"
58 #include "detect-engine-state.h"
59 #include "stream.h"
60 
61 #include "app-layer-parser.h"
62 #include "app-layer.h"
63 
64 #include "util-profiling.h"
65 
66 /**
67  * \internal
68  * \brief Pseudo packet setup for flow forced reassembly.
69  *
70  * \param direction Direction of the packet. 0 indicates toserver and 1
71  * indicates toclient.
72  * \param f Pointer to the flow.
73  * \param ssn Pointer to the tcp session.
74  * \param dummy Indicates to create a dummy pseudo packet. Not all pseudo
75  * packets need to force reassembly, in which case we just
76  * set dummy ack/seq values.
77  */
78 static inline Packet *FlowForceReassemblyPseudoPacketSetup(Packet *p,
79  int direction,
80  Flow *f,
81  TcpSession *ssn)
82 {
83  p->tenant_id = f->tenant_id;
84  p->datalink = DLT_RAW;
85  p->proto = IPPROTO_TCP;
86  FlowReference(&p->flow, f);
87  p->flags |= PKT_STREAM_EST;
88  p->flags |= PKT_STREAM_EOF;
89  p->flags |= PKT_HAS_FLOW;
91  p->vlan_id[0] = f->vlan_id[0];
92  p->vlan_id[1] = f->vlan_id[1];
93  p->vlan_idx = f->vlan_idx;
94  p->livedev = (struct LiveDevice_ *)f->livedev;
95 
98  }
101  }
102 
103  if (direction == 0)
105  else
108  p->payload = NULL;
109  p->payload_len = 0;
110 
111  /* apply reversed flow logic after setting direction to the packet */
112  direction ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
113 
114  if (FLOW_IS_IPV4(f)) {
115  if (direction == 0) {
118  p->sp = f->sp;
119  p->dp = f->dp;
120  } else {
123  p->sp = f->dp;
124  p->dp = f->sp;
125  }
126 
127  /* Check if we have enough room in direct data. We need ipv4 hdr + tcp hdr.
128  * Force an allocation if it is not the case.
129  */
130  if (GET_PKT_DIRECT_MAX_SIZE(p) < 40) {
131  if (PacketCallocExtPkt(p, 40) == -1) {
132  goto error;
133  }
134  }
135  /* set the ip header */
136  p->ip4h = (IPV4Hdr *)GET_PKT_DATA(p);
137  /* version 4 and length 20 bytes for the tcp header */
138  p->ip4h->ip_verhl = 0x45;
139  p->ip4h->ip_tos = 0;
140  p->ip4h->ip_len = htons(40);
141  p->ip4h->ip_id = 0;
142  p->ip4h->ip_off = 0;
143  p->ip4h->ip_ttl = 64;
144  p->ip4h->ip_proto = IPPROTO_TCP;
145  //p->ip4h->ip_csum =
146  if (direction == 0) {
147  p->ip4h->s_ip_src.s_addr = f->src.addr_data32[0];
148  p->ip4h->s_ip_dst.s_addr = f->dst.addr_data32[0];
149  } else {
150  p->ip4h->s_ip_src.s_addr = f->dst.addr_data32[0];
151  p->ip4h->s_ip_dst.s_addr = f->src.addr_data32[0];
152  }
153 
154  /* set the tcp header */
155  p->tcph = (TCPHdr *)((uint8_t *)GET_PKT_DATA(p) + 20);
156 
157  SET_PKT_LEN(p, 40); /* ipv4 hdr + tcp hdr */
158 
159  } else if (FLOW_IS_IPV6(f)) {
160  if (direction == 0) {
163  p->sp = f->sp;
164  p->dp = f->dp;
165  } else {
168  p->sp = f->dp;
169  p->dp = f->sp;
170  }
171 
172  /* Check if we have enough room in direct data. We need ipv6 hdr + tcp hdr.
173  * Force an allocation if it is not the case.
174  */
175  if (GET_PKT_DIRECT_MAX_SIZE(p) < 60) {
176  if (PacketCallocExtPkt(p, 60) == -1) {
177  goto error;
178  }
179  }
180  /* set the ip header */
181  p->ip6h = (IPV6Hdr *)GET_PKT_DATA(p);
182  /* version 6 */
183  p->ip6h->s_ip6_vfc = 0x60;
184  p->ip6h->s_ip6_flow = 0;
185  p->ip6h->s_ip6_nxt = IPPROTO_TCP;
186  p->ip6h->s_ip6_plen = htons(20);
187  p->ip6h->s_ip6_hlim = 64;
188  if (direction == 0) {
189  p->ip6h->s_ip6_src[0] = f->src.addr_data32[0];
190  p->ip6h->s_ip6_src[1] = f->src.addr_data32[1];
191  p->ip6h->s_ip6_src[2] = f->src.addr_data32[2];
192  p->ip6h->s_ip6_src[3] = f->src.addr_data32[3];
193  p->ip6h->s_ip6_dst[0] = f->dst.addr_data32[0];
194  p->ip6h->s_ip6_dst[1] = f->dst.addr_data32[1];
195  p->ip6h->s_ip6_dst[2] = f->dst.addr_data32[2];
196  p->ip6h->s_ip6_dst[3] = f->dst.addr_data32[3];
197  } else {
198  p->ip6h->s_ip6_src[0] = f->dst.addr_data32[0];
199  p->ip6h->s_ip6_src[1] = f->dst.addr_data32[1];
200  p->ip6h->s_ip6_src[2] = f->dst.addr_data32[2];
201  p->ip6h->s_ip6_src[3] = f->dst.addr_data32[3];
202  p->ip6h->s_ip6_dst[0] = f->src.addr_data32[0];
203  p->ip6h->s_ip6_dst[1] = f->src.addr_data32[1];
204  p->ip6h->s_ip6_dst[2] = f->src.addr_data32[2];
205  p->ip6h->s_ip6_dst[3] = f->src.addr_data32[3];
206  }
207 
208  /* set the tcp header */
209  p->tcph = (TCPHdr *)((uint8_t *)GET_PKT_DATA(p) + 40);
210 
211  SET_PKT_LEN(p, 60); /* ipv6 hdr + tcp hdr */
212  }
213 
214  p->tcph->th_offx2 = 0x50;
215  p->tcph->th_flags |= TH_ACK;
216  p->tcph->th_win = 10;
217  p->tcph->th_urp = 0;
218 
219  /* to server */
220  if (direction == 0) {
221  p->tcph->th_sport = htons(f->sp);
222  p->tcph->th_dport = htons(f->dp);
223 
224  p->tcph->th_seq = htonl(ssn->client.next_seq);
225  p->tcph->th_ack = htonl(ssn->server.last_ack);
226 
227  /* to client */
228  } else {
229  p->tcph->th_sport = htons(f->dp);
230  p->tcph->th_dport = htons(f->sp);
231 
232  p->tcph->th_seq = htonl(ssn->server.next_seq);
233  p->tcph->th_ack = htonl(ssn->client.last_ack);
234  }
235 
236  if (FLOW_IS_IPV4(f)) {
237  p->tcph->th_sum = TCPChecksum(p->ip4h->s_ip_addrs,
238  (uint16_t *)p->tcph, 20, 0);
239  /* calc ipv4 csum as we may log it and barnyard might reject
240  * a wrong checksum */
241  p->ip4h->ip_csum = IPV4Checksum((uint16_t *)p->ip4h,
242  IPV4_GET_RAW_HLEN(p->ip4h), 0);
243  } else if (FLOW_IS_IPV6(f)) {
244  p->tcph->th_sum = TCPChecksum(p->ip6h->s_ip6_addrs,
245  (uint16_t *)p->tcph, 20, 0);
246  }
247 
248  memset(&p->ts, 0, sizeof(struct timeval));
249  TimeGet(&p->ts);
250 
252 
253  return p;
254 
255 error:
256  FlowDeReference(&p->flow);
257  return NULL;
258 }
259 
260 static inline Packet *FlowForceReassemblyPseudoPacketGet(int direction,
261  Flow *f,
262  TcpSession *ssn)
263 {
264  PacketPoolWait();
266  if (p == NULL) {
267  return NULL;
268  }
269 
271 
272  return FlowForceReassemblyPseudoPacketSetup(p, direction, f, ssn);
273 }
274 
275 /**
276  * \brief Check if a flow needs forced reassembly, or any other processing
277  *
278  * \param f *LOCKED* flow
279  * \param server ptr to int that should be set to 1 or 2 if we return 1
280  * \param client ptr to int that should be set to 1 or 2 if we return 1
281  *
282  * \retval 0 no
283  * \retval 1 yes
284  */
285 int FlowForceReassemblyNeedReassembly(Flow *f, int *server, int *client)
286 {
287  if (f == NULL || f->protoctx == NULL) {
288  *server = *client = STREAM_HAS_UNPROCESSED_SEGMENTS_NONE;
289  SCReturnInt(0);
290  }
291 
292  TcpSession *ssn = (TcpSession *)f->protoctx;
293  *client = StreamNeedsReassembly(ssn, STREAM_TOSERVER);
294  *server = StreamNeedsReassembly(ssn, STREAM_TOCLIENT);
295 
296  /* if state is not fully closed we assume that we haven't fully
297  * inspected the app layer state yet */
298  if (ssn->state >= TCP_ESTABLISHED && ssn->state != TCP_CLOSED)
299  {
302  }
303 
304  /* if app layer still needs some love, push through */
305  if (f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL) {
306  const uint64_t total_txs = AppLayerParserGetTxCnt(f, f->alstate);
307 
309  {
311  }
313  {
315  }
316  }
317 
318  /* nothing to do */
319  if (*client == STREAM_HAS_UNPROCESSED_SEGMENTS_NONE &&
321  SCReturnInt(0);
322  }
323 
324  SCReturnInt(1);
325 }
326 
327 /**
328  * \internal
329  * \brief Forces reassembly for flow if it needs it.
330  *
331  * The function requires flow to be locked beforehand.
332  *
333  * \param f Pointer to the flow.
334  * \param server action required for server: 1 or 2
335  * \param client action required for client: 1 or 2
336  *
337  * \retval 0 This flow doesn't need any reassembly processing; 1 otherwise.
338  */
339 int FlowForceReassemblyForFlow(Flow *f, int server, int client)
340 {
341  Packet *p1 = NULL, *p2 = NULL;
342 
343  /* looks like we have no flows in this queue */
344  if (f == NULL || f->protoctx == NULL) {
345  return 0;
346  }
347 
348  /* Get the tcp session for the flow */
349  TcpSession *ssn = (TcpSession *)f->protoctx;
350 
351  /* The packets we use are based on what segments in what direction are
352  * unprocessed.
353  * p1 if we have client segments for reassembly purpose only. If we
354  * have no server segments p2 can be a toserver packet with dummy
355  * seq/ack, and if we have server segments p2 has to carry out reassembly
356  * for server segment as well, in which case we will also need a p3 in the
357  * toclient which is now dummy since all we need it for is detection */
358 
359  /* insert a pseudo packet in the toserver direction */
361  p1 = FlowForceReassemblyPseudoPacketGet(0, f, ssn);
362  if (p1 == NULL) {
363  goto done;
364  }
366 
368  p2 = FlowForceReassemblyPseudoPacketGet(1, f, ssn);
369  if (p2 == NULL) {
370  FlowDeReference(&p1->flow);
371  TmqhOutputPacketpool(NULL, p1);
372  goto done;
373  }
375  }
376  } else {
378  p1 = FlowForceReassemblyPseudoPacketGet(1, f, ssn);
379  if (p1 == NULL) {
380  goto done;
381  }
383  } else {
384  /* impossible */
385  BUG_ON(1);
386  }
387  }
388 
389  /* inject the packet(s) into the appropriate thread */
390  int thread_id = (int)f->thread_id;
391  Packet *packets[3] = { p1, p2 ? p2 : NULL, NULL }; /**< null terminated array of packets */
392  if (unlikely(!(TmThreadsInjectPacketsById(packets, thread_id)))) {
393  FlowDeReference(&p1->flow);
394  TmqhOutputPacketpool(NULL, p1);
395  if (p2) {
396  FlowDeReference(&p2->flow);
397  TmqhOutputPacketpool(NULL, p2);
398  }
399  }
400 
401  /* done, in case of error (no packet) we still tag flow as complete
402  * as we're probably resource stress if we couldn't get packets */
403 done:
405  return 1;
406 }
407 
408 /**
409  * \internal
410  * \brief Forces reassembly for flows that need it.
411  *
412  * When this function is called we're running in virtually dead engine,
413  * so locking the flows is not strictly required. The reasons it is still
414  * done are:
415  * - code consistency
416  * - silence complaining profilers
417  * - allow us to aggressively check using debug valdation assertions
418  * - be robust in case of future changes
419  * - locking overhead if neglectable when no other thread fights us
420  *
421  * \param q The queue to process flows from.
422  */
423 static inline void FlowForceReassemblyForHash(void)
424 {
425  for (uint32_t idx = 0; idx < flow_config.hash_size; idx++) {
426  FlowBucket *fb = &flow_hash[idx];
427 
429  FBLOCK_LOCK(fb);
430 
431  /* get the topmost flow from the QUEUE */
432  Flow *f = fb->head;
433 
434  /* we need to loop through all the flows in the queue */
435  while (f != NULL) {
437 
438  FLOWLOCK_WRLOCK(f);
439 
440  /* Get the tcp session for the flow */
441  TcpSession *ssn = (TcpSession *)f->protoctx;
442  /* \todo Also skip flows that shouldn't be inspected */
443  if (ssn == NULL) {
444  FLOWLOCK_UNLOCK(f);
445  f = f->hnext;
446  continue;
447  }
448 
449  int client_ok = 0;
450  int server_ok = 0;
451  if (FlowForceReassemblyNeedReassembly(f, &server_ok, &client_ok) == 1) {
452  FlowForceReassemblyForFlow(f, server_ok, client_ok);
453  }
454 
455  FLOWLOCK_UNLOCK(f);
456 
457  /* next flow in the queue */
458  f = f->hnext;
459  }
460  FBLOCK_UNLOCK(fb);
461  }
462  return;
463 }
464 
465 /**
466  * \brief Force reassembly for all the flows that have unprocessed segments.
467  */
469 {
470  /* Carry out flow reassembly for unattended flows */
471  FlowForceReassemblyForHash();
472  return;
473 }
#define FLOW_IS_IPV4(f)
Definition: flow.h:134
#define GET_PKT_DIRECT_MAX_SIZE(p)
Definition: decode.h:225
#define FBLOCK_UNLOCK(fb)
Definition: flow-hash.h:70
#define FLOW_TIMEOUT_REASSEMBLY_DONE
Definition: flow.h:91
uint8_t ip_proto
Definition: decode-ipv4.h:78
struct Flow_ * flow
Definition: decode.h:443
void FlowForceReassembly(void)
Force reassembly for all the flows that have unprocessed segments.
Definition: flow-timeout.c:468
struct Flow_ * hnext
Definition: flow.h:451
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
#define BUG_ON(x)
#define SET_PKT_LEN(p, len)
Definition: decode.h:227
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:243
Port sp
Definition: flow.h:331
#define FLOW_COPY_IPV6_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:149
int TmThreadsInjectPacketsById(Packet **packets, const int id)
Definition: tm-threads.c:2370
#define unlikely(expr)
Definition: util-optimize.h:35
Port sp
Definition: decode.h:413
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
void PacketPoolWaitForN(int n)
Wait until we have the requested amount of packets in the pool.
int FlowForceReassemblyNeedReassembly(Flow *f, int *server, int *client)
Check if a flow needs forced reassembly, or any other processing.
Definition: flow-timeout.c:285
uint32_t tenant_id
Definition: flow.h:374
Port dp
Definition: decode.h:421
#define FLOW_COPY_IPV4_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:144
Address dst
Definition: decode.h:411
#define FLOW_PKT_ESTABLISHED
Definition: flow.h:203
#define FLOW_DIR_REVERSED
Definition: flow.h:106
uint16_t ip_id
Definition: decode-ipv4.h:75
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:240
TCPHdr * tcph
Definition: decode.h:520
uint8_t ip_tos
Definition: decode-ipv4.h:73
#define DecodeSetNoPacketInspectionFlag(p)
Set the No packet inspection Flag for the packet.
Definition: decode.h:983
Packet * PacketPoolGetPacket(void)
Get a new packet from the packet pool.
void * protoctx
Definition: flow.h:400
uint16_t vlan_id[2]
Definition: decode.h:433
FlowConfig flow_config
Definition: flow-private.h:97
uint8_t ip_ttl
Definition: decode-ipv4.h:77
#define PKT_SET_SRC(p, src_val)
Definition: decode.h:1134
FlowAddress dst
Definition: flow.h:329
void * alstate
Definition: flow.h:438
uint16_t vlan_id[2]
Definition: flow.h:346
#define FLOW_NOPACKET_INSPECTION
Definition: flow.h:59
#define DecodeSetNoPayloadInspectionFlag(p)
Set the No payload inspection Flag for the packet.
Definition: decode.h:971
IPV6Hdr * ip6h
Definition: decode.h:500
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
int datalink
Definition: decode.h:574
uint8_t proto
Definition: decode.h:428
void TimeGet(struct timeval *tv)
Definition: util-time.c:146
Data structures and function prototypes for keeping state for the detection engine.
#define FLOW_NOPAYLOAD_INSPECTION
Definition: flow.h:61
void AppLayerParserSetEOF(AppLayerParserState *pstate)
uint64_t AppLayerParserGetTransactionActive(const Flow *f, AppLayerParserState *pstate, uint8_t direction)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:437
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:201
uint8_t vlan_idx
Definition: flow.h:347
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1093
#define SCReturnInt(x)
Definition: util-debug.h:341
struct LiveDevice_ * livedev
Definition: flow.h:350
int FlowForceReassemblyForFlow(Flow *f, int server, int client)
Definition: flow-timeout.c:339
IPV4Hdr * ip4h
Definition: decode.h:498
uint8_t vlan_idx
Definition: decode.h:434
uint16_t ip_len
Definition: decode-ipv4.h:74
void PacketPoolWait(void)
#define IPV4_GET_RAW_HLEN(ip4h)
Definition: decode-ipv4.h:95
FlowThreadId thread_id
Definition: flow.h:426
FlowBucket * flow_hash
Definition: flow-private.h:96
#define PKT_STREAM_EOF
Definition: decode.h:1091
Port dp
Definition: flow.h:338
#define FLOW_IS_IPV6(f)
Definition: flow.h:136
int PacketCallocExtPkt(Packet *p, int datalen)
Definition: decode.c:192
#define PACKET_PROFILING_START(p)
#define STREAM_TOSERVER
Definition: stream.h:31
#define GET_PKT_DATA(p)
Definition: decode.h:223
#define PKT_HAS_FLOW
Definition: decode.h:1092
FlowAddress src
Definition: flow.h:329
uint16_t ip_csum
Definition: decode-ipv4.h:79
struct LiveDevice_ * livedev
Definition: decode.h:553
uint32_t tenant_id
Definition: decode.h:594
struct timeval ts
Definition: decode.h:449
#define FLOW_PKT_TOCLIENT
Definition: flow.h:202
AppProto alproto
application level protocol
Definition: flow.h:409
uint32_t flags
Definition: decode.h:441
uint16_t payload_len
Definition: decode.h:541
uint16_t ip_off
Definition: decode-ipv4.h:76
Flow data structure.
Definition: flow.h:325
uint8_t * payload
Definition: decode.h:540
uint32_t flags
Definition: flow.h:379
#define PKT_STREAM_EST
Definition: decode.h:1090
#define FBLOCK_LOCK(fb)
Definition: flow-hash.h:68
uint8_t ip_verhl
Definition: decode-ipv4.h:72
AppLayerParserState * alparser
Definition: flow.h:437
Address src
Definition: decode.h:410
uint32_t hash_size
Definition: flow.h:264