suricata
flow-timeout.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2017 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "decode.h"
27 #include "conf.h"
28 #include "threadvars.h"
29 #include "tm-threads.h"
30 #include "runmodes.h"
31 
32 #include "util-random.h"
33 #include "util-time.h"
34 
35 #include "flow.h"
36 #include "flow-queue.h"
37 #include "flow-hash.h"
38 #include "flow-util.h"
39 #include "flow-var.h"
40 #include "flow-private.h"
41 #include "flow-manager.h"
42 #include "flow-timeout.h"
43 #include "pkt-var.h"
44 #include "host.h"
45 
46 #include "stream-tcp-private.h"
47 #include "stream-tcp-reassemble.h"
48 #include "stream-tcp.h"
49 
50 #include "util-unittest.h"
51 #include "util-unittest-helper.h"
52 #include "util-byte.h"
53 
54 #include "util-debug.h"
55 #include "util-privs.h"
56 
57 #include "detect.h"
58 #include "detect-engine-state.h"
59 #include "stream.h"
60 
61 #include "app-layer-parser.h"
62 #include "app-layer.h"
63 
64 #include "util-profiling.h"
65 
66 /**
67  * \internal
68  * \brief Pseudo packet setup for flow forced reassembly.
69  *
70  * \param direction Direction of the packet. 0 indicates toserver and 1
71  * indicates toclient.
72  * \param f Pointer to the flow.
73  * \param ssn Pointer to the tcp session.
74  * \param dummy Indicates to create a dummy pseudo packet. Not all pseudo
75  * packets need to force reassembly, in which case we just
76  * set dummy ack/seq values.
77  */
78 static inline Packet *FlowForceReassemblyPseudoPacketSetup(Packet *p,
79  int direction,
80  Flow *f,
81  TcpSession *ssn)
82 {
83  p->tenant_id = f->tenant_id;
84  p->datalink = DLT_RAW;
85  p->proto = IPPROTO_TCP;
86  FlowReference(&p->flow, f);
87  p->flags |= PKT_STREAM_EST;
88  p->flags |= PKT_STREAM_EOF;
89  p->flags |= PKT_HAS_FLOW;
91 
94  }
97  }
98 
99  if (direction == 0)
101  else
104  p->payload = NULL;
105  p->payload_len = 0;
106 
107  /* apply reversed flow logic after setting direction to the packet */
108  direction ^= ((f->flags & FLOW_DIR_REVERSED) != 0);
109 
110  if (FLOW_IS_IPV4(f)) {
111  if (direction == 0) {
114  p->sp = f->sp;
115  p->dp = f->dp;
116  } else {
119  p->sp = f->dp;
120  p->dp = f->sp;
121  }
122 
123  /* Check if we have enough room in direct data. We need ipv4 hdr + tcp hdr.
124  * Force an allocation if it is not the case.
125  */
126  if (GET_PKT_DIRECT_MAX_SIZE(p) < 40) {
127  if (PacketCallocExtPkt(p, 40) == -1) {
128  goto error;
129  }
130  }
131  /* set the ip header */
132  p->ip4h = (IPV4Hdr *)GET_PKT_DATA(p);
133  /* version 4 and length 20 bytes for the tcp header */
134  p->ip4h->ip_verhl = 0x45;
135  p->ip4h->ip_tos = 0;
136  p->ip4h->ip_len = htons(40);
137  p->ip4h->ip_id = 0;
138  p->ip4h->ip_off = 0;
139  p->ip4h->ip_ttl = 64;
140  p->ip4h->ip_proto = IPPROTO_TCP;
141  //p->ip4h->ip_csum =
142  if (direction == 0) {
143  p->ip4h->s_ip_src.s_addr = f->src.addr_data32[0];
144  p->ip4h->s_ip_dst.s_addr = f->dst.addr_data32[0];
145  } else {
146  p->ip4h->s_ip_src.s_addr = f->dst.addr_data32[0];
147  p->ip4h->s_ip_dst.s_addr = f->src.addr_data32[0];
148  }
149 
150  /* set the tcp header */
151  p->tcph = (TCPHdr *)((uint8_t *)GET_PKT_DATA(p) + 20);
152 
153  SET_PKT_LEN(p, 40); /* ipv4 hdr + tcp hdr */
154 
155  } else if (FLOW_IS_IPV6(f)) {
156  if (direction == 0) {
159  p->sp = f->sp;
160  p->dp = f->dp;
161  } else {
164  p->sp = f->dp;
165  p->dp = f->sp;
166  }
167 
168  /* Check if we have enough room in direct data. We need ipv6 hdr + tcp hdr.
169  * Force an allocation if it is not the case.
170  */
171  if (GET_PKT_DIRECT_MAX_SIZE(p) < 60) {
172  if (PacketCallocExtPkt(p, 60) == -1) {
173  goto error;
174  }
175  }
176  /* set the ip header */
177  p->ip6h = (IPV6Hdr *)GET_PKT_DATA(p);
178  /* version 6 */
179  p->ip6h->s_ip6_vfc = 0x60;
180  p->ip6h->s_ip6_flow = 0;
181  p->ip6h->s_ip6_nxt = IPPROTO_TCP;
182  p->ip6h->s_ip6_plen = htons(20);
183  p->ip6h->s_ip6_hlim = 64;
184  if (direction == 0) {
185  p->ip6h->s_ip6_src[0] = f->src.addr_data32[0];
186  p->ip6h->s_ip6_src[1] = f->src.addr_data32[1];
187  p->ip6h->s_ip6_src[2] = f->src.addr_data32[2];
188  p->ip6h->s_ip6_src[3] = f->src.addr_data32[3];
189  p->ip6h->s_ip6_dst[0] = f->dst.addr_data32[0];
190  p->ip6h->s_ip6_dst[1] = f->dst.addr_data32[1];
191  p->ip6h->s_ip6_dst[2] = f->dst.addr_data32[2];
192  p->ip6h->s_ip6_dst[3] = f->dst.addr_data32[3];
193  } else {
194  p->ip6h->s_ip6_src[0] = f->dst.addr_data32[0];
195  p->ip6h->s_ip6_src[1] = f->dst.addr_data32[1];
196  p->ip6h->s_ip6_src[2] = f->dst.addr_data32[2];
197  p->ip6h->s_ip6_src[3] = f->dst.addr_data32[3];
198  p->ip6h->s_ip6_dst[0] = f->src.addr_data32[0];
199  p->ip6h->s_ip6_dst[1] = f->src.addr_data32[1];
200  p->ip6h->s_ip6_dst[2] = f->src.addr_data32[2];
201  p->ip6h->s_ip6_dst[3] = f->src.addr_data32[3];
202  }
203 
204  /* set the tcp header */
205  p->tcph = (TCPHdr *)((uint8_t *)GET_PKT_DATA(p) + 40);
206 
207  SET_PKT_LEN(p, 60); /* ipv6 hdr + tcp hdr */
208  }
209 
210  p->tcph->th_offx2 = 0x50;
211  p->tcph->th_flags |= TH_ACK;
212  p->tcph->th_win = 10;
213  p->tcph->th_urp = 0;
214 
215  /* to server */
216  if (direction == 0) {
217  p->tcph->th_sport = htons(f->sp);
218  p->tcph->th_dport = htons(f->dp);
219 
220  p->tcph->th_seq = htonl(ssn->client.next_seq);
221  p->tcph->th_ack = htonl(ssn->server.last_ack);
222 
223  /* to client */
224  } else {
225  p->tcph->th_sport = htons(f->dp);
226  p->tcph->th_dport = htons(f->sp);
227 
228  p->tcph->th_seq = htonl(ssn->server.next_seq);
229  p->tcph->th_ack = htonl(ssn->client.last_ack);
230  }
231 
232  if (FLOW_IS_IPV4(f)) {
233  p->tcph->th_sum = TCPChecksum(p->ip4h->s_ip_addrs,
234  (uint16_t *)p->tcph, 20, 0);
235  /* calc ipv4 csum as we may log it and barnyard might reject
236  * a wrong checksum */
237  p->ip4h->ip_csum = IPV4Checksum((uint16_t *)p->ip4h,
238  IPV4_GET_RAW_HLEN(p->ip4h), 0);
239  } else if (FLOW_IS_IPV6(f)) {
240  p->tcph->th_sum = TCPChecksum(p->ip6h->s_ip6_addrs,
241  (uint16_t *)p->tcph, 20, 0);
242  }
243 
244  memset(&p->ts, 0, sizeof(struct timeval));
245  TimeGet(&p->ts);
246 
248 
249  return p;
250 
251 error:
252  FlowDeReference(&p->flow);
253  return NULL;
254 }
255 
256 static inline Packet *FlowForceReassemblyPseudoPacketGet(int direction,
257  Flow *f,
258  TcpSession *ssn)
259 {
260  PacketPoolWait();
262  if (p == NULL) {
263  return NULL;
264  }
265 
267 
268  return FlowForceReassemblyPseudoPacketSetup(p, direction, f, ssn);
269 }
270 
271 /**
272  * \brief Check if a flow needs forced reassembly, or any other processing
273  *
274  * \param f *LOCKED* flow
275  * \param server ptr to int that should be set to 1 or 2 if we return 1
276  * \param client ptr to int that should be set to 1 or 2 if we return 1
277  *
278  * \retval 0 no
279  * \retval 1 yes
280  */
281 int FlowForceReassemblyNeedReassembly(Flow *f, int *server, int *client)
282 {
283  if (f == NULL || f->protoctx == NULL) {
284  *server = *client = STREAM_HAS_UNPROCESSED_SEGMENTS_NONE;
285  SCReturnInt(0);
286  }
287 
288  TcpSession *ssn = (TcpSession *)f->protoctx;
289  *client = StreamNeedsReassembly(ssn, STREAM_TOSERVER);
290  *server = StreamNeedsReassembly(ssn, STREAM_TOCLIENT);
291 
292  /* if state is not fully closed we assume that we haven't fully
293  * inspected the app layer state yet */
294  if (ssn->state >= TCP_ESTABLISHED && ssn->state != TCP_CLOSED)
295  {
298  }
299 
300  /* if app layer still needs some love, push through */
301  if (f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL &&
303  {
304  const uint64_t total_txs = AppLayerParserGetTxCnt(f, f->alstate);
305 
307  {
309  }
311  {
313  }
314  }
315 
316  /* nothing to do */
317  if (*client == STREAM_HAS_UNPROCESSED_SEGMENTS_NONE &&
319  SCReturnInt(0);
320  }
321 
322  SCReturnInt(1);
323 }
324 
325 /**
326  * \internal
327  * \brief Forces reassembly for flow if it needs it.
328  *
329  * The function requires flow to be locked beforehand.
330  *
331  * \param f Pointer to the flow.
332  * \param server action required for server: 1 or 2
333  * \param client action required for client: 1 or 2
334  *
335  * \retval 0 This flow doesn't need any reassembly processing; 1 otherwise.
336  */
337 int FlowForceReassemblyForFlow(Flow *f, int server, int client)
338 {
339  Packet *p1 = NULL, *p2 = NULL;
340 
341  /* looks like we have no flows in this queue */
342  if (f == NULL || f->protoctx == NULL) {
343  return 0;
344  }
345 
346  /* Get the tcp session for the flow */
347  TcpSession *ssn = (TcpSession *)f->protoctx;
348 
349  /* The packets we use are based on what segments in what direction are
350  * unprocessed.
351  * p1 if we have client segments for reassembly purpose only. If we
352  * have no server segments p2 can be a toserver packet with dummy
353  * seq/ack, and if we have server segments p2 has to carry out reassembly
354  * for server segment as well, in which case we will also need a p3 in the
355  * toclient which is now dummy since all we need it for is detection */
356 
357  /* insert a pseudo packet in the toserver direction */
359  p1 = FlowForceReassemblyPseudoPacketGet(0, f, ssn);
360  if (p1 == NULL) {
361  goto done;
362  }
364 
366  p2 = FlowForceReassemblyPseudoPacketGet(1, f, ssn);
367  if (p2 == NULL) {
368  FlowDeReference(&p1->flow);
369  TmqhOutputPacketpool(NULL, p1);
370  goto done;
371  }
373  }
374  } else {
376  p1 = FlowForceReassemblyPseudoPacketGet(1, f, ssn);
377  if (p1 == NULL) {
378  goto done;
379  }
381  } else {
382  /* impossible */
383  BUG_ON(1);
384  }
385  }
386 
387  /* inject the packet(s) into the appropriate thread */
388  int thread_id = (int)f->thread_id;
389  Packet *packets[3] = { p1, p2 ? p2 : NULL, NULL }; /**< null terminated array of packets */
390  if (unlikely(!(TmThreadsInjectPacketsById(packets, thread_id)))) {
391  FlowDeReference(&p1->flow);
392  TmqhOutputPacketpool(NULL, p1);
393  if (p2) {
394  FlowDeReference(&p2->flow);
395  TmqhOutputPacketpool(NULL, p2);
396  }
397  }
398 
399  /* done, in case of error (no packet) we still tag flow as complete
400  * as we're probably resource stress if we couldn't get packets */
401 done:
403  return 1;
404 }
405 
406 /**
407  * \internal
408  * \brief Forces reassembly for flows that need it.
409  *
410  * When this function is called we're running in virtually dead engine,
411  * so locking the flows is not strictly required. The reasons it is still
412  * done are:
413  * - code consistency
414  * - silence complaining profilers
415  * - allow us to aggressively check using debug valdation assertions
416  * - be robust in case of future changes
417  * - locking overhead if neglectable when no other thread fights us
418  *
419  * \param q The queue to process flows from.
420  */
421 static inline void FlowForceReassemblyForHash(void)
422 {
423  for (uint32_t idx = 0; idx < flow_config.hash_size; idx++) {
424  FlowBucket *fb = &flow_hash[idx];
425 
427  FBLOCK_LOCK(fb);
428 
429  /* get the topmost flow from the QUEUE */
430  Flow *f = fb->head;
431 
432  /* we need to loop through all the flows in the queue */
433  while (f != NULL) {
435 
436  FLOWLOCK_WRLOCK(f);
437 
438  /* Get the tcp session for the flow */
439  TcpSession *ssn = (TcpSession *)f->protoctx;
440  /* \todo Also skip flows that shouldn't be inspected */
441  if (ssn == NULL) {
442  FLOWLOCK_UNLOCK(f);
443  f = f->hnext;
444  continue;
445  }
446 
447  int client_ok = 0;
448  int server_ok = 0;
449  if (FlowForceReassemblyNeedReassembly(f, &server_ok, &client_ok) == 1) {
450  FlowForceReassemblyForFlow(f, server_ok, client_ok);
451  }
452 
453  FLOWLOCK_UNLOCK(f);
454 
455  /* next flow in the queue */
456  f = f->hnext;
457  }
458  FBLOCK_UNLOCK(fb);
459  }
460  return;
461 }
462 
463 /**
464  * \brief Force reassembly for all the flows that have unprocessed segments.
465  */
467 {
468  /* Carry out flow reassembly for unattended flows */
469  FlowForceReassemblyForHash();
470  return;
471 }
472 
#define FLOW_IS_IPV4(f)
Definition: flow.h:133
#define GET_PKT_DIRECT_MAX_SIZE(p)
Definition: decode.h:225
#define FBLOCK_UNLOCK(fb)
Definition: flow-hash.h:70
#define FLOW_TIMEOUT_REASSEMBLY_DONE
Definition: flow.h:90
uint8_t ip_proto
Definition: decode-ipv4.h:78
struct Flow_ * flow
Definition: decode.h:443
void FlowForceReassembly(void)
Force reassembly for all the flows that have unprocessed segments.
Definition: flow-timeout.c:466
struct Flow_ * hnext
Definition: flow.h:446
int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
see what if any work the TCP session still needs
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:343
#define SET_PKT_LEN(p, len)
Definition: decode.h:227
#define FLOWLOCK_UNLOCK(fb)
Definition: flow.h:242
Port sp
Definition: flow.h:330
#define FLOW_COPY_IPV6_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:148
int TmThreadsInjectPacketsById(Packet **packets, const int id)
Definition: tm-threads.c:2382
#define unlikely(expr)
Definition: util-optimize.h:35
Port sp
Definition: decode.h:413
uint64_t AppLayerParserGetTxCnt(const Flow *f, void *alstate)
void PacketPoolWaitForN(int n)
Wait until we have the requested amount of packets in the pool.
int FlowForceReassemblyNeedReassembly(Flow *f, int *server, int *client)
Check if a flow needs forced reassembly, or any other processing.
Definition: flow-timeout.c:281
uint32_t tenant_id
Definition: flow.h:369
Port dp
Definition: decode.h:421
#define FLOW_COPY_IPV4_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:143
Address dst
Definition: decode.h:411
#define FLOW_PKT_ESTABLISHED
Definition: flow.h:202
#define FLOW_DIR_REVERSED
Definition: flow.h:105
uint16_t ip_id
Definition: decode-ipv4.h:75
#define FLOWLOCK_WRLOCK(fb)
Definition: flow.h:239
TCPHdr * tcph
Definition: decode.h:520
uint8_t ip_tos
Definition: decode-ipv4.h:73
#define DecodeSetNoPacketInspectionFlag(p)
Set the No packet inspection Flag for the packet.
Definition: decode.h:983
Packet * PacketPoolGetPacket(void)
Get a new packet from the packet pool.
void * protoctx
Definition: flow.h:395
FlowConfig flow_config
Definition: flow-private.h:97
uint8_t ip_ttl
Definition: decode-ipv4.h:77
#define PKT_SET_SRC(p, src_val)
Definition: decode.h:1134
FlowAddress dst
Definition: flow.h:328
void * alstate
Definition: flow.h:433
#define FLOW_NOPACKET_INSPECTION
Definition: flow.h:58
#define DecodeSetNoPayloadInspectionFlag(p)
Set the No payload inspection Flag for the packet.
Definition: decode.h:971
IPV6Hdr * ip6h
Definition: decode.h:500
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
int datalink
Definition: decode.h:574
uint8_t proto
Definition: decode.h:428
void TimeGet(struct timeval *tv)
Definition: util-time.c:138
Data structures and function prototypes for keeping state for the detection engine.
#define FLOW_NOPAYLOAD_INSPECTION
Definition: flow.h:60
void AppLayerParserSetEOF(AppLayerParserState *pstate)
uint64_t AppLayerParserGetTransactionActive(const Flow *f, AppLayerParserState *pstate, uint8_t direction)
#define TH_ACK
Definition: decode-tcp.h:39
uint8_t flowflags
Definition: decode.h:437
#define STREAM_TOCLIENT
Definition: stream.h:32
#define FLOW_PKT_TOSERVER
Definition: flow.h:200
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1093
#define SCReturnInt(x)
Definition: util-debug.h:341
int FlowForceReassemblyForFlow(Flow *f, int server, int client)
Definition: flow-timeout.c:337
IPV4Hdr * ip4h
Definition: decode.h:498
uint16_t ip_len
Definition: decode-ipv4.h:74
void PacketPoolWait(void)
#define IPV4_GET_RAW_HLEN(ip4h)
Definition: decode-ipv4.h:95
FlowThreadId thread_id
Definition: flow.h:421
FlowBucket * flow_hash
Definition: flow-private.h:96
#define PKT_STREAM_EOF
Definition: decode.h:1091
Port dp
Definition: flow.h:337
#define FLOW_IS_IPV6(f)
Definition: flow.h:135
int PacketCallocExtPkt(Packet *p, int datalen)
Definition: decode.c:191
#define PACKET_PROFILING_START(p)
#define STREAM_TOSERVER
Definition: stream.h:31
#define GET_PKT_DATA(p)
Definition: decode.h:223
#define PKT_HAS_FLOW
Definition: decode.h:1092
FlowAddress src
Definition: flow.h:328
uint16_t ip_csum
Definition: decode-ipv4.h:79
uint32_t tenant_id
Definition: decode.h:594
struct timeval ts
Definition: decode.h:449
#define FLOW_PKT_TOCLIENT
Definition: flow.h:201
AppProto alproto
application level protocol
Definition: flow.h:404
uint32_t flags
Definition: decode.h:441
uint16_t payload_len
Definition: decode.h:541
uint16_t ip_off
Definition: decode-ipv4.h:76
Flow data structure.
Definition: flow.h:324
uint8_t * payload
Definition: decode.h:540
uint32_t flags
Definition: flow.h:374
int AppLayerParserProtocolSupportsTxs(uint8_t ipproto, AppProto alproto)
#define PKT_STREAM_EST
Definition: decode.h:1090
#define FBLOCK_LOCK(fb)
Definition: flow-hash.h:68
uint8_t ip_verhl
Definition: decode-ipv4.h:72
AppLayerParserState * alparser
Definition: flow.h:432
Address src
Definition: decode.h:410
uint32_t hash_size
Definition: flow.h:263