suricata
defrag-hash.c
Go to the documentation of this file.
1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 #include "suricata-common.h"
19 #include "conf.h"
20 #include "defrag-hash.h"
21 #include "defrag-queue.h"
22 #include "defrag-config.h"
23 #include "util-random.h"
24 #include "util-byte.h"
25 #include "util-misc.h"
26 #include "util-hash-lookup3.h"
27 
28 /** defrag tracker hash table */
31 SC_ATOMIC_DECLARE(uint64_t,defrag_memuse);
32 SC_ATOMIC_DECLARE(unsigned int,defragtracker_counter);
33 SC_ATOMIC_DECLARE(unsigned int,defragtracker_prune_idx);
34 
35 static DefragTracker *DefragTrackerGetUsedDefragTracker(void);
36 
37 /** queue with spare tracker */
38 static DefragTrackerQueue defragtracker_spare_q;
39 
40 /**
41  * \brief Update memcap value
42  *
43  * \param size new memcap value
44  */
45 int DefragTrackerSetMemcap(uint64_t size)
46 {
47  if ((uint64_t)SC_ATOMIC_GET(defrag_memuse) < size) {
48  SC_ATOMIC_SET(defrag_config.memcap, size);
49  return 1;
50  }
51 
52  return 0;
53 }
54 
55 /**
56  * \brief Return memcap value
57  *
58  * \retval memcap value
59  */
60 uint64_t DefragTrackerGetMemcap(void)
61 {
62  uint64_t memcapcopy = SC_ATOMIC_GET(defrag_config.memcap);
63  return memcapcopy;
64 }
65 
66 /**
67  * \brief Return memuse value
68  *
69  * \retval memuse value
70  */
71 uint64_t DefragTrackerGetMemuse(void)
72 {
73  uint64_t memusecopy = (uint64_t)SC_ATOMIC_GET(defrag_memuse);
74  return memusecopy;
75 }
76 
78 {
79  return DefragTrackerQueueLen(&defragtracker_spare_q);
80 }
81 
83 {
84  DefragTrackerEnqueue(&defragtracker_spare_q, h);
85  (void) SC_ATOMIC_SUB(defragtracker_counter, 1);
86 }
87 
88 static DefragTracker *DefragTrackerAlloc(void)
89 {
90  if (!(DEFRAG_CHECK_MEMCAP(sizeof(DefragTracker)))) {
91  return NULL;
92  }
93 
94  (void) SC_ATOMIC_ADD(defrag_memuse, sizeof(DefragTracker));
95 
96  DefragTracker *dt = SCMalloc(sizeof(DefragTracker));
97  if (unlikely(dt == NULL))
98  goto error;
99 
100  memset(dt, 0x00, sizeof(DefragTracker));
101 
102  SCMutexInit(&dt->lock, NULL);
103  SC_ATOMIC_INIT(dt->use_cnt);
104  return dt;
105 
106 error:
107  return NULL;
108 }
109 
110 static void DefragTrackerFree(DefragTracker *dt)
111 {
112  if (dt != NULL) {
114 
115  SCMutexDestroy(&dt->lock);
116  SCFree(dt);
117  (void) SC_ATOMIC_SUB(defrag_memuse, sizeof(DefragTracker));
118  }
119 }
120 
121 #define DefragTrackerIncrUsecnt(dt) \
122  SC_ATOMIC_ADD((dt)->use_cnt, 1)
123 #define DefragTrackerDecrUsecnt(dt) \
124  SC_ATOMIC_SUB((dt)->use_cnt, 1)
125 
126 static void DefragTrackerInit(DefragTracker *dt, Packet *p)
127 {
128  /* copy address */
129  COPY_ADDRESS(&p->src, &dt->src_addr);
130  COPY_ADDRESS(&p->dst, &dt->dst_addr);
131 
132  if (PKT_IS_IPV4(p)) {
133  dt->id = (int32_t)IPV4_GET_IPID(p);
134  dt->af = AF_INET;
135  } else {
136  dt->id = (int32_t)IPV6_EXTHDR_GET_FH_ID(p);
137  dt->af = AF_INET6;
138  }
139  dt->proto = IP_GET_IPPROTO(p);
140  dt->vlan_id[0] = p->vlan_id[0];
141  dt->vlan_id[1] = p->vlan_id[1];
142  dt->policy = DefragGetOsPolicy(p);
144  dt->remove = 0;
145  dt->seen_last = 0;
146 
147  (void) DefragTrackerIncrUsecnt(dt);
148 }
149 
151 {
152  (void) DefragTrackerDecrUsecnt(t);
153  SCMutexUnlock(&t->lock);
154 }
155 
157 {
159 }
160 
161 #define DEFRAG_DEFAULT_HASHSIZE 4096
162 #define DEFRAG_DEFAULT_MEMCAP 16777216
163 #define DEFRAG_DEFAULT_PREALLOC 1000
164 
165 /** \brief initialize the configuration
166  * \warning Not thread safe */
167 void DefragInitConfig(bool quiet)
168 {
169  SCLogDebug("initializing defrag engine...");
170 
171  memset(&defrag_config, 0, sizeof(defrag_config));
172  //SC_ATOMIC_INIT(flow_flags);
173  SC_ATOMIC_INIT(defragtracker_counter);
174  SC_ATOMIC_INIT(defrag_memuse);
175  SC_ATOMIC_INIT(defragtracker_prune_idx);
177  DefragTrackerQueueInit(&defragtracker_spare_q);
178 
179  /* set defaults */
180  defrag_config.hash_rand = (uint32_t)RandomGet();
184  defrag_config.memcap_policy = ExceptionPolicyParse("defrag.memcap-policy", false);
185 
186  /* Check if we have memcap and hash_size defined at config */
187  const char *conf_val;
188  uint32_t configval = 0;
189 
190  uint64_t defrag_memcap;
191  /** set config values for memcap, prealloc and hash_size */
192  if ((ConfGet("defrag.memcap", &conf_val)) == 1)
193  {
194  if (ParseSizeStringU64(conf_val, &defrag_memcap) < 0) {
195  SCLogError(SC_ERR_SIZE_PARSE, "Error parsing defrag.memcap "
196  "from conf file - %s. Killing engine",
197  conf_val);
198  exit(EXIT_FAILURE);
199  } else {
200  SC_ATOMIC_SET(defrag_config.memcap, defrag_memcap);
201  }
202  }
203  if ((ConfGet("defrag.hash-size", &conf_val)) == 1)
204  {
205  if (StringParseUint32(&configval, 10, strlen(conf_val),
206  conf_val) > 0) {
207  defrag_config.hash_size = configval;
208  } else {
209  WarnInvalidConfEntry("defrag.hash-size", "%"PRIu32, defrag_config.hash_size);
210  }
211  }
212 
213 
214  if ((ConfGet("defrag.trackers", &conf_val)) == 1)
215  {
216  if (StringParseUint32(&configval, 10, strlen(conf_val),
217  conf_val) > 0) {
218  defrag_config.prealloc = configval;
219  } else {
220  WarnInvalidConfEntry("defrag.trackers", "%"PRIu32, defrag_config.prealloc);
221  }
222  }
223  SCLogDebug("DefragTracker config from suricata.yaml: memcap: %"PRIu64", hash-size: "
224  "%"PRIu32", prealloc: %"PRIu32, SC_ATOMIC_GET(defrag_config.memcap),
226 
227  /* alloc hash memory */
228  uint64_t hash_size = defrag_config.hash_size * sizeof(DefragTrackerHashRow);
229  if (!(DEFRAG_CHECK_MEMCAP(hash_size))) {
230  SCLogError(SC_ERR_DEFRAG_INIT, "allocating defrag hash failed: "
231  "max defrag memcap is smaller than projected hash size. "
232  "Memcap: %"PRIu64", Hash table size %"PRIu64". Calculate "
233  "total hash size by multiplying \"defrag.hash-size\" with %"PRIuMAX", "
234  "which is the hash bucket size.", SC_ATOMIC_GET(defrag_config.memcap), hash_size,
235  (uintmax_t)sizeof(DefragTrackerHashRow));
236  exit(EXIT_FAILURE);
237  }
239  if (unlikely(defragtracker_hash == NULL)) {
241  "Fatal error encountered in DefragTrackerInitConfig. Exiting...");
242  }
244 
245  uint32_t i = 0;
246  for (i = 0; i < defrag_config.hash_size; i++) {
248  }
249  (void) SC_ATOMIC_ADD(defrag_memuse, (defrag_config.hash_size * sizeof(DefragTrackerHashRow)));
250 
251  if (!quiet) {
252  SCLogConfig("allocated %"PRIu64" bytes of memory for the defrag hash... "
253  "%" PRIu32 " buckets of size %" PRIuMAX "",
254  SC_ATOMIC_GET(defrag_memuse), defrag_config.hash_size,
255  (uintmax_t)sizeof(DefragTrackerHashRow));
256  }
257 
258  if ((ConfGet("defrag.prealloc", &conf_val)) == 1)
259  {
260  if (ConfValIsTrue(conf_val)) {
261  /* pre allocate defrag trackers */
262  for (i = 0; i < defrag_config.prealloc; i++) {
263  if (!(DEFRAG_CHECK_MEMCAP(sizeof(DefragTracker)))) {
264  SCLogError(SC_ERR_DEFRAG_INIT, "preallocating defrag trackers failed: "
265  "max defrag memcap reached. Memcap %"PRIu64", "
266  "Memuse %"PRIu64".", SC_ATOMIC_GET(defrag_config.memcap),
267  ((uint64_t)SC_ATOMIC_GET(defrag_memuse) + (uint64_t)sizeof(DefragTracker)));
268  exit(EXIT_FAILURE);
269  }
270 
271  DefragTracker *h = DefragTrackerAlloc();
272  if (h == NULL) {
273  SCLogError(SC_ERR_DEFRAG_INIT, "preallocating defrag failed: %s", strerror(errno));
274  exit(EXIT_FAILURE);
275  }
276  DefragTrackerEnqueue(&defragtracker_spare_q,h);
277  }
278  if (!quiet) {
279  SCLogConfig("preallocated %" PRIu32 " defrag trackers of size %" PRIuMAX "",
280  defragtracker_spare_q.len, (uintmax_t)sizeof(DefragTracker));
281  }
282  }
283  }
284 
285  if (!quiet) {
286  SCLogConfig("defrag memory usage: %"PRIu64" bytes, maximum: %"PRIu64,
287  SC_ATOMIC_GET(defrag_memuse), SC_ATOMIC_GET(defrag_config.memcap));
288  }
289 
290  return;
291 }
292 
293 /** \brief print some defrag stats
294  * \warning Not thread safe */
295 static void DefragTrackerPrintStats (void)
296 {
297 }
298 
299 /** \brief shutdown the flow engine
300  * \warning Not thread safe */
302 {
303  DefragTracker *dt;
304  uint32_t u;
305 
306  DefragTrackerPrintStats();
307 
308  /* free spare queue */
309  while((dt = DefragTrackerDequeue(&defragtracker_spare_q))) {
310  BUG_ON(SC_ATOMIC_GET(dt->use_cnt) > 0);
311  DefragTrackerFree(dt);
312  }
313 
314  /* clear and free the hash */
315  if (defragtracker_hash != NULL) {
316  for (u = 0; u < defrag_config.hash_size; u++) {
317  dt = defragtracker_hash[u].head;
318  while (dt) {
319  DefragTracker *n = dt->hnext;
321  DefragTrackerFree(dt);
322  dt = n;
323  }
324 
326  }
328  defragtracker_hash = NULL;
329  }
330  (void) SC_ATOMIC_SUB(defrag_memuse, defrag_config.hash_size * sizeof(DefragTrackerHashRow));
331  DefragTrackerQueueDestroy(&defragtracker_spare_q);
332  return;
333 }
334 
335 /** \brief compare two raw ipv6 addrs
336  *
337  * \note we don't care about the real ipv6 ip's, this is just
338  * to consistently fill the DefragHashKey6 struct, without all
339  * the SCNtohl calls.
340  *
341  * \warning do not use elsewhere unless you know what you're doing.
342  * detect-engine-address-ipv6.c's AddressIPv6GtU32 is likely
343  * what you are looking for.
344  */
345 static inline int DefragHashRawAddressIPv6GtU32(const uint32_t *a, const uint32_t *b)
346 {
347  for (int i = 0; i < 4; i++) {
348  if (a[i] > b[i])
349  return 1;
350  if (a[i] < b[i])
351  break;
352  }
353 
354  return 0;
355 }
356 
357 typedef struct DefragHashKey4_ {
358  union {
359  struct {
360  uint32_t src, dst;
361  uint32_t id;
362  uint16_t vlan_id[2];
363  };
364  uint32_t u32[4];
365  };
367 
368 typedef struct DefragHashKey6_ {
369  union {
370  struct {
371  uint32_t src[4], dst[4];
372  uint32_t id;
373  uint16_t vlan_id[2];
374  };
375  uint32_t u32[10];
376  };
378 
379 /* calculate the hash key for this packet
380  *
381  * we're using:
382  * hash_rand -- set at init time
383  * source address
384  * destination address
385  * id
386  * vlan_id
387  */
388 static inline uint32_t DefragHashGetKey(Packet *p)
389 {
390  uint32_t key;
391 
392  if (p->ip4h != NULL) {
393  DefragHashKey4 dhk;
394  if (p->src.addr_data32[0] > p->dst.addr_data32[0]) {
395  dhk.src = p->src.addr_data32[0];
396  dhk.dst = p->dst.addr_data32[0];
397  } else {
398  dhk.src = p->dst.addr_data32[0];
399  dhk.dst = p->src.addr_data32[0];
400  }
401  dhk.id = (uint32_t)IPV4_GET_IPID(p);
402  dhk.vlan_id[0] = p->vlan_id[0];
403  dhk.vlan_id[1] = p->vlan_id[1];
404 
405  uint32_t hash = hashword(dhk.u32, 4, defrag_config.hash_rand);
406  key = hash % defrag_config.hash_size;
407  } else if (p->ip6h != NULL) {
408  DefragHashKey6 dhk;
409  if (DefragHashRawAddressIPv6GtU32(p->src.addr_data32, p->dst.addr_data32)) {
410  dhk.src[0] = p->src.addr_data32[0];
411  dhk.src[1] = p->src.addr_data32[1];
412  dhk.src[2] = p->src.addr_data32[2];
413  dhk.src[3] = p->src.addr_data32[3];
414  dhk.dst[0] = p->dst.addr_data32[0];
415  dhk.dst[1] = p->dst.addr_data32[1];
416  dhk.dst[2] = p->dst.addr_data32[2];
417  dhk.dst[3] = p->dst.addr_data32[3];
418  } else {
419  dhk.src[0] = p->dst.addr_data32[0];
420  dhk.src[1] = p->dst.addr_data32[1];
421  dhk.src[2] = p->dst.addr_data32[2];
422  dhk.src[3] = p->dst.addr_data32[3];
423  dhk.dst[0] = p->src.addr_data32[0];
424  dhk.dst[1] = p->src.addr_data32[1];
425  dhk.dst[2] = p->src.addr_data32[2];
426  dhk.dst[3] = p->src.addr_data32[3];
427  }
428  dhk.id = IPV6_EXTHDR_GET_FH_ID(p);
429  dhk.vlan_id[0] = p->vlan_id[0];
430  dhk.vlan_id[1] = p->vlan_id[1];
431 
432  uint32_t hash = hashword(dhk.u32, 10, defrag_config.hash_rand);
433  key = hash % defrag_config.hash_size;
434  } else
435  key = 0;
436 
437  return key;
438 }
439 
440 /* Since two or more trackers can have the same hash key, we need to compare
441  * the tracker with the current tracker key. */
442 #define CMP_DEFRAGTRACKER(d1,d2,id) \
443  (((CMP_ADDR(&(d1)->src_addr, &(d2)->src) && \
444  CMP_ADDR(&(d1)->dst_addr, &(d2)->dst)) || \
445  (CMP_ADDR(&(d1)->src_addr, &(d2)->dst) && \
446  CMP_ADDR(&(d1)->dst_addr, &(d2)->src))) && \
447  (d1)->proto == IP_GET_IPPROTO(d2) && \
448  (d1)->id == (id) && \
449  (d1)->vlan_id[0] == (d2)->vlan_id[0] && \
450  (d1)->vlan_id[1] == (d2)->vlan_id[1])
451 
452 static inline int DefragTrackerCompare(DefragTracker *t, Packet *p)
453 {
454  uint32_t id;
455  if (PKT_IS_IPV4(p)) {
456  id = (uint32_t)IPV4_GET_IPID(p);
457  } else {
458  id = IPV6_EXTHDR_GET_FH_ID(p);
459  }
460 
461  return CMP_DEFRAGTRACKER(t, p, id);
462 }
463 
464 /**
465  * \brief Get a new defrag tracker
466  *
467  * Get a new defrag tracker. We're checking memcap first and will try to make room
468  * if the memcap is reached.
469  *
470  * \retval dt *LOCKED* tracker on succes, NULL on error.
471  */
472 static DefragTracker *DefragTrackerGetNew(Packet *p)
473 {
474 #ifdef DEBUG
475  if (g_eps_defrag_memcap != UINT64_MAX && g_eps_defrag_memcap == p->pcap_cnt) {
476  SCLogNotice("simulating memcap hit for packet %" PRIu64, p->pcap_cnt);
478  return NULL;
479  }
480 #endif
481 
482  DefragTracker *dt = NULL;
483 
484  /* get a tracker from the spare queue */
485  dt = DefragTrackerDequeue(&defragtracker_spare_q);
486  if (dt == NULL) {
487  /* If we reached the max memcap, we get a used tracker */
488  if (!(DEFRAG_CHECK_MEMCAP(sizeof(DefragTracker)))) {
489  /* declare state of emergency */
490  //if (!(SC_ATOMIC_GET(defragtracker_flags) & DEFRAG_EMERGENCY)) {
491  // SC_ATOMIC_OR(defragtracker_flags, DEFRAG_EMERGENCY);
492 
493  /* under high load, waking up the flow mgr each time leads
494  * to high cpu usage. Flows are not timed out much faster if
495  * we check a 1000 times a second. */
496  // FlowWakeupFlowManagerThread();
497  //}
498 
499  dt = DefragTrackerGetUsedDefragTracker();
500  if (dt == NULL) {
502  return NULL;
503  }
504 
505  /* freed a tracker, but it's unlocked */
506  } else {
507  /* now see if we can alloc a new tracker */
508  dt = DefragTrackerAlloc();
509  if (dt == NULL) {
511  return NULL;
512  }
513 
514  /* tracker is initialized but *unlocked* */
515  }
516  } else {
517  /* tracker has been recycled before it went into the spare queue */
518 
519  /* tracker is initialized (recylced) but *unlocked* */
520  }
521 
522  (void) SC_ATOMIC_ADD(defragtracker_counter, 1);
523  SCMutexLock(&dt->lock);
524  return dt;
525 }
526 
527 /* DefragGetTrackerFromHash
528  *
529  * Hash retrieval function for trackers. Looks up the hash bucket containing the
530  * tracker pointer. Then compares the packet with the found tracker to see if it is
531  * the tracker we need. If it isn't, walk the list until the right tracker is found.
532  *
533  * returns a *LOCKED* tracker or NULL
534  */
536 {
537  DefragTracker *dt = NULL;
538 
539  /* get the key to our bucket */
540  uint32_t key = DefragHashGetKey(p);
541  /* get our hash bucket and lock it */
543  DRLOCK_LOCK(hb);
544 
545  /* see if the bucket already has a tracker */
546  if (hb->head == NULL) {
547  dt = DefragTrackerGetNew(p);
548  if (dt == NULL) {
549  DRLOCK_UNLOCK(hb);
550  return NULL;
551  }
552 
553  /* tracker is locked */
554  hb->head = dt;
555  hb->tail = dt;
556 
557  /* got one, now lock, initialize and return */
558  DefragTrackerInit(dt,p);
559 
560  DRLOCK_UNLOCK(hb);
561  return dt;
562  }
563 
564  /* ok, we have a tracker in the bucket. Let's find out if it is our tracker */
565  dt = hb->head;
566 
567  /* see if this is the tracker we are looking for */
568  if (dt->remove || DefragTrackerCompare(dt, p) == 0) {
569  DefragTracker *pdt = NULL; /* previous tracker */
570 
571  while (dt) {
572  pdt = dt;
573  dt = dt->hnext;
574 
575  if (dt == NULL) {
576  dt = pdt->hnext = DefragTrackerGetNew(p);
577  if (dt == NULL) {
578  DRLOCK_UNLOCK(hb);
579  return NULL;
580  }
581  hb->tail = dt;
582 
583  /* tracker is locked */
584 
585  dt->hprev = pdt;
586 
587  /* initialize and return */
588  DefragTrackerInit(dt,p);
589 
590  DRLOCK_UNLOCK(hb);
591  return dt;
592  }
593 
594  if (DefragTrackerCompare(dt, p) != 0) {
595  /* we found our tracker, lets put it on top of the
596  * hash list -- this rewards active trackers */
597  if (dt->hnext) {
598  dt->hnext->hprev = dt->hprev;
599  }
600  if (dt->hprev) {
601  dt->hprev->hnext = dt->hnext;
602  }
603  if (dt == hb->tail) {
604  hb->tail = dt->hprev;
605  }
606 
607  dt->hnext = hb->head;
608  dt->hprev = NULL;
609  hb->head->hprev = dt;
610  hb->head = dt;
611 
612  /* found our tracker, lock & return */
613  SCMutexLock(&dt->lock);
614  (void) DefragTrackerIncrUsecnt(dt);
615  DRLOCK_UNLOCK(hb);
616  return dt;
617  }
618  }
619  }
620 
621  /* lock & return */
622  SCMutexLock(&dt->lock);
623  (void) DefragTrackerIncrUsecnt(dt);
624  DRLOCK_UNLOCK(hb);
625  return dt;
626 }
627 
628 /** \brief look up a tracker in the hash
629  *
630  * \param a address to look up
631  *
632  * \retval h *LOCKED* tracker or NULL
633  */
635 {
636  DefragTracker *dt = NULL;
637 
638  /* get the key to our bucket */
639  uint32_t key = DefragHashGetKey(p);
640  /* get our hash bucket and lock it */
642  DRLOCK_LOCK(hb);
643 
644  /* see if the bucket already has a tracker */
645  if (hb->head == NULL) {
646  DRLOCK_UNLOCK(hb);
647  return dt;
648  }
649 
650  /* ok, we have a tracker in the bucket. Let's find out if it is our tracker */
651  dt = hb->head;
652 
653  /* see if this is the tracker we are looking for */
654  if (DefragTrackerCompare(dt, p) == 0) {
655  while (dt) {
656  dt = dt->hnext;
657 
658  if (dt == NULL) {
659  DRLOCK_UNLOCK(hb);
660  return dt;
661  }
662 
663  if (DefragTrackerCompare(dt, p) != 0) {
664  /* we found our tracker, lets put it on top of the
665  * hash list -- this rewards active tracker */
666  if (dt->hnext) {
667  dt->hnext->hprev = dt->hprev;
668  }
669  if (dt->hprev) {
670  dt->hprev->hnext = dt->hnext;
671  }
672  if (dt == hb->tail) {
673  hb->tail = dt->hprev;
674  }
675 
676  dt->hnext = hb->head;
677  dt->hprev = NULL;
678  hb->head->hprev = dt;
679  hb->head = dt;
680 
681  /* found our tracker, lock & return */
682  SCMutexLock(&dt->lock);
683  (void) DefragTrackerIncrUsecnt(dt);
684  DRLOCK_UNLOCK(hb);
685  return dt;
686  }
687  }
688  }
689 
690  /* lock & return */
691  SCMutexLock(&dt->lock);
692  (void) DefragTrackerIncrUsecnt(dt);
693  DRLOCK_UNLOCK(hb);
694  return dt;
695 }
696 
697 /** \internal
698  * \brief Get a tracker from the hash directly.
699  *
700  * Called in conditions where the spare queue is empty and memcap is reached.
701  *
702  * Walks the hash until a tracker can be freed. "defragtracker_prune_idx" atomic int makes
703  * sure we don't start at the top each time since that would clear the top of
704  * the hash leading to longer and longer search times under high pressure (observed).
705  *
706  * \retval dt tracker or NULL
707  */
708 static DefragTracker *DefragTrackerGetUsedDefragTracker(void)
709 {
710  uint32_t idx = SC_ATOMIC_GET(defragtracker_prune_idx) % defrag_config.hash_size;
711  uint32_t cnt = defrag_config.hash_size;
712 
713  while (cnt--) {
714  if (++idx >= defrag_config.hash_size)
715  idx = 0;
716 
718 
719  if (DRLOCK_TRYLOCK(hb) != 0)
720  continue;
721 
722  DefragTracker *dt = hb->tail;
723  if (dt == NULL) {
724  DRLOCK_UNLOCK(hb);
725  continue;
726  }
727 
728  if (SCMutexTrylock(&dt->lock) != 0) {
729  DRLOCK_UNLOCK(hb);
730  continue;
731  }
732 
733  /** never prune a tracker that is used by a packets
734  * we are currently processing in one of the threads */
735  if (SC_ATOMIC_GET(dt->use_cnt) > 0) {
736  DRLOCK_UNLOCK(hb);
737  SCMutexUnlock(&dt->lock);
738  continue;
739  }
740 
741  /* remove from the hash */
742  if (dt->hprev != NULL)
743  dt->hprev->hnext = dt->hnext;
744  if (dt->hnext != NULL)
745  dt->hnext->hprev = dt->hprev;
746  if (hb->head == dt)
747  hb->head = dt->hnext;
748  if (hb->tail == dt)
749  hb->tail = dt->hprev;
750 
751  dt->hnext = NULL;
752  dt->hprev = NULL;
753  DRLOCK_UNLOCK(hb);
754 
756 
757  SCMutexUnlock(&dt->lock);
758 
759  (void) SC_ATOMIC_ADD(defragtracker_prune_idx, (defrag_config.hash_size - cnt));
760  return dt;
761  }
762 
763  return NULL;
764 }
765 
766 
PKT_DROP_REASON_DEFRAG_MEMCAP
@ PKT_DROP_REASON_DEFRAG_MEMCAP
Definition: decode.h:402
util-byte.h
DefragTrackerFreeFrags
void DefragTrackerFreeFrags(DefragTracker *tracker)
Free all frags associated with a tracker.
Definition: defrag.c:155
DefragLookupTrackerFromHash
DefragTracker * DefragLookupTrackerFromHash(Packet *p)
look up a tracker in the hash
Definition: defrag-hash.c:634
ExceptionPolicyApply
void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
Definition: util-exception-policy.c:27
DefragHashKey4
struct DefragHashKey4_ DefragHashKey4
IPV6_EXTHDR_GET_FH_ID
#define IPV6_EXTHDR_GET_FH_ID(p)
Definition: decode-ipv6.h:130
DefragHashKey4_::src
uint32_t src
Definition: defrag-hash.c:360
hashword
uint32_t hashword(const uint32_t *k, size_t length, uint32_t initval)
Definition: util-hash-lookup3.c:174
DefragTracker_::hnext
struct DefragTracker_ * hnext
Definition: defrag.h:115
defragtracker_hash
DefragTrackerHashRow * defragtracker_hash
Definition: defrag-hash.c:29
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
DefragTrackerIncrUsecnt
#define DefragTrackerIncrUsecnt(dt)
Definition: defrag-hash.c:121
Packet_::vlan_id
uint16_t vlan_id[2]
Definition: decode.h:460
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
SC_ATOMIC_SET
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
Definition: util-atomic.h:387
DefragTrackerHashRow_
Definition: defrag-hash.h:58
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
Packet_::pcap_cnt
uint64_t pcap_cnt
Definition: decode.h:594
ParseSizeStringU64
int ParseSizeStringU64(const char *size, uint64_t *res)
Definition: util-misc.c:200
DefragTrackerSpareQueueGetSize
uint32_t DefragTrackerSpareQueueGetSize(void)
Definition: defrag-hash.c:77
DefragTrackerQueueDestroy
void DefragTrackerQueueDestroy(DefragTrackerQueue *q)
Destroy a tracker queue.
Definition: defrag-queue.c:59
DefragHashKey4_::id
uint32_t id
Definition: defrag-hash.c:361
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
DefragHashKey6_::vlan_id
uint16_t vlan_id[2]
Definition: defrag-hash.c:373
DefragTrackerHashRow
struct DefragTrackerHashRow_ DefragTrackerHashRow
DefragTrackerClearMemory
void DefragTrackerClearMemory(DefragTracker *dt)
Definition: defrag-hash.c:156
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
DefragTracker_::host_timeout
uint32_t host_timeout
Definition: defrag.h:107
DefragTracker_::vlan_id
uint16_t vlan_id[2]
Definition: defrag.h:87
RandomGet
long int RandomGet(void)
Definition: util-random.c:130
DefragHashKey6_::u32
uint32_t u32[10]
Definition: defrag-hash.c:375
DefragTrackerGetMemuse
uint64_t DefragTrackerGetMemuse(void)
Return memuse value.
Definition: defrag-hash.c:71
defrag-config.h
DefragPolicyGetHostTimeout
int DefragPolicyGetHostTimeout(Packet *p)
Definition: defrag-config.c:90
DefragTrackerMoveToSpare
void DefragTrackerMoveToSpare(DefragTracker *h)
Definition: defrag-hash.c:82
ConfValIsTrue
int ConfValIsTrue(const char *val)
Check if a value is true.
Definition: conf.c:521
DefragTracker_::hprev
struct DefragTracker_ * hprev
Definition: defrag.h:116
DefragTracker_::lock
SCMutex lock
Definition: defrag.h:84
DefragConfig_::hash_size
uint32_t hash_size
Definition: defrag-hash.h:73
DefragGetOsPolicy
uint8_t DefragGetOsPolicy(Packet *p)
Get the defrag policy based on the destination address of the packet.
Definition: defrag.c:929
SC_ERR_SIZE_PARSE
@ SC_ERR_SIZE_PARSE
Definition: util-error.h:230
DefragConfig_::prealloc
uint32_t prealloc
Definition: defrag-hash.h:74
DefragHashKey6_::src
uint32_t src[4]
Definition: defrag-hash.c:371
ConfGet
int ConfGet(const char *name, const char **vptr)
Retrieve the value of a configuration node.
Definition: conf.c:330
DefragTrackerHashRow_::head
DefragTracker * head
Definition: defrag-hash.h:60
DefragTracker_::remove
uint8_t remove
Definition: defrag.h:101
defrag_config
DefragConfig defrag_config
Definition: defrag-hash.c:30
DefragTrackerQueueLen
uint32_t DefragTrackerQueueLen(DefragTrackerQueue *q)
Definition: defrag-queue.c:136
DefragTracker_::seen_last
uint8_t seen_last
Definition: defrag.h:99
DefragTracker_::policy
uint8_t policy
Definition: defrag.h:94
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
DefragTracker_
Definition: defrag.h:83
DEFRAG_CHECK_MEMCAP
#define DEFRAG_CHECK_MEMCAP(size)
check if a memory alloc would fit in the memcap
Definition: defrag-hash.h:85
DefragTrackerHashRow_::tail
DefragTracker * tail
Definition: defrag-hash.h:61
DEFRAG_DEFAULT_HASHSIZE
#define DEFRAG_DEFAULT_HASHSIZE
Definition: defrag-hash.c:161
StringParseUint32
int StringParseUint32(uint32_t *res, int base, size_t len, const char *str)
Definition: util-byte.c:313
DefragHashKey6_::dst
uint32_t dst[4]
Definition: defrag-hash.c:371
defrag-queue.h
DefragTrackerGetMemcap
uint64_t DefragTrackerGetMemcap(void)
Return memcap value.
Definition: defrag-hash.c:60
DefragTracker_::dst_addr
Address dst_addr
Definition: defrag.h:104
DefragTrackerDequeue
DefragTracker * DefragTrackerDequeue(DefragTrackerQueue *q)
remove a tracker from the queue
Definition: defrag-queue.c:103
IPV4_GET_IPID
#define IPV4_GET_IPID(p)
Definition: decode-ipv4.h:128
DRLOCK_LOCK
#define DRLOCK_LOCK(fb)
Definition: defrag-hash.h:51
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:281
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
DefragHashKey6
struct DefragHashKey6_ DefragHashKey6
Packet_
Definition: decode.h:433
DefragTracker_::id
uint32_t id
Definition: defrag.h:89
DefragTracker_::src_addr
Address src_addr
Definition: defrag.h:103
DRLOCK_UNLOCK
#define DRLOCK_UNLOCK(fb)
Definition: defrag-hash.h:53
conf.h
Packet_::ip4h
IPV4Hdr * ip4h
Definition: decode.h:531
DefragInitConfig
void DefragInitConfig(bool quiet)
initialize the configuration
Definition: defrag-hash.c:167
SC_ERR_DEFRAG_INIT
@ SC_ERR_DEFRAG_INIT
Definition: util-error.h:245
DefragTrackerQueue_
Definition: defrag-queue.h:42
DEFRAG_DEFAULT_MEMCAP
#define DEFRAG_DEFAULT_MEMCAP
Definition: defrag-hash.c:162
DefragTrackerRelease
void DefragTrackerRelease(DefragTracker *t)
Definition: defrag-hash.c:150
DefragTracker_::proto
uint8_t proto
Definition: defrag.h:92
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
WarnInvalidConfEntry
#define WarnInvalidConfEntry(param_name, format, value)
Generic API that can be used by all to log an invalid conf entry.
Definition: util-misc.h:36
DefragConfig_::memcap_policy
enum ExceptionPolicy memcap_policy
Definition: defrag-hash.h:75
DRLOCK_DESTROY
#define DRLOCK_DESTROY(fb)
Definition: defrag-hash.h:50
DefragTrackerQueue_::len
uint32_t len
Definition: defrag-queue.h:45
ExceptionPolicyParse
enum ExceptionPolicy ExceptionPolicyParse(const char *option, const bool support_flow)
Definition: util-exception-policy.c:67
suricata-common.h
DefragGetTrackerFromHash
DefragTracker * DefragGetTrackerFromHash(Packet *p)
Definition: defrag-hash.c:535
DefragTrackerQueueInit
DefragTrackerQueue * DefragTrackerQueueInit(DefragTrackerQueue *q)
Definition: defrag-queue.c:34
DefragHashKey4_
Definition: defrag-hash.c:357
DefragHashKey6_
Definition: defrag-hash.c:368
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:255
DefragConfig_
Definition: defrag-hash.h:70
CMP_DEFRAGTRACKER
#define CMP_DEFRAGTRACKER(d1, d2, id)
Definition: defrag-hash.c:442
FatalError
#define FatalError(x,...)
Definition: util-debug.h:530
util-hash-lookup3.h
DRLOCK_TRYLOCK
#define DRLOCK_TRYLOCK(fb)
Definition: defrag-hash.h:52
DefragTrackerSetMemcap
int DefragTrackerSetMemcap(uint64_t size)
Update memcap value.
Definition: defrag-hash.c:45
DefragConfig_::hash_rand
uint32_t hash_rand
Definition: defrag-hash.h:72
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogConfig
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
DefragTrackerEnqueue
void DefragTrackerEnqueue(DefragTrackerQueue *q, DefragTracker *dt)
add a tracker to a queue
Definition: defrag-queue.c:70
SCFree
#define SCFree(p)
Definition: util-mem.h:61
DRLOCK_INIT
#define DRLOCK_INIT(fb)
Definition: defrag-hash.h:49
DefragTracker_::af
uint8_t af
Definition: defrag.h:96
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
DefragHashKey4_::vlan_id
uint16_t vlan_id[2]
Definition: defrag-hash.c:362
IP_GET_IPPROTO
#define IP_GET_IPPROTO(p)
Definition: decode.h:265
util-random.h
DEFRAG_DEFAULT_PREALLOC
#define DEFRAG_DEFAULT_PREALLOC
Definition: defrag-hash.c:163
DefragTrackerDecrUsecnt
#define DefragTrackerDecrUsecnt(dt)
Definition: defrag-hash.c:123
defrag-hash.h
Packet_::dst
Address dst
Definition: decode.h:438
DefragHashKey6_::id
uint32_t id
Definition: defrag-hash.c:372
DefragHashShutdown
void DefragHashShutdown(void)
shutdown the flow engine
Definition: defrag-hash.c:301
Packet_::ip6h
IPV6Hdr * ip6h
Definition: decode.h:533
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
util-misc.h
COPY_ADDRESS
#define COPY_ADDRESS(a, b)
Definition: decode.h:128
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint64_t, defrag_memuse)
SCLogNotice
#define SCLogNotice(...)
Macro used to log NOTICE messages.
Definition: util-debug.h:230
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
PKT_IS_IPV4
#define PKT_IS_IPV4(p)
Definition: decode.h:253
DefragHashKey4_::dst
uint32_t dst
Definition: defrag-hash.c:360
DefragHashKey4_::u32
uint32_t u32[4]
Definition: defrag-hash.c:364
Packet_::src
Address src
Definition: decode.h:437
SCMutexTrylock
#define SCMutexTrylock(mut)
Definition: threads-debug.h:118