suricata
tm-threads.h
Go to the documentation of this file.
1 /* Copyright (C) 2007-2011 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  * \author Anoop Saldanha <anoopsaldanha@gmail.com>
23  */
24 
25 #ifndef __TM_THREADS_H__
26 #define __TM_THREADS_H__
27 
28 #include "tmqh-packetpool.h"
29 #include "tm-threads-common.h"
30 #include "tm-modules.h"
31 #include "flow.h" // for the FlowQueue
32 
33 #ifdef OS_WIN32
34 static inline void SleepUsec(uint64_t usec)
35 {
36  uint64_t msec = 1;
37  if (usec > 1000) {
38  msec = usec / 1000;
39  }
40  Sleep(msec);
41 }
42 #define SleepMsec(msec) Sleep((msec))
43 #else
44 #define SleepUsec(usec) usleep((usec))
45 #define SleepMsec(msec) usleep((msec) * 1000)
46 #endif
47 
48 #define TM_QUEUE_NAME_MAX 16
49 #define TM_THREAD_NAME_MAX 16
50 
51 typedef TmEcode (*TmSlotFunc)(ThreadVars *, Packet *, void *);
52 
53 typedef struct TmSlot_ {
54  /* function pointers */
55  union {
57  TmEcode (*PktAcqLoop)(ThreadVars *, void *, void *);
58  TmEcode (*Management)(ThreadVars *, void *);
59  };
60  /** linked list of slots, used when a pipeline has multiple slots
61  * in a single thread. */
62  struct TmSlot_ *slot_next;
63 
64  SC_ATOMIC_DECLARE(void *, slot_data);
65 
66  TmEcode (*SlotThreadInit)(ThreadVars *, const void *, void **);
69 
70  /* data storage */
71  const void *slot_initdata;
72  /* store the thread module id */
73  int tm_id;
74 
76 
77 extern ThreadVars *tv_root[TVT_MAX];
78 
79 extern SCMutex tv_root_lock;
80 
81 void TmSlotSetFuncAppend(ThreadVars *, TmModule *, const void *);
82 
83 ThreadVars *TmThreadCreate(const char *, const char *, const char *, const char *, const char *, const char *,
84  void *(fn_p)(void *), int);
85 ThreadVars *TmThreadCreatePacketHandler(const char *, const char *, const char *, const char *, const char *,
86  const char *);
87 ThreadVars *TmThreadCreateMgmtThread(const char *name, void *(fn_p)(void *), int);
88 ThreadVars *TmThreadCreateMgmtThreadByName(const char *name, const char *module,
89  int mucond);
90 ThreadVars *TmThreadCreateCmdThreadByName(const char *name, const char *module,
91  int mucond);
93 void TmThreadSetFlags(ThreadVars *, uint8_t);
94 void TmThreadKillThreadsFamily(int family);
95 void TmThreadKillThreads(void);
96 void TmThreadClearThreadsFamily(int family);
97 void TmThreadAppend(ThreadVars *, int);
98 void TmThreadSetGroupName(ThreadVars *tv, const char *name);
99 
102 TmEcode TmThreadSetCPU(ThreadVars *, uint8_t);
105 int TmThreadGetNbThreads(uint8_t type);
106 
107 void TmThreadInitMC(ThreadVars *);
110 void TmThreadContinueThreads(void);
111 void TmThreadCheckThreadState(void);
113 
114 int TmThreadsCheckFlag(ThreadVars *, uint32_t);
115 void TmThreadsSetFlag(ThreadVars *, uint32_t);
116 void TmThreadsUnsetFlag(ThreadVars *, uint32_t);
117 void TmThreadWaitForFlag(ThreadVars *, uint32_t);
118 
120 
123 
124 uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags);
125 
127 
128 static inline void TmThreadsCleanDecodePQ(PacketQueueNoLock *pq)
129 {
130  while (1) {
131  Packet *p = PacketDequeueNoLock(pq);
132  if (unlikely(p == NULL))
133  break;
134  TmqhOutputPacketpool(NULL, p);
135  }
136 }
137 
138 static inline void TmThreadsSlotProcessPktFail(ThreadVars *tv, TmSlot *s, Packet *p)
139 {
140  if (p != NULL) {
142  }
143  TmThreadsCleanDecodePQ(&tv->decode_pq);
144  if (tv->stream_pq_local) {
148  }
150 }
151 
152 /**
153  * \brief Handle timeout from the capture layer. Checks
154  * stream_pq which may have been filled by the flow
155  * manager.
156  * \param s pipeline to run on these packets.
157  */
158 static inline bool TmThreadsHandleInjectedPackets(ThreadVars *tv)
159 {
161  if (pq && pq->len > 0) {
162  while (1) {
163  SCMutexLock(&pq->mutex_q);
164  Packet *extra_p = PacketDequeue(pq);
165  SCMutexUnlock(&pq->mutex_q);
166  if (extra_p == NULL)
167  break;
168 #ifdef DEBUG_VALIDATION
169  BUG_ON(extra_p->flow != NULL);
170 #endif
172  if (r == TM_ECODE_FAILED) {
173  TmThreadsSlotProcessPktFail(tv, tv->tm_flowworker, extra_p);
174  break;
175  }
176  tv->tmqh_out(tv, extra_p);
177  }
178  return true;
179  } else {
180  return false;
181  }
182 }
183 
184 /**
185  * \brief Process the rest of the functions (if any) and queue.
186  */
187 static inline TmEcode TmThreadsSlotProcessPkt(ThreadVars *tv, TmSlot *s, Packet *p)
188 {
189  if (s == NULL) {
190  tv->tmqh_out(tv, p);
191  return TM_ECODE_OK;
192  }
193 
194  TmEcode r = TmThreadsSlotVarRun(tv, p, s);
195  if (unlikely(r == TM_ECODE_FAILED)) {
196  TmThreadsSlotProcessPktFail(tv, s, p);
197  return TM_ECODE_FAILED;
198  }
199 
200  tv->tmqh_out(tv, p);
201 
202  TmThreadsHandleInjectedPackets(tv);
203 
204  return TM_ECODE_OK;
205 }
206 
207 /** \brief inject packet if THV_CAPTURE_INJECT_PKT is set
208  * Allow caller to supply their own packet
209  *
210  * Meant for detect reload process that interupts an sleeping capture thread
211  * to force a packet through the engine to complete a reload */
212 static inline void TmThreadsCaptureInjectPacket(ThreadVars *tv, Packet *p)
213 {
215  if (p == NULL)
217  if (p != NULL) {
220  if (TmThreadsSlotProcessPkt(tv, tv->tm_flowworker, p) != TM_ECODE_OK) {
222  }
223  }
224 }
225 
226 /** \brief handle capture timeout
227  * When a capture method times out we check for house keeping
228  * tasks in the capture thread.
229  *
230  * \param p packet. Capture method may have taken a packet from
231  * the pool prior to the timing out call. We will then
232  * use that packet. Otherwise we can get our own.
233  */
234 static inline void TmThreadsCaptureHandleTimeout(ThreadVars *tv, Packet *p)
235 {
237  TmThreadsCaptureInjectPacket(tv, p); /* consumes 'p' */
238  return;
239 
240  } else {
241  if (TmThreadsHandleInjectedPackets(tv) == false) {
242  /* see if we have to do some house keeping */
243  if (tv->flow_queue && SC_ATOMIC_GET(tv->flow_queue->non_empty) == true) {
244  TmThreadsCaptureInjectPacket(tv, p); /* consumes 'p' */
245  return;
246  }
247  }
248  }
249 
250  /* packet could have been passed to us that we won't use
251  * return it to the pool. */
252  if (p != NULL)
253  tv->tmqh_out(tv, p);
254 }
255 
256 void TmThreadsListThreads(void);
257 int TmThreadsRegisterThread(ThreadVars *tv, const int type);
258 void TmThreadsUnregisterThread(const int id);
259 void TmThreadsInjectFlowById(Flow *f, const int id);
260 
262 void TmThreadsSetThreadTimestamp(const int id, const SCTime_t ts);
263 void TmThreadsGetMinimalTimestamp(struct timeval *ts);
264 uint16_t TmThreadsGetWorkerThreadMax(void);
265 bool TmThreadsTimeSubsysIsReady(void);
266 
267 #endif /* __TM_THREADS_H__ */
TVT_MAX
@ TVT_MAX
Definition: tm-threads-common.h:94
tv_root
ThreadVars * tv_root[TVT_MAX]
Definition: tm-threads.c:80
TmSlot_::tm_id
int tm_id
Definition: tm-threads.h:73
ThreadVars_::flow_queue
struct FlowQueue_ * flow_queue
Definition: threadvars.h:134
ts
uint64_t ts
Definition: source-erf-file.c:55
TmThreadsInitThreadsTimestamp
void TmThreadsInitThreadsTimestamp(const SCTime_t ts)
Definition: tm-threads.c:2208
TmThreadSetPrio
void TmThreadSetPrio(ThreadVars *)
Adjusting nice value for threads.
Definition: tm-threads.c:765
TmThreadContinue
void TmThreadContinue(ThreadVars *)
Unpauses a thread.
Definition: tm-threads.c:1779
TmThreadSetCPU
TmEcode TmThreadSetCPU(ThreadVars *, uint8_t)
Definition: tm-threads.c:811
TmThreadsListThreads
void TmThreadsListThreads(void)
Definition: tm-threads.c:2074
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
PacketQueue_
simple fifo queue for packets with mutex and cond Calling the mutex or triggering the cond is respons...
Definition: packet-queue.h:49
TmSlot_::SlotFunc
TmSlotFunc SlotFunc
Definition: tm-threads.h:56
PKT_SRC_CAPTURE_TIMEOUT
@ PKT_SRC_CAPTURE_TIMEOUT
Definition: decode.h:64
Packet_::flags
uint32_t flags
Definition: decode.h:463
Flow_
Flow data structure.
Definition: flow.h:357
TmThreadsSlotVarRun
TmEcode TmThreadsSlotVarRun(ThreadVars *tv, Packet *p, TmSlot *slot)
Separate run function so we can call it recursively.
Definition: tm-threads.c:115
TmThreadsInjectFlowById
void TmThreadsInjectFlowById(Flow *f, const int id)
inject a flow into a threads flow queue
Definition: tm-threads.c:2294
tm-threads-common.h
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
TmThreadKillThreadsFamily
void TmThreadKillThreadsFamily(int family)
Definition: tm-threads.c:1520
ThreadVars_::stream_pq_local
struct PacketQueue_ * stream_pq_local
Definition: threadvars.h:116
TmThreadClearThreadsFamily
void TmThreadClearThreadsFamily(int family)
Definition: tm-threads.c:1621
tm-modules.h
TmThreadSpawn
TmEcode TmThreadSpawn(ThreadVars *)
Spawns a thread associated with the ThreadVars instance tv.
Definition: tm-threads.c:1646
SleepUsec
#define SleepUsec(usec)
Definition: tm-threads.h:44
TmqhOutputPacketpool
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
Definition: tmqh-packetpool.c:356
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
PacketQueueNoLock_
simple fifo queue for packets
Definition: packet-queue.h:34
tmqh-packetpool.h
TmqhReleasePacketsToPacketPool
void TmqhReleasePacketsToPacketPool(PacketQueue *pq)
Release all the packets in the queue back to the packetpool. Mainly used by threads that have failed,...
Definition: tmqh-packetpool.c:466
TmThreadsGetWorkerThreadMax
uint16_t TmThreadsGetWorkerThreadMax(void)
Definition: tm-threads.c:2259
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
TmThreadsUnregisterThread
void TmThreadsUnregisterThread(const int id)
Definition: tm-threads.c:2141
PacketQueue_::mutex_q
SCMutex mutex_q
Definition: packet-queue.h:56
PKT_SET_SRC
#define PKT_SET_SRC(p, src_val)
Definition: decode.h:1062
TmThreadDisablePacketThreads
void TmThreadDisablePacketThreads(void)
Disable all packet threads.
Definition: tm-threads.c:1471
TmThreadsGetMinimalTimestamp
void TmThreadsGetMinimalTimestamp(struct timeval *ts)
Definition: tm-threads.c:2223
TmThreadsSetThreadTimestamp
void TmThreadsSetThreadTimestamp(const int id, const SCTime_t ts)
Definition: tm-threads.c:2174
TmSlot_::PktAcqLoop
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-threads.h:57
type
uint8_t type
Definition: decode-icmpv4.h:0
PacketDequeueNoLock
Packet * PacketDequeueNoLock(PacketQueueNoLock *qnl)
Definition: packet-queue.c:206
TmSlot_::Management
TmEcode(* Management)(ThreadVars *, void *)
Definition: tm-threads.h:58
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
PKT_PSEUDO_STREAM_END
#define PKT_PSEUDO_STREAM_END
Definition: decode.h:1005
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
TmThreadDisableReceiveThreads
void TmThreadDisableReceiveThreads(void)
Disable all threads having the specified TMs.
Definition: tm-threads.c:1338
TmSlotSetFuncAppend
void TmSlotSetFuncAppend(ThreadVars *, TmModule *, const void *)
Appends a new entry to the slots.
Definition: tm-threads.c:637
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
TmThreadCreateCmdThreadByName
ThreadVars * TmThreadCreateCmdThreadByName(const char *name, const char *module, int mucond)
Creates and returns the TV instance for a Command thread (CMD). This function supports only custom sl...
Definition: tm-threads.c:1122
TmThreadSetGroupName
void TmThreadSetGroupName(ThreadVars *tv, const char *name)
Definition: tm-threads.c:1603
Packet_
Definition: decode.h:428
TmThreadsSetFlag
void TmThreadsSetFlag(ThreadVars *, uint32_t)
Set a thread flag.
Definition: tm-threads.c:99
TmThreadWaitOnThreadInit
TmEcode TmThreadWaitOnThreadInit(void)
Used to check if all threads have finished their initialization. On finding an un-initialized thread,...
Definition: tm-threads.c:1933
THV_CAPTURE_INJECT_PKT
#define THV_CAPTURE_INJECT_PKT
Definition: threadvars.h:52
TmThreadCreate
ThreadVars * TmThreadCreate(const char *, const char *, const char *, const char *, const char *, const char *, void *(fn_p)(void *), int)
TmSlot_
Definition: tm-threads.h:53
SCTime_t
Definition: util-time.h:40
TmEcode
TmEcode
Definition: tm-threads-common.h:83
TmThreadGetNbThreads
int TmThreadGetNbThreads(uint8_t type)
Definition: tm-threads.c:827
TmThreadsRegisterThread
int TmThreadsRegisterThread(ThreadVars *tv, const int type)
Definition: tm-threads.c:2098
TmModule_
Definition: tm-modules.h:43
TmThreadCreateMgmtThread
ThreadVars * TmThreadCreateMgmtThread(const char *name, void *(fn_p)(void *), int)
Creates and returns the TV instance for a Management thread(MGMT). This function supports only custom...
Definition: tm-threads.c:1061
TmThreadCreateMgmtThreadByName
ThreadVars * TmThreadCreateMgmtThreadByName(const char *name, const char *module, int mucond)
Creates and returns the TV instance for a Management thread(MGMT). This function supports only custom...
Definition: tm-threads.c:1089
TmSlot
struct TmSlot_ TmSlot
tv_root_lock
SCMutex tv_root_lock
Definition: tm-threads.c:83
Packet_::flow
struct Flow_ * flow
Definition: decode.h:465
TmThreadKillThreads
void TmThreadKillThreads(void)
Definition: tm-threads.c:1549
flags
uint8_t flags
Definition: decode-gre.h:0
TmThreadAppend
void TmThreadAppend(ThreadVars *, int)
Appends this TV to tv_root based on its type.
Definition: tm-threads.c:1148
ThreadVars_::tmqh_out
void(* tmqh_out)(struct ThreadVars_ *, struct Packet_ *)
Definition: threadvars.h:105
THV_FAILED
#define THV_FAILED
Definition: threadvars.h:40
ThreadVars_::tm_flowworker
struct TmSlot_ * tm_flowworker
Definition: threadvars.h:100
TmThreadInitMC
void TmThreadInitMC(ThreadVars *)
Initializes the mutex and condition variables for this TV.
Definition: tm-threads.c:1701
PacketQueue_::len
uint32_t len
Definition: packet-queue.h:52
TmSlot_::SlotThreadInit
TmEcode(* SlotThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-threads.h:66
TmThreadWaitForFlag
void TmThreadWaitForFlag(ThreadVars *, uint32_t)
Waits till the specified flag(s) is(are) set. We don't bother if the kill flag has been set or not on...
Definition: tm-threads.c:1765
TmSlot_::slot_initdata
const void * slot_initdata
Definition: tm-threads.h:71
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
PacketDequeue
Packet * PacketDequeue(PacketQueue *q)
Definition: packet-queue.c:212
TmThreadSetCPUAffinity
TmEcode TmThreadSetCPUAffinity(ThreadVars *, uint16_t)
Set the thread options (cpu affinity).
Definition: tm-threads.c:802
TmSlot_::SlotThreadDeinit
TmEcode(* SlotThreadDeinit)(ThreadVars *, void *)
Definition: tm-threads.h:68
TmSlot_::SlotThreadExitPrintStats
void(* SlotThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-threads.h:67
TmSlot_::SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(void *, slot_data)
ThreadVars_::decode_pq
PacketQueueNoLock decode_pq
Definition: threadvars.h:111
TmThreadCheckThreadState
void TmThreadCheckThreadState(void)
Used to check the thread for certain conditions of failure.
Definition: tm-threads.c:1909
TmThreadTestThreadUnPaused
void TmThreadTestThreadUnPaused(ThreadVars *)
Tests if the thread represented in the arg has been unpaused or not.
Definition: tm-threads.c:1747
TmSlot_::slot_next
struct TmSlot_ * slot_next
Definition: tm-threads.h:62
TmThreadsCheckFlag
int TmThreadsCheckFlag(ThreadVars *, uint32_t)
Check if a thread flag is set.
Definition: tm-threads.c:91
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
TmThreadsUnsetFlag
void TmThreadsUnsetFlag(ThreadVars *, uint32_t)
Unset a thread flag.
Definition: tm-threads.c:107
TmThreadSetThreadPriority
TmEcode TmThreadSetThreadPriority(ThreadVars *, int)
Set the thread options (thread priority).
Definition: tm-threads.c:754
TmThreadWaitOnThreadRunning
TmEcode TmThreadWaitOnThreadRunning(void)
Waits for all threads to be in a running state.
Definition: tm-threads.c:1790
flow.h
TmSlotFunc
TmEcode(* TmSlotFunc)(ThreadVars *, Packet *, void *)
Definition: tm-threads.h:51
TmThreadContinueThreads
void TmThreadContinueThreads(void)
Unpauses all threads present in tv_root.
Definition: tm-threads.c:1892
TmThreadSetFlags
void TmThreadSetFlags(ThreadVars *, uint8_t)
SCMutex
#define SCMutex
Definition: threads-debug.h:114
PacketGetFromQueueOrAlloc
Packet * PacketGetFromQueueOrAlloc(void)
Get a packet. We try to get a packet from the packetpool first, but if that is empty we alloc a packe...
Definition: decode.c:208
TmThreadSetupOptions
TmEcode TmThreadSetupOptions(ThreadVars *)
Set the thread options (cpu affinitythread). Priority should be already set by pthread_create.
Definition: tm-threads.c:843
TmThreadsTimeSubsysIsReady
bool TmThreadsTimeSubsysIsReady(void)
Definition: tm-threads.c:2191
TmThreadCountThreadsByTmmFlags
uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags)
returns a count of all the threads that match the flag
Definition: tm-threads.c:1996
TmThreadCreatePacketHandler
ThreadVars * TmThreadCreatePacketHandler(const char *, const char *, const char *, const char *, const char *, const char *)
Creates and returns a TV instance for a Packet Processing Thread. This function doesn't support custo...
Definition: tm-threads.c:1032